diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/ci.yml similarity index 70% rename from .azure-pipelines/bazel.yml rename to .azure-pipelines/ci.yml index 66b737caf2bf..1d959856d503 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/ci.yml @@ -176,31 +176,68 @@ steps: tmpfsDockerDisabled: "${{ parameters.tmpfsDockerDisabled }}" - script: | - if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then - unset GOOGLE_BES_PROJECT_ID + ENVOY_SHARED_TMP_DIR=/tmp/bazel-shared + mkdir -p "$ENVOY_SHARED_TMP_DIR" + BAZEL_BUILD_EXTRA_OPTIONS="${{ parameters.bazelBuildExtraOptions }}" + if [[ "${{ parameters.rbe }}" == "True" ]]; then + # mktemp will create a tempfile with u+rw permission minus umask, it will not be readable by all + # users by default. + GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -p "${ENVOY_SHARED_TMP_DIR}" -t gcp_service_account.XXXXXX.json) + bash -c 'echo "$(GcpServiceAccountKey)"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" + BAZEL_BUILD_EXTRA_OPTIONS+=" ${{ parameters.bazelConfigRBE }} --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}" + ENVOY_RBE=1 + if [[ "${{ parameters.bazelUseBES }}" == "True" && -n "${GOOGLE_BES_PROJECT_ID}" ]]; then + BAZEL_BUILD_EXTRA_OPTIONS+=" --config=rbe-google-bes --bes_instance_name=${GOOGLE_BES_PROJECT_ID}" + fi + else + echo "using local build cache." + # Normalize branches - `release/vX.xx`, `vX.xx`, `vX.xx.x` -> `vX.xx` + TARGET_BRANCH=$(echo "${CI_TARGET_BRANCH}" | cut -d/ -f2-) + BRANCH_NAME="$(echo "${TARGET_BRANCH}" | cut -d/ -f2 | cut -d. -f-2)" + if [[ "$BRANCH_NAME" == "merge" ]]; then + # Manually run PR commit - there is no easy way of telling which branch + # it is, so just set it to `main` - otherwise it tries to cache as `branch/merge` + BRANCH_NAME=main + fi + BAZEL_REMOTE_INSTANCE="branch/${BRANCH_NAME}" + echo "instance_name: ${BAZEL_REMOTE_INSTANCE}." + BAZEL_BUILD_EXTRA_OPTIONS+=" --config=ci --config=cache-local --remote_instance_name=${BAZEL_REMOTE_INSTANCE} --remote_timeout=600" fi - ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}' - condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true')) + if [[ "${{ parameters.cacheTestResults }}" != "True" ]]; then + VERSION_DEV="$(cut -d- -f2 "VERSION.txt")" + # Use uncached test results for non-release scheduledruns. + if [[ $VERSION_DEV == "dev" ]]; then + BAZEL_EXTRA_TEST_OPTIONS+=" --nocache_test_results" + fi + fi + # Any PR or CI run in envoy-presubmit uses the fake SCM hash + if [[ "${{ variables['Build.Reason'] }}" == "PullRequest" || "${{ variables['Build.DefinitionName'] }}" == 'envoy-presubmit' ]]; then + # sha1sum of `ENVOY_PULL_REQUEST` + BAZEL_FAKE_SCM_REVISION=e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9 + fi + echo "##vso[task.setvariable variable=BAZEL_BUILD_EXTRA_OPTIONS]${BAZEL_BUILD_EXTRA_OPTIONS}" + echo "##vso[task.setvariable variable=BAZEL_EXTRA_TEST_OPTIONS]${BAZEL_EXTRA_TEST_OPTIONS}" + echo "##vso[task.setvariable variable=BAZEL_FAKE_SCM_REVISION]${BAZEL_FAKE_SCM_REVISION}" + echo "##vso[task.setvariable variable=BAZEL_STARTUP_EXTRA_OPTIONS]${{ parameters.bazelStartupExtraOptions }}" + echo "##vso[task.setvariable variable=CI_TARGET_BRANCH]${CI_TARGET_BRANCH}" + echo "##vso[task.setvariable variable=ENVOY_BUILD_FILTER_EXAMPLE]${{ parameters.envoyBuildFilterExample }}" + echo "##vso[task.setvariable variable=ENVOY_DOCKER_BUILD_DIR]$(Build.StagingDirectory)" + echo "##vso[task.setvariable variable=ENVOY_RBE]${ENVOY_RBE}" + echo "##vso[task.setvariable variable=ENVOY_SHARED_TMP_DIR]${ENVOY_SHARED_TMP_DIR}" + echo "##vso[task.setvariable variable=GCP_SERVICE_ACCOUNT_KEY_PATH]${GCP_SERVICE_ACCOUNT_KEY_PATH}" + echo "##vso[task.setvariable variable=GITHUB_TOKEN]${{ parameters.authGithub }}" workingDirectory: $(Build.SourcesDirectory) env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GITHUB_TOKEN: "${{ parameters.authGithub }}" - BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}" ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)" ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)" - # Any PR or CI run in envoy-presubmit uses the fake SCM hash - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}: - # sha1sum of `ENVOY_PULL_REQUEST` - BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9 - ${{ if parameters.rbe }}: - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}" - ${{ if eq(parameters.rbe, false) }}: - BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}" - BAZEL_REMOTE_CACHE: $(LocalBuildCache) + displayName: "CI env ${{ parameters.ciTarget }}" + +- script: ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}' + condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true')) + workingDirectory: $(Build.SourcesDirectory) + env: ${{ each var in parameters.env }}: ${{ var.key }}: ${{ var.value }} displayName: "Fetch assets (${{ parameters.ciTarget }})" @@ -231,34 +268,10 @@ steps: displayName: "Enable IPv6" condition: ${{ parameters.managedAgent }} -- script: | - if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then - unset GOOGLE_BES_PROJECT_ID - fi - ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}' +- script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}' workingDirectory: $(Build.SourcesDirectory) env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) ENVOY_BUILD_FILTER_EXAMPLE: ${{ parameters.envoyBuildFilterExample }} - GITHUB_TOKEN: "${{ parameters.authGithub }}" - BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}" - ${{ if ne(parameters['cacheTestResults'], true) }}: - BAZEL_NO_CACHE_TEST_RESULTS: 1 - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)" - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)" - # Any PR or CI run in envoy-presubmit uses the fake SCM hash - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}: - # sha1sum of `ENVOY_PULL_REQUEST` - BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9 - ${{ if parameters.rbe }}: - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}" - ${{ if eq(parameters.rbe, false) }}: - BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}" - BAZEL_REMOTE_CACHE: $(LocalBuildCache) ${{ each var in parameters.env }}: ${{ var.key }}: ${{ var.value }} displayName: "Run CI script ${{ parameters.ciTarget }}" @@ -296,6 +309,13 @@ steps: - ${{ each pair in step }}: ${{ pair.key }}: ${{ pair.value }} +- bash: | + if [[ -n "$GCP_SERVICE_ACCOUNT_KEY_PATH" && -e "$GCP_SERVICE_ACCOUNT_KEY_PATH" ]]; then + echo "Removed key: ${GCP_SERVICE_ACCOUNT_KEY_PATH}" + rm -rf "$GCP_SERVICE_ACCOUNT_KEY_PATH" + fi + condition: not(canceled()) + - script: | set -e sudo .azure-pipelines/docker/save_cache.sh "$(Build.StagingDirectory)" /mnt/cache/all true true diff --git a/.azure-pipelines/env.yml b/.azure-pipelines/env.yml index be55bdbf4962..3b3ebf6d2eeb 100644 --- a/.azure-pipelines/env.yml +++ b/.azure-pipelines/env.yml @@ -135,19 +135,12 @@ jobs: # TODO(phlax): move this to a script to ensure proper linting etc set -e - # Run everything in postsubmit - if [[ "$(Build.Reason)" != "PullRequest" ]]; then - echo "##vso[task.setvariable variable=build;isoutput=true]true" - echo "##vso[task.setvariable variable=checks;isoutput=true]true" - echo "##vso[task.setvariable variable=docker;isoutput=true]true" - echo "##vso[task.setvariable variable=packaging;isoutput=true]true" - exit 0 - fi - RUN_BUILD=true RUN_CHECKS=true RUN_DOCKER=true RUN_PACKAGING=true + RUN_RELEASE_TESTS=true + if [[ "$(changed.mobileOnly)" == true || "$(changed.docsOnly)" == true ]]; then RUN_BUILD=false RUN_DOCKER=false @@ -159,10 +152,25 @@ jobs: if [[ "$(changed.examplesOnly)" == true ]]; then RUN_CHECKS=false fi + if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$(state.isDev)" == false ]]; then + RUN_RELEASE_TESTS=false + fi + + # Run ~everything in postsubmit + if [[ "$(Build.Reason)" != "PullRequest" ]]; then + echo "##vso[task.setvariable variable=build;isoutput=true]true" + echo "##vso[task.setvariable variable=checks;isoutput=true]true" + echo "##vso[task.setvariable variable=docker;isoutput=true]true" + echo "##vso[task.setvariable variable=packaging;isoutput=true]true" + echo "##vso[task.setvariable variable=releaseTests;isoutput=true]${RUN_RELEASE_TESTS}" + exit 0 + fi + echo "##vso[task.setvariable variable=build;isoutput=true]${RUN_BUILD}" echo "##vso[task.setvariable variable=checks;isoutput=true]${RUN_CHECKS}" echo "##vso[task.setvariable variable=docker;isoutput=true]${RUN_DOCKER}" echo "##vso[task.setvariable variable=packaging;isoutput=true]${RUN_PACKAGING}" + echo "##vso[task.setvariable variable=releaseTests;isoutput=true]${RUN_RELEASE_TESTS}" displayName: "Decide what to run" workingDirectory: $(Build.SourcesDirectory) @@ -174,32 +182,18 @@ jobs: PUBLISH_GITHUB_RELEASE=$(run.packaging) PUBLISH_DOCKERHUB=false - PUBLISH_DOCS=false - PUBLISH_DOCS_LATEST=false - PUBLISH_DOCS_RELEASE=false if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then - # Build docs for publishing either latest or a release build - PUBLISH_DOCS=true # main if [[ "$ISMAIN" == True ]]; then # Update the Dockerhub README PUBLISH_DOCKERHUB=true - if [[ "$(state.isDev)" == true ]]; then - # Postsubmit on `main` trigger rebuild of latest docs - PUBLISH_DOCS_LATEST=true - fi # Not main, and not -dev elif [[ "$(state.isDev)" == false ]]; then if [[ "$(state.versionPatch)" -eq 0 ]]; then # A just-forked branch PUBLISH_GITHUB_RELEASE=false fi - # A stable release, publish docs to the release - PUBLISH_DOCS_RELEASE=true - else - # Postsubmit for non-main/release, skip publishing docs in this case - PUBLISH_DOCS=false fi fi @@ -210,9 +204,6 @@ jobs: echo "##vso[task.setvariable variable=githubRelease;isoutput=true]${PUBLISH_GITHUB_RELEASE}" echo "##vso[task.setvariable variable=dockerhub;isoutput=true]${PUBLISH_DOCKERHUB}" - echo "##vso[task.setvariable variable=docs;isoutput=true]${PUBLISH_DOCS}" - echo "##vso[task.setvariable variable=docsLatest;isoutput=true]${PUBLISH_DOCS_LATEST}" - echo "##vso[task.setvariable variable=docsRelease;isoutput=true]${PUBLISH_DOCS_RELEASE}" displayName: "Decide what to publish" workingDirectory: $(Build.SourcesDirectory) @@ -231,12 +222,10 @@ jobs: echo "env.outputs['run.build']: $(run.build)" echo "env.outputs['run.checks']: $(run.checks)" echo "env.outputs['run.packaging']: $(run.packaging)" + echo "env.outputs['run.releaseTests']: $(run.releaseTests)" echo echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)" echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)" - echo "env.outputs['publish.docs]: $(publish.docs)" - echo "env.outputs['publish.docsLatest]: $(publish.docsLatest)" - echo "env.outputs['publish.docsRelease]: $(publish.docsRelease)" displayName: "Print build environment" diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 6cb7ac6fff03..0b55d0a6219a 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -45,7 +45,7 @@ variables: ## Variable settings # Caches (tip: append a version suffix while testing caches) - name: cacheKeyVersion - value: v2 + value: v3 - name: cacheKeyBazel value: '.bazelversion | ./WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**' - name: cacheKeyDocker @@ -80,10 +80,6 @@ stages: - env checkStageDeps: - env - macBuildStageDeps: - - env - windowsBuildStageDeps: - - env # Postsubmit main/release branches - ${{ if eq(variables.pipelinePostsubmit, true) }}: @@ -96,7 +92,3 @@ stages: - env checkStageDeps: - env - macBuildStageDeps: - - env - windowsBuildStageDeps: - - env diff --git a/.azure-pipelines/stage/checks.yml b/.azure-pipelines/stage/checks.yml index 50fdb0956cda..8c03249e227b 100644 --- a/.azure-pipelines/stage/checks.yml +++ b/.azure-pipelines/stage/checks.yml @@ -77,7 +77,7 @@ jobs: timeoutInMinutes: 180 pool: envoy-x64-small steps: - - template: ../bazel.yml + - template: ../ci.yml parameters: ciTarget: $(CI_TARGET) cacheName: $(CI_TARGET) @@ -101,15 +101,7 @@ jobs: displayName: "Upload $(CI_TARGET) Report to GCS" condition: and(not(canceled()), or(eq(variables['CI_TARGET'], 'coverage'), eq(variables['CI_TARGET'], 'fuzz_coverage'))) env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci --config=rbe-google --jobs=$(RbeJobs)" - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)" - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)" - job: complete displayName: "Checks complete" diff --git a/.azure-pipelines/stage/linux.yml b/.azure-pipelines/stage/linux.yml index be4e4a6ada14..5c3caa11d3ab 100644 --- a/.azure-pipelines/stage/linux.yml +++ b/.azure-pipelines/stage/linux.yml @@ -11,6 +11,10 @@ parameters: displayName: "Artifact suffix" type: string default: +- name: runTests + displayName: "Run release tests" + type: string + default: true - name: rbe displayName: "Use RBE" type: boolean @@ -22,6 +26,9 @@ parameters: - name: bazelBuildExtraOptions type: string default: "" +- name: bazelConfigRBE + type: string + default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs) - name: managedAgent type: boolean @@ -45,11 +52,21 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutBuild }} pool: ${{ parameters.pool }} steps: - - template: ../bazel.yml + - bash: | + if [[ "${{ parameters.runTests }}" == "false" ]]; then + CI_TARGET="release.server_only" + else + CI_TARGET="release" + fi + echo "${CI_TARGET}" + echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}" + name: target + - template: ../ci.yml parameters: managedAgent: ${{ parameters.managedAgent }} - ciTarget: release + ciTarget: $(target.value) cacheName: "release" + bazelConfigRBE: ${{ parameters.bazelConfigRBE }} bazelBuildExtraOptions: ${{ parameters.bazelBuildExtraOptions }} cacheTestResults: ${{ parameters.cacheTestResults }} cacheVersion: $(cacheKeyBazel) diff --git a/.azure-pipelines/stage/macos.yml b/.azure-pipelines/stage/macos.yml deleted file mode 100644 index 6089bd89ee89..000000000000 --- a/.azure-pipelines/stage/macos.yml +++ /dev/null @@ -1,56 +0,0 @@ - -parameters: - -# Auth -- name: authGCP - type: string - default: "" - -- name: runBuild - displayName: "Run build" - type: string - default: true - -jobs: -- job: test - displayName: Build and test - condition: | - and(not(canceled()), - eq(${{ parameters.runBuild }}, 'true')) - timeoutInMinutes: 180 - pool: - vmImage: "macos-11" - steps: - - script: ./ci/mac_ci_setup.sh - displayName: "Install dependencies" - - - script: ./ci/mac_ci_steps.sh - displayName: "Run Mac CI" - env: - BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - ENVOY_RBE: 1 - - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-testlogs/**/test.xml" - testRunTitle: "macOS" - timeoutInMinutes: 10 - condition: not(canceled()) - -- job: tested - displayName: Complete - dependsOn: ["test"] - pool: - vmImage: $(agentUbuntu) - # This condition ensures that this (required) job passes if all of - # the preceeding jobs either pass or are skipped - # adapted from: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage - condition: and(eq(variables['Build.Reason'], 'PullRequest'), in(dependencies.test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) - steps: - - checkout: none - - bash: | - echo "macos tested" diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index 09ff21fe05d5..b699a960eace 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -32,11 +32,18 @@ parameters: # a lot of change - eg protobuf changed, or a primitve proto changed. default: 40 +- name: runPrechecks + displayName: "Run prechecks" + type: string + default: true jobs: - job: prechecks displayName: Precheck timeoutInMinutes: ${{ parameters.timeoutPrechecks }} + condition: | + and(not(canceled()), + eq(${{ parameters.runPrechecks }}, 'true')) pool: vmImage: $(agentUbuntu) variables: @@ -48,11 +55,13 @@ jobs: CI_TARGET: "format" protobuf: CI_TARGET: "check_and_fix_proto_format" - publishing: - CI_TARGET: docs + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + publishing: + CI_TARGET: docs steps: - - template: ../bazel.yml + - template: ../ci.yml parameters: + bazelBuildExtraOptions: --config=docs-ci ciTarget: $(CI_TARGET) cacheName: $(CI_TARGET) cacheTestResults: ${{ parameters.cacheTestResults }} @@ -90,7 +99,7 @@ jobs: authGPGKey: ${{ parameters.authGPGKey }} # GNUPGHOME inside the container pathGPGConfiguredHome: /build/.gnupg - pathGPGHome: /tmp/envoy-docker-build/.gnupg + pathGPGHome: $(Build.StagingDirectory)/.gnupg - bash: | set -e ci/run_envoy_docker.sh " @@ -98,7 +107,7 @@ jobs: && gpg --clearsign /tmp/authority \ && cat /tmp/authority.asc \ && gpg --verify /tmp/authority.asc" - rm -rf /tmp/envoy-docker-build/.gnupg + rm -rf $(Build.StagingDirectory)/.gnupg displayName: "Ensure container CI can sign with GPG" condition: and(not(canceled()), eq(variables['CI_TARGET'], 'docs')) @@ -120,10 +129,6 @@ jobs: ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-readme' displayName: "Dockerhub publishing test" env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)" - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['CI_TARGET'], 'docs') @@ -146,14 +151,9 @@ jobs: condition: and(failed(), eq(variables['CI_TARGET'], 'check_and_fix_proto_format')) # Publish docs - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' displayName: "Upload Docs to GCS" env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)" - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['CI_TARGET'], 'docs') diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index b04013b69721..30e62ebc362c 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -39,24 +39,12 @@ parameters: - name: authGPGKey type: string default: "" -- name: authNetlifyURL - type: string - default: "" - name: authDockerUser type: string default: "" - name: authDockerPassword type: string default: "" -- name: authSSHDocsKey - type: string - default: "" -- name: authSSHDocsKeyPublic - type: string - default: "" -- name: authSSHKeyPassphrase - type: string - default: "" - name: runDocker displayName: "Run Docker" @@ -71,18 +59,6 @@ parameters: displayName: "Publish Dockerhub" type: string default: false -- name: publishDocs - displayName: "Publish Docs" - type: string - default: false -- name: publishDocsLatest - displayName: "Publish latest docs" - type: string - default: false -- name: publishDocsRelease - displayName: "Publish release docs" - type: string - default: false - name: publishGithubRelease displayName: "Publish Github release" type: string @@ -104,7 +80,7 @@ jobs: artifactName: "release" itemPattern: "release/**/bin/*" targetPath: $(Build.StagingDirectory) - - template: ../bazel.yml + - template: ../ci.yml parameters: ciTarget: docker-upload cacheName: docker-upload @@ -120,32 +96,36 @@ jobs: echo "disk space at beginning of Docker build:" df -h displayName: "Check disk space before Docker build" + # TODO(phlax): switch docker <> docker-upload as main task - bash: | set -e - - mkdir -p linux/amd64 linux/arm64 - - # x64 - cp -a $(Build.StagingDirectory)/release/x64/bin/release.tar.zst linux/amd64/release.tar.zst - cp -a $(Build.StagingDirectory)/release/x64/bin/schema_validator_tool linux/amd64/schema_validator_tool - - # arm64 - cp -a $(Build.StagingDirectory)/release/arm64/bin/release.tar.zst linux/arm64/release.tar.zst - cp -a $(Build.StagingDirectory)/release/arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool - - # Debug what files appear to have been downloaded - find linux -type f -name "*" | xargs ls -l - - ci/docker_ci.sh + mkdir -p $(Build.StagingDirectory)/envoy + rm -rf $(Build.StagingDirectory)/envoy/* + mv $(Build.StagingDirectory)/release/* $(Build.StagingDirectory)/envoy + ./ci/run_envoy_docker.sh 'ci/do_ci.sh docker' displayName: Build Docker images timeoutInMinutes: ${{ parameters.timeoutDockerPublish }} workingDirectory: $(Build.SourcesDirectory) env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) + CI_BRANCH: $(Build.SourceBranch) + CI_SHA1: $(Build.SourceVersion) DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} DOCKER_BUILD_TIMEOUT: ${{ parameters.timeoutDockerBuild }} + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + ENVOY_DOCKER_IN_DOCKER: 1 + + stepsPost: + - script: | + ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' + condition: | + and(not(canceled()), succeeded(), + eq(${{ parameters.publishDockerhub }}, 'true')) + displayName: "Publish Dockerhub description and README" + env: + GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} + DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} + DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - job: package_x64 displayName: Linux debs (x64) @@ -163,7 +143,7 @@ jobs: artifactName: "release" itemPattern: "release/x64/bin/*" targetPath: $(Build.StagingDirectory) - - template: ../bazel.yml + - template: ../ci.yml parameters: ciTarget: distribution cacheName: distribution @@ -197,7 +177,7 @@ jobs: itemPattern: "release/arm64/bin/*" targetPath: $(Build.StagingDirectory) - - template: ../bazel.yml + - template: ../ci.yml parameters: managedAgent: false ciTarget: distribution @@ -220,80 +200,6 @@ jobs: set -e rm -rf $(Build.StagingDirectory)/.gnupg -- job: docs - displayName: Publish docs - dependsOn: [] - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocs }}, 'true')) - pool: - vmImage: $(agentUbuntu) - steps: - - template: ../bazel.yml - parameters: - ciTarget: docs - cacheName: docs - cacheVersion: $(cacheKeyBazel) - publishEnvoy: false - publishTestResults: false - env: - AZP_BRANCH: $(Build.SourceBranch) - stepsPost: - - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh dockerhub-publish' - condition: | - and(not(canceled()), - eq(${{ parameters.publishDockerhub }}, 'true')) - displayName: "Publish Dockerhub description and README" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)" - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - DOCKERHUB_USERNAME: ${{ parameters.authDockerUser }} - DOCKERHUB_PASSWORD: ${{ parameters.authDockerPassword }} - - # Trigger Netlify rebuild of latest docs - - script: | - ci/run_envoy_docker.sh 'ci/do_ci.sh docs-upload' - displayName: "Upload Docs to GCS" - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsLatest }}, 'true')) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs)" - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs-publish-latest' - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsLatest }}, 'true')) - displayName: "Publish latest docs" - workingDirectory: $(Build.SourcesDirectory) - env: - NETLIFY_TRIGGER_URL: ${{ parameters.authNetlifyURL }} - - # Publish docs to the website - - task: InstallSSHKey@0 - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsRelease }}, 'true')) - inputs: - hostName: $(authGithubSSHKeyPublic) - sshPublicKey: "${{ parameters.authSSHDocsKeyPublic }}" - sshPassphrase: "${{ parameters.authSSHKeyPassphrase }}" - sshKeySecureFile: "${{ parameters.authSSHDocsKey }}" - - script: docs/publish.sh - condition: | - and(not(canceled()), - eq(${{ parameters.publishDocsRelease }}, 'true')) - displayName: "Publish release docs" - workingDirectory: $(Build.SourcesDirectory) - - job: signed_release displayName: Signed binaries dependsOn: @@ -318,7 +224,7 @@ jobs: artifactName: "distribution" itemPattern: "distribution/**/packages.*.tar.gz" targetPath: $(Build.StagingDirectory) - - template: ../bazel.yml + - template: ../ci.yml parameters: ciTarget: release.signed cacheName: release-signed @@ -333,8 +239,14 @@ jobs: authGPGKey: ${{ parameters.authGPGKey }} pathGPGConfiguredHome: /build/.gnupg pathGPGHome: $(Build.StagingDirectory)/.gnupg + - bash: | + set -e -o pipefail + mkdir -p distribution/custom + cp -a $(Build.StagingDirectory)/*/*64 distribution/custom/ + workingDirectory: $(Build.SourcesDirectory) + - job: success - dependsOn: ["docker", "docs", "signed_release"] + dependsOn: ["docker", "signed_release"] displayName: Success (linux artefacts) pool: vmImage: $(agentUbuntu) @@ -345,7 +257,6 @@ jobs: condition: | and( in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.docs.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), in(dependencies.signed_release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) steps: - checkout: none @@ -362,7 +273,17 @@ jobs: pool: vmImage: $(agentUbuntu) steps: - - template: ../bazel.yml + - task: DownloadSecureFile@1 + name: WorkflowTriggerKey + displayName: 'Download workflow trigger key' + inputs: + secureFile: '${{ parameters.authGithubWorkflow }}' + - bash: | + set -e + KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" + echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" + name: key + - template: ../ci.yml parameters: ciTarget: verify.trigger cacheName: verify-trigger @@ -371,6 +292,7 @@ jobs: publishEnvoy: false publishTestResults: false env: + ENVOY_REPO: $(Build.Repository.Name) ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: ENVOY_HEAD_REF: "$(Build.SourceBranch)" ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" @@ -395,13 +317,3 @@ jobs: mkdir -p $(Build.StagingDirectory)/release.signed mv release.signed.tar.zst $(Build.StagingDirectory)/release.signed displayName: Fetch signed release - - task: DownloadSecureFile@1 - name: WorkflowTriggerKey - displayName: 'Download workflow trigger key' - inputs: - secureFile: '${{ parameters.authGithubWorkflow }}' - - bash: | - set -e - KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" - echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" - name: key diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml index 2214bee7971e..f429feb4ff44 100644 --- a/.azure-pipelines/stage/verify.yml +++ b/.azure-pipelines/stage/verify.yml @@ -12,8 +12,7 @@ jobs: displayName: Debs (x64) condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) timeoutInMinutes: 120 - pool: - vmImage: $(agentUbuntu) + pool: envoy-x64-small steps: - task: DownloadBuildArtifacts@0 inputs: @@ -22,11 +21,12 @@ jobs: itemPattern: "distribution/x64/packages.x64.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) - - template: ../bazel.yml + - template: ../ci.yml parameters: ciTarget: verify_distro cacheName: verify_distro publishTestResults: false + tmpfsDockerDisabled: true env: ENVOY_DOCKER_IN_DOCKER: 1 @@ -43,7 +43,7 @@ jobs: itemPattern: "distribution/arm64/packages.arm64.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) - - template: ../bazel.yml + - template: ../ci.yml parameters: managedAgent: false ciTarget: verify_distro diff --git a/.azure-pipelines/stage/windows.yml b/.azure-pipelines/stage/windows.yml deleted file mode 100644 index e9e400da52e7..000000000000 --- a/.azure-pipelines/stage/windows.yml +++ /dev/null @@ -1,112 +0,0 @@ - -parameters: - -# Auth -- name: authGCP - type: string - default: "" - -- name: runBuild - displayName: "Run build" - type: string - default: true - -jobs: -- job: release - displayName: Build and test - condition: | - and(not(canceled()), - eq(${{ parameters.runBuild }}, 'true')) - timeoutInMinutes: 180 - pool: - vmImage: "windows-2019" - steps: - - task: Cache@2 - inputs: - key: '"windows.release" | $(cacheKeyBazel)' - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh - displayName: "Run Windows msvc-cl CI" - env: - CI_TARGET: "windows" - ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2" - GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} - - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" - testRunTitle: "windows" - searchFolder: $(Build.StagingDirectory)/tmp - timeoutInMinutes: 10 - condition: not(canceled()) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: windows.release - timeoutInMinutes: 10 - condition: not(canceled()) - -- job: docker - displayName: Build Docker image - condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) - strategy: - matrix: - windows2019: - imageName: 'windows-2019' - windowsBuildType: "windows" - windowsImageBase: "mcr.microsoft.com/windows/servercore" - windowsImageTag: "ltsc2019" - windows2022: - imageName: 'windows-2022' - windowsBuildType: "windows-ltsc2022" - windowsImageBase: "mcr.microsoft.com/windows/nanoserver" - windowsImageTag: "ltsc2022" - dependsOn: ["release"] - timeoutInMinutes: 120 - pool: - vmImage: $(imageName) - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "windows.release" - itemPattern: "windows.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - # Convert to Unix-style path so tar doesn't think drive letter is a hostname - STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" - mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - WINDOWS_BUILD_TYPE: $(windowsBuildType) - WINDOWS_IMAGE_BASE: $(windowsImageBase) - WINDOWS_IMAGE_TAG: $(windowsImageTag) - -- job: released - displayName: Complete - dependsOn: ["release", "docker"] - pool: - vmImage: $(agentUbuntu) - # This condition ensures that this (required) job passes if all of - # the preceeding jobs either pass or are skipped - # adapted from: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage - condition: | - and( - eq(variables['Build.Reason'], 'PullRequest'), - in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) - steps: - - checkout: none - - bash: | - echo "windows released" diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 1d070158ec98..aff98c359839 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -8,18 +8,6 @@ parameters: default: - env - prechecks -- name: macBuildStageDeps - displayName: "macOS stage dependencies" - type: object - default: - - env - - prechecks -- name: windowsBuildStageDeps - displayName: "Windows stage dependencies" - type: object - default: - - env - - prechecks - name: checkStageDeps displayName: "Check stage dependencies" type: object @@ -57,7 +45,6 @@ stages: jobs: - template: env.yml - - stage: prechecks displayName: Prechecks dependsOn: ["env"] @@ -71,24 +58,30 @@ stages: authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) authGPGPath: $(MaintainerGPGKey.secureFilePath) bucketGCP: $(GcsArtifactBucket) + runPrechecks: stageDependencies.env.repo.outputs['run.releaseTests'] - stage: linux_x64 displayName: Linux x64 dependsOn: ${{ parameters.buildStageDeps }} variables: RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] + RUN_TESTS: $[stageDependencies.env.repo.outputs['run.releaseTests']] jobs: - template: stage/linux.yml parameters: cacheTestResults: ${{ parameters.cacheTestResults }} + bazelConfigRBE: --config=remote-ci --config=rbe-google --jobs=200 + pool: envoy-x64-large + # these are parsed differently and _must_ be expressed in this way runBuild: variables['RUN_BUILD'] - tmpfsDockerDisabled: true + runTests: $(RUN_TESTS) - stage: linux_arm64 displayName: Linux arm64 dependsOn: ${{ parameters.buildStageDeps }} variables: RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] + RUN_TESTS: $[stageDependencies.env.repo.outputs['run.releaseTests']] jobs: - template: stage/linux.yml parameters: @@ -98,6 +91,7 @@ stages: timeoutBuild: 180 pool: envoy-arm-large runBuild: variables['RUN_BUILD'] + runTests: $(RUN_TESTS) bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - stage: check @@ -123,9 +117,6 @@ stages: RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']] PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']] PUBLISH_DOCKERHUB: $[stageDependencies.env.repo.outputs['publish.dockerhub']] - PUBLISH_DOCS: $[stageDependencies.env.repo.outputs['publish.docs']] - PUBLISH_DOCS_LATEST: $[stageDependencies.env.repo.outputs['publish.docsLatest']] - PUBLISH_DOCS_RELEASE: $[stageDependencies.env.repo.outputs['publish.docsRelease']] jobs: - template: stage/publish.yml parameters: @@ -138,18 +129,11 @@ stages: authGPGPassphrase: $(MaintainerGPGKeyPassphrase) authGPGKey: $(MaintainerGPGKeySecureFileDownloadPath) authGPGPath: $(MaintainerGPGKey.secureFilePath) - authNetlifyURL: $(NetlifyTriggerURL) - authSSHDocsKeyPublic: $(DocsPublicKey) - authSSHDocsKey: $(DocsPrivateKey) - authSSHKeyPassphrase: $(SshDeployKeyPassphrase) bucketGCP: $(GcsArtifactBucket) timeoutDockerBuild: ${{ parameters.timeoutDockerBuild }} timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }} runDocker: variables['RUN_DOCKER'] runPackaging: variables['RUN_PACKAGING'] - publishDocs: variables['PUBLISH_DOCS'] - publishDocsLatest: variables['PUBLISH_DOCS_LATEST'] - publishDocsRelease: variables['PUBLISH_DOCS_RELEASE'] publishDockerhub: variables['PUBLISH_DOCKERHUB'] publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE'] @@ -162,25 +146,3 @@ stages: - template: stage/verify.yml parameters: authGCP: $(GcpServiceAccountKey) - -- stage: macos - displayName: macOS - dependsOn: ${{ parameters.macBuildStageDeps }} - variables: - RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] - jobs: - - template: stage/macos.yml - parameters: - authGCP: $(GcpServiceAccountKey) - runBuild: variables['RUN_BUILD'] - -- stage: windows - displayName: Windows - dependsOn: ${{ parameters.windowsBuildStageDeps }} - variables: - RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] - jobs: - - template: stage/windows.yml - parameters: - authGCP: $(GcpServiceAccountKey) - runBuild: variables['RUN_BUILD'] diff --git a/.bazelrc b/.bazelrc index 8167ab650f21..dedce6437731 100644 --- a/.bazelrc +++ b/.bazelrc @@ -48,6 +48,8 @@ build --action_env=BAZEL_FAKE_SCM_REVISION --host_action_env=BAZEL_FAKE_SCM_REVI build --test_summary=terse +build:docs-ci --action_env=DOCS_RST_CHECK=1 --host_action_env=DOCS_RST_CHECK=1 + # TODO(keith): Remove once these 2 are the default build --incompatible_config_setting_private_default_visibility build --incompatible_enforce_config_setting_visibility @@ -59,6 +61,7 @@ common --experimental_allow_tags_propagation # Enable position independent code (this is the default on macOS and Windows) # (Workaround for https://github.com/bazelbuild/rules_foreign_cc/issues/421) +build:linux --copt=-fdebug-types-section build:linux --copt=-fPIC build:linux --copt=-Wno-deprecated-declarations build:linux --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 @@ -90,6 +93,14 @@ build:clang-pch --define=ENVOY_CLANG_PCH=1 # Use gold linker for gcc compiler. build:gcc --linkopt=-fuse-ld=gold +# Clang-tidy +# TODO(phlax): enable this, its throwing some errors as well as finding more issues +# build:clang-tidy --@envoy_toolshed//format/clang_tidy:executable=@envoy//tools/clang-tidy +build:clang-tidy --@envoy_toolshed//format/clang_tidy:config=//:clang_tidy_config +build:clang-tidy --aspects @envoy_toolshed//format/clang_tidy:clang_tidy.bzl%clang_tidy_aspect +build:clang-tidy --output_groups=report +build:clang-tidy --build_tag_filters=-notidy + # Basic ASAN/UBSAN that works for gcc build:asan --action_env=ENVOY_ASAN=1 build:asan --config=sanitizer @@ -225,6 +236,8 @@ build:fuzz-coverage --config=plain-fuzzer build:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh build:fuzz-coverage --test_tag_filters=-nocoverage +build:cache-local --remote_cache=grpc://localhost:9092 + # Remote execution: https://docs.bazel.build/versions/master/remote-execution.html build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 @@ -289,6 +302,7 @@ build:remote-windows --spawn_strategy=remote,local build:remote-windows --strategy=Javac=remote,local build:remote-windows --strategy=Closure=remote,local build:remote-windows --strategy=Genrule=remote,local +build:remote-windows --strategy=CppLink=local build:remote-windows --remote_timeout=7200 build:remote-windows --google_default_credentials=true build:remote-windows --remote_download_toplevel @@ -343,7 +357,7 @@ build:compile-time-options --@envoy//source/extensions/filters/http/kill_request # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:56f235b141079013e64912d676fe7da981368402@sha256:d44499c6fd28a8a6a75dc61668b8a9e7bc3d99db11f9a61e8ea1d1f39c20a236 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:fd9ec000fdd72d5c5e4e4ef16db4f9103058779e@sha256:1386a26f687826850ba488d66a6cd5337c5941b3b8793d08cfa6f9df12aa2fcf build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker @@ -429,7 +443,7 @@ build:windows --define hot_restart=disabled build:windows --define tcmalloc=disabled build:windows --define wasm=disabled build:windows --define manual_stamp=manual_stamp -build:windows --cxxopt="/std:c++17" +build:windows --cxxopt="/std:c++20" build:windows --output_groups=+pdb_file # TODO(wrowe,sunjayBhatia): Resolve bugs upstream in curl and rules_foreign_cc @@ -472,11 +486,12 @@ build:windows --features=static_link_msvcrt build:windows --dynamic_mode=off # RBE (Google) -build:rbe-google --google_default_credentials=true -build:rbe-google --remote_cache=grpcs://remotebuildexecution.googleapis.com +build:cache-google --google_default_credentials=true +build:cache-google --remote_cache=grpcs://remotebuildexecution.googleapis.com +build:cache-google --remote_instance_name=projects/envoy-ci/instances/default_instance +build:cache-google --remote_timeout=7200 build:rbe-google --remote_executor=grpcs://remotebuildexecution.googleapis.com -build:rbe-google --remote_timeout=7200 -build:rbe-google --remote_instance_name=projects/envoy-ci/instances/default_instance +build:rbe-google --config=cache-google build:rbe-google-bes --bes_backend=grpcs://buildeventservice.googleapis.com build:rbe-google-bes --bes_results_url=https://source.cloud.google.com/results/invocations/ @@ -493,6 +508,18 @@ build:rbe-engflow --remote_timeout=3600s build:rbe-engflow --bes_timeout=3600s build:rbe-engflow --bes_upload_mode=fully_async +build:rbe-envoy-engflow --google_default_credentials=false +build:rbe-envoy-engflow --remote_cache=grpcs://morganite.cluster.engflow.com +build:rbe-envoy-engflow --remote_executor=grpcs://morganite.cluster.engflow.com +build:rbe-envoy-engflow --bes_backend=grpcs://morganite.cluster.engflow.com/ +build:rbe-envoy-engflow --bes_results_url=https://morganite.cluster.engflow.com/invocation/ +build:rbe-envoy-engflow --credential_helper=*.engflow.com=%workspace%/bazel/engflow-bazel-credential-helper.sh +build:rbe-envoy-engflow --grpc_keepalive_time=30s +build:rbe-envoy-engflow --remote_timeout=3600s +build:rbe-envoy-engflow --bes_timeout=3600s +build:rbe-envoy-engflow --bes_upload_mode=fully_async +build:rbe-envoy-engflow --remote_default_exec_properties=container-image=docker://docker.io/envoyproxy/envoy-build-ubuntu:fd9ec000fdd72d5c5e4e4ef16db4f9103058779e@sha256:1386a26f687826850ba488d66a6cd5337c5941b3b8793d08cfa6f9df12aa2fcf + ############################################################################# # debug: Various Bazel debugging flags ############################################################################# diff --git a/.clang-tidy b/.clang-tidy index 7e9d197c9921..72f533b39c11 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -56,9 +56,39 @@ CheckOptions: value: 'CamelCase' # Ignore GoogleTest function macros. - key: readability-identifier-naming.FunctionIgnoredRegexp - value: '(TEST|TEST_F|TEST_P|INSTANTIATE_TEST_SUITE_P|MOCK_METHOD|TYPED_TEST)' + # To have the regex chomped correctly fence all items with `|` (other than first/last) + value: >- + (^AbslHashValue$| + |^called_count$| + |^case_sensitive$| + |^Create$| + |^envoy_resolve_dns$| + |^evconnlistener_free$| + |^event_base_free$| + |^(get|set)EVP_PKEY$| + |^has_value$| + |^Ip6(ntohl|htonl)$| + |^get_$| + |^HeaderHasValue(Ref)?$| + |^HeaderValueOf$| + |^Is(Superset|Subset)OfHeaders$| + |^LLVMFuzzerInitialize$| + |^LLVMFuzzerTestOneInput$| + |^Locality$| + |^MOCK_METHOD$| + |^PrepareCall$| + |^PrintTo$| + |^resolve_dns$| + |^result_type$| + |Returns(Default)?WorkerId$| + |^sched_getaffinity$| + |^shutdownThread_$| + |TEST| + |^use_count$) - key: readability-identifier-naming.ParameterCase value: 'lower_case' +- key: readability-identifier-naming.ParameterIgnoredRegexp + value: (^cname_ttl_$) - key: readability-identifier-naming.PrivateMemberCase value: 'lower_case' - key: readability-identifier-naming.PrivateMemberSuffix @@ -67,11 +97,21 @@ CheckOptions: value: 'CamelCase' - key: readability-identifier-naming.TypeAliasCase value: 'CamelCase' +- key: readability-identifier-naming.TypeAliasIgnoredRegexp + value: '(result_type)' - key: readability-identifier-naming.UnionCase value: 'CamelCase' - key: readability-identifier-naming.FunctionCase value: 'camelBack' +HeaderFilterRegex: '^./source/.*|^./contrib/.*|^./test/.*|^./envoy/.*' + UseColor: true WarningsAsErrors: '*' + +## The version here is arbitrary since any change to this file will +## trigger a full run of clang-tidy against all files. +## It can be useful as it seems some header changes may not trigger the +## expected rerun. +# v0 diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ae03bdd50ba1..cdf3e9815ce9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/envoy-ci/envoy-build:56f235b141079013e64912d676fe7da981368402@sha256:6e3e8bd34ba568befa3f9c2fd067a1d82c1e55f0f597bcc5fddebbb644930761 +FROM gcr.io/envoy-ci/envoy-build:fd9ec000fdd72d5c5e4e4ef16db4f9103058779e@sha256:b0c0e07c97337fdd56423ddd6749a4250adaea9f66f85763a8d9cec36162c972 ARG USERNAME=vscode ARG USER_UID=501 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 77852d506a02..f24d57d22ede 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,34 +16,38 @@ "containerEnv": { "ENVOY_SRCDIR": "${containerWorkspaceFolder}", }, - "settings": { - "terminal.integrated.shell.linux": "/bin/bash", - "bazel.buildifierFixOnFormat": true, - "clangd.path": "/opt/llvm/bin/clangd", - "python.pythonPath": "/usr/bin/python3", - "python.formatting.provider": "yapf", - "python.formatting.yapfArgs": [ - "--style=${workspaceFolder}/.style.yapf" - ], - "files.exclude": { - "**/.clangd/**": true, - "**/bazel-*/**": true - }, - "files.watcherExclude": { - "**/.clangd/**": true, - "**/bazel-*/**": true - } - }, "remoteUser": "vscode", "containerUser": "vscode", "postCreateCommand": ".devcontainer/setup.sh", - "extensions": [ - "github.vscode-pull-request-github", - "zxh404.vscode-proto3", - "bazelbuild.vscode-bazel", - "llvm-vs-code-extensions.vscode-clangd", - "vadimcn.vscode-lldb", - "webfreak.debug", - "ms-python.python" - ] + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "bazel.buildifierFixOnFormat": true, + "clangd.path": "/opt/llvm/bin/clangd", + "python.pythonPath": "/usr/bin/python3", + "python.formatting.provider": "yapf", + "python.formatting.yapfArgs": [ + "--style=${workspaceFolder}/.style.yapf" + ], + "files.exclude": { + "**/.clangd/**": true, + "**/bazel-*/**": true + }, + "files.watcherExclude": { + "**/.clangd/**": true, + "**/bazel-*/**": true + } + }, + "extensions": [ + "github.vscode-pull-request-github", + "zxh404.vscode-proto3", + "bazelbuild.vscode-bazel", + "llvm-vs-code-extensions.vscode-clangd", + "vadimcn.vscode-lldb", + "webfreak.debug", + "ms-python.python" + ] + } + }, } diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index d2a54b474bb1..b50bb1190d66 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,10 +1,8 @@ #!/usr/bin/env bash -. ci/setup_cache.sh -trap - EXIT # Don't remove the key file written into a temporary file - BAZELRC_FILE=~/.bazelrc bazel/setup_clang.sh /opt/llvm +# TODO(phlax): use user.bazelrc # Use generated toolchain config because we know the base container is the one we're using in RBE. # Not using libc++ here because clangd will raise some tidy issue in libc++ header as of version 9. echo "build --config=rbe-toolchain-clang" >> ~/.bazelrc diff --git a/.github/actions/do_ci/action.yml b/.github/actions/do_ci/action.yml deleted file mode 100644 index 5a024feede03..000000000000 --- a/.github/actions/do_ci/action.yml +++ /dev/null @@ -1,90 +0,0 @@ -inputs: - target: - required: true - type: string - rbe: - type: boolean - default: true - managed: - type: boolean - default: true - - auth_bazel_rbe: - type: string - default: '' - - bazel_extra: - type: string - default: - bazel_local_cache: - type: string - default: - bazel_rbe_cache: - type: string - default: grpcs://remotebuildexecution.googleapis.com - bazel_rbe_instance: - type: string - default: projects/envoy-ci/instances/default_instance - bazel_rbe_jobs: - type: number - default: 75 - - command_prefix: - type: string - default: ./ci/run_envoy_docker.sh - command_ci: - type: string - default: ./ci/do_ci.sh - - env: - type: string - - GITHUB_TOKEN: - required: true - -runs: - using: composite - steps: - - id: do_ci - name: 'Run CI target ${{ inputs.target }}' - run: | - if [[ "${#INPUT_ENV}" -ne 0 ]]; then - SOURCETMP="$(mktemp)" - # TODO(phlax): Fix escaping - echo "${{ inputs.env }}" > "$SOURCETMP" - . "$SOURCETMP" - rm -rf "$SOURCETMP" - fi - if [[ "${{ inputs.rbe }}" == 'true' ]]; then - export ENVOY_RBE=1 - export GCP_SERVICE_ACCOUNT_KEY=${{ inputs.auth_bazel_rbe }} - export BAZEL_BUILD_EXTRA_OPTIONS="--config=remote-ci --jobs=${{ inputs.bazel_rbe_jobs }} ${{ inputs.bazel_extra }}" - export BAZEL_REMOTE_CACHE=${{ inputs.bazel_rbe_cache }}" - export BAZEL_REMOTE_INSTANCE=${{ inputs.bazel_rbe_instance }}" - else - export BAZEL_BUILD_EXTRA_OPTIONS="--config=ci ${{ inputs.bazel_extra }}" - export BAZEL_REMOTE_CACHE="${{ inputs.bazel_local_cache }}" - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - export BAZEL_REMOTE_INSTANCE_BRANCH="${{ github.event.base.ref }}" - else - export BAZEL_REMOTE_INSTANCE_BRANCH="${{ github.ref }}" - fi - fi - - if [[ -n "${{ inputs.command_prefix }}" ]]; then - ${{ inputs.command_prefix }} '${{ inputs.command_ci }} ${{ inputs.target }}' - else - ${{ inputs.command_ci }} ${{ inputs.target }} - fi - - if [[ ${{ github.event_name }} == "pull_request" ]]; then - export BAZEL_FAKE_SCM_REVISION=e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9 - export CI_TARGET_BRANCH="${{ github.event.base.ref }}" - else - export CI_TARGET_BRANCH="${{ github.ref }}" - fi - shell: bash - env: - GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} - ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }} - INPUT_ENV: ${{ inputs.env }} diff --git a/.github/actions/env/action.yml b/.github/actions/env/action.yml deleted file mode 100644 index b5d44c56d24f..000000000000 --- a/.github/actions/env/action.yml +++ /dev/null @@ -1,175 +0,0 @@ -inputs: - build_image_tag: - type: string - required: true - build_image_repo: - type: string - required: true - build_image_mobile_sha: - type: string - required: true - build_image_sha: - type: string - required: true - - repo_ref: - type: string - repo_ref_sha: - type: string - repo_ref_name: - type: string - - trusted_bots: - type: string - default: | - trigger-release-envoy[bot] - - check_mobile_run: - type: boolean - default: true - -outputs: - build_image_ubuntu: - value: ${{ steps.build.outputs.build_image_ubuntu }} - build_image_ubuntu_mobile: - value: ${{ steps.build.outputs.build_image_ubuntu_mobile }} - - mobile_android_build: - value: ${{ steps.should_run.outputs.mobile_android_build }} - mobile_android_build_all: - value: ${{ steps.should_run.outputs.mobile_android_build_all }} - mobile_android_tests: - value: ${{ steps.should_run.outputs.mobile_android_tests }} - mobile_asan: - value: ${{ steps.should_run.outputs.mobile_asan }} - mobile_cc_tests: - value: ${{ steps.should_run.outputs.mobile_cc_tests }} - mobile_compile_time_options: - value: ${{ steps.should_run.outputs.mobile_compile_time_options }} - mobile_coverage: - value: ${{ steps.should_run.outputs.mobile_coverage }} - mobile_formatting: - value: ${{ steps.should_run.outputs.mobile_formatting }} - mobile_ios_build: - value: ${{ steps.should_run.outputs.mobile_ios_build }} - mobile_ios_build_all: - value: ${{ steps.should_run.outputs.mobile_ios_build_all }} - mobile_ios_tests: - value: ${{ steps.should_run.outputs.mobile_ios_tests }} - mobile_release_validation: - value: ${{ steps.should_run.outputs.mobile_release_validation }} - mobile_tsan: - value: ${{ steps.should_run.outputs.mobile_tsan }} - repo_ref: - value: ${{ steps.context.outputs.repo_ref }} - repo_ref_name: - value: ${{ steps.context.outputs.repo_ref_name }} - repo_ref_pr_number: - value: ${{ steps.context.outputs.repo_ref_pr_number }} - repo_ref_sha: - value: ${{ steps.context.outputs.repo_ref_sha }} - repo_ref_sha_short: - value: ${{ steps.context.outputs.repo_ref_sha_short }} - repo_ref_title: - value: ${{ steps.context.outputs.repo_ref_title }} - trusted: - value: ${{ steps.trusted.outputs.trusted }} - version_dev: - value: ${{ steps.context.outputs.version_dev }} - version_patch: - value: ${{ steps.context.outputs.version_patch }} - -runs: - using: composite - steps: - - - if: ${{ inputs.check_mobile_run != 'false' }} - id: should_run - name: 'Check what to run' - run: ./mobile/tools/what_to_run.sh - shell: bash - - - id: trusted - name: 'Check if its a trusted run' - run: | - TRUSTED=1 - ACTOR="${{ github.actor }}" - if [[ "$ACTOR" =~ \[bot\] ]]; then - TRUSTED_BOT= - TRUSTED_BOTS=(${{ inputs.trusted_bots }}) - for bot in ${TRUSTED_BOTS[@]}; do - if [[ "$bot" == "$ACTOR" ]]; then - # Trusted bot account, ie non-PR - TRUSTED_BOT=1 - break - fi - done - if [[ -z "$TRUSTED_BOT" ]]; then - echo "Not trusted bot account" - TRUSTED= - fi - fi - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "Not trusted pull_request event" - TRUSTED= - fi - if [[ -n "$TRUSTED" ]]; then - echo "trusted=true" >> "$GITHUB_OUTPUT" - else - echo "trusted=false" >> "$GITHUB_OUTPUT" - fi - shell: bash - - - id: context - name: 'CI context' - run: | - if grep dev VERSION.txt; then - VERSION_DEV="$(cat VERSION.txt | cut -d- -f2)" - else - VERSION_DEV="" - fi - VERSION_PATCH="$(cat VERSION.txt | cut -d- -f1 | rev | cut -d. -f1 | rev)" - # TODO: strip merge from pr names - REF_NAME=${{ inputs.repo_ref_name || github.ref_name }} - if [[ "$REF_NAME" =~ ^refs/pull/ ]]; then - REF_NAME="${REF_NAME:10}" - REF_PR_NUMBER="$(echo "${REF_NAME}" | cut -d/ -f1)" - elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then - REF_PR_NUMBER="$(echo "${REF_NAME}" | cut -d/ -f1)" - fi - echo "SET PR NUMBER: ${REF_PR_NUMBER}" - - REF="${{ steps.trusted.outputs.trusted != 'true' && inputs.repo_ref || '' }}" - REF_SHA=${{ inputs.repo_ref_sha || github.event.pull_request.head.sha || github.sha }} - REF_SHA_SHORT="${REF_SHA:0:7}" - REF_TITLE=( - "${{ steps.trusted.outputs.trusted == 'true' && 'postsubmit' || 'pr' }}/" - "${REF_NAME}" - "@${REF_SHA_SHORT}") - REF_TITLE="$(printf %s "${REF_TITLE[@]}" $'\n')" - { - echo "repo_ref=$REF" - echo "repo_ref_name=$REF_NAME" - echo "repo_ref_pr_number=$REF_PR_NUMBER" - echo "repo_ref_sha=$REF_SHA" - echo "repo_ref_title=$REF_TITLE" - echo "repo_ref_sha_short=$REF_SHA_SHORT" - echo "version_dev=$VERSION_DEV" - echo "version_patch=$VERSION_PATCH" - } >> "$GITHUB_OUTPUT" - shell: bash - - - id: build - name: 'Check current build images' - run: | - { - echo "build_image_ubuntu=${BUILD_IMAGE_UBUNTU_REPO}:${BUILD_IMAGE_UBUNTU}@sha256:${BUILD_IMAGE_UBUNTU_SHA}" - echo "build_image_ubuntu_mobile=${BUILD_IMAGE_UBUNTU_REPO}:mobile-${BUILD_IMAGE_UBUNTU}@sha256:${BUILD_IMAGE_UBUNTU_MOBILE_SHA}" - } >> "$GITHUB_OUTPUT" - env: - # TODO(phlax): derive these from a config file - BUILD_IMAGE_UBUNTU_REPO: ${{ inputs.build_image_repo }} - BUILD_IMAGE_UBUNTU: ${{ inputs.build_image_tag }} - BUILD_IMAGE_UBUNTU_SHA: ${{ inputs.build_image_sha }} - BUILD_IMAGE_UBUNTU_MOBILE_SHA: ${{ inputs.build_image_mobile_sha }} - shell: bash diff --git a/.github/actions/pr_notifier/pr_notifier.py b/.github/actions/pr_notifier/pr_notifier.py deleted file mode 100644 index 4366327d2888..000000000000 --- a/.github/actions/pr_notifier/pr_notifier.py +++ /dev/null @@ -1,267 +0,0 @@ -# Script for collecting PRs in need of review, and informing maintainers via -# slack. -# -# By default this runs in "developer mode" which means that it collects PRs -# associated with maintainers and API reviewers, and spits them out (badly -# formatted) to the command line. -# -# .github/workflows/pr_notifier.yml runs the script with --cron_job -# which instead sends the collected PRs to the various slack channels. -# -# NOTE: Slack IDs can be found in the user's full profile from within Slack. - -from __future__ import print_function - -import argparse -import datetime -import os -import sys - -import github -from slack_sdk import WebClient -from slack_sdk.errors import SlackApiError - -MAINTAINERS = { - 'alyssawilk': 'U78RP48V9', - 'mattklein123': 'U5CALEVSL', - 'lizan': 'U79E51EQ6', - 'snowp': 'U93KTPQP6', - 'ggreenway': 'U78MBV869', - 'htuch': 'U78E7055Z', - 'zuercher': 'U78J72Q82', - 'phlax': 'U017PLM0GNQ', - 'jmarantz': 'U80HPLBPG', - 'ravenblackx': 'U02MJHFEX35', - 'yanavlasov': 'UJHLR5KFS', - 'RyanTheOptimist': 'U01SW3JC8GP', - 'adisuissa': 'UT17EMMTP', - 'KBaichoo': 'U016ZPU8KBK', - 'wbpcode': 'U017KF5C0Q6', - 'kyessenov': 'U7KTRAA8M', - 'keith': 'UGS5P90CF', - 'abeyad': 'U03CVM7GPM1', - 'soulxu': 'U01GNQ3B8AY', -} - -# First pass reviewers who are not maintainers should get -# notifications but not result in a PR not getting assigned a -# maintainer owner. -FIRST_PASS = { - 'dmitri-d': 'UB1883Q5S', - 'tonya11en': 'U989BG2CW', - 'esmet': 'U01BCGBUUAE', - 'mathetake': 'UG9TD2FSB', -} - -# Only notify API reviewers who aren't maintainers. -# Maintainers are already notified of pending PRs. -API_REVIEWERS = { - 'markdroth': 'UMN8K55A6', - 'adisuissa': 'UT17EMMTP', -} - - -def get_slo_hours(): - # on Monday, allow for 24h + 48h - if datetime.date.today().weekday() == 0: - return 72 - return 24 - - -# Return true if the PR has a waiting tag, false otherwise. -def is_waiting(labels): - for label in labels: - if label.name == 'waiting' or label.name == 'waiting:any': - return True - return False - - -def is_contrib(labels): - return any(label.name == "contrib" for label in labels) - - -# Return true if the PR has an API tag, false otherwise. -def is_api(labels): - for label in labels: - if label.name == 'api': - return True - return False - - -# Generate a pr message, bolding the time if it's out-SLO -def pr_message(pr_age, pr_url, pr_title, delta_days, delta_hours): - if pr_age < datetime.timedelta(hours=get_slo_hours()): - return "<%s|%s> has been waiting %s days %s hours\n" % ( - pr_url, pr_title, delta_days, delta_hours) - else: - return "<%s|%s> has been waiting *%s days %s hours*\n" % ( - pr_url, pr_title, delta_days, delta_hours) - - -# Adds reminder lines to the appropriate assignee to review the assigned PRs -# Returns true if one of the assignees is in the primary_assignee_map, false otherwise. -def add_reminders( - assignees, assignees_and_prs, message, primary_assignee_map, first_pass_assignee_map): - has_primary_assignee = False - for assignee_info in assignees: - assignee = assignee_info.login - if assignee in primary_assignee_map: - has_primary_assignee = True - elif assignee not in first_pass_assignee_map: - continue - if assignee not in assignees_and_prs.keys(): - assignees_and_prs[ - assignee] = "Hello, %s, here are your PR reminders for the day \n" % assignee - assignees_and_prs[assignee] = assignees_and_prs[assignee] + message - return has_primary_assignee - - -# Returns true if the PR needs an LGTM from an API shephard. -def needs_api_review(labels, repo, pr_info): - # API reviews should always have the label, so don't bother doing an RPC if - # it's not tagged (this helps avoid github rate limiting) - if not (is_api(labels)): - return False - # repokitten tags each commit as pending unless there has been an API LGTM - # since the latest API changes. If this PR is tagged pendding it needs an - # API review, otherwise it's set. - status = repo.get_commit(pr_info.head.sha).get_statuses() - return status[0].state == "pending" if status.totalCount else False - - -def track_prs(github_token): - git = github.Github(github_token) - - repo = git.get_repo('envoyproxy/envoy') - - # The list of PRs which are not waiting, but are well within review SLO - recent_prs = [] - # A dict of maintainer : outstanding_pr_string to be sent to slack - maintainers_and_prs = {} - # A placeholder for unassigned PRs, to be sent to #maintainers eventually - maintainers_and_prs['unassigned'] = "" - # A dict of shephard : outstanding_pr_string to be sent to slack - api_review_and_prs = {} - # Out-SLO PRs to be sent to #envoy-maintainer-oncall - stalled_prs = "" - - # Snag all PRs, including drafts - for pr_info in repo.get_pulls("open", "updated", "desc"): - labels = pr_info.labels - assignees = pr_info.assignees - # If the PR is waiting, continue. - if is_waiting(labels): - continue - # Drafts are not covered by our SLO (repokitteh warns of this) - if pr_info.draft: - continue - # Don't warn for dependabot. - if pr_info.user.login == 'dependabot[bot]': - continue - - # Update the time based on the time zone delta from github's - pr_age = pr_info.updated_at - datetime.timedelta(hours=4) - delta = datetime.datetime.now() - pr_age - delta_days = delta.days - delta_hours = delta.seconds // 3600 - - # If we get to this point, the review may be in SLO - nudge if it's in - # SLO, nudge in bold if not. - message = pr_message(delta, pr_info.html_url, pr_info.title, delta_days, delta_hours) - - if (needs_api_review(labels, repo, pr_info)): - add_reminders(pr_info.assignees, api_review_and_prs, message, API_REVIEWERS, []) - - # If the PR has been out-SLO for over a day, inform on-call - if delta > datetime.timedelta(hours=get_slo_hours() + 36): - stalled_prs = stalled_prs + message - - # Add a reminder to each maintainer-assigner on the PR. - has_maintainer_assignee = add_reminders( - pr_info.assignees, maintainers_and_prs, message, MAINTAINERS, FIRST_PASS) - - # If there was no maintainer, track it as unassigned. - if not has_maintainer_assignee and not is_contrib(labels): - maintainers_and_prs['unassigned'] = maintainers_and_prs['unassigned'] + message - - # Return the dict of {maintainers : PR notifications}, - # the dict of {api-shephards-who-are-not-maintainers: PR notifications}, - # and stalled PRs - return maintainers_and_prs, api_review_and_prs, stalled_prs - - -def post_to_assignee(client, assignees_and_messages, assignees_map): - # Post updates to individual assignees - for key in assignees_and_messages: - message = assignees_and_messages[key] - - # Only send messages if we have the slack UID - if key not in assignees_map: - continue - uid = assignees_map[key] - - # Ship messages off to slack. - try: - print(assignees_and_messages[key]) - response = client.conversations_open(users=uid, text="hello") - channel_id = response["channel"]["id"] - client.chat_postMessage(channel=channel_id, text=message) - except SlackApiError as e: - print("Unexpected error %s", e.response["error"]) - - -def post_to_oncall(client, unassigned_prs, out_slo_prs): - # Post updates to #envoy-maintainer-oncall - unassigned_prs = maintainers_and_messages['unassigned'] - try: - client.chat_postMessage( - channel='#envoy-maintainer-oncall', - text=("*'Unassigned' PRs* (PRs with no maintainer assigned)\n%s" % unassigned_prs)) - client.chat_postMessage( - channel='#envoy-maintainer-oncall', - text=("*Stalled PRs* (PRs with review out-SLO, please address)\n%s" % out_slo_prs)) - issue_link = "https://github.com/envoyproxy/envoy/issues?q=is%3Aissue+is%3Aopen+label%3Atriage" - client.chat_postMessage( - channel='#envoy-maintainer-oncall', - text=( - "*Untriaged Issues* (please tag and cc area experts)\n<%s|%s>" % - (issue_link, issue_link))) - except SlackApiError as e: - print("Unexpected error %s", e.response["error"]) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--cron_job', - action="store_true", - help="true if this is run by the daily cron job, false if run manually by a developer") - args = parser.parse_args() - - github_token = os.getenv('GITHUB_TOKEN') - if not github_token: - print('Missing GITHUB_TOKEN: please check github workflow configuration') - sys.exit(1) - - slack_bot_token = os.getenv('SLACK_BOT_TOKEN') - if not slack_bot_token: - print( - 'Missing SLACK_BOT_TOKEN: please export token from https://api.slack.com/apps/A023NPQQ33K/oauth?' - ) - sys.exit(1) - - maintainers_and_messages, shephards_and_messages, stalled_prs = track_prs(github_token) - - if not args.cron_job: - print(maintainers_and_messages) - print("\n\n\n") - print(shephards_and_messages) - print("\n\n\n") - print(stalled_prs) - exit(0) - - client = WebClient(token=slack_bot_token) - post_to_oncall(client, maintainers_and_messages['unassigned'], stalled_prs) - post_to_assignee(client, shephards_and_messages, API_REVIEWERS) - post_to_assignee(client, maintainers_and_messages, MAINTAINERS) - post_to_assignee(client, maintainers_and_messages, FIRST_PASS) diff --git a/.github/actions/pr_notifier/requirements.in b/.github/actions/pr_notifier/requirements.in deleted file mode 100644 index b27ccacba25a..000000000000 --- a/.github/actions/pr_notifier/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -pygithub -slack_sdk diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt deleted file mode 100644 index 4dfea44d1807..000000000000 --- a/.github/actions/pr_notifier/requirements.txt +++ /dev/null @@ -1,224 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes .github/actions/pr_notifier/requirements.txt -# -certifi==2023.7.22 \ - --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ - --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 - # via requests -cffi==1.14.5 \ - --hash=sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813 \ - --hash=sha256:04c468b622ed31d408fea2346bec5bbffba2cc44226302a0de1ade9f5ea3d373 \ - --hash=sha256:06d7cd1abac2ffd92e65c0609661866709b4b2d82dd15f611e602b9b188b0b69 \ - --hash=sha256:06db6321b7a68b2bd6df96d08a5adadc1fa0e8f419226e25b2a5fbf6ccc7350f \ - --hash=sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06 \ - --hash=sha256:0f861a89e0043afec2a51fd177a567005847973be86f709bbb044d7f42fc4e05 \ - --hash=sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea \ - --hash=sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee \ - --hash=sha256:1bf1ac1984eaa7675ca8d5745a8cb87ef7abecb5592178406e55858d411eadc0 \ - --hash=sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396 \ - --hash=sha256:24a570cd11895b60829e941f2613a4f79df1a27344cbbb82164ef2e0116f09c7 \ - --hash=sha256:24ec4ff2c5c0c8f9c6b87d5bb53555bf267e1e6f70e52e5a9740d32861d36b6f \ - --hash=sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73 \ - --hash=sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315 \ - --hash=sha256:293e7ea41280cb28c6fcaaa0b1aa1f533b8ce060b9e701d78511e1e6c4a1de76 \ - --hash=sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1 \ - --hash=sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49 \ - --hash=sha256:3c3f39fa737542161d8b0d680df2ec249334cd70a8f420f71c9304bd83c3cbed \ - --hash=sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892 \ - --hash=sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482 \ - --hash=sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058 \ - --hash=sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5 \ - --hash=sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53 \ - --hash=sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045 \ - --hash=sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3 \ - --hash=sha256:681d07b0d1e3c462dd15585ef5e33cb021321588bebd910124ef4f4fb71aef55 \ - --hash=sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5 \ - --hash=sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e \ - --hash=sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c \ - --hash=sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369 \ - --hash=sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827 \ - --hash=sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053 \ - --hash=sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa \ - --hash=sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4 \ - --hash=sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322 \ - --hash=sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132 \ - --hash=sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62 \ - --hash=sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa \ - --hash=sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0 \ - --hash=sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396 \ - --hash=sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e \ - --hash=sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991 \ - --hash=sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6 \ - --hash=sha256:cc5a8e069b9ebfa22e26d0e6b97d6f9781302fe7f4f2b8776c3e1daea35f1adc \ - --hash=sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1 \ - --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ - --hash=sha256:df5052c5d867c1ea0b311fb7c3cd28b19df469c056f7fdcfe88c7473aa63e333 \ - --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ - --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c - # via - # cryptography - # pynacl -charset-normalizer==3.1.0 \ - --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ - --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ - --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ - --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ - --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ - --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ - --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ - --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ - --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ - --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ - --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ - --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ - --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ - --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ - --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ - --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ - --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ - --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ - --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ - --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ - --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ - --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ - --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ - --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ - --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ - --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ - --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ - --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ - --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ - --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ - --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ - --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ - --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ - --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ - --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ - --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ - --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ - --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ - --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ - --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ - --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ - --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ - --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ - --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ - --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ - --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ - --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ - --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ - --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ - --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ - --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ - --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ - --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ - --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ - --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ - --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ - --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ - --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ - --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ - --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ - --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ - --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ - --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ - --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ - --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ - --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ - --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ - --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ - --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ - --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ - --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ - --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ - --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ - --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ - --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab - # via requests -cryptography==41.0.3 \ - --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ - --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ - --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ - --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ - --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ - --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ - --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ - --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ - --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ - --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ - --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ - --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ - --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ - --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ - --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ - --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ - --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ - --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ - --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ - --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ - --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ - --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ - --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de - # via pyjwt -deprecated==1.2.13 \ - --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \ - --hash=sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d - # via pygithub -idna==2.10 \ - --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ - --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 - # via requests -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 - # via cffi -pygithub==1.59.1 \ - --hash=sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9 \ - --hash=sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217 - # via -r requirements.in -pyjwt[crypto]==2.4.0 \ - --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ - --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba - # via pygithub -pynacl==1.4.0 \ - --hash=sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4 \ - --hash=sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4 \ - --hash=sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574 \ - --hash=sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d \ - --hash=sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634 \ - --hash=sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25 \ - --hash=sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f \ - --hash=sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505 \ - --hash=sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122 \ - --hash=sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7 \ - --hash=sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420 \ - --hash=sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f \ - --hash=sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96 \ - --hash=sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6 \ - --hash=sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6 \ - --hash=sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514 \ - --hash=sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff \ - --hash=sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80 - # via pygithub -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 - # via pygithub -six==1.16.0 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via pynacl -slack-sdk==3.22.0 \ - --hash=sha256:6eacce0fa4f8cfb4d84eac0d7d7e1b1926040a2df654ae86b94179bdf2bc4d8c \ - --hash=sha256:f102a4902115dff3b97c3e8883ad4e22d54732221886fc5ef29bfc290f063b4a - # via -r requirements.in -urllib3==1.26.6 \ - --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ - --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f - # via requests -wrapt==1.12.1 \ - --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 - # via deprecated diff --git a/.github/actions/publish/release/setup/action.yml b/.github/actions/publish/release/setup/action.yml deleted file mode 100644 index 4e0935710d2d..000000000000 --- a/.github/actions/publish/release/setup/action.yml +++ /dev/null @@ -1,26 +0,0 @@ -inputs: - ref: - type: string - required: true - bucket: - type: string - required: true - -runs: - using: composite - steps: - - id: url - run: | - echo "base=https://storage.googleapis.com/${{ inputs.bucket }}/${REF:0:7}/release" \ - >> "$GITHUB_OUTPUT" - env: - REF: ${{ inputs.ref }} - shell: bash - - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.0.10 - id: fetch - with: - url: "${{ steps.url.outputs.base }}/release.signed.tar.zst" - - run: | - mkdir -p ${{ runner.temp }}/release.signed - mv ${{ steps.fetch.outputs.path }} ${{ runner.temp }}/release.signed - shell: bash diff --git a/.github/actions/verify/examples/setup/action.yml b/.github/actions/verify/examples/setup/action.yml deleted file mode 100644 index 18f3205721ce..000000000000 --- a/.github/actions/verify/examples/setup/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -inputs: - ref: - type: string - required: true - bucket: - type: string - default: envoy-pr - -runs: - using: composite - steps: - - id: url - run: | - echo "base=https://storage.googleapis.com/${{ inputs.bucket }}/${REF:0:7}/docker" \ - >> "$GITHUB_OUTPUT" - env: - REF: ${{ inputs.ref }} - shell: bash - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.0.10 - with: - url: "${{ steps.url.outputs.base }}/envoy.tar" - variant: dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.0.10 - with: - url: "${{ steps.url.outputs.base }}/envoy-contrib.tar" - variant: contrib-dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.0.10 - with: - url: "${{ steps.url.outputs.base }}/envoy-google-vrp.tar" - variant: google-vrp-dev - - run: docker images | grep envoy - shell: bash - - run: | - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update -y - sudo apt-get -qq install -y --no-install-recommends expect - shell: bash diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 000000000000..b7b51adaad95 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,413 @@ +agent-ubuntu: ubuntu-22.04 +build-image: + # Authoritative configuration for build image/s + repo: envoyproxy/envoy-build-ubuntu + sha: 1386a26f687826850ba488d66a6cd5337c5941b3b8793d08cfa6f9df12aa2fcf + mobile-sha: 1db9bac6578115179fe686cd0e58ee340e2cb3737d19d9136a264d5d94351961 + # this is authoritative, but is not currently used in github ci + gcr-sha: b0c0e07c97337fdd56423ddd6749a4250adaea9f66f85763a8d9cec36162c972 + tag: fd9ec000fdd72d5c5e4e4ef16db4f9103058779e + +config: + envoy: + icon: >- + [![](https://avatars.githubusercontent.com/u/30125649?s=24&v=4)](#) + +checks: + # Checks: this configures which _checks_ will be activated or skipped + # + # The configured _names_ need to match the checks configured for the repo + # + # Any check that is marked as `required` but is not triggered by the run + # config above in a given CI run is marked as `skipped` + # + # For example if macos is marked as `required: true` but then has a path + # selection that means its doesnt run the check will be `skipped` and pass + macos: + name: Envoy/macOS + required: true + on-run: + - build-macos + mobile-android: + name: Mobile/Android + required: true + on-run: + - mobile-android + mobile-android-tests: + name: Mobile/Android tests + required: true + on-run: + - mobile-android-tests + mobile-asan: + name: Mobile/ASAN + required: true + on-run: + - mobile-asan + mobile-cc: + name: Mobile/CC + required: true + on-run: + - mobile-cc + mobile-compile-time-options: + name: Mobile/Compile time options + on-run: + - mobile-compile-time-options + mobile-core: + name: Mobile/Core + required: true + on-run: + - mobile-core + mobile-coverage: + name: Mobile/Coverage + required: true + on-run: + - mobile-coverage + mobile-docs: + name: Mobile/Docs + on-run: + - mobile-docs + mobile-format: + name: Mobile/Format + required: true + on-run: + - mobile-format + mobile-ios: + name: Mobile/iOS + required: true + cache: + on-run: + - mobile-ios + mobile-ios-tests: + name: Mobile/iOS tests + required: true + on-run: + - mobile-ios-tests + mobile-perf: + name: Mobile/Perf + required: true + on-run: + - mobile-perf + mobile-release-validation: + name: Mobile/Release validation + required: true + on-run: + - mobile-release-validation + mobile-tsan: + name: Mobile/TSAN + required: true + on-run: + - mobile-tsan + prechecks: + name: Envoy/Prechecks + on-run: + - precheck-deps + required: true + # yamllint disable rule:line-length + advice: + general: | + ### Ensuring your commits are signed off + + You can set up DCO using Envoy's git hooks. + + ### Git hooks + + To set this up, do the following: + + ```console + $ ./support/bootstrap + ``` + + If you only want the DCO check you can do the following to disable the + other hooks + + ```console + $ echo NO_VERIFY=1 > .env + ``` + deps: | + ### Advice on updating dependencies + + General information about Envoy's depdendencies [can be found here](https://github.com/envoyproxy/envoy/blob/main/DEPENDENCY_POLICY.md) + format: | + ### Advice on correct formatting + + Envoy ensures a minimum standard for all files in the repository. + + You are strongly advised to heed the following CI notice: + + ```console + Please fix your editor to ensure: + + - no trailing whitespace + - no preceding mixed tabs/spaces + - all files end with a newline + ``` + # yamllint enable rule:line-length + publish: + name: >- + Envoy/Publish and verify + on-run: + - publish + - verify + required: true + windows: + name: Envoy/Windows + required: true + on-run: + - build-windows + +run: + build-windows: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - api/**/* + - bazel/**/* + - ci/**/* + - configs/**/* + - contrib/**/* + - envoy/**/* + - source/**/* + - test/**/* + build-macos: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - api/**/* + - bazel/**/* + - ci/**/* + - configs/**/* + - contrib/**/* + - envoy/**/* + - source/**/* + - test/**/* + mobile-android: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-android-all: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + push: never + mobile-android-tests: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-asan: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-cc: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-compile-time-options: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-coverage: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-core: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-format: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-ios: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-ios-all: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + push: never + mobile-ios-tests: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-perf: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-release-validation: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + mobile-tsan: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - bazel/external/quiche.BUILD + - bazel/repository_locations.bzl + - mobile/**/* + - tools/code_format/check_format.py + precheck-deps: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - .github/dependabot.yml + - bazel/BUILD + - tools/dependency/* + - "**/*.bzl" + - "**/requirements.txt" + - "**/go.mod" + - "**/Dockerfile*" + publish: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - api/**/* + - bazel/**/* + - ci/**/* + - contrib/**/* + - envoy/**/* + - examples/**/* + - source/**/* + - tools/**/* + verify: + paths: + - .bazelrc + - .bazelversion + - .github/config.yml + - api/**/* + - bazel/**/* + - ci/**/* + - contrib/**/* + - envoy/**/* + - examples/**/* + - source/**/* + - tools/**/* + push: paths + +tables: + env: + collapse: true + title: Environment + table-title: Request variables + filter: | + .request + | del(.["build-image" as $prefix | keys[] | select(startswith($prefix))]) + | del(.["version" as $prefix | keys[] | select(startswith($prefix))]) + | .actor = "\"\(.actor.name)\" @\(.actor.name)" + build-image: + collapse: true + title: Build image + table-title: Container image/s (as used in this CI run) + filter: | + "https://hub.docker.com/r/envoyproxy/envoy-build-ubuntu/tags?page=1&name=" as $dockerLink + | .request["build-image"] + | del(.changed) + | with_entries( + .value as $v + | ($v | split(":") | .[1] | split("@") | .[0]) as $tag + | .value = "[\($v | split("@") | .[0])](\($dockerLink)\($tag))") + build-image-current: + collapse: true + title: Build image (current) + table-title: Current or previous container image + filter: | + "https://hub.docker.com/r/envoyproxy/envoy-build-ubuntu/tags?page=1&name=" as $dockerLink + | if .request["build-image"].changed then + .request["build-image-current"] + | with_entries( + .value as $v + | ($v | split(":") | .[1] | split("@") | .[0]) as $tag + | .value = "[\($v | split("@") | .[0])](\($dockerLink)\($tag))") + else {} end + version: + collapse: true + title: Version + table-title: Envoy version (as used in this CI run) + filter: | + .request.version + | del(.changed) + version-current: + collapse: true + title: Version (current) + table-title: Current or previous version + filter: | + if .request.version.changed then + .request["version-current"] + else + {} + end diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a9462ca3b88d..03641ba10b21 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,44 +9,49 @@ updates: # # Please ensure any new ones are added here, and any that are removed are removed here also. -- package-ecosystem: "pip" - directory: "/.github/actions/pr_notifier" - schedule: - interval: "daily" - time: "06:00" - - package-ecosystem: "pip" directory: "/examples/grpc-bridge/client" + groups: + examples-grpc-bridge: + patterns: + - "*" schedule: interval: "daily" time: "06:00" - package-ecosystem: "pip" directory: "/examples/cache" + groups: + examples-cache: + patterns: + - "*" schedule: interval: "daily" time: "06:00" - package-ecosystem: "pip" directory: "/examples/shared/python/aiohttp" + groups: + examples-shared-python: + patterns: + - "*" schedule: interval: "daily" time: "06:00" - package-ecosystem: "pip" directory: "/examples/shared/python/postgres" + groups: + examples-postgres: + patterns: + - "*" schedule: interval: "daily" time: "06:00" - package-ecosystem: "pip" directory: "/tools/base" - schedule: - interval: "daily" - time: "06:00" - -- package-ecosystem: "pip" - directory: "/tools/code_format" + open-pull-requests-limit: 20 schedule: interval: "daily" time: "06:00" @@ -65,6 +70,10 @@ updates: - package-ecosystem: "docker" directory: "/examples/ext_authz" + groups: + examples-ext-authz: + patterns: + - "*" schedule: interval: daily time: "06:00" @@ -77,24 +86,40 @@ updates: - package-ecosystem: "docker" directory: "/examples/golang-network" + groups: + examples-golang-network: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "docker" directory: "/examples/grpc-bridge" + groups: + examples-grpc-bridge: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "docker" directory: "/examples/kafka" + groups: + examples-kafka: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "docker" directory: "/examples/local_ratelimit" + groups: + examples-local-ratelimit: + patterns: + - "*" schedule: interval: daily time: "06:00" @@ -204,42 +229,90 @@ updates: - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/basic" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/dummy" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/echo" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/metric" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/passthrough" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/access_log" + groups: + contrib-golang: + patterns: + - "*" + schedule: + interval: daily + time: "06:00" + +- package-ecosystem: "gomod" + directory: "/contrib/golang/filters/http/test/test_data/action" + groups: + contrib-golang: + patterns: + - "*" + schedule: + interval: daily + time: "06:00" + +- package-ecosystem: "gomod" + directory: "/contrib/golang/filters/http/test/test_data/buffer" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/contrib/golang/filters/http/test/test_data/routeconfig" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" @@ -251,43 +324,71 @@ updates: time: "06:00" - package-ecosystem: "gomod" - directory: "/contrib/golang/router/cluster_specifier/test/test_data/simple" + directory: "/contrib/golang/filters/network/test/test_data" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" - directory: "/contrib/golang/filters/network/test/test_data" + directory: "/contrib/golang/router/cluster_specifier/test/test_data/simple" + groups: + contrib-golang: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/examples/ext_authz/auth/grpc-service" + groups: + examples-ext-authz: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/examples/load-reporting-service" + groups: + examples-load-reporting: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/examples/grpc-bridge/server" + groups: + examples-grpc-bridge: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/examples/golang-http/simple" + groups: + examples-golang-http: + patterns: + - "*" schedule: interval: daily time: "06:00" - package-ecosystem: "gomod" directory: "/examples/golang-network/simple" + groups: + examples-golang-network: + patterns: + - "*" schedule: interval: daily time: "06:00" diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000000..743c7f39acdd --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,198 @@ +## CI configuration + +CI is configured in .github/config.yml. + +The configuration is per-branch and in this way different branches can have a different +runtime configuration. + +In a pull request only 2 things are read from the config.yml submitted in the request: + +- version +- build image + +As these can change the way the CI runs they are allowed to change. No other configuration +is read from the pull request itself. + +### Checks + +Which checks should run against a commit or PR is configured under the `checks` key. + +The names of these checks should match any checks that are set to required for the repo, +and if a check is required this should be set in the config to ensure the check is marked +as skipped if the related runs are skipped. + +### Runs + +This controls which workflows run, and where necessary which jobs in the workflows. + +This paths can be configured with glob matches to match changed files. + +Paths are always matched for PRs. + +For push requests the config can be set to: + +- always (default): Always runs +- paths: Runs when paths match +- never: Doesnt run on pushes + +## CI requests + +### All CI is requested + +Whether triggered by push event or a pull_request all CI should be viewed as "requested". + +This is very important as it means we can treat incoming triggers in much the same way +as we might handle an incoming web request. + +Much like a web request, CI requests may be "trusted" or "untrusted" and as a consequence +have more or less capability or access. + +Again, much like web requests, CI requests cannot be assumed to be safe. + +Any incoming data - critically data over which a user has the capability to change should +be treated in the same way that user data is handled in a web request. + +Failure to do this opens our CI up to many of the same attacks you might expect in a web scenario +- mostly injection attacks of various sorts. + +### Requests are always made _from_ the triggering branch + +The only CI workflow that is required/used on any branch other than `main` is `request.yml`. + +This file contains any custom configurations required by the branch - for example, build images. + +The request workflow on any branch always delegates to the `_request.yml` on `main`. + +The `_request.yml` workflow contains all required configuration for handling an incoming request. + +All other CI listens for the request workflow to run, and then runs with the requested/parsed data. + +### CI is always run _in_ the context of main + +Other than updating configurations in any given `request.yml` - no CI workflows are parsed +anywhere other than in the context of `main`. + +This means that **all** changes must be made to the `main` workflows for _any_ branch _and_ for PRs. + +Like branch CI, PRs also run in the context of `main` - making changes to these files in a PR will have +no effect until/unless they are landed on the `main` branch. + +### Lifecycle of a CI request + +#### Incoming request: + +Requests can be triggered by a `push` to `main` or a release branch or from a +`pull_request_target` to those branches. + +The `request.yml` file handles this and *must* live on every branch. + +This wf then calls the reusable `_request.yml` workflow, typically on `main`, but +branches can pin this if required. + +#### Request is handled by `_request.yml` workflow: + +This workflow initially reads the `.github/config.yml` from the target branch. + +It uses this to decide which CI and which checks need to be run, and collects information +about the CI request. + +This can be configured on a per-branch basis, by editing the file on the branch. + +This also holds the authoritative build image information. + +Users can request a CI run in a PR with custom build images by editing the config.yml file +on the relevant branch. CI will allow this but flag the change. + +Likewise the version is checked at this stage, and CI flags if it has changed. + +No other CI vars should be editable by users in a PR. + +#### CI check runs *on main* listen for incoming requests and run if required: + +These checks *always* run on `main` but with the repo checked out for the branch or the PR. + +If branches require custom CI this can be added in the relevant file *on main* with +a condition to only trigger for relevant target branch. + +#### Checks are completed at the end of each CI run: + +Currently this reports only on the overall outcome of the CI run and updates the check. + +We can add eg Slack reporting here to notify on failed `main` runs. + +#### Retesting + +PR CI can be retested by issuing `/retest` on the PR. + +This finds the checks related to the latest request and restarts them if they are +failed or cancelled. + +Links on the request page link to the original checks, but the checks themselves will +offer a `reload` button to refresh to the latest version. + +## Branch CI + +All CI is run on `main` - branch CI included. + +The CI will checkout the correct commits and run the CI at that point. + +This means that the CI on `main` should always be able to run the current supported branches. + +There are possible workaround for custom branch CI but the better path is to ensure legacy support +in current `main` or backport any required changes. + +## CI caching + +Currently only x86 Docker images are cached. + +Github has a hard per-repo limit of 10GB cache for CI which is LRU cycled when exceeded. + +This should just be enough to store x86 and arm Docker images for most of our release branches +but will not leave anything to spare. + +We can probably set up a bucket cache for bazel and other caching but this will need to be +done separately for un/trusted CI. + +### Cache mutex + +Due to shortcomings in Github's concurrency algorithm we are using a mutex lock that +is currently stored in the (private) https://github.com/envoyproxy/ci-mutex repository. + +The lock allows CI jobs to wait while the cache is being primed rather than all jobs attempting +to prime the cache simultaneously. + +## Development, testing and CI + +Any Github workflows that use the repository context (`pull_request_target`, `workflow_run`, etc) +**are not tested in Pull Requests** + +This means that changes to CI must be tested/verified in the (private) staging repository. + +### CI enabling vars + +The CI workflows and actions are receptive to certain environment variables being set. + +`ENVOY_CI`: this allows CI to run in non-`envoyproxy/envoy` repos +`ENVOY_MOBILE_CI`: this allows mobile CI to be run in non-`envoyproxy/envoy` repos +`ENVOY_MACOS_CI`: this allows macOS CI to be run in non-`envoyproxy/envoy` repos +`ENVOY_WINDOWS_CI`: this allows Windows CI to be run in non-`envoyproxy/envoy` repos + +With these flags activated the CI runs will respect the normal conditions for running. + +### CI override vars + +The CI workflows will also trigger for specific run settings. + +For example: + +`ENVOY_CI_RUN_MOBILE_ANDROID` would trigger the android CI irrespective of files changed, etc. + +These correspond to the run names as configured in config.yml - for example: + +`ENVOY_CI_RUN_BUILD_MACOS` would ensure the `build-macos` run is triggered. + +### Debugging CI + +Setting `CI_DEBUG` will provide a large amount of runtime information. + +Generally this does not want to be set in a production context. diff --git a/.github/workflows/_cache.yml b/.github/workflows/_cache.yml new file mode 100644 index 000000000000..b2fa26c3e2f6 --- /dev/null +++ b/.github/workflows/_cache.yml @@ -0,0 +1,69 @@ +name: Cache prime (docker) + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + inputs: + image-tag: + type: string + required: true + request: + type: string + required: true + lock-repository: + type: string + default: envoyproxy/ci-mutex + +## Docker cache +# +# This workflow will only prime the cache, and should be done separately first, prior +# to any jobs that require it. +# +# For a job that does, you can restore with something like: +# +# steps: +# - uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 +# with: +# key: "${{ needs.env.outputs.build-image }}" +# + + +jobs: + docker: + runs-on: ubuntu-22.04 + steps: + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: appauth + name: Appauth (mutex lock) + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/docker/cache/prime@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: docker + name: Prime Docker cache (${{ inputs.image-tag }}) + with: + image-tag: ${{ inputs.image-tag }} + lock-token: ${{ steps.appauth.outputs.token }} + lock-repository: ${{ inputs.lock-repository }} + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: data + name: Cache data + with: + input-format: yaml + input: | + cached: ${{ steps.docker.outputs.cached }} + key: ${{ inputs.image-tag }} + - uses: envoyproxy/toolshed/gh-actions/json/table@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Summary + with: + json: ${{ steps.data.outputs.value }} + output-path: GITHUB_STEP_SUMMARY + title: >- + Cache (Docker x64) diff --git a/.github/workflows/_cache_docker.yml b/.github/workflows/_cache_docker.yml deleted file mode 100644 index 84386d5aedfc..000000000000 --- a/.github/workflows/_cache_docker.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Cache prime (docker) - -permissions: - contents: read - -on: - workflow_call: - inputs: - image_tag: - type: string - required: true - image_repo: - type: string - required: true - image_sha: - type: string - required: true - -concurrency: - group: cache_docker-${{ inputs.image_tag }} - cancel-in-progress: false - -## Docker cache -# -# This workflow will only prime the cache, and should be done separately first, prior -# to any jobs that require it. -# -# For a job that does, you can restore with something like: -# -# steps: -# - uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.0.5 -# with: -# key: "${{ needs.env.outputs.build_image_ubuntu }}" -# - -jobs: - docker: - runs-on: ubuntu-22.04 - steps: - - uses: envoyproxy/toolshed/gh-actions/docker/cache/prime@actions-v0.0.18 - name: Prime Docker cache (${{ inputs.image_repo }}:${{ inputs.image_tag }}@sha256:${{ inputs.image_sha }}) - with: - image_tag: "${{ inputs.image_repo }}:${{ inputs.image_tag }}@sha256:${{ inputs.image_sha }}" diff --git a/.github/workflows/_ci.yml b/.github/workflows/_ci.yml deleted file mode 100644 index 057d4d462012..000000000000 --- a/.github/workflows/_ci.yml +++ /dev/null @@ -1,160 +0,0 @@ -name: Envoy CI - -on: - workflow_call: - inputs: - target: - required: true - type: string - rbe: - type: boolean - default: true - managed: - type: boolean - default: true - - auth_bazel_rbe: - type: string - default: '' - - bazel_extra: - type: string - default: - bazel_local_cache: - type: string - default: - bazel_rbe_cache: - type: string - default: grpcs://remotebuildexecution.googleapis.com - bazel_rbe_instance: - type: string - default: projects/envoy-ci/instances/default_instance - bazel_rbe_jobs: - type: number - default: 75 - - cache_build_image: - type: string - - command_prefix: - type: string - default: ./ci/run_envoy_docker.sh - command_ci: - type: string - default: ./ci/do_ci.sh - - diskspace_hack: - type: boolean - default: false - - run_pre: - type: string - default: - run_pre_with: - type: string - default: - - run_post: - type: string - default: - run_post_with: - type: string - default: - - repo_fetch_depth: - type: number - default: 1 - repo_ref: - type: string - skip: - type: boolean - default: false - trusted: - type: boolean - default: false - - env: - type: string - -concurrency: - group: | - ${{ github.actor != 'trigger-release-envoy[bot]' - && github.event.inputs.head_ref - || github.run_id - }}-${{ github.workflow }}-${{ inputs.target }} - cancel-in-progress: true - -jobs: - do_ci: - if: ${{ ! inputs.skip }} - runs-on: ubuntu-22.04 - name: ${{ inputs.command_ci }} ${{ inputs.target }} - steps: - - if: ${{ inputs.cache_build_image }} - name: Restore Docker cache (${{ inputs.cache_build_image }}) - uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.0.18 - with: - image_tag: ${{ inputs.cache_build_image }} - - # If the run is "trusted" (ie has access to secrets) then it should - # **not** set the ref and should use the code of the calling context. - - if: ${{ inputs.repo_ref && inputs.trusted }} - run: | - echo '`repo_ref` should not be set for trusted CI runs' - exit 1 - - - uses: actions/checkout@v4 - name: Checkout Envoy repository - with: - fetch-depth: ${{ inputs.repo_fetch_depth }} - # WARNING: This allows untrusted code to run!!! - # If this is set, then anything before or after in the job should be regarded as - # compromised. - ref: ${{ ! inputs.trusted && inputs.repo_ref || '' }} - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - - if: ${{ inputs.diskspace_hack }} - uses: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.0.18 - - run: | - echo "disk space at beginning of build:" - df -h - name: "Check disk space at beginning" - - - if: ${{ inputs.run_pre }} - name: Run pre action ${{ inputs.run_pre && format('({0})', inputs.run_pre) || '' }} - uses: envoyproxy/toolshed/gh-actions/using/recurse@actions-v0.0.18 - with: - uses: ${{ inputs.run_pre }} - with: ${{ inputs.run_pre_with }} - - - uses: ./.github/actions/do_ci - name: Do CI - with: - target: ${{ inputs.target }} - rbe: ${{ inputs.rbe }} - managed: ${{ inputs.managed }} - auth_bazel_rbe: ${{ inputs.auth_bazel_rbe }} - bazel_extra: ${{ inputs.bazel_extra }} - bazel_local_cache: ${{ inputs.bazel_local_cache }} - bazel_rbe_cache: ${{ inputs.bazel_rbe_cache }} - bazel_rbe_instance: ${{ inputs.bazel_rbe_instance }} - bazel_rbe_jobs: ${{ inputs.bazel_rbe_jobs }} - command_prefix: ${{ inputs.command_prefix }} - command_ci: ${{ inputs.command_ci }} - env: ${{ inputs.env }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - if: ${{ inputs.run_post }} - name: Run post action ${{ inputs.run_pre && format('({0})', inputs.run_post) || '' }} - uses: envoyproxy/toolshed/gh-actions/using/recurse@actions-v0.0.18 - with: - uses: ${{ inputs.run_post }} - with: ${{ inputs.run_post_with }} - - - run: | - echo "disk space at end of build:" - df -h - echo - du -ch "${{ runner.temp }}" | grep -E "[0-9]{2,}M|[0-9]G" - name: "Check disk space at end" diff --git a/.github/workflows/_env.yml b/.github/workflows/_env.yml deleted file mode 100644 index 3ec7f4082c7a..000000000000 --- a/.github/workflows/_env.yml +++ /dev/null @@ -1,187 +0,0 @@ -name: Environment - -permissions: - contents: read - -on: - workflow_call: - inputs: - # Authoritative configuration for build image/s - build_image_repo: - type: string - default: envoyproxy/envoy-build-ubuntu - build_image_sha: - type: string - default: d44499c6fd28a8a6a75dc61668b8a9e7bc3d99db11f9a61e8ea1d1f39c20a236 - build_image_mobile_sha: - type: string - default: b3cfc59c2fd1a86a2b12d303324f33d7f7248458233f3be2959fab119b11fa6f - build_image_tag: - type: string - default: 56f235b141079013e64912d676fe7da981368402 - - check_mobile_run: - type: boolean - default: true - prime_build_image: - type: boolean - default: false - - start_check_status: - type: string - default: - - repo_ref: - type: string - default: - repo_ref_sha: - type: string - default: - repo_ref_name: - type: string - default: - - outputs: - debug: - value: false - agent_ubuntu: - value: ubuntu-22.04 - build_image_ubuntu: - value: ${{ jobs.repo.outputs.build_image_ubuntu }} - build_image_ubuntu_mobile: - value: ${{ jobs.repo.outputs.build_image_ubuntu_mobile }} - mobile_android_build: - value: ${{ jobs.repo.outputs.mobile_android_build }} - mobile_android_build_all: - value: ${{ jobs.repo.outputs.mobile_android_build_all }} - mobile_android_tests: - value: ${{ jobs.repo.outputs.mobile_android_tests }} - mobile_asan: - value: ${{ jobs.repo.outputs.mobile_asan }} - mobile_cc_tests: - value: ${{ jobs.repo.outputs.mobile_cc_tests }} - mobile_compile_time_options: - value: ${{ jobs.repo.outputs.mobile_compile_time_options }} - mobile_coverage: - value: ${{ jobs.repo.outputs.mobile_coverage }} - mobile_formatting: - value: ${{ jobs.repo.outputs.mobile_formatting }} - mobile_ios_build: - value: ${{ jobs.repo.outputs.mobile_ios_build }} - mobile_ios_build_all: - value: ${{ jobs.repo.outputs.mobile_ios_build_all }} - mobile_ios_tests: - value: ${{ jobs.repo.outputs.mobile_ios_tests }} - mobile_release_validation: - value: ${{ jobs.repo.outputs.mobile_release_validation }} - mobile_tsan: - value: ${{ jobs.repo.outputs.mobile_tsan }} - - repo_ref: - value: ${{ jobs.repo.outputs.repo_ref }} - repo_ref_name: - value: ${{ jobs.repo.outputs.repo_ref_name }} - repo_ref_sha: - value: ${{ jobs.repo.outputs.repo_ref_sha }} - repo_ref_sha_short: - value: ${{ jobs.repo.outputs.repo_ref_sha_short }} - repo_ref_title: - value: ${{ jobs.repo.outputs.repo_ref_title }} - - trusted: - value: ${{ jobs.repo.outputs.trusted }} - - version_dev: - value: ${{ jobs.repo.outputs.version_dev }} - version_patch: - value: ${{ jobs.repo.outputs.version_patch }} - -concurrency: - group: | - ${{ github.actor != 'trigger-release-envoy[bot]' - && github.event.inputs.head_ref - || github.run_id - }}-${{ github.workflow }}-env - cancel-in-progress: true - -jobs: - repo: - if: github.repository == 'envoyproxy/envoy' - runs-on: ubuntu-22.04 - outputs: - build_image_ubuntu: ${{ steps.env.outputs.build_image_ubuntu }} - build_image_ubuntu_mobile: ${{ steps.env.outputs.build_image_ubuntu_mobile }} - mobile_android_build: ${{ steps.env.outputs.mobile_android_build }} - mobile_android_build_all: ${{ steps.env.outputs.mobile_android_build_all }} - mobile_android_tests: ${{ steps.env.outputs.mobile_android_tests }} - mobile_asan: ${{ steps.env.outputs.mobile_asan }} - mobile_cc_tests: ${{ steps.env.outputs.mobile_cc_tests }} - mobile_compile_time_options: ${{ steps.env.outputs.mobile_compile_time_options }} - mobile_coverage: ${{ steps.env.outputs.mobile_coverage }} - mobile_formatting: ${{ steps.env.outputs.mobile_formatting }} - mobile_ios_build: ${{ steps.env.outputs.mobile_ios_build }} - mobile_ios_build_all: ${{ steps.env.outputs.mobile_ios_build_all }} - mobile_ios_tests: ${{ steps.env.outputs.mobile_ios_tests }} - mobile_release_validation: ${{ steps.env.outputs.mobile_release_validation }} - mobile_tsan: ${{ steps.env.outputs.mobile_tsan }} - repo_ref: ${{ steps.env.outputs.repo_ref }} - repo_ref_name: ${{ steps.env.outputs.repo_ref_name }} - repo_ref_sha: ${{ steps.env.outputs.repo_ref_sha }} - repo_ref_sha_short: ${{ steps.env.outputs.repo_ref_sha_short }} - repo_ref_title: ${{ steps.env.outputs.repo_ref_title }} - trusted: ${{ steps.env.outputs.trusted }} - version_dev: ${{ steps.env.outputs.version_dev }} - version_patch: ${{ steps.env.outputs.version_patch }} - steps: - - uses: actions/checkout@v4 - name: Checkout Envoy repository - with: - fetch-depth: ${{ ! inputs.check_mobile_run && 1 || 0 }} - - uses: ./.github/actions/env - name: Generate environment variables - id: env - with: - check_mobile_run: ${{ inputs.check_mobile_run }} - repo_ref: ${{ inputs.repo_ref }} - repo_ref_name: ${{ inputs.repo_ref_name }} - repo_ref_sha: ${{ inputs.repo_ref_sha }} - build_image_repo: ${{ inputs.build_image_repo }} - build_image_tag: ${{ inputs.build_image_tag }} - build_image_mobile_sha: ${{ inputs.build_image_mobile_sha }} - build_image_sha: ${{ inputs.build_image_sha }} - - - name: 'Print env' - run: | - echo "version_dev=${{ steps.env.outputs.version_dev }}" - echo "version_patch=${{ steps.env.outputs.version_patch }}" - echo "trusted=${{ steps.env.outputs.trusted }}" - echo "repo_ref=${{ steps.env.outputs.repo_ref }}" - echo "repo_ref_name=${{ steps.env.outputs.repo_ref_name }}" - echo "repo_ref_pr_number=${{ steps.env.outputs.repo_ref_pr_number }}" - echo "repo_ref_sha=${{ steps.env.outputs.repo_ref_sha }}" - echo "repo_ref_sha_short=${{ steps.env.outputs.repo_ref_sha_short }}" - echo "repo_ref_title=${{ steps.env.outputs.repo_ref_title }}" - echo "build_image_ubuntu=${{ steps.env.outputs.build_image_ubuntu }}" - echo "build_image_ubuntu_mobile=${{ steps.env.outputs.build_image_ubuntu_mobile }}" - echo - if [[ -n "${{ steps.env.outputs.repo_ref_pr_number }}" ]]; then - echo "PR: https://github.com/envoyproxy/envoy/pull/${{ steps.env.outputs.repo_ref_pr_number }}" - fi - - check: - if: ${{ inputs.start_check_status && github.event_name != 'pull_request' }} - uses: ./.github/workflows/_workflow-start.yml - permissions: - contents: read - statuses: write - with: - workflow_name: ${{ inputs.start_check_status }} - sha: ${{ inputs.repo_ref_sha }} - - cache: - if: ${{ inputs.prime_build_image }} - uses: ./.github/workflows/_cache_docker.yml - with: - image_repo: ${{ inputs.build_image_repo }} - image_tag: ${{ inputs.build_image_tag }} - image_sha: ${{ inputs.build_image_sha }} diff --git a/.github/workflows/_finish.yml b/.github/workflows/_finish.yml new file mode 100644 index 000000000000..5aae4ba0c9aa --- /dev/null +++ b/.github/workflows/_finish.yml @@ -0,0 +1,115 @@ +name: Workflow/complete + +permissions: + contents: read + + +on: + # Do not run untrusted code here + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + inputs: + needs: + type: string + required: true + template-check-text: + type: string + default: | + ## \($icon) Check run finished (\($outcome.name) \($outcome.icon)) + + ## The check run can be viewed here: + + # \($icon) \($run_link) + +env: + CI_DEBUG: ${{ vars.CI_DEBUG && true || false }} + + +jobs: + complete: + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: read + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Incoming data + id: needs + with: + input: | + check_name: ${{ fromJSON(inputs.needs).load.outputs.check-name }} + repo: ${{ github.repository }} + run_id: ${{ github.run_id }} + outcomes: ${{ toJSON(fromJSON(inputs.needs).*.result) }} + load: ${{ toJSON(fromJSON(inputs.needs).load.outputs) }} + input-format: yaml + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + filter: | + .repo as $repo + | .run_id as $run_id + | .needs as $result + | .check_name as $check_name + | .load as $load + | $load["check-id"] as $check_id + | $load["run-id"] as $workflow_id + | (.load.request | fromjson) as $request + | $request.config.envoy.icon as $icon + | .outcomes + | if any(. == "failure") then + {name: "failure", icon: ":x:"} + elif any(. == "cancelled") then + {name: "cancelled", icon: ""} + elif all(. == "skipped") then + {name: "skipped", icon: ""} + else + {name: "success", icon: ":heavy_check_mark:"} + end + | . as $outcome + | "\($request.check.name) (\($request.summary.title))" as $run_link_text + | "[\($run_link_text)](https://github.com/\($repo)/actions/runs/\($run_id))" as $run_link + | "${{ inputs.template-check-text }}" as $text + | {"summary-title": "\($icon) \($request.check.name) complete (\($outcome.name))", + "check-id": $check_id, + conclusion: $outcome.name, + checks: { + ($check_name): { + name: $request.check.name, + head_sha: $request.request.sha, + status: "completed", + conclusion: $outcome.name, + external_id: "\($run_id)", + output: { + title: "\($request.check.name) (\($outcome.name))", + summary: "Check has finished", + text: $text}}}} + + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Print summary + with: + input: ${{ toJSON(steps.needs.outputs.value).summary-title }} + filter: | + "## \(.)" + options: -Rr + output-path: GITHUB_STEP_SUMMARY + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Appauth + id: appauth + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/github/checks@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Update check + with: + action: update + checks: ${{ toJSON(fromJSON(steps.needs.outputs.value).checks) }} + token: ${{ steps.appauth.outputs.token }} + + # This is necessary to ensure that any retests have their checks updated + - name: Fail the job + if: ${{ fromJSON(steps.needs.outputs.value).conclusion != 'success' }} + run: | + exit 1 diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml new file mode 100644 index 000000000000..ae8b21646189 --- /dev/null +++ b/.github/workflows/_load.yml @@ -0,0 +1,165 @@ +name: Request/load + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + lock-app-id: + required: true + lock-app-key: + required: true + + inputs: + agent-ubuntu: + type: string + default: ubuntu-22.04 + cache-docker: + type: boolean + default: true + check-name: + type: string + required: true + check-title: + type: string + default: + head-sha: + type: string + default: + run-id: + type: string + default: ${{ github.event.workflow_run.id }} + template-request-summary: + type: string + default: | + ## \($linkedTitle) + + \($summary) + + \($extra) + + outputs: + build-image: + value: ${{ jobs.request.outputs.build-image }} + build-image-mobile: + value: ${{ jobs.request.outputs.build-image-mobile }} + check-id: + value: ${{ jobs.request.outputs.check-id }} + check-name: + value: ${{ inputs.check-name }} + request: + value: ${{ jobs.request.outputs.request }} + run-id: + value: ${{ inputs.run-id }} + trusted: + value: ${{ jobs.request.outputs.trusted }} + +concurrency: + group: | + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-env + cancel-in-progress: true + +env: + CI_DEBUG: ${{ vars.CI_DEBUG && true || false }} + + +jobs: + request: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: read + pull-requests: read + outputs: + build-image: ${{ toJSON(fromJSON(steps.request-output.outputs.value).request.build-image) }} + build-image-mobile: ${{ fromJSON(steps.request-output.outputs.value).request.build-image-mobile }} + check-id: ${{ fromJSON(steps.request-output.outputs.value).check.check-id }} + request: ${{ steps.request-output.outputs.value }} + trusted: ${{ fromJSON(steps.request-output.outputs.value).request.trusted }} + skip: ${{ fromJSON(steps.request-output.outputs.value).check.action != 'RUN' }} + steps: + # Load env data + # Handle any failure in triggering job + # Remove any `checks` we dont care about + # Prepare a check request + - uses: envoyproxy/toolshed/gh-actions/github/env/load@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Load env + id: data + with: + run-id: ${{ inputs.run-id }} + check-name: ${{ inputs.check-name }} + head-sha: ${{ inputs.head-sha }} + env: + GH_TOKEN: ${{ github.token }} + + # Update the check + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Appauth + id: appauth + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/github/checks@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Update check + if: ${{ fromJSON(steps.data.outputs.data).data.check.action == 'RUN' }} + with: + action: update + checks: ${{ toJSON(fromJSON(steps.data.outputs.data).checks) }} + token: ${{ steps.appauth.outputs.token }} + + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Print request summary + with: + input: | + action: ${{ fromJSON(steps.data.outputs.data).data.check.action }} + summary: ${{ toJSON(fromJSON(steps.data.outputs.data).data.summary) }} + input-format: yaml + output-path: GITHUB_STEP_SUMMARY + options: -r + filter: | + .action as $action + | .summary as $summary + | if ($action != "RUN") then + "### ${{ github.workflow }} was skipped" + else "" end + | . as $extra + | $summary["linked-title"] as $linkedTitle + | $summary.summary as $summary + | "${{ inputs.template-request-summary }}" + + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: request-output + name: Load request + with: + input: | + check: ${{ toJSON(fromJSON(steps.data.outputs.data).data.check) }} + config: ${{ toJSON(fromJSON(steps.data.outputs.data).data.config) }} + request: ${{ toJSON(fromJSON(steps.data.outputs.data).data.request) }} + run: ${{ toJSON(fromJSON(steps.data.outputs.data).data.run) }} + summary_title: ${{ fromJSON(steps.data.outputs.data).data.summary.title }} + input-format: yaml + filter: | + . + | .summary = {title: .summary_title} + | del(.request.message, .summary_title) + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + + cache: + secrets: + app-id: ${{ secrets.lock-app-id }} + app-key: ${{ secrets.lock-app-key }} + uses: ./.github/workflows/_cache.yml + needs: request + if: ${{ inputs.cache-docker && ! fromJSON(needs.request.outputs.skip) }} + with: + request: ${{ toJSON(needs.request.outputs) }} + image-tag: ${{ fromJSON(needs.request.outputs.build-image).default }} diff --git a/.github/workflows/_load_env.yml b/.github/workflows/_load_env.yml new file mode 100644 index 000000000000..b5d19f3ab7e2 --- /dev/null +++ b/.github/workflows/_load_env.yml @@ -0,0 +1,114 @@ +name: Request/load + +permissions: + contents: read + +on: + workflow_call: + secrets: + lock-app-id: + required: true + lock-app-key: + required: true + + inputs: + branch-name: + type: string + default: main + cache-docker: + type: boolean + default: true + config-file: + type: string + default: ./.github/config.yml + event-name: + type: string + default: ${{ github.workflow }} + event-type: + type: string + default: ${{ github.event_name == 'workflow_dispatch' && 'dispatch' || 'scheduled' }} + trusted: + type: boolean + default: true + + outputs: + build-image: + value: ${{ jobs.request.outputs.build-image }} + build-image-mobile: + value: ${{ jobs.request.outputs.build-image-mobile }} + request: + value: ${{ jobs.request.outputs.request }} + trusted: + value: ${{ jobs.request.outputs.trusted }} + +concurrency: + group: | + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-env + cancel-in-progress: true + +env: + CI_DEBUG: ${{ vars.CI_DEBUG && true || false }} + + +jobs: + request: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + runs-on: ubuntu-22.04 + outputs: + build-image: ${{ toJSON(fromJSON(steps.env.outputs.data).request.build-image) }} + build-image-mobile: ${{ fromJSON(steps.env.outputs.data).request.build-image-mobile }} + request: ${{ steps.env.outputs.data }} + trusted: true + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: started + name: Create timestamp + with: + options: -r + filter: | + now + - uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: checkout + name: Checkout Envoy repository + - name: Generate environment variables + uses: envoyproxy/toolshed/gh-actions/envoy/ci/env@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: env + with: + branch-name: ${{ inputs.branch-name }} + config-file: ${{ inputs.config-file }} + started: ${{ steps.started.outputs.value }} + token: ${{ secrets.GITHUB_TOKEN }} + vars: ${{ toJSON(vars) }} + trusted: ${{ inputs.trusted }} + + - name: Request summary + id: summary + uses: envoyproxy/toolshed/gh-actions/github/env/summary@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + actor: ${{ toJSON(fromJSON(steps.env.outputs.data).request.actor) }} + base-sha: ${{ fromJSON(steps.env.outputs.data).request.base-sha }} + event-name: ${{ inputs.event-name }} + event-type: ${{ inputs.event-type }} + link: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} + output-path: GITHUB_STEP_SUMMARY + data: ${{ steps.env.outputs.data }} + tables: ${{ toJSON(fromJSON(steps.env.outputs.data).config.tables) }} + icon: ${{ fromJSON(steps.env.outputs.data).config.envoy.icon }} + message: ${{ fromJSON(steps.env.outputs.data).request.message }} + ref: ${{ fromJSON(steps.env.outputs.data).request.ref }} + sha: ${{ fromJSON(steps.env.outputs.data).request.sha }} + target-branch: ${{ fromJSON(steps.env.outputs.data).request.target-branch }} + + cache: + secrets: + app-id: ${{ secrets.lock-app-id }} + app-key: ${{ secrets.lock-app-key }} + uses: ./.github/workflows/_cache.yml + needs: request + if: ${{ inputs.cache-docker }} + with: + request: ${{ toJSON(needs.request.outputs) }} + image-tag: ${{ fromJSON(needs.request.outputs.build-image).default }} diff --git a/.github/workflows/_mobile_container_ci.yml b/.github/workflows/_mobile_container_ci.yml new file mode 100644 index 000000000000..da6b777c46e7 --- /dev/null +++ b/.github/workflows/_mobile_container_ci.yml @@ -0,0 +1,153 @@ +name: Mobile CI + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + app-key: + rbe-key: + ssh-key-extra: + inputs: + args: + type: string + catch-errors: + type: boolean + default: false + checkout-extra: + type: string + default: + command: + type: string + default: ./bazelw + container: + type: string + container-output: + type: string + default: + container-command: + type: string + default: >- + docker run + --volume=${PWD}:/source + --volume=${TMP_ENTRYPOINT}:/tmp/mobile-entrypoint.sh + --volume=/tmp/cache:/root/.cache + --volume=/tmp/container-output:/tmp/container-output + --workdir=/source/mobile + --entrypoint=/tmp/mobile-entrypoint.sh + -e GITHUB_TOKEN + -e CC + -e CXX + -e COVERAGE_THRESHOLD + -e BAZEL_BUILD_OPTION_LIST + -e MOBILE_DOCS_CHECKOUT_DIR + diskspace-hack: + type: boolean + default: false + downloads: + type: string + default: + entrypoint: + type: string + default: + entrypoint-DEFAULT: + type: string + default: | + #!/bin/bash -e + export PATH=/opt/llvm/bin:$PATH + exec "$@" + error-match: + type: string + default: | + ERROR + error: + Error: + notice-match: + type: string + default: | + NOTICE + Streaming build results + output-path: + type: string + default: /tmp/container-output + rbe: + type: boolean + default: true + ref: + type: string + request: + type: string + required: true + runs-on: + type: string + skip: + type: boolean + default: false + source: + type: string + default: + steps-pre: + type: string + steps-pre-name: + type: string + steps-post: + type: string + default: + steps-post-name: + type: string + target: + type: string + required: true + temp-dir: + type: string + timeout-minutes: + type: number + trusted: + type: boolean + default: false + upload-name: + type: string + upload-path: + type: string + warning-match: + type: string + default: | + WARNING + warning: + Warning: + + +jobs: + ci: + uses: ./.github/workflows/_run.yml + name: ${{ inputs.target }} + permissions: + contents: read + packages: read + secrets: + ssh-key-extra: ${{ secrets.ssh-key-extra }} + with: + args: ${{ inputs.args }} + rbe: ${{ inputs.rbe }} + # This always just caches the main build image, the mobile one is layered on top + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + catch-errors: ${{ inputs.catch-errors }} + container-command: ${{ inputs.container-command }} ${{ inputs.container || fromJSON(inputs.request).request.build-image.default }} + container-output: ${{ inputs.container-output }} + command: ${{ inputs.command }} + entrypoint: ${{ inputs.entrypoint || inputs.entrypoint-DEFAULT }} + downloads: ${{ inputs.downloads }} + error-match: ${{ inputs.error-match }} + notice-match: ${{ inputs.notice-match }} + output-path: ${{ inputs.output-path }} + request: ${{ inputs.request }} + source: ${{ inputs.source }} + steps-pre: ${{ inputs.steps-pre }} + steps-post: ${{ inputs.steps-post }} + target: ${{ inputs.target }} + trusted: ${{ fromJSON(inputs.request).request.trusted }} + upload-name: ${{ inputs.upload-name }} + upload-path: ${{ inputs.upload-path }} + warning-match: ${{ inputs.warning-match }} diff --git a/.github/workflows/_precheck_deps.yml b/.github/workflows/_precheck_deps.yml new file mode 100644 index 000000000000..df9157fe84c5 --- /dev/null +++ b/.github/workflows/_precheck_deps.yml @@ -0,0 +1,58 @@ +name: Precheck/deps + +permissions: + contents: read + +on: + workflow_call: + inputs: + dependency-review: + type: boolean + default: false + request: + type: string + required: true + trusted: + type: boolean + required: true + +concurrency: + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-publish + cancel-in-progress: true + + +jobs: + prechecks: + permissions: + contents: read + packages: read + uses: ./.github/workflows/_run.yml + name: ${{ matrix.target }} + with: + bazel-extra: '--config=rbe-envoy-engflow' + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + request: ${{ inputs.request }} + error-match: | + ERROR + error: + Error: + rbe: ${{ matrix.rbe }} + target: ${{ matrix.target }} + trusted: ${{ inputs.trusted }} + strategy: + matrix: + include: + - target: deps + rbe: false + + dependency-review: + runs-on: ubuntu-22.04 + if: ${{ inputs.dependency-review }} + steps: + - name: Checkout Repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ fromJSON(inputs.request).request.sha }} + persist-credentials: false + - name: Dependency Review + uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4 # v3.1.4 diff --git a/.github/workflows/_request.yml b/.github/workflows/_request.yml new file mode 100644 index 000000000000..11b1b945767d --- /dev/null +++ b/.github/workflows/_request.yml @@ -0,0 +1,121 @@ +name: Request/incoming + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + + # Defaults are set .github/config.yml on the `main` branch. + inputs: + config-file: + type: string + default: ./.github/config.yml + +concurrency: + group: | + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-env-prime + cancel-in-progress: true + +env: + CI_DEBUG: ${{ (vars.CI_DEBUG || vars.RUNNER_DEBUG) && true || false }} + + +jobs: + incoming: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + runs-on: ubuntu-22.04 + permissions: + contents: read + pull-requests: read + outputs: + env: ${{ steps.data.outputs.value }} + config: ${{ steps.config.outputs.config }} + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: started + name: Create timestamp + with: + options: -r + filter: | + now + - uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: checkout + name: Checkout Envoy repository + with: + pr: ${{ github.event.number }} + branch: ${{ github.ref_name }} + config: | + fetch-depth: ${{ startsWith(github.event_name, 'pull_request') && 1 || 2 }} + # This step *LOOKS AT* the repo at the point requested + # Its essential that this _job_ *MUST NOT EXECUTE ANY CODE FROM THE CHECKED OUT REPO* + # *ALL* variables collected should be treated as untrusted and should be sanitized before + # use + - name: Generate environment variables from commit + uses: envoyproxy/toolshed/gh-actions/envoy/ci/request@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: env + with: + branch-name: ${{ steps.checkout.outputs.branch-name }} + config-file: ${{ inputs.config-file }} + merge-commit: ${{ steps.checkout.outputs.merge-commit }} + started: ${{ steps.started.outputs.value }} + token: ${{ secrets.GITHUB_TOKEN }} + vars: ${{ toJSON(vars) }} + - name: Request summary + id: summary + uses: envoyproxy/toolshed/gh-actions/github/env/summary@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + actor: ${{ toJSON(fromJSON(steps.env.outputs.data).request.actor) }} + base-sha: ${{ fromJSON(steps.env.outputs.data).request.base-sha }} + link: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} + output-path: GITHUB_STEP_SUMMARY + pr: ${{ github.event.number }} + data: ${{ steps.env.outputs.data }} + tables: ${{ toJSON(fromJSON(steps.env.outputs.data).config.tables) }} + icon: ${{ fromJSON(steps.env.outputs.data).config.envoy.icon }} + message: ${{ fromJSON(steps.env.outputs.data).request.message }} + ref: ${{ fromJSON(steps.env.outputs.data).request.ref }} + sha: ${{ fromJSON(steps.env.outputs.data).request.sha }} + target-branch: ${{ fromJSON(steps.env.outputs.data).request.target-branch }} + + - name: Environment data + uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: data + with: + input: | + env: ${{ steps.env.outputs.data }} + title: ${{ steps.summary.outputs.title }} + link: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} + summary: ${{ steps.summary.outputs.summary }} + input-format: yaml + filter: | + .title as $title + | .env.config.envoy.icon as $icon + | .link as $link + | "\($icon) Request ([\($title)](\($link)))" as $linkedTitle + | .summary as $summary + | .env + | .summary = { + $summary, + $title, + $link, + "linked-title": $linkedTitle} + | del(.config.tables) + + checks: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + needs: incoming + uses: ./.github/workflows/_start.yml + secrets: + app-id: ${{ secrets.app-id }} + app-key: ${{ secrets.app-key }} + with: + env: ${{ needs.incoming.outputs.env }} diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml new file mode 100644 index 000000000000..2670d17a7ae7 --- /dev/null +++ b/.github/workflows/_run.yml @@ -0,0 +1,265 @@ +name: Envoy CI + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + app-key: + rbe-key: + ssh-key: + ssh-key-extra: + inputs: + args: + type: string + bazel-extra: + type: string + bazel-rbe-jobs: + type: number + default: 75 + cache-build-image: + type: string + catch-errors: + type: boolean + default: false + checkout-extra: + type: string + container-command: + type: string + default: ./ci/run_envoy_docker.sh + container-output: + type: string + default: + command: + type: string + default: ./ci/do_ci.sh + diskspace-hack: + type: boolean + default: false + diskspace-hack-paths: + type: string + default: + downloads: + type: string + entrypoint: + type: string + default: + error-match: + type: string + default: | + ERROR + error: + Error: + notice-match: + type: string + default: | + NOTICE + Streaming build results + output-path: + type: string + default: + rbe: + type: boolean + default: true + repo-fetch-depth: + type: number + default: 1 + report-pre: + type: string + default: | + - run: | + echo "disk space at beginning of build:" + df -h + shell: bash + report-post: + type: string + default: | + - run: | + echo "disk space at end of build:" + df -h + shell: bash + request: + type: string + required: true + runs-on: + type: string + default: + skip: + type: boolean + default: false + source: + type: string + summary-post: + type: string + default: | + - uses: envoyproxy/toolshed/gh-actions/envoy/run/summary@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + context: %{{ inputs.context }} + steps-pre: + type: string + steps-pre-name: + type: string + steps-post: + type: string + default: | + - run: | + du -ch "%{{ inputs.temp-dir || runner.temp }}" | grep -E "[0-9]{2,}M|[0-9]G" || : + shell: bash + steps-post-name: + type: string + target: + type: string + required: true + temp-dir: + type: string + timeout-minutes: + type: number + trusted: + type: boolean + required: true + upload-name: + type: string + upload-path: + type: string + warning-match: + type: string + default: | + WARNING + warning: + Warning: + working-directory: + type: string + default: . + +concurrency: + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.head_ref + || github.run_id + }}-${{ github.workflow }}-${{ inputs.target }} + cancel-in-progress: true + +env: + CI_DEBUG: ${{ vars.CI_DEBUG }} + + +jobs: + ci: + permissions: + contents: read + packages: read + if: ${{ ! inputs.skip }} + runs-on: ${{ inputs.runs-on || fromJSON(inputs.request).config.ci.agent-ubuntu }} + name: ${{ inputs.command }} ${{ inputs.target }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: started + name: Create timestamp + with: + options: -r + filter: | + now + # This controls which input vars are exposed to the run action (and related steps) + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Context + id: context + with: + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + input: ${{ inputs.request }} + filter: | + . + | (.check // {name: "${{ github.workflow }}"}) as $check + | .config as $config + | if "${{ inputs.runs-on }}" != "" then + "${{ inputs.runs-on }}" + else .config.ci["agent-ubuntu"] end + | . as $runsOn + | {"target": "${{ inputs.target }}", + "catch-errors": ${{ inputs.catch-errors }}, + "runs-on": $runsOn, + "job-started": ${{ steps.started.outputs.value }}} + | . * {$config, $check} + - if: ${{ inputs.cache-build-image }} + name: Restore Docker cache ${{ inputs.cache-build-image && format('({0})', inputs.cache-build-image) || '' }} + uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + image_tag: ${{ inputs.cache-build-image }} + + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: appauth + name: Appauth + if: ${{ inputs.trusted }} + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + # You cant use a secret as a condition so this always runs even if the app id/key are empty + # - the workaround is to allow the token to be passed through. + token: ${{ github.token }} + token-ok: true + - uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: checkout + name: Checkout Envoy repository + with: + branch: ${{ fromJSON(inputs.request).request.target-branch }} + config: | + fetch-depth: ${{ inputs.repo-fetch-depth }} + # WARNING: This allows untrusted code to run!!! + # If this is set to run untrusted code, then anything before or after in the job should be regarded as + # compromisable. + ref: ${{ inputs.trusted && fromJSON(inputs.request).request.sha || fromJSON(inputs.request).request.ref }} + fetch-merge-commit: false + pr: ${{ fromJSON(inputs.request).request.pr }} + ssh-key: ${{ inputs.trusted && inputs.ssh-key || '' }} + token: ${{ inputs.trusted && steps.appauth.outputs.token || github.token }} + + # This is currently only use by mobile-docs and can be removed once they are updated to the newer website + - uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: checkout-extra + name: Checkout extra repository (for publishing) + if: ${{ inputs.checkout-extra }} + with: + config: ${{ inputs.checkout-extra }} + ssh-key: ${{ inputs.trusted && inputs.ssh-key-extra || '' }} + + - uses: envoyproxy/toolshed/gh-actions/github/run@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Run CI ${{ inputs.command }} ${{ inputs.target }} + with: + args: ${{ inputs.args != '--' && inputs.args || inputs.target }} + catch-errors: ${{ inputs.catch-errors }} + command: ${{ inputs.command }} + container-command: ${{ env.CONTAINER_COMMAND || inputs.container-command }} + container-output: ${{ inputs.container-output }} + context: ${{ steps.context.outputs.value }} + diskspace-hack: ${{ inputs.diskspace-hack }} + diskspace-hack-paths: ${{ inputs.diskspace-hack-paths }} + downloads: ${{ inputs.downloads }} + entrypoint: ${{ inputs.entrypoint }} + error-match: ${{ inputs.error-match }} + notice-match: ${{ inputs.notice-match }} + output-path: ${{ inputs.output-path }} + report-pre: ${{ inputs.report-pre }} + report-post: ${{ inputs.report-post }} + source: ${{ inputs.source }} + steps-pre: ${{ inputs.steps-pre }} + steps-pre-name: ${{ inputs.steps-pre-name }} + steps-post: ${{ inputs.steps-post }} + steps-post-name: ${{ inputs.steps-post-name }} + summary-post: ${{ inputs.summary-post }} + upload-name: ${{ inputs.upload-name }} + upload-path: ${{ inputs.upload-path }} + warning-match: ${{ inputs.warning-match }} + working-directory: ${{ inputs.working-directory }} + env: + GITHUB_TOKEN: ${{ steps.checkout.outputs.token && steps.checkout.outputs.token || secrets.GITHUB_TOKEN }} + ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }} + ENVOY_RBE: ${{ inputs.rbe != 'false' && 1 || '' }} + RBE_KEY: ${{ secrets.rbe-key }} + BAZEL_BUILD_EXTRA_OPTIONS: >- + --config=remote-ci + ${{ inputs.bazel-extra }} + ${{ inputs.rbe != 'false' && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }} + BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }} + CI_TARGET_BRANCH: ${{ fromJSON(inputs.request).request.target-branch }} diff --git a/.github/workflows/_stage_publish.yml b/.github/workflows/_stage_publish.yml index 2b0dcca963cc..374cf0e6e1fa 100644 --- a/.github/workflows/_stage_publish.yml +++ b/.github/workflows/_stage_publish.yml @@ -11,82 +11,103 @@ permissions: on: workflow_call: + secrets: + ENVOY_CI_SYNC_APP_ID: + ENVOY_CI_SYNC_APP_KEY: + ENVOY_CI_PUBLISH_APP_ID: + ENVOY_CI_PUBLISH_APP_KEY: inputs: + request: + type: string + required: true trusted: type: boolean - default: false - build_image_ubuntu: - type: string - default: '' - version_dev: - type: string - default: '' - head_ref: - type: string - default: '' - repo_ref: - type: string - given_ref: - type: string + required: true concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-publish + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.event.inputs.head_ref + || github.run_id + }}-${{ github.event.workflow.id }}-publish cancel-in-progress: true + jobs: - publish_ci: - if: ${{ ! inputs.trusted }} + publish: + secrets: + app-id: ${{ inputs.trusted && secrets.ENVOY_CI_PUBLISH_APP_ID || '' }} + app-key: ${{ inputs.trusted && secrets.ENVOY_CI_PUBLISH_APP_KEY || '' }} + permissions: + contents: read + packages: read name: ${{ matrix.name || matrix.target }} - strategy: - fail-fast: false - matrix: - include: - - target: publish - name: github - run_pre: ./.github/actions/publish/release/setup - run_pre_with: | - ref: ${{ inputs.given_ref }} - bucket: envoy-pr - env: | - export ENVOY_PUBLISH_DRY_RUN=1 - uses: ./.github/workflows/_ci.yml + uses: ./.github/workflows/_run.yml with: target: ${{ matrix.target }} rbe: false - managed: true - cache_build_image: ${{ inputs.build_image_ubuntu }} - run_pre: ${{ matrix.run_pre }} - run_pre_with: ${{ matrix.run_pre_with }} - env: ${{ matrix.env }} - trusted: false - repo_ref: ${{ inputs.repo_ref }} - - publish: - if: ${{ inputs.trusted }} - name: ${{ matrix.name || matrix.target }} - permissions: - contents: write + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + source: ${{ matrix.source }} + request: ${{ inputs.request }} + steps-pre: ${{ matrix.steps-pre }} + trusted: ${{ inputs.trusted }} strategy: fail-fast: false matrix: include: - target: publish name: github - run_pre: ./.github/actions/publish/release/setup - run_pre_with: | - ref: ${{ inputs.given_ref }} - bucket: envoy-postsubmit - env: | - if [[ '${{ inputs.version_dev }}' != '' ]]; then - export ENVOY_PUBLISH_DRY_RUN=1 - fi - uses: ./.github/workflows/_ci.yml - with: - target: ${{ matrix.target }} - rbe: false - managed: true - cache_build_image: ${{ inputs.build_image_ubuntu }} - run_pre: ${{ matrix.run_pre }} - run_pre_with: ${{ matrix.run_pre_with }} - env: ${{ matrix.env }} - trusted: true + source: | + export ENVOY_COMMIT=${{ fromJSON(inputs.request).request.sha }} + export ENVOY_REPO=${{ github.repository }} + export ENVOY_PUBLISH_DRY_RUN=${{ (fromJSON(inputs.request).request.version.dev || ! inputs.trusted) && 1 || '' }} + steps-pre: | + - id: url + uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + options: -Rr + input: >- + ${{ inputs.trusted + && fromJSON(inputs.request).request.sha + || fromJSON(inputs.request).request.ref }} + filter: | + .[:7] as $sha + | "release/release.signed.tar.zst" as $path + | if ${{ inputs.trusted }} then + "envoy-postsubmit" + else + "envoy-pr" + end + | . as $bucket + | "https://storage.googleapis.com/\($bucket)/\($sha)/\($path)" + - uses: envoyproxy/toolshed/gh-actions/fetch@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + url: %{{ steps.url.outputs.value }} + path: %{{ runner.temp }}/release.signed + + publish_docs: + # For normal commits to Envoy main this will trigger an update in the website repo, + # which will update its envoy dep shas, and rebuild the website for the latest docs + # + # For commits that create a release, it instead triggers an update in the archive repo, + # which builds a static version of the docs for the release and commits it to the archive. + # In turn the archive repo triggers an update in the website so the new release docs are + # included in the published site + if: ${{ inputs.trusted && github.repository == 'envoyproxy/envoy' }} + runs-on: ubuntu-22.04 + needs: + - publish + steps: + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: appauth + with: + app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} + key: ${{ secrets.ENVOY_CI_SYNC_APP_KEY }} + - uses: envoyproxy/toolshed/gh-actions/dispatch@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + ref: main + repository: ${{ fromJSON(inputs.request).request.version.dev && 'envoyproxy/envoy-website' || 'envoyproxy/archive' }} + token: ${{ steps.appauth.outputs.token }} + workflow: envoy-sync.yaml + inputs: | + commit_sha: ${{ fromJSON(inputs.request).request.version.dev && github.sha || '' }} diff --git a/.github/workflows/_stage_verify.yml b/.github/workflows/_stage_verify.yml index a9dcf195c5db..31b12d98666b 100644 --- a/.github/workflows/_stage_verify.yml +++ b/.github/workflows/_stage_verify.yml @@ -6,48 +6,84 @@ permissions: on: workflow_call: inputs: + request: + type: string + required: true trusted: type: boolean - default: false - repo_ref: - type: string - given_ref: - type: string + required: true concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }}-verify + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.event.inputs.head_ref + || github.run_id + }}-${{ github.event.workflow.id }}-verify cancel-in-progress: true + jobs: verify: + permissions: + contents: read + packages: read name: ${{ matrix.name || matrix.target }} + uses: ./.github/workflows/_run.yml + with: + cache-build-image: + container-command: + rbe: ${{ matrix.rbe }} + request: ${{ inputs.request }} + runs-on: envoy-x64-small + steps-pre: ${{ matrix.steps-pre }} + source: ${{ matrix.source }} + target: ${{ matrix.target }} + trusted: ${{ inputs.trusted }} strategy: fail-fast: false matrix: include: - - target: verify_examples - name: examples - rbe: false - managed: true - cache_build_image: "" - command_prefix: "" - diskspace_hack: true - run_pre: ./.github/actions/verify/examples/setup - run_pre_with: | - bucket: envoy-${{ inputs.trusted && 'postsubmit' || 'pr' }} - ref: ${{ inputs.given_ref }} - env: | + - name: examples + target: verify_examples + source: | export NO_BUILD_SETUP=1 - uses: ./.github/workflows/_ci.yml - with: - target: ${{ matrix.target }} - rbe: ${{ matrix.rbe }} - managed: ${{ matrix.managed }} - cache_build_image: ${{ matrix.cache_build_image }} - diskspace_hack: ${{ matrix.diskspace_hack }} - command_prefix: ${{ matrix.command_prefix }} - run_pre: ${{ matrix.run_pre }} - run_pre_with: ${{ matrix.run_pre_with }} - env: ${{ matrix.env }} - trusted: ${{ inputs.trusted }} - repo_ref: ${{ ! inputs.trusted && inputs.repo_ref || '' }} + rbe: false + steps-pre: | + - id: url + uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + options: -Rr + input: >- + ${{ inputs.trusted + && fromJSON(inputs.request).request.sha + || fromJSON(inputs.request).request.ref }} + filter: | + .[:7] as $sha + | if ${{ inputs.trusted }} then + "envoy-postsubmit" + else + "envoy-pr" + end + | . as $bucket + | "https://storage.googleapis.com/\($bucket)/\($sha)" + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + url: %{{ steps.url.outputs.value }}/docker/envoy.tar + variant: dev + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar + variant: contrib-dev + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar + variant: google-vrp-dev + - run: docker images | grep envoy + shell: bash + - run: | + # Install expected host packages + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -qq update -y + sudo apt-get -qq install -y --no-install-recommends expect gettext whois + pip install -r ./.github/workflows/verify-requirements.txt + shell: bash diff --git a/.github/workflows/_start.yml b/.github/workflows/_start.yml new file mode 100644 index 000000000000..7f084737130d --- /dev/null +++ b/.github/workflows/_start.yml @@ -0,0 +1,132 @@ +name: Workflow start +# This workflow is only required for externally triggered jobs that need to manually +# set the check status for a commit/PR + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + inputs: + details-url: + type: string + default: >- + https://github.com/envoyproxy/envoy/tree/main/.github/workflows + env: + type: string + required: true + run-summary: + type: string + default: >- + The check will start once any required jobs have completed and a VM becomes available + run-title: + type: string + default: >- + Waiting for check ... + skipped-summary: + type: string + default: >- + This check was not triggered in this CI run + skipped-title: + type: string + default: >- + Check was skipped + template-run-text: + type: string + default: | + ## \($icon) Check run pending + + ## Details of the check run will be provided here once it has started. + + ### Check started by + + +env: + CI_DEBUG: ${{ (vars.CI_DEBUG || vars.RUNNER_DEBUG) && true || false }} + + +jobs: + start: + runs-on: ubuntu-22.04 + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: check-config + name: Prepare check data + with: + print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} + input: ${{ inputs.env }} + filter: | + . as $env + | .config.envoy.icon as $icon + | {} + | .["head_sha"] = $env.request.sha + | .details_url = "${{ inputs.details-url }}" + | {run: ., skipped: ., request: $env.summary.summary} + | .run.output.title = "${{ inputs.run-title }}" + | .run.output.summary = "${{ inputs.run-summary }}" + | .run.output.text = "${{ inputs.template-run-text }}" + | .run.status = "queued" + | .skipped.status = "completed" + | .skipped.conclusion = "skipped" + | .skipped.output.title = "${{ inputs.skipped-title }}" + | .skipped.output.summary = "${{ inputs.skipped-summary }}" + | .skipped.output.text = "" + + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Appauth + id: appauth + with: + app_id: ${{ secrets.app-id }} + key: ${{ secrets.app-key }} + - uses: envoyproxy/toolshed/gh-actions/github/checks@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Start checks + id: checks + with: + checks: ${{ toJSON(fromJSON(inputs.env).checks) }} + config: ${{ steps.check-config.outputs.value }} + text-extra: | + ## ${{ fromJSON(inputs.env).summary.linked-title }} + + ${{ fromJSON(inputs.env).summary.summary }} + token: ${{ steps.appauth.outputs.token }} + - uses: envoyproxy/toolshed/gh-actions/json/table@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Summary + with: + collapse-open: true + json: | + {"checks": ${{ steps.checks.outputs.checks }}, + "config": ${{ toJSON(fromJSON(inputs.env).checks) }}} + filter: | + .checks + heading: >- + ${{ fromJSON(inputs.env).config.envoy.icon }} Checks + mutate-cells: | + .cell as $cell + | .row as $row + | .table as $table + | $cell + | if ($row | index($cell) == 0) then + $table.data.config[$cell].name + elif ($table.data.config[$row[0]].action != "SKIP") then + "[started](http://github.com/${{ github.repository }}/runs/\($cell))" + else "skipped" end + output-path: GITHUB_STEP_SUMMARY + title: Checks started/skipped + + - uses: envoyproxy/toolshed/gh-actions/github/env/save@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Save env + id: data + with: + env: ${{ inputs.env }} + env-filter: | + ${{ steps.checks.outputs.checks }} as $checksStarted + | .checks + |= with_entries( + if $checksStarted[.key] != "skipped" then + .value["check-id"] = $checksStarted[.key] + else . end) diff --git a/.github/workflows/_workflow-start.yml b/.github/workflows/_workflow-start.yml deleted file mode 100644 index 0a8944c25c0e..000000000000 --- a/.github/workflows/_workflow-start.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Workflow start -# This workflow is only required for externally triggered jobs that need to manually -# set the check status for a commit/PR - -permissions: - contents: read - -on: - workflow_call: - inputs: - workflow_name: - required: true - type: string - sha: - required: true - type: string - -jobs: - start: - runs-on: ubuntu-22.04 - permissions: - statuses: write - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/env - id: env - with: - check_mobile_run: false - - - if: ${{ steps.env.outputs.trusted != 'true' }} - name: Start status check - uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.18 - with: - authToken: ${{ secrets.GITHUB_TOKEN }} - context: ${{ inputs.workflow_name }} - state: 'pending' - sha: ${{ inputs.sha }} - target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - if: ${{ steps.env.outputs.trusted != 'true' }} - name: Save the SHA - env: - STATE_SHA: ${{ inputs.sha }} - run: | - mkdir -p ./sha - echo $STATE_SHA > ./sha/state_sha - - if: ${{ steps.env.outputs.trusted != 'true' }} - uses: actions/upload-artifact@v3 - with: - name: state_sha - path: sha/ diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml deleted file mode 100644 index eae216b3a1a0..000000000000 --- a/.github/workflows/check-deps.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Check dependencies - -permissions: - contents: read - -on: - schedule: - - cron: '0 8 * * *' - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-22.04 - if: >- - ${{ - github.repository == 'envoyproxy/envoy' - && (github.event.schedule - || !contains(github.actor, '[bot]')) - }} - permissions: - contents: read # to fetch code (actions/checkout) - issues: write # required to open/close dependency issues - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - name: Set up Python (3.10) - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 - with: - python-version: "3.10" - - - name: Run dependency checker - run: | - TODAY_DATE=$(date -u -I"date") - export TODAY_DATE - bazel run //tools/dependency:check --action_env=TODAY_DATE -- -c release_issues --fix - bazel run //tools/dependency:check --action_env=TODAY_DATE -- -c cves -w error - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index cd05ce9ac045..9eba30588dac 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -1,3 +1,8 @@ +name: CodeQL/daily + +permissions: + contents: read + on: schedule: - cron: '0 12 * * 4' @@ -6,11 +11,13 @@ concurrency: group: ${{ github.head_ref-github.workflow || github.run_id }} cancel-in-progress: true + jobs: CodeQL-Build: permissions: security-events: write # for github/codeql-action/analyze to upload SARIF results + pull-requests: read strategy: fail-fast: false @@ -20,20 +27,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a + uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # codeql-bundle-v2.13.4 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -64,4 +62,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a + uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a # codeql-bundle-v2.13.4 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 0ce0abf6bd33..a3773944f4b2 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -1,5 +1,8 @@ name: CodeQL +permissions: + contents: read + on: push: paths: @@ -12,11 +15,13 @@ concurrency: group: ${{ github.head_ref-github.workflow || github.run_id }} cancel-in-progress: true + jobs: CodeQL-Build: permissions: security-events: write # for github/codeql-action/analyze to upload SARIF results + pull-requests: read strategy: fail-fast: false @@ -25,12 +30,14 @@ jobs: if: github.repository == 'envoyproxy/envoy' steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Pre-cleanup + uses: envoyproxy/toolshed/gh-actions/diskspace@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 + to_remove: | + /usr/local/lib/android + + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Get build targets run: | @@ -38,14 +45,10 @@ jobs: echo 'BUILD_TARGETS<> $GITHUB_ENV echo $BUILD_TARGETS_LOCAL >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a + uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # codeql-bundle-v2.13.4 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -78,4 +81,4 @@ jobs: - name: Perform CodeQL Analysis if: env.BUILD_TARGETS != '' - uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a + uses: github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a # codeql-bundle-v2.13.4 diff --git a/.github/workflows/command.yml b/.github/workflows/command.yml new file mode 100644 index 000000000000..fb8d3e144117 --- /dev/null +++ b/.github/workflows/command.yml @@ -0,0 +1,57 @@ +name: Command + +# NB: **ALL** commands should be permissionless and only use an app token or relevant secrets +# specific to their requirements! +permissions: + contents: read + +on: + issue_comment: + types: + - created + +env: + CI_DEBUG: ${{ vars.CI_DEBUG }} + + +jobs: + # For speed and _security_ only a single command (first matching) will be parsed/run from a comment + command: + name: Parse and run command + runs-on: ubuntu-22.04 + if: >- + ${{ + github.event.issue.pull_request + && (vars.ENVOY_CI + || github.repository == 'envoyproxy/envoy') + && github.actor != 'repokitteh-read-only[bot]' + && github.actor != 'dependabot[bot]' + }} + steps: + - uses: envoyproxy/toolshed/gh-actions/github/command@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Parse command from comment + id: command + with: + text: ${{ github.event.comment.body }} + matching: >- + ^/(retest) + + # /retest + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + if: ${{ steps.command.outputs.command == 'retest' }} + id: appauth-retest + name: Appauth (retest) + with: + key: ${{ secrets.ENVOY_CI_APP_KEY }} + app_id: ${{ secrets.ENVOY_CI_APP_ID }} + - uses: envoyproxy/toolshed/gh-actions/retest@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + if: ${{ steps.command.outputs.command == 'retest' }} + name: Retest + with: + token: ${{ steps.appauth-retest.outputs.token }} + azp_org: cncf + azp_token: ${{ secrets.AZP_TOKEN }} + comment-id: ${{ github.event.comment.id }} + pr-url: ${{ github.event.issue.pull_request.url }} + args: ${{ steps.command.outputs.args }} + app-owner: ci-envoy diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml deleted file mode 100644 index 6b964d4bb147..000000000000 --- a/.github/workflows/commands.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: commands - -on: - issue_comment: - types: - - created - -permissions: - contents: read - -jobs: - retest: - if: >- - ${{ - github.event.issue.pull_request - && github.repository == 'envoyproxy/envoy' - && github.actor != 'repokitteh-read-only[bot]' - && github.actor != 'dependabot[bot]' - }} - name: Retest - runs-on: ubuntu-22.04 - permissions: - pull-requests: write - actions: write - checks: read - steps: - - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.0.18 - with: - token: ${{ secrets.GITHUB_TOKEN }} - azp_org: cncf - azp_token: ${{ secrets.AZP_TOKEN }} diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml deleted file mode 100644 index 3890070d58d5..000000000000 --- a/.github/workflows/depsreview.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: 'Dependency Review' -on: [pull_request] - -concurrency: - group: ${{ github.head_ref-github.workflow || github.run_id }} - cancel-in-progress: true - -jobs: - dependency-review: - runs-on: ubuntu-22.04 - if: github.repository == 'envoyproxy/envoy' - steps: - - name: 'Checkout Repository' - uses: actions/checkout@v4 - - name: 'Dependency Review' - uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 diff --git a/.github/workflows/envoy-dependency.yml b/.github/workflows/envoy-dependency.yml new file mode 100644 index 000000000000..623aa1a2a586 --- /dev/null +++ b/.github/workflows/envoy-dependency.yml @@ -0,0 +1,246 @@ +name: Envoy/dependency + +permissions: + contents: read + +on: + schedule: + - cron: '0 8 * * *' + workflow_dispatch: + inputs: + task: + description: Select a task + required: true + default: bazel + type: choice + options: + - bazel + - bazel-api + - build-image + - check + dependency: + description: Dependency to update (if applicable) + version: + description: Version to set (optional) + pr: + type: boolean + default: true + pr-message: + description: Additional message for PR, eg to fix an issue (optional) + +concurrency: + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + +env: + COMMITTER_NAME: dependency-envoy[bot] + COMMITTER_EMAIL: 148525496+dependency-envoy[bot]@users.noreply.github.com + +jobs: + update-bazel: + if: >- + ${{ + github.event_name == 'workflow_dispatch' + && startsWith(inputs.task, 'bazel') + }} + name: >- + Update dep + (${{ inputs.pr && 'PR/' || '' }}${{ inputs.task == 'bazel' && 'bazel' || 'bazel/api' }}/${{ inputs.dependency }}/${{ inputs.version }}) + runs-on: ubuntu-22.04 + steps: + - id: appauth + name: Appauth + uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app_id: ${{ secrets.ENVOY_CI_DEP_APP_ID }} + key: ${{ secrets.ENVOY_CI_DEP_APP_KEY }} + - id: checkout + name: Checkout Envoy repository + uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + token: ${{ steps.appauth.outputs.token }} + - uses: envoyproxy/toolshed/gh-actions/bson@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: update + name: Update dependency (${{ inputs.dependency }}) + with: + input: | + dependency: ${{ inputs.dependency }} + task: ${{ inputs.task }} + version: "${{ inputs.version }}" + input-format: yaml + filter: | + .version as $version + | .dependency as $dependency + | .task as $task + | (try ($version | validate::sha(40) | .[:7]) + catch $version) as $version_short + | {} + | if $task == "bazel" then + . + | .task = "bazel" + | .target = "update" + else + . + | .task = "api/bazel" + | .target = "api-update" + end + | .task as $task + | .target as $target + | (" + echo \"Updating(\($task)): \($dependency) -> \($version_short)\" + bazel run --config=ci //bazel:\($target) \($dependency) \($version) + OUTPUT=\($version_short) + " | bash::output) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: envoyproxy/toolshed/gh-actions/upload/diff@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Upload diff + with: + name: ${{ inputs.dependency }}-${{ steps.update.outputs.output }} + - name: Create a PR + if: ${{ inputs.pr }} + uses: envoyproxy/toolshed/gh-actions/github/pr@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + base: main + body: | + Created by Envoy dependency bot for @${{ github.actor }} + + ${{ inputs.pr-message }} + branch: >- + dependency/${{ inputs.task }}/${{ inputs.dependency }}/${{ steps.update.outputs.output }} + commit-message: | + ${{ inputs.task == 'bazel' && 'deps' || 'deps/api' }}: Bump `${{ inputs.dependency }}` -> ${{ steps.update.outputs.output }} + + Signed-off-by: ${{ env.COMMITTER_NAME }} <${{ env.COMMITTER_EMAIL }}> + committer-name: ${{ env.COMMITTER_NAME }} + committer-email: ${{ env.COMMITTER_EMAIL }} + title: >- + ${{ inputs.task == 'bazel' && 'deps' || 'deps/api' }}: Bump `${{ inputs.dependency }}` + -> ${{ steps.update.outputs.output }} + GITHUB_TOKEN: ${{ steps.appauth.outputs.token }} + + update-build-image: + if: >- + ${{ + github.event_name == 'workflow_dispatch' + && github.event.inputs.task == 'build-image' + }} + name: Update build image (PR) + runs-on: ubuntu-22.04 + steps: + - id: appauth + name: Appauth + uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app_id: ${{ secrets.ENVOY_CI_DEP_APP_ID }} + key: ${{ secrets.ENVOY_CI_DEP_APP_KEY }} + - uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: checkout + name: Checkout Envoy repository + with: + config: | + path: envoy + fetch-depth: 0 + token: ${{ steps.appauth.outputs.token }} + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + name: Checkout Envoy build tools repository + with: + repository: envoyproxy/envoy-build-tools + path: build-tools + fetch-depth: 0 + - run: | + shas=( + tag + sha + mobile-sha + gcr-sha) + for sha in "${shas[@]}"; do + current_sha=$(bazel run --config=ci //tools/dependency:build-image-sha "$sha") + echo "${sha}=${current_sha}" >> "$GITHUB_OUTPUT" + done + id: current + name: Current SHAs + working-directory: envoy + - run: | + if [[ -z "$CONTAINER_TAG" ]]; then + # get current build image version + CONTAINER_TAG=$(git log -1 --pretty=format:"%H" "./docker") + fi + echo "tag=${CONTAINER_TAG}" >> "$GITHUB_OUTPUT" + echo "tag_short=${CONTAINER_TAG::7}" >> "$GITHUB_OUTPUT" + env: + CONTAINER_TAG: ${{ inputs.version }} + id: build-tools + name: Build image SHA + working-directory: build-tools + + - name: Check Docker SHAs + id: build-images + uses: envoyproxy/toolshed/gh-actions/docker/shas@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + images: | + sha: envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }} + mobile-sha: envoyproxy/envoy-build-ubuntu:mobile-${{ steps.build-tools.outputs.tag }} + gcr-sha: gcr.io/envoy-ci/envoy-build:${{ steps.build-tools.outputs.tag }} + + - run: | + SHA_REPLACE=( + "$CURRENT_ENVOY_TAG:$ENVOY_TAG" + "$CURRENT_ENVOY_SHA:${{ fromJSON(steps.build-images.outputs.shas).sha }}" + "$CURRENT_ENVOY_MOBILE_SHA:${{ fromJSON(steps.build-images.outputs.shas).mobile-sha }}" + "$CURRENT_ENVOY_GCR_SHA:${{ fromJSON(steps.build-images.outputs.shas).gcr-sha }}") + echo "replace=${SHA_REPLACE[*]}" >> "$GITHUB_OUTPUT" + name: Find SHAs to replace + id: shas + env: + ENVOY_TAG: ${{ steps.build-tools.outputs.tag }} + CURRENT_ENVOY_TAG: ${{ steps.current.outputs.tag }} + CURRENT_ENVOY_SHA: ${{ steps.current.outputs.sha }} + CURRENT_ENVOY_MOBILE_SHA: ${{ steps.current.outputs.mobile-sha }} + CURRENT_ENVOY_GCR_SHA: ${{ steps.current.outputs.gcr-sha }} + - run: | + echo "${SHA_REPLACE}" | xargs bazel run --config=ci @envoy_toolshed//sha:replace "${PWD}" + env: + SHA_REPLACE: ${{ steps.shas.outputs.replace }} + name: Update SHAs + working-directory: envoy + - name: Create a PR + uses: envoyproxy/toolshed/gh-actions/github/pr@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + base: main + body: Created by Envoy dependency bot + branch: dependency-envoy/build-image/latest + committer-name: ${{ env.COMMITTER_NAME }} + committer-email: ${{ env.COMMITTER_EMAIL }} + commit-message: | + deps: Bump build images -> `${{ steps.build-tools.outputs.tag_short }}` + + Signed-off-by: ${{ env.COMMITTER_NAME }} <${{ env.COMMITTER_EMAIL }}> + title: 'deps: Bump build images -> `${{ steps.build-tools.outputs.tag_short }}`' + GITHUB_TOKEN: ${{ steps.appauth.outputs.token }} + working-directory: envoy + + scheduled: + runs-on: ubuntu-22.04 + if: >- + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || (!contains(github.actor, '[bot]') + && inputs.task == 'check')) + }} + permissions: + contents: read + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Run dependency checker + run: | + TODAY_DATE=$(date -u -I"date") + export TODAY_DATE + bazel run --config=ci //tools/dependency:check --action_env=TODAY_DATE -- -c release_issues --fix + bazel run --config=ci //tools/dependency:check --action_env=TODAY_DATE -- -c cves -w error + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/envoy-macos.yml b/.github/workflows/envoy-macos.yml new file mode 100644 index 000000000000..f432d866ee82 --- /dev/null +++ b/.github/workflows/envoy-macos.yml @@ -0,0 +1,94 @@ +name: Envoy/macOS + +permissions: + contents: read + +on: + workflow_run: + workflows: + - Request + types: + - completed + +concurrency: + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + + +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} + permissions: + actions: read + contents: read + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: macos + + macos: + permissions: + contents: read + packages: read + secrets: + rbe-key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + if: ${{ fromJSON(needs.load.outputs.request).run.build-macos }} + needs: + - load + uses: ./.github/workflows/_run.yml + name: CI ${{ matrix.name || matrix.target }} + with: + command: + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12-xl + steps-post: + steps-pre: ${{ matrix.steps-pre }} + target: ${{ matrix.target }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + strategy: + fail-fast: false + matrix: + include: + - target: ci/mac_ci_steps.sh + name: macOS + steps-pre: | + - run: ./ci/mac_ci_setup.sh + shell: bash + name: Setup macos + source: | + GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) + bash -c "echo \"${RBE_KEY}\" | base64 --decode > \"${GCP_SERVICE_ACCOUNT_KEY_PATH}\"" + _BAZEL_BUILD_EXTRA_OPTIONS=( + --remote_download_toplevel + --flaky_test_attempts=2 + --config=cache-google + --config=ci + --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}) + export BAZEL_BUILD_EXTRA_OPTIONS=${_BAZEL_BUILD_EXTRA_OPTIONS[*]} + + request: + permissions: + actions: read + contents: read + pull-requests: read + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.build-macos }} + needs: + - load + - macos + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/envoy-prechecks.yml b/.github/workflows/envoy-prechecks.yml index 67fff9920a8e..bd8400a1a4ae 100644 --- a/.github/workflows/envoy-prechecks.yml +++ b/.github/workflows/envoy-prechecks.yml @@ -1,50 +1,71 @@ -name: Envoy/prechecks +name: Envoy/Prechecks permissions: contents: read on: - push: - branches: - - main - - release/v* - pull_request: - paths: - - '**/requirements*.txt' - - '**/go.mod' - - '**/*.bzl' - - 'WORKSPACE' - - '.github/workflows/envoy-prechecks.yml' - - '.github/workflows/_*.yml' + workflow_run: + workflows: + - Request + types: + - completed concurrency: - group: ${{ github.event.inputs.head_ref || github.run_id }}-${{ github.workflow }} + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true +env: + CI_DEBUG: ${{ vars.CI_DEBUG }} + + jobs: - env: - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} + permissions: + actions: read + contents: read + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml with: - prime_build_image: true - check_mobile_run: false + check-name: prechecks + + deps: permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + name: Precheck (${{ fromJSON(needs.load.outputs.request).summary.title }}) + uses: ./.github/workflows/_precheck_deps.yml + if: ${{ fromJSON(needs.load.outputs.request).run.precheck-deps }} + needs: + - load + with: + dependency-review: ${{ github.event_name == 'pull_request_target' && github.repository == 'envoyproxy/envoy' }} + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} - prechecks: + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.precheck-deps }} needs: - - env - strategy: - fail-fast: false - matrix: - include: - - target: deps - rbe: false - managed: true - uses: ./.github/workflows/_ci.yml - name: CI ${{ matrix.target }} + - load + - deps + uses: ./.github/workflows/_finish.yml with: - target: ${{ matrix.target }} - rbe: ${{ matrix.rbe }} - managed: ${{ matrix.managed }} - cache_build_image: ${{ needs.env.outputs.build_image_ubuntu }} + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/envoy-publish.yml b/.github/workflows/envoy-publish.yml index 8bc5d511b01e..593f8ce00455 100644 --- a/.github/workflows/envoy-publish.yml +++ b/.github/workflows/envoy-publish.yml @@ -1,4 +1,7 @@ -name: Publish & verify +# This workflow is triggered by azp currently +# Once arm/x64 build jobs are shifted to github, this can be triggered +# by on: workflow_run +name: Envoy/Publish & verify permissions: contents: read @@ -15,55 +18,85 @@ on: description: "Ref for grouping PRs" concurrency: - group: | + group: >- ${{ github.actor != 'trigger-release-envoy[bot]' && github.event.inputs.head_ref || github.run_id - }}-${{ github.workflow }} + }}-${{ github.event.workflow.id }} cancel-in-progress: true jobs: - env: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} + permissions: + actions: read + contents: read + packages: read + pull-requests: read if: >- ${{ - github.repository == 'envoyproxy/envoy' + (github.repository == 'envoyproxy/envoy' + || vars.ENVOY_CI) && (!contains(github.actor, '[bot]') || github.actor == 'trigger-workflow-envoy[bot]' || github.actor == 'trigger-release-envoy[bot]') }} - uses: ./.github/workflows/_env.yml + uses: ./.github/workflows/_load.yml with: - check_mobile_run: false - prime_build_image: true - start_check_status: Verify/examples - repo_ref: ${{ inputs.ref }} - repo_ref_sha: ${{ inputs.sha }} - repo_ref_name: ${{ inputs.head_ref }} + check-name: publish + head-sha: ${{ inputs.sha }} + publish: + secrets: + ENVOY_CI_SYNC_APP_ID: ${{ fromJSON(needs.load.outputs.trusted) && secrets.ENVOY_CI_SYNC_APP_ID || '' }} + ENVOY_CI_SYNC_APP_KEY: ${{ fromJSON(needs.load.outputs.trusted) && secrets.ENVOY_CI_SYNC_APP_KEY || '' }} + ENVOY_CI_PUBLISH_APP_ID: ${{ fromJSON(needs.load.outputs.trusted) && secrets.ENVOY_CI_PUBLISH_APP_ID || '' }} + ENVOY_CI_PUBLISH_APP_KEY: ${{ fromJSON(needs.load.outputs.trusted) && secrets.ENVOY_CI_PUBLISH_APP_KEY || '' }} permissions: contents: read - statuses: write - - publish: + packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.publish }} needs: - - env + - load uses: ./.github/workflows/_stage_publish.yml - name: Publish ${{ needs.env.outputs.repo_ref_title }} + name: Publish with: - build_image_ubuntu: ${{ needs.env.outputs.build_image_ubuntu }} - trusted: ${{ needs.env.outputs.trusted == 'true' && true || false }} - version_dev: ${{ needs.env.outputs.version_dev }} - given_ref: ${{ inputs.ref }} - repo_ref: ${{ needs.env.outputs.trusted != 'true' && inputs.ref || '' }} - permissions: - contents: write + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} verify: + permissions: + contents: read + packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.verify }} + needs: + - load uses: ./.github/workflows/_stage_verify.yml - name: Verify ${{ needs.env.outputs.repo_ref_title }} + name: Verify + with: + request: ${{ needs.load.outputs.request }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && (fromJSON(needs.load.outputs.request).run.publish + || fromJSON(needs.load.outputs.request).run.verify) }} needs: - - env + - load + - publish + - verify + uses: ./.github/workflows/_finish.yml with: - trusted: ${{ needs.env.outputs.trusted == 'true' && true || false }} - given_ref: ${{ inputs.ref }} - repo_ref: ${{ needs.env.outputs.trusted != 'true' && needs.env.outputs.repo_ref || '' }} + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/envoy-release.yml b/.github/workflows/envoy-release.yml new file mode 100644 index 000000000000..d4743a7cf45e --- /dev/null +++ b/.github/workflows/envoy-release.yml @@ -0,0 +1,196 @@ +name: Envoy/release + +permissions: + contents: read + +on: + release: + types: + - published + branches: + - main + - release/v* + workflow_dispatch: + inputs: + task: + description: Select a task + required: true + default: create-release + type: choice + options: + - create-release + - sync-version-histories + pr: + type: boolean + default: true + description: Create a PR + pr-message: + description: Additional message for PR, eg to fix an issue or additional signoff (optional) + wip: + type: boolean + default: false + description: WIP + author: + description: >- + Author: User/email, eg 'Myname ' + (used by create-release, default: `changelogs/summary.md` last committer) + summary: + type: boolean + default: true + description: Use changelog summary (required to publish release) + +env: + COMMITTER_NAME: publish-envoy[bot] + COMMITTER_EMAIL: 140627008+publish-envoy[bot]@users.noreply.github.com + + +jobs: + ## Triggerable actions + + # Create a release commit, when landed this will publish. + create_release: + runs-on: ubuntu-22.04 + if: github.event_name == 'workflow_dispatch' && inputs.task == 'create-release' + name: Create release + steps: + - id: checkout + name: Checkout Envoy repository + uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app_id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} + app_key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} + committer-name: ${{ env.COMMITTER_NAME }} + committer-email: ${{ env.COMMITTER_EMAIL }} + strip-prefix: release/ + - run: | + if [[ ! -s "changelogs/summary.md" ]]; then + if [[ "${{ inputs.summary }}" == "false" ]]; then + echo "::warning::Changelog summary (changelogs/summary.md) is empty!" + exit 0 + fi + echo "::error::Changelog summary (changelogs/summary.md) is empty!" + exit 1 + fi + COMMITTER=$(git log -n 1 --format='%an <%ae>' -- changelogs/summary.md) + echo "committer=${COMMITTER}" >> $GITHUB_OUTPUT + id: changelog + name: Check changelog summary + - if: ${{ inputs.author }} + name: Validate signoff email + uses: envoyproxy/toolshed/gh-actions/email/validate@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + email: ${{ inputs.author }} + - uses: envoyproxy/toolshed/gh-actions/github/run@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Create release + with: + source: | + BAZEL_ARGS=(--) + BAZEL_RUN_ARGS=(--config=ci) + if [[ -n "${{ inputs.author }}" ]]; then + BAZEL_ARGS+=( + "--release-author=${{ inputs.author }}" + "--signoff=${{ steps.changelog.outputs.committer }}") + else + BAZEL_ARGS+=("--release-author=${{ steps.changelog.outputs.committer }}") + fi + command: >- + bazel + run + "${BAZEL_RUN_ARGS[@]}" + @envoy_repo//:release + "${BAZEL_ARGS[@]}" + - run: | + VERSION=$(cat VERSION.txt) + echo "version=v${VERSION}" >> $GITHUB_OUTPUT + name: Release version + id: release + - name: Create a PR + uses: envoyproxy/toolshed/gh-actions/github/pr@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + base: ${{ github.ref_name }} + commit: false + append-commit-message: true + body: | + Created by Envoy publish bot for @${{ github.actor }} + ${{ ! inputs.summary && ':warning: Created without changelog summary, this will need to be updated before publishing' || '' }} + branch: release/create/${{ steps.checkout.outputs.branch-name }} + diff-upload: release-${{ steps.checkout.outputs.branch-name }} + diff-show: true + dry-run: ${{ ! inputs.pr }} + wip: ${{ ! inputs.summary || inputs.wip }} + title: >- + [${{ (! inputs.summary || inputs.wip) && 'WIP/' || '' }}release/${{ steps.branch.outputs.name }}] + repo: Release ${{ steps.release.outputs.version }} + GITHUB_TOKEN: ${{ steps.checkout.outputs.token }} + + sync_version_histories: + runs-on: ubuntu-22.04 + if: github.event_name == 'workflow_dispatch' && inputs.task == 'sync-version-histories' + name: Sync version histories + steps: + - id: checkout + name: Checkout Envoy repository + uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app_id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} + app_key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} + committer-name: ${{ env.COMMITTER_NAME }} + committer-email: ${{ env.COMMITTER_EMAIL }} + - uses: envoyproxy/toolshed/gh-actions/github/run@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + name: Sync version histories + with: + command: >- + bazel + run + --config=ci @envoy_repo//:sync + -- + --signoff="${{ env.COMMITTER_NAME }} <${{ env.COMMITTER_EMAIL }}>" + - name: Create a PR + uses: envoyproxy/toolshed/gh-actions/github/pr@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + append-commit-message: true + base: ${{ github.ref_name }} + commit: false + body: | + Created by Envoy publish bot for @${{ github.actor }} + branch: release/sync/${{ steps.checkout.outputs.branch-name }} + diff-upload: version-histories-${{ steps.checkout.outputs.branch-name }} + diff-show: true + dry-run: ${{ ! inputs.pr }} + GITHUB_TOKEN: ${{ steps.checkout.outputs.token }} + title: >- + ${{ steps.branch.outputs.name != 'main' && '[${{ steps.branch.outputs.name }}]' || '' }} + repo: Sync version histories + + + ## Triggered actions + + # On release to `main`: + # - fork the branch to a release branch + # - add an initial dev commit + # - remove anything unwanted + # - push branch + create_release_branch: + runs-on: ubuntu-22.04 + if: github.event_name == 'release' && endsWith(github.ref, '.0') + name: Create release branch + steps: + - name: Checkout repository + uses: envoyproxy/toolshed/gh-actions/github/checkout@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app_id: ${{ secrets.ENVOY_CI_PUBLISH_APP_ID }} + app_key: ${{ secrets.ENVOY_CI_PUBLISH_APP_KEY }} + committer-name: ${{ env.COMMITTER_NAME }} + committer-email: ${{ env.COMMITTER_EMAIL }} + - name: Create release branch + run: | + version="$(cut -d- -f1 < VERSION.txt | cut -d. -f-2)" + release_branch="release/v${version}" + commit_sha="$(git rev-parse HEAD)" + echo "Creating ${release_branch} from ${commit_sha}" + git checkout -b "$release_branch" + bazel run @envoy_repo//:dev -- --patch + git rm -rf .github/workflows/mobile*yml + git commit . -m "repo: Remove mobile ci for release branch" + git log + git push origin "$release_branch" diff --git a/.github/workflows/envoy-sync.yml b/.github/workflows/envoy-sync.yml index 8bb631066f85..d24ad17f2253 100644 --- a/.github/workflows/envoy-sync.yml +++ b/.github/workflows/envoy-sync.yml @@ -1,5 +1,8 @@ name: 'Sync downstream' +permissions: + contents: read + on: push: branches: @@ -26,11 +29,16 @@ jobs: - go-control-plane - envoy-filter-example - data-plane-api + - mobile-website steps: - - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.18 + - uses: envoyproxy/toolshed/gh-actions/appauth@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + id: appauth + with: + app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} + key: ${{ secrets.ENVOY_CI_SYNC_APP_KEY }} + - uses: envoyproxy/toolshed/gh-actions/dispatch@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 with: repository: "envoyproxy/${{ matrix.downstream }}" ref: main - key: "${{ secrets.ENVOY_CI_SYNC_APP_KEY }}" + token: ${{ steps.appauth.outputs.token }} workflow: envoy-sync.yaml - app_id: ${{ secrets.ENVOY_CI_SYNC_APP_ID }} diff --git a/.github/workflows/envoy-windows.yml b/.github/workflows/envoy-windows.yml new file mode 100644 index 000000000000..17be3ff309e8 --- /dev/null +++ b/.github/workflows/envoy-windows.yml @@ -0,0 +1,139 @@ +name: Envoy/Windows + +permissions: + contents: read + +on: + workflow_run: + workflows: + - Request + types: + - completed + +concurrency: + group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + + +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} + permissions: + actions: read + contents: read + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: windows + + windows: + permissions: + contents: read + packages: read + secrets: + rbe-key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + if: ${{ fromJSON(needs.load.outputs.request).run.build-windows }} + needs: + - load + uses: ./.github/workflows/_run.yml + name: CI ${{ matrix.name || matrix.target }} + with: + command: + request: ${{ needs.load.outputs.request }} + runs-on: envoy-win19-small + source: | + export ENVOY_SHARED_TMP_DIR="C:\Users\runner\AppData\Local\Temp\bazel-shared" + export ENVOY_DOCKER_BUILD_DIR="C:\Users\runner\AppData\Local\Temp" + mkdir -p "$ENVOY_SHARED_TMP_DIR" + GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -p "${ENVOY_SHARED_TMP_DIR}" -t gcp_service_account.XXXXXX.json) + bash -c "echo \"${RBE_KEY}\" | base64 --decode > \"${GCP_SERVICE_ACCOUNT_KEY_PATH}\"" + _BAZEL_BUILD_EXTRA_OPTIONS=( + --config=remote-ci + --config=rbe-google + --config=remote-msvc-cl + --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH} + --jobs=75 + --flaky_test_attempts=2) + export BAZEL_BUILD_EXTRA_OPTIONS=${_BAZEL_BUILD_EXTRA_OPTIONS[*]} + steps-post: + target: ${{ matrix.target }} + temp-dir: 'C:\Users\runner\AppData\Local\Temp\bazel-shared' + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + upload-name: windows.release + upload-path: 'C:\Users\runner\AppData\Local\Temp\envoy' + strategy: + fail-fast: false + matrix: + include: + - target: ci/windows_ci_steps.sh + name: Windows 2019 + + docker: + needs: + - load + - windows + strategy: + fail-fast: false + matrix: + include: + - target: windows2019 + name: Windows 2019 + runs-on: envoy-win19-small + build-type: windows + image-base: mcr.microsoft.com/windows/servercore + image-tag: ltsc2019 + - target: windows2022 + name: Windows 2022 + runs-on: envoy-win22-small + build-type: windows-ltsc2022 + image-base: mcr.microsoft.com/windows/nanoserver + image-tag: ltsc2022 + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ needs.load.outputs.repo_ref }} + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: windows.release + - run: | + # Convert to Unix-style path so tar doesn't think drive letter is a hostname + STAGING_DIR="$(echo $PWD | tr -d ':' | tr '\\' '/')" + mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/envoy_binary.tar.gz" -C ./windows/amd64 + CI_SHA1=$(git rev-parse head) + export CI_SHA1 + ci/docker_ci.sh + shell: bash + env: + CI_BRANCH: ${{ github.ref }} + DOCKERHUB_USERNAME: ${{ fromJSON(needs.load.outputs.trusted) && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_PASSWORD: ${{ fromJSON(needs.load.outputs.trusted) && secrets.DOCKERHUB_PASSWORD || '' }} + WINDOWS_BUILD_TYPE: ${{ matrix.build-type }} + WINDOWS_IMAGE_BASE: ${{ matrix.image-base }} + WINDOWS_IMAGE_TAG: ${{ matrix.image-tag }} + request: + permissions: + actions: read + contents: read + pull-requests: read + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.build-windows }} + needs: + - load + - windows + - docker + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml index 24840a28f653..76d8b368fbcc 100644 --- a/.github/workflows/mobile-android_build.yml +++ b/.github/workflows/mobile-android_build.yml @@ -1,264 +1,178 @@ -name: android_build +name: Mobile/Android build permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true -jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml - permissions: - contents: read - statuses: write - androidbuild: - if: ${{ needs.env.outputs.mobile_android_build == 'true' }} - needs: env +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - name: android_build - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build envoy.aar distributable' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci \ - --fat_apk_cpu=x86_64 \ - --linkopt=-fuse-ld=lld \ - //:android_dist + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-android - javahelloworld: - if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} - needs: - - env - - androidbuild + build: permissions: contents: read packages: read - name: java_helloworld - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - run: | - cd mobile - ./ci/mac_ci_setup.sh --android - name: 'Install dependencies' - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 10 - max_attempts: 3 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # cd mobile && ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/java/hello_world:hello_envoy - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start java app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - --fat_apk_cpu=x86_64 \ - //examples/java/hello_world:hello_envoy - adb install -r --no-incremental bazel-bin/examples/java/hello_world/hello_envoy.apk - adb shell am start -n io.envoyproxy.envoymobile.helloenvoy/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 adb logcat -e "received headers with status 301" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android }} + needs: load + name: Build envoy.aar distributable + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + build + --config=mobile-remote-release-clang-android + //:android_dist + container: ${{ fromJSON(needs.load.outputs.build-image).mobile }} + diskspace-hack: true + request: ${{ needs.load.outputs.request }} + timeout-minutes: 90 + target: build - kotlinhelloworld: - if: ${{ needs.env.outputs.mobile_android_build == 'true' }} - needs: - - env - - androidbuild + kotlin-hello-world: permissions: contents: read packages: read - name: kotlin_helloworld - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh --android - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 10 - max_attempts: 3 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start kotlin app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - --fat_apk_cpu=x86_64 \ - //examples/kotlin/hello_world:hello_envoy_kt - adb install -r --no-incremental bazel-bin/examples/kotlin/hello_world/hello_envoy_kt.apk - adb shell am start -n io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 adb logcat -e "received headers with status 200" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } - - kotlinbaselineapp: - if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} + name: kotlin-hello-world + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android }} needs: - - env - - androidbuild + - load + - build + with: + command: ./bazelw + container-command: + # Return to using: + # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt + # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. + args: >- + build + --config=mobile-remote-release-clang-android + //examples/kotlin/hello_world:hello_envoy_kt + request: ${{ needs.load.outputs.request }} + target: kotlin-hello-world + runs-on: envoy-x64-small + steps-pre: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/pre@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + apk: bazel-bin/examples/kotlin/hello_world/hello_envoy_kt.apk + app: io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity + status: 200 + timeout-minutes: 50 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + + apps: permissions: contents: read packages: read - name: kotlin_baseline_app - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh --android - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 10 - max_attempts: 3 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start kotlin app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - --fat_apk_cpu=x86_64 \ + name: Android apps + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android-all }} + needs: + - load + - build + with: + command: ./bazelw + container-command: + args: ${{ matrix.args }} + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + runs-on: envoy-x64-small + steps-pre: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/pre@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + steps-post: ${{ matrix.steps-post }} + timeout-minutes: 50 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: java-hello-world + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + apk: bazel-bin/examples/java/hello_world/hello_envoy.apk + app: io.envoyproxy.envoymobile.helloenvoy/.MainActivity + status: 301 + target: java-hello-world + args: >- + build + --config=mobile-remote-release-clang-android + //examples/java/hello_world:hello_envoy + - name: kotlin-baseline-app + # Return to using: + # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt + # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. + args: >- + build + --config=mobile-remote-release-clang-android //test/kotlin/apps/baseline:hello_envoy_kt - adb install -r --no-incremental bazel-bin/test/kotlin/apps/baseline/hello_envoy_kt.apk - adb shell am start -n io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 adb logcat -e "received headers with status 301" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + apk: bazel-bin/test/kotlin/apps/baseline/hello_envoy_kt.apk + app: io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity + status: 301 + target: kotlin-baseline-app + - name: kotlin-experimental-app + # Return to using: + # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt + # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. + args: >- + build + --config=mobile-remote-release-clang-android + //test/kotlin/apps/experimental:hello_envoy_kt + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/android/post@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + apk: bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk + app: io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity + status: 200 + target: kotlin-experimental-app - kotlinexperimentalapp: - if: ${{ needs.env.outputs.mobile_android_build_all == 'true' }} - needs: - - env - - androidbuild + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: kotlin_experimental_app - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh --android - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start emulator' - with: - timeout_minutes: 10 - max_attempts: 3 - command: ./mobile/ci/start_android_emulator.sh - # Return to using: - # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt - # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - - name: 'Start kotlin app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - --fat_apk_cpu=x86_64 \ - --define envoy_mobile_listener=enabled \ - //test/kotlin/apps/experimental:hello_envoy_kt - adb install -r --no-incremental bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk - adb shell am start -n io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity - - name: 'Check connectivity' - run: | - timeout 30 adb logcat -e "received headers with status 200" -m 1 || { - echo "Failed checking for headers in adb logcat" >&2 - timeout 30 adb logcat || { - echo "Failed dumping adb logcat" >&2 - } - exit 1 - } + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-android }} + needs: + - load + - build + - kotlin-hello-world + - apps + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml index 7f9fde9d270a..d7f4a23b10f7 100644 --- a/.github/workflows/mobile-android_tests.yml +++ b/.github/workflows/mobile-android_tests.yml @@ -1,124 +1,84 @@ -name: android_tests +name: Mobile/Android tests permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true -jobs: - env: - uses: ./.github/workflows/_env.yml - permissions: - contents: read - statuses: write - kotlintestsmac: - if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} - needs: env +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - # revert to //test/kotlin/... once fixed - # https://github.com/envoyproxy/envoy-mobile/issues/1932 - name: kotlin_tests_mac - runs-on: macos-12 - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - - name: 'Java setup' - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh - - name: 'Run Kotlin library tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --build_tests_only \ - --config=mobile-remote-ci-macos \ - --define=signal_trace=disabled \ - //test/kotlin/io/... + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-android-tests - javatestsmac: - if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} - needs: env + linux: permissions: contents: read packages: read - name: java_tests_mac - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - name: 'Java setup' - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh - - name: 'Run Java library tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --build_tests_only \ - --config test-android \ - --define envoy_mobile_listener=enabled \ - --config=mobile-remote-ci-macos \ - --define=signal_trace=disabled \ - --define=system-helper=android \ + name: Android linux tests + uses: ./.github/workflows/_mobile_container_ci.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-android-tests }} + needs: load + with: + args: ${{ matrix.args }} + container: ${{ fromJSON(needs.load.outputs.build-image).mobile }} + diskspace-hack: true + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - name: java + target: java_tests_linux + args: >- + test + --config=mobile-remote-ci-android //test/java/... + - name: kotlin + target: kotlin_tests_linux + args: >- + test + --config=mobile-remote-ci-android + //test/kotlin/... - kotlintestslinux: - if: ${{ needs.env.outputs.mobile_android_tests == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - # Only kotlin tests are executed since with linux: - # https://github.com/envoyproxy/envoy-mobile/issues/1418. - name: kotlin_tests_linux - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run Kotlin library integration tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --build_tests_only \ - --config test-android \ - --config=mobile-remote-ci \ - --define=signal_trace=disabled \ - //test/kotlin/... + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-android-tests }} + needs: + - load + - linux + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml index c129661893cd..7d630b70a65c 100644 --- a/.github/workflows/mobile-asan.yml +++ b/.github/workflows/mobile-asan.yml @@ -1,50 +1,68 @@ -name: mobile_asan +name: Mobile/ASAN permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-asan asan: - if: ${{ needs.env.outputs.mobile_asan == 'true' }} - needs: env permissions: contents: read packages: read name: asan - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 180 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - --config=mobile-remote-ci-linux-asan \ - //test/common/... + uses: ./.github/workflows/_mobile_container_ci.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-asan }} + needs: load + with: + args: >- + test + --config=mobile-remote-ci-linux-asan + //test/common/... + request: ${{ needs.load.outputs.request }} + target: asan + timeout-minutes: 180 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-asan }} + needs: + - load + - asan + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml index fdf1f1ed2ce3..506e150440d2 100644 --- a/.github/workflows/mobile-cc_tests.yml +++ b/.github/workflows/mobile-cc_tests.yml @@ -1,49 +1,68 @@ -name: mobile_cc_tests +name: Mobile/CC permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-cc - cctests: - if: ${{ needs.env.outputs.mobile_cc_tests == 'true' }} - needs: env + cc-tests: permissions: contents: read packages: read - name: cc_tests - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: 'Run tests' - # Regression test using the new API listener. TODO(#2711) clean up. - run: | - cd mobile - ./bazelw test \ - --action_env=LD_LIBRARY_PATH \ - --copt=-DUSE_API_LISTENER \ - --config=mobile-remote-ci \ - //test/cc/... + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-cc }} + needs: load + name: cc-tests + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + test + --config=mobile-remote-ci-cc + //test/cc/... + request: ${{ needs.load.outputs.request }} + target: cc-tests + timeout-minutes: 120 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-cc }} + needs: + - load + - cc-tests + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index 4d646072cfb7..9ea03ea16079 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -1,150 +1,143 @@ -name: mobile_compile_time_options +name: Mobile/Compile time options permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true -jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml - permissions: - contents: read - statuses: write - cc_test_no_yaml: - needs: env +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - name: cc_test_no_yaml - runs-on: ubuntu-20.04 - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Running C++ test with YAML disabled' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Envoy Mobile build which verifies that the build configuration where YAML is disabled. - run: | - cd mobile - ./bazelw test \ - --config=mobile-remote-ci \ - --define=envoy_yaml=disabled \ - --define=envoy_full_protos=disabled \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - //test/common/integration:client_integration_test + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-compile-time-options - cc_test: - needs: env + cc: permissions: contents: read packages: read - name: cc_test - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Running C++ tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - TARGETS=$(bazel query --noshow_progress --noshow_loading_progress //test/cc/... + //test/common/... except //test/common/integration:client_integration_test) - ./bazelw test \ - --test_output=all \ - --config=mobile-remote-ci \ - --define=signal_trace=disabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - $TARGETS + uses: ./.github/workflows/_mobile_container_ci.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-compile-time-options }} + needs: load + with: + args: ${{ matrix.args }} + command: ./bazelw + entrypoint: ${{ matrix.entrypoint }} + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - name: Running C++ test with YAML disabled + target: cc-test-no-yaml + args: >- + build + --config=mobile-remote-ci-cc-no-yaml + //test/common/integration:client_integration_test + - name: Running C++ build with exceptions disabled + target: cc-no-build-exceptions + args: >- + build + --config=mobile-remote-ci-cc-no-exceptions + //test/performance:test_binary_size //library/cc/... + - name: Running C++ test + target: cc-test + args: >- + test + --config=mobile-remote-ci-cc-test + entrypoint: | + #!/bin/bash -e + export PATH=/opt/llvm/bin:$PATH + cd /source/mobile + EXTRA_ARGS=$(bazel query //test/cc/... + //test/common/... except //test/common/integration:client_integration_test) + exec "$@" $EXTRA_ARGS - swift_build: - if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} - needs: env + build: permissions: contents: read packages: read - name: swift_build - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build Swift library' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw shutdown - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - --define=signal_trace=disabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_mobile_stats_reporting=disabled \ - --define=envoy_mobile_swift_cxx_interop=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --@envoy//bazel:http3=False \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-compile-time-options }} + needs: load + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: ${{ matrix.source }} + steps-pre: ${{ matrix.steps-pre }} + target: ${{ matrix.target || matrix.name }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + timeout-minutes: 120 + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: kotlin-build + args: >- + build + --config=mobile-remote-ci-macos-kotlin + //:android_dist + source: | + . ./ci/mac_ci_setup.sh --android + echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" >> $GITHUB_ENV + export ANDROID_NDK_HOME + steps-pre: | + - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 + with: + java-version: '8' + java-package: jdk + architecture: x64 + distribution: zulu + - name: swift-build + args: >- + build + --config=mobile-remote-ci-macos-swift //library/swift:ios_framework + source: | + ./ci/mac_ci_setup.sh + ./bazelw shutdown - kotlin_build: - if: ${{ needs.env.outputs.mobile_compile_time_options == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: kotlin_build - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh --android - - name: 'Build Kotlin library' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - --fat_apk_cpu=x86_64 \ - --define=signal_trace=disabled \ - --define=envoy_mobile_request_compression=disabled \ - --define=envoy_enable_http_datagrams=disabled \ - --define=google_grpc=disabled \ - --define=envoy_yaml=disabled \ - --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= \ - //:android_dist + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-compile-time-options }} + needs: + - load + - cc + - build + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml index 02c16ec1fc7b..70de7033c9a0 100644 --- a/.github/workflows/mobile-core.yml +++ b/.github/workflows/mobile-core.yml @@ -1,52 +1,67 @@ -name: mobile_core +name: Mobile/Core permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-core - unittests: - if: ${{ github.repository == 'envoyproxy/envoy' }} - needs: env + unit-tests: permissions: contents: read packages: read - name: unit_tests - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Ensure no listener leaks - run: rm source/extensions/listener_managers/listener_manager/listener_manager_impl.h - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --build_tests_only \ - --action_env=LD_LIBRARY_PATH \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - --define envoy_mobile_listener=disabled \ - --config=mobile-remote-ci \ - //test/common/... + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-core }} + needs: load + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + test + --config=mobile-remote-ci-core + //test/common/... + request: ${{ needs.load.outputs.request }} + target: unit-tests + timeout-minutes: 120 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-core }} + needs: + - load + - unit-tests + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml index 6dd105b56c87..160d36e91ed0 100644 --- a/.github/workflows/mobile-coverage.yml +++ b/.github/workflows/mobile-coverage.yml @@ -1,59 +1,79 @@ -name: mobile_coverage +name: Mobile/Coverage permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-coverage coverage: - if: ${{ needs.env.outputs.mobile_coverage == 'true' }} - needs: env permissions: contents: read packages: read - name: coverage - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - defaults: - run: - shell: bash - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run coverage' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - export BAZEL_BUILD_OPTION_LIST="--config=mobile-remote-ci-linux-coverage" \ - PATH=/opt/llvm/bin:${PATH} \ - COVERAGE_THRESHOLD=76 - ../test/run_envoy_bazel_coverage.sh //test/common/... //test/cc/... - - name: 'Package coverage' - run: | - cd mobile - tar -czf coverage.tar.gz generated/coverage - - name: 'Upload report' - uses: actions/upload-artifact@v3 - with: - name: coverage.tar.gz - path: mobile/coverage.tar.gz + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-coverage }} + needs: load + name: Running mobile coverage + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + //test/common/... + //test/cc/... + command: ../test/run_envoy_bazel_coverage.sh + request: ${{ needs.load.outputs.request }} + source: + export COVERAGE_THRESHOLD=76 + export BAZEL_BUILD_OPTION_LIST=--config=mobile-remote-ci-linux-coverage + steps-post: | + - name: Package coverage + shell: bash + run: | + cd mobile + tar -czf coverage.tar.gz generated/coverage + target: mobile-coverage + timeout-minutes: 120 + upload-name: coverage.tar.gz + upload-path: mobile/coverage.tar.gz + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-coverage }} + needs: + - load + - coverage + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml index d6a3f4e4d82d..0c3268de6bc1 100644 --- a/.github/workflows/mobile-docs.yml +++ b/.github/workflows/mobile-docs.yml @@ -1,50 +1,96 @@ -name: mobile_docs +name: Mobile/Docs permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-docs docs: - if: ${{ github.repository == 'envoyproxy/envoy' }} - needs: env + secrets: + ssh-key-extra: ${{ needs.load.outputs.trusted && secrets.ENVOY_MOBILE_WEBSITE_DEPLOY_KEY || '' }} permissions: contents: read packages: read - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - name: Generate docs - run: ./ci/run_envoy_docker.sh 'cd mobile && docs/build.sh' - - name: Set up deploy key - if: github.ref == 'refs/heads/main' - uses: shimataro/ssh-key-action@v2.5.1 - with: - key: ${{ secrets.ENVOY_MOBILE_WEBSITE_DEPLOY_KEY }} - known_hosts: unnecessary - - name: Publish docs - if: github.ref == 'refs/heads/main' - run: ./ci/run_envoy_docker.sh 'cd mobile && docs/publish.sh' - - uses: actions/upload-artifact@v3 - with: - name: docs - path: generated/docs + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-docs }} + needs: load + uses: ./.github/workflows/_run.yml + with: + args: >- + -- + command: ./docs/build.sh + request: ${{ needs.load.outputs.request }} + target: mobile-docs + cache-build-image: ${{ fromJSON(needs.load.outputs.build-image).build-image }} + checkout-extra: | + repository: envoy-mobile/envoy-mobile.github.io + path: mobile-docs + fetch-depth: 0 + branch: master + source: | + echo "ENVOY_DOCKER_SOURCE_DIR=/source/mobile" >> $GITHUB_ENV + # Path relative to ./mobile directory + echo "MOBILE_DOCS_CHECKOUT_DIR=../mobile-docs" >> $GITHUB_ENV + steps-post: | + - name: Publish docs + shell: bash + run: | + ./ci/run_envoy_docker.sh docs/publish.sh + # This step needs to be done outside the container to access ssh creds + - name: Push changes + # && github.repository == 'envoyproxy/envoy' + if: ${{ needs.load.outputs.trusted }} + shell: bash + run: | + git -C mobile-docs status + exit 0 + git -C mobile-docs push origin master + timeout-minutes: 20 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + upload-name: docs + upload-path: mobile/generated/docs + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-docs }} + needs: + - load + - docs + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml index 9a979da04ea4..3f09ce951087 100644 --- a/.github/workflows/mobile-format.yml +++ b/.github/workflows/mobile-format.yml @@ -1,130 +1,158 @@ -name: mobile_format +name: Mobile/Format permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true -jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml - permissions: - contents: read - statuses: write - formatall: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env +jobs: + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - name: format_all - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 45 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run formatters' - run: | - cd mobile - ./tools/check_format.sh + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-format - precommit: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env + container: permissions: contents: read packages: read - name: precommit - runs-on: macos-12 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - - name: 'Install precommit' - run: brew install pre-commit - - name: 'Run precommit' - run: | - cd mobile - find mobile/* | pre-commit run --files + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-format }} + needs: load + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: ${{ matrix.args }} + command: ${{ matrix.command }} + container: ${{ matrix.container }} + request: ${{ needs.load.outputs.request }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + strategy: + fail-fast: false + matrix: + include: + - name: Format all + args: check + command: ./tools/check_format.sh + target: format-all + timeout-minutes: 45 + - name: Swift lint + args: >- + lint + --strict + command: swiftlint + container: >- + ghcr.io/realm/swiftlint:0.50.3 + target: swift-lint + timeout-minutes: 45 - swiftlint: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env + host: permissions: contents: read packages: read - name: swift_lint - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 5 - container: - image: ghcr.io/realm/swiftlint:0.50.3 - steps: - - uses: actions/checkout@v4 - - name: 'Run Swift Lint (SwiftLint)' - run: swiftlint lint --strict + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-format }} + needs: load + uses: ./.github/workflows/_run.yml + with: + args: ${{ matrix.args }} + command: ${{ matrix.command }} + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: ${{ matrix.runs-on }} + source: ${{ matrix.source }} + steps-pre: ${{ matrix.steps-pre }} + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Run drstring + args: >- + run + --config=remote-ci + @DrString//:drstring check + command: ./bazelw + runs-on: macos-12 + source: | + export DEVELOPER_DIR=/Applications/Xcode_14.1.app + target: drstring + timeout-minutes: 10 + - name: Pre-commit + command: pre-commit + args: >- + run + --files + mobile/* + source: | + pip install pre-commit + target: pre-commit + timeout-minutes: 45 + - name: Kotlin lint + args: >- + build + --config=mobile-remote-ci-macos + //library/kotlin/io/envoyproxy/envoymobile:envoy_lib_lint + //examples/kotlin/hello_world:hello_envoy_kt_lint + command: ./bazelw + runs-on: macos-12 + source: | + export DEVELOPER_DIR=/Applications/Xcode_14.1.app + ./ci/mac_ci_setup.sh + steps-pre: | + - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 + with: + java-version: '8' + java-package: jdk + architecture: x64 + distribution: zulu + steps-post: | + - name: Run Kotlin Formatter (ktlint) + shell: bash + run: | + cd mobile + ./bazelw build --config=remote-ci kotlin_format + target: kotlin-lint + timeout-minutes: 10 - drstring: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env - permissions: - contents: read - packages: read - name: drstring - runs-on: macos-12 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - name: 'Run DrString' - env: - DEVELOPER_DIR: /Applications/Xcode_14.1.app - run: | - cd mobile - ./bazelw run --config=remote-ci @DrString//:drstring check - - kotlinlint: - if: ${{ needs.env.outputs.mobile_formatting == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: kotlin_lint - runs-on: macos-12 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 - with: - java-version: '8' - java-package: jdk - architecture: x64 - distribution: zulu - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Run Kotlin Lint (Detekt)' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-ci-macos \ - //library/kotlin/io/envoyproxy/envoymobile:envoy_lib_lint \ - //examples/kotlin/hello_world:hello_envoy_kt_lint - - name: 'Run Kotlin Formatter (ktlint)' - run: | - cd mobile - ./bazelw build --config=remote-ci kotlin_format + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-format }} + needs: + - load + - container + - host + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-ios_build.yml b/.github/workflows/mobile-ios_build.yml index 3a6ca86f5dba..11625c4a1430 100644 --- a/.github/workflows/mobile-ios_build.yml +++ b/.github/workflows/mobile-ios_build.yml @@ -1,312 +1,184 @@ -name: ios_build +name: Mobile/iOS build permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed -concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} - cancel-in-progress: true jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: mobile-ios - iosbuild: - if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} - needs: env + build: permissions: contents: read packages: read - name: ios_build - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios }} + needs: load + name: ios-build + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build Envoy.framework distributable' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile ./bazelw shutdown - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Build Envoy.framework distributable + args: >- + build + --config=mobile-remote-ci-macos-ios //library/swift:ios_framework + target: ios + timeout-minutes: 120 - swifthelloworld: - if: ${{ needs.env.outputs.mobile_ios_build == 'true' }} - name: swift_helloworld - needs: - - env - - iosbuild + hello-world: permissions: contents: read packages: read - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/swift/hello_world:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/swift/hello_world:app &> /tmp/envoy.log & - - run: | - sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' - - swiftbaselineapp: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios }} needs: - - env - - iosbuild - permissions: - contents: read - packages: read - name: swift_baseline_app - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + - load + - build + name: ios-hello-world + with: + args: >- + build + ${{ matrix.args || '--config=mobile-remote-ci-macos-ios' }} + ${{ matrix.app }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //test/swift/apps/baseline:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //test/swift/apps/baseline:app &> /tmp/envoy.log & - - run: | - sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' + ./bazelw shutdown + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app: ${{ matrix.app }} + args: ${{ matrix.args || '--config=mobile-remote-ci-macos-ios' }} + expected: received headers with status ${{ matrix.expected-status }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Build swift hello world + app: //examples/swift/hello_world:app + expected-status: 200 + target: swift-hello-world + timeout-minutes: 50 - swiftexperimentalapp: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: - - env - - iosbuild + apps: permissions: contents: read packages: read - name: swift_experimental_app - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - --define=admin_functionality=enabled \ - --define envoy_mobile_listener=enabled \ - //test/swift/apps/experimental:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - --define=admin_functionality=enabled \ - --define envoy_mobile_listener=enabled \ - //test/swift/apps/experimental:app &> /tmp/envoy.log & - - run: | - sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' - - swiftasyncawait: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios-all }} needs: - - env - - iosbuild - permissions: - contents: read - packages: read - name: swift_async_await - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + - load + - build + name: ios-apps + with: + args: >- + build + ${{ matrix.args || '--config=mobile-remote-ci-macos-ios' }} + ${{ matrix.app }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/swift/async_await:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/swift/async_await:app &> /tmp/envoy.log & - - run: | - checklogs () { - sed '/\[2\] Uploaded 7 MB of data/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - } - export -f checklogs - # TODO(phlax): figure if this needs this long - timeout 5m bash -c checklogs || { - retcode=$? - if [[ "$retcode" != 124 ]]; then - echo "Command failed" >&2 - elif grep -q "Upload failed" /tmp/envoy.log; then - echo "Upload failed" >&2 - else - echo "Upload timed out" >&2 - fi - exit 1 - } - if: steps.should_run.outputs.run_ci_job == 'true' - name: 'Check upload succeeded' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' + steps-post: | + - uses: envoyproxy/toolshed/gh-actions/envoy/ios/post@6b3ddd1e42c252d68fb98973760c0ee1943c9c21 # actions-v0.2.20 + with: + app: ${{ matrix.app }} + args: ${{ matrix.args || '--config=mobile-remote-ci-macos-ios' }} + expected: >- + ${{ matrix.expected + || format('received headers with status {0}', matrix.expected-status) }} + target: ${{ matrix.target }} + timeout-minutes: 50 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Build swift baseline app + app: //test/swift/apps/baseline:app + expected-status: 301 + target: swift-baseline-app + - name: Build swift experimental app + args: >- + --config=mobile-remote-ci-macos-ios-admin + app: //test/swift/apps/experimental:app + expected-status: 200 + target: swift-experimental-app + - name: Build swift async await + app: //examples/swift/async_await:app + expected: >- + \[2\] Uploaded 7 MB of data + target: swift-async-await + - name: Build objc hello world + app: //examples/objective-c/hello_world:app + expected-status: 301 + target: objc-hello-world - objchelloworld: - if: ${{ needs.env.outputs.mobile_ios_build_all == 'true' }} - needs: - - env - - iosbuild + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: objc_helloworld - runs-on: macos-12 - timeout-minutes: 50 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile - ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/objective-c/hello_world:app - - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd - name: 'Start simulator' - with: - timeout_minutes: 5 - max_attempts: 3 - command: ./mobile/ci/start_ios_simulator.sh - # Run the app in the background and redirect logs. - - name: 'Run app' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw run \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //examples/objective-c/hello_world:app &> /tmp/envoy.log & - - run: sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) - name: 'Check connectivity' - - run: cat /tmp/envoy.log - if: ${{ failure() || cancelled() }} - name: 'Log app run' + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-ios }} + needs: + - load + - build + - hello-world + - apps + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-ios_tests.yml b/.github/workflows/mobile-ios_tests.yml index 9329d26a6133..2c7a0ae173a1 100644 --- a/.github/workflows/mobile-ios_tests.yml +++ b/.github/workflows/mobile-ios_tests.yml @@ -1,78 +1,90 @@ -name: ios_tests +name: Mobile/iOS tests permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed -concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} - cancel-in-progress: true jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: mobile-ios-tests - swifttests: - if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }} - needs: env + tests: permissions: contents: read packages: read - name: swift_tests - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - name: 'Install dependencies' - run: | - cd mobile + uses: ./.github/workflows/_run.yml + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-ios-tests }} + needs: load + name: ios-tests + with: + args: ${{ matrix.args }} + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - - name: 'Run swift library tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # runs with the listener enabled due to IdleTimeoutTest not setting up a test backend. - run: | - cd mobile - ./bazelw test \ - --experimental_ui_max_stdouterr_bytes=10485760 \ - --config=ios \ - --define envoy_mobile_listener=enabled \ - --build_tests_only \ - --config=mobile-remote-ci-macos \ + steps-post: ${{ matrix.steps-post }} + target: ${{ matrix.target }} + timeout-minutes: ${{ matrix.timeout-minutes }} + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + strategy: + fail-fast: false + matrix: + include: + - name: Run swift library tests + args: >- + test + --config=mobile-remote-ci-macos-ios-swift //test/swift/... + target: swift-tests + timeout-minutes: 120 + - name: Run Objective-C library tests + args: >- + test + --config=mobile-remote-ci-macos-ios-obj-c + //test/objective-c/... + //test/cc/unit:envoy_config_test + target: c-and-objc-tests + timeout-minutes: 120 - objctests: - if: ${{ needs.env.outputs.mobile_ios_tests == 'true' }} - needs: env + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} permissions: + actions: read contents: read - packages: read - name: c_and_objc_tests - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - name: 'Install dependencies' - run: | - cd mobile - ./ci/mac_ci_setup.sh - - name: 'Run Objective-C library tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --config=ios \ - --build_tests_only \ - --config=mobile-remote-ci-macos \ - //test/objective-c/... \ - //test/cc/unit:envoy_config_test + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-ios-tests }} + needs: + - load + - tests + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml index be61176bc5ad..807f1eda861b 100644 --- a/.github/workflows/mobile-perf.yml +++ b/.github/workflows/mobile-perf.yml @@ -1,113 +1,128 @@ -name: mobile_perf +name: Mobile/Perf permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - sizecurrent: - if: ${{ github.repository == 'envoyproxy/envoy' }} + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read packages: read - name: size_current - runs-on: ubuntu-22.04 - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build test binary' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ - //test/performance:test_binary_size - - uses: actions/upload-artifact@v3 - with: - name: sizecurrent - path: mobile/bazel-bin/test/performance/test_binary_size + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-perf - sizemain: - if: ${{ github.repository == 'envoyproxy/envoy' }} + build: permissions: contents: read packages: read - name: size_main - runs-on: ubuntu-22.04 - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - with: - ref: main - - name: Add safe directory - run: | - git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build test binary' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=mobile-remote-release-clang \ + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-perf }} + needs: load + name: Build test binaries + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: ${{ matrix.args }} + command: ./bazelw + ref: ${{ matrix.ref }} + request: ${{ needs.load.outputs.request }} + source: ${{ matrix.source }} + container-output: | + "bazel-bin/test/performance/test_binary_size": test_binary_size/${{ matrix.target }} + target: ${{ matrix.target }} + upload-name: ${{ matrix.target }} + upload-path: /tmp/container-output/test_binary_size + timeout-minutes: 90 + strategy: + matrix: + include: + - name: Current size + args: >- + build + --config=mobile-remote-release-clang + //test/performance:test_binary_size + # Ensure files don't leak back into the main binary + source: >- + rm + source/common/listener_manager/listener_manager_impl.h + source/server/overload_manager_impl.cc + source/common/network/listen_socket_impl.h + source/common/network/tcp_listener_impl.h + source/server/guarddog_impl.h + source/server/watchdog_impl.h + source/server/options_impl.cc + target: size-current + - name: Main size + args: >- + build + --config=mobile-remote-release-clang //test/performance:test_binary_size - - uses: actions/upload-artifact@v3 - with: - name: sizemain - path: mobile/bazel-bin/test/performance/test_binary_size + ref: main + target: size-main - sizecompare: - if: ${{ github.repository == 'envoyproxy/envoy' }} - needs: - - sizecurrent - - sizemain + compare: permissions: contents: read packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-perf }} + needs: + - load + - build name: size_compare - runs-on: ubuntu-22.04 - timeout-minutes: 30 - container: - image: ${{ needs.env.outputs.build_image_ubuntu }} - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: sizecurrent - path: dist/sizecurrent - - uses: actions/download-artifact@v3 - with: - name: sizemain - path: dist/sizemain - - name: 'Strip and Zip binary' - run: | - ls -lh dist/ - strip -s -o dist/main.stripped dist/sizemain/test_binary_size - strip -s -o dist/current.stripped dist/sizecurrent/test_binary_size + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + ../dist/main.zip + ../dist/current.zip + command: ./ci/test_size_regression.sh + downloads: | + size-current: dist/sizecurrent + size-main: dist/sizemain + request: ${{ needs.load.outputs.request }} + runs-on: ubuntu-22.04 + source: | + strip -s -o dist/main.stripped dist/sizemain/size-main + strip -s -o dist/current.stripped dist/sizecurrent/size-current zip -9 dist/main.zip dist/main.stripped zip -9 dist/current.zip dist/current.stripped - - name: 'Test size regression' - run: | - cd mobile - ./ci/test_size_regression.sh ../dist/main.zip ../dist/current.zip + target: size-compare + timeout-minutes: 30 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-perf }} + needs: + - load + - build + - compare + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-release.yml b/.github/workflows/mobile-release.yml index effa6cd4192c..d051821814f0 100644 --- a/.github/workflows/mobile-release.yml +++ b/.github/workflows/mobile-release.yml @@ -1,4 +1,4 @@ -name: mobile_release +name: Mobile/Release permissions: contents: read @@ -11,86 +11,102 @@ on: jobs: env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + secrets: + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: contents: read - statuses: write + uses: ./.github/workflows/_load_env.yml - android_release_artifacts: + release: + permissions: + contents: read + packages: read if: >- ${{ - github.repository == 'envoyproxy/envoy' + (github.repository == 'envoyproxy/envoy' + || vars.ENVOY_CI) && (github.event.schedule || !contains(github.actor, '[bot]')) }} needs: env - permissions: - contents: read - packages: read - name: android_release_artifacts - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 120 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Build envoy.aar distributable' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: mobile - run: | - version="0.5.0.$(date '+%Y%m%d')" - ./bazelw build \ - --config=mobile-remote-release-clang \ - --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ - --fat_apk_cpu=x86,x86_64,armeabi-v7a,arm64-v8a \ - --define=pom_version="$version" \ - --config=mobile-release-android \ - --linkopt=-fuse-ld=lld \ - //:android_dist - - name: 'Tar artifacts' - run: | - tar -czvf envoy_android_aar_sources.tar.gz \ - bazel-bin/library/kotlin/io/envoyproxy/envoymobile/envoy.aar \ - bazel-bin/library/kotlin/io/envoyproxy/envoymobile/envoy-pom.xml \ - bazel-bin/library/kotlin/io/envoyproxy/envoymobile/envoy-sources.jar \ - bazel-bin/library/kotlin/io/envoyproxy/envoymobile/envoy-javadoc.jar - working-directory: mobile - - uses: actions/upload-artifact@v3 - with: - name: envoy_android_aar_sources - path: mobile/envoy_android_aar_sources.tar.gz + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: ${{ matrix.args }} + container: ${{ fromJSON(needs.env.outputs.build-image).mobile }} + container-output: | + "bazel-bin/library/kotlin/io/envoyproxy/envoymobile/${{ matrix.output }}.aar": build/ + "bazel-bin/library/kotlin/io/envoyproxy/envoymobile/${{ matrix.output }}-pom.xml": build/ + "bazel-bin/library/kotlin/io/envoyproxy/envoymobile/${{ matrix.output }}-sources.jar": build/ + "bazel-bin/library/kotlin/io/envoyproxy/envoymobile/${{ matrix.output }}-javadoc.jar": build/ + request: ${{ needs.env.outputs.request }} + steps-pre: | + - run: | + mkdir /tmp/mobile + VERSION="0.5.0.$(date '+%Y%m%d')" + echo "VERSION=${VERSION}" >> $GITHUB_ENV + shell: bash + steps-post: | + - run: | + mkdir /tmp/output + shell: bash + - name: Tar artifacts + run: >- + tar + -czhf + /tmp/output/${{ matrix.output }}_android_aar_sources.tar.gz + -C + /tmp/container-output/build + . + shell: bash + target: ${{ matrix.target }} + upload-name: ${{ matrix.output }}_android_aar_sources + upload-path: /tmp/output/${{ matrix.output }}_android_aar_sources.tar.gz + strategy: + fail-fast: false + matrix: + include: + - target: android-release + args: >- + build + --config=mobile-remote-release-clang-android-publish + --define=pom_version=$VERSION + //:android_dist + output: envoy + - target: xds-release + args: >- + build + --config=mobile-remote-release-clang-android-publish-xds + --define=pom_version=$VERSION + //:android_xds_dist + output: envoy_xds - android_release_deploy: - name: android_release_deploy - needs: android_release_artifacts + deploy: + needs: release permissions: contents: read packages: read runs-on: ubuntu-22.04 timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - output: envoy + - output: envoy_xds steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: fetch-depth: 0 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: - name: envoy_android_aar_sources + name: ${{ matrix.output }}_android_aar_sources path: . - name: Expand archive run: | - tar -xvf envoy_android_aar_sources.tar.gz - mv bazel-bin/library/kotlin/io/envoyproxy/envoymobile/* . + tar -xf ${{ matrix.output }}_android_aar_sources.tar.gz - name: 'Configure gpg signing' env: GPG_KEY: ${{ secrets.EM_GPG_KEY }} @@ -107,10 +123,10 @@ jobs: gpg --passphrase $GPG_PASSPHRASE --batch --import signing-key shred signing-key - gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab envoy.aar - gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab envoy-pom.xml - gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab envoy-javadoc.jar - gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab envoy-sources.jar + gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab ${{ matrix.output }}.aar + gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab ${{ matrix.output }}-pom.xml + gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab ${{ matrix.output }}-javadoc.jar + gpg --pinentry-mode=loopback --passphrase $GPG_PASSPHRASE -ab ${{ matrix.output }}-sources.jar - name: 'Release to sonatype repository' env: READWRITE_USER: ${{ secrets.EM_SONATYPE_USER }} @@ -120,14 +136,15 @@ jobs: version="0.5.0.$(date '+%Y%m%d')" python mobile/ci/sonatype_nexus_upload.py \ --profile_id=$SONATYPE_PROFILE_ID \ + --artifact_id=${{ matrix.output }} \ --version=$version \ --files \ - envoy.aar \ - envoy-pom.xml \ - envoy-sources.jar \ - envoy-javadoc.jar \ + ${{ matrix.output }}.aar \ + ${{ matrix.output }}-pom.xml \ + ${{ matrix.output }}-sources.jar \ + ${{ matrix.output }}-javadoc.jar \ --signed_files \ - envoy.aar.asc \ - envoy-pom.xml.asc \ - envoy-sources.jar.asc \ - envoy-javadoc.jar.asc + ${{ matrix.output }}.aar.asc \ + ${{ matrix.output }}-pom.xml.asc \ + ${{ matrix.output }}-sources.jar.asc \ + ${{ matrix.output }}-javadoc.jar.asc diff --git a/.github/workflows/mobile-release_validation.yml b/.github/workflows/mobile-release_validation.yml index d55af440e54a..94331c4f3ec7 100644 --- a/.github/workflows/mobile-release_validation.yml +++ b/.github/workflows/mobile-release_validation.yml @@ -1,59 +1,92 @@ -name: mobile_release_validation +name: Mobile/Release validation permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + cache-docker: false + check-name: mobile-release-validation - validate_swiftpm_example: - if: ${{ needs.env.outputs.mobile_release_validation == 'true' }} - needs: env + validate-swiftpm-example: permissions: contents: read packages: read - name: validate_swiftpm_example - runs-on: macos-12 - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - - run: | - cd mobile + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-release-validation }} + needs: load + uses: ./.github/workflows/_run.yml + name: Build xframework + with: + args: >- + build + --config=mobile-remote-ci-macos-ios + //:ios_xcframework + command: ./bazelw + container-command: + request: ${{ needs.load.outputs.request }} + runs-on: macos-12 + source: | ./ci/mac_ci_setup.sh - name: 'Install dependencies' - - name: 'Build xcframework' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw build \ - --config=ios \ - --config=mobile-remote-ci-macos \ - //:ios_xcframework - # Ignore errors: Bad CRC when unzipping large files: https://bbs.archlinux.org/viewtopic.php?id=153011 - - run: | - unzip mobile/bazel-bin/library/swift/Envoy.xcframework.zip \ - -d mobile/examples/swift/swiftpm/Packages \ - || : - name: 'Unzip xcframework' - - run: | - xcodebuild -project mobile/examples/swift/swiftpm/EnvoySwiftPMExample.xcodeproj \ - -scheme EnvoySwiftPMExample \ - -destination platform="iOS Simulator,name=iPhone 14 Pro Max,OS=16.1" - name: 'Build app' - # TODO(jpsim): Run app and inspect logs to validate + # Ignore errors: Bad CRC when unzipping large files: https://bbs.archlinux.org/viewtopic.php?id=153011 + steps-post: | + - run: | + unzip mobile/bazel-bin/library/swift/Envoy.xcframework.zip \ + -d mobile/examples/swift/swiftpm/Packages \ + || : + shell: bash + name: Unzip xcframework + - run: | + xcodebuild -project mobile/examples/swift/swiftpm/EnvoySwiftPMExample.xcodeproj \ + -scheme EnvoySwiftPMExample \ + -destination platform="iOS Simulator,name=iPhone 14 Pro Max,OS=16.1" + shell: bash + name: Build app + # TODO(jpsim): Run app and inspect logs to validate + target: validate-swiftpm-example + timeout-minutes: 120 + trusted: ${{ fromJSON(needs.load.outputs.trusted) }} + working-directory: mobile + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + pull-requests: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-release-validation }} + needs: + - load + - validate-swiftpm-example + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/mobile-traffic_director.yml b/.github/workflows/mobile-traffic_director.yml index 7243138c6cf8..4458469f0636 100644 --- a/.github/workflows/mobile-traffic_director.yml +++ b/.github/workflows/mobile-traffic_director.yml @@ -4,9 +4,9 @@ permissions: contents: read on: - schedule: - # Once a day at midnight. - - cron: '0 0 * * *' + # Disabled scheduled runs until https://github.com/envoyproxy/envoy/issues/31416 is resolved. + # schedule: + # - cron: '0 0 * * *' # Allows manual triggering in the UI. Makes it easier to test. workflow_dispatch: @@ -30,12 +30,15 @@ jobs: timeout-minutes: 120 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - name: 'Run GcpTrafficDirectorIntegrationTest' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GCP_TEST_PROJECT_PROD_API_KEY: ${{ secrets.GCP_TEST_PROJECT_PROD_API_KEY }} + # TODO(abeyad): remove this key once we're fully switched over to using the + # prod consumer mesh shard. GCP_TEST_PROJECT_API_KEY: ${{ secrets.GCP_TEST_PROJECT_API_KEY }} ENVOY_IP_TEST_VERSIONS: v4only run: | diff --git a/.github/workflows/mobile-tsan.yml b/.github/workflows/mobile-tsan.yml index 281445793e70..70dd452bb9c9 100644 --- a/.github/workflows/mobile-tsan.yml +++ b/.github/workflows/mobile-tsan.yml @@ -1,50 +1,70 @@ -name: mobile_tsan +name: Mobile/TSAN permissions: contents: read on: - push: - branches: - - main - pull_request: + workflow_run: + workflows: + - Request + types: + - completed concurrency: group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} cancel-in-progress: true + jobs: - env: - if: ${{ github.repository == 'envoyproxy/envoy' }} - uses: ./.github/workflows/_env.yml + load: + secrets: + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} permissions: + actions: read contents: read - statuses: write + packages: read + pull-requests: read + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/_load.yml + with: + check-name: mobile-tsan + run-id: ${{ github.event.workflow_run.id }} tsan: - if: ${{ needs.env.outputs.mobile_tsan == 'true' }} - needs: env permissions: contents: read packages: read + if: ${{ fromJSON(needs.load.outputs.request).run.mobile-tsan }} + needs: load name: tsan - runs-on: ${{ needs.env.outputs.agent_ubuntu }} - timeout-minutes: 90 - container: - image: ${{ needs.env.outputs.build_image_ubuntu_mobile }} - env: - CC: /opt/llvm/bin/clang - CXX: /opt/llvm/bin/clang++ - steps: - - uses: actions/checkout@v4 - - name: Add safe directory - run: git config --global --add safe.directory /__w/envoy/envoy - - name: 'Run tests' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd mobile - ./bazelw test \ - --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ - --config=mobile-remote-ci-linux-tsan \ - //test/common/... + uses: ./.github/workflows/_mobile_container_ci.yml + with: + args: >- + test + --config=mobile-remote-ci-linux-tsan + //test/common/... + //test/cc/... + request: ${{ needs.load.outputs.request }} + target: tsan + timeout-minutes: 90 + + request: + secrets: + app-id: ${{ secrets.ENVOY_CI_APP_ID }} + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + permissions: + actions: read + contents: read + if: >- + ${{ always() + && github.event.workflow_run.conclusion == 'success' + && fromJSON(needs.load.outputs.request).run.mobile-tsan }} + needs: + - load + - tsan + uses: ./.github/workflows/_finish.yml + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml index fbb841977a83..95083e527889 100644 --- a/.github/workflows/pr_notifier.yml +++ b/.github/workflows/pr_notifier.yml @@ -1,4 +1,5 @@ on: + pull_request: workflow_dispatch: schedule: - cron: '0 5 * * 1,2,3,4,5' @@ -21,18 +22,14 @@ jobs: || !contains(github.actor, '[bot]')) }} steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.8 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 - with: - python-version: '3.8' - architecture: 'x64' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./.github/actions/pr_notifier/requirements.txt + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Notify about PRs - run: python ./.github/actions/pr_notifier/pr_notifier.py --cron_job + run: | + ARGS=() + if [[ "${{ github.event_name }}" == 'pull_request' ]]; then + ARGS+=(--dry_run) + fi + bazel run //tools/repo:notify -- "${ARGS[@]}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release_branch.yml b/.github/workflows/release_branch.yml deleted file mode 100644 index 50dd144d6141..000000000000 --- a/.github/workflows/release_branch.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Create release branch - -on: - release: - types: - - published - branches: - - main - -permissions: read-all - -jobs: - fork_release_branch: - runs-on: ubuntu-22.04 - if: github.repository == 'envoyproxy/envoy' - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - name: Create release branch - run: ./ci/create_release_branch.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - reopen_branch: - runs-on: ubuntu-22.04 - if: github.repository == 'envoyproxy/envoy' - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: refs/heads/main - - - name: Reopen branch - run: ./ci/reopen_branch.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/request.yml b/.github/workflows/request.yml new file mode 100644 index 000000000000..a245052db14a --- /dev/null +++ b/.github/workflows/request.yml @@ -0,0 +1,39 @@ +# This file must live on every branch and pass necessary secrets and permissions +# to initiate the request +name: Request + +permissions: + contents: read + +on: + pull_request_target: + push: + branches: + - main + - release/v* + +concurrency: + group: | + ${{ github.head_ref + || github.run_id + }}-${{ github.workflow }}-request + cancel-in-progress: true + + +jobs: + request: + # For branches this can be pinned to a specific version if required + # NB: `uses` cannot be dynamic so it _must_ be hardcoded anywhere it is read + uses: envoyproxy/envoy/.github/workflows/_request.yml@main + if: ${{ vars.ENVOY_CI || github.repository == 'envoyproxy/envoy' }} + permissions: + actions: read + contents: read + # required for engflow/bazel caching (not yet used) + packages: read + # required to fetch merge commit + pull-requests: read + secrets: + # these are required to start checks + app-key: ${{ secrets.ENVOY_CI_APP_KEY }} + app-id: ${{ secrets.ENVOY_CI_APP_ID }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 000000000000..8c4bb04b2ba0 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,45 @@ +name: Scorecard supply-chain security +on: + branch_protection_rule: + schedule: + - cron: '33 13 * * 5' + push: + branches: + - "main" + +permissions: + contents: read + + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-22.04 + permissions: + security-events: write + id-token: write + + steps: + - name: "Checkout code" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: "Upload artifact" + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 + with: + sarif_file: results.sarif diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d8f8986bae8a..aed93868a897 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Prune Stale - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 + uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Different amounts of days for issues/PRs are not currently supported but there is a PR diff --git a/.github/workflows/verify-requirements.in b/.github/workflows/verify-requirements.in new file mode 100644 index 000000000000..87de2e955af3 --- /dev/null +++ b/.github/workflows/verify-requirements.in @@ -0,0 +1 @@ +yq diff --git a/.github/workflows/verify-requirements.txt b/.github/workflows/verify-requirements.txt new file mode 100644 index 000000000000..2c6e79d55e41 --- /dev/null +++ b/.github/workflows/verify-requirements.txt @@ -0,0 +1,74 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes verify-requirements.in +# +argcomplete==3.2.1 \ + --hash=sha256:30891d87f3c1abe091f2142613c9d33cac84a5e15404489f033b20399b691fec \ + --hash=sha256:437f67fb9b058da5a090df505ef9be0297c4883993f3f56cb186ff087778cfb4 + # via yq +pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ + --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ + --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ + --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ + --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ + --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ + --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ + --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ + --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ + --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ + --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ + --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ + --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ + --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ + --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ + --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ + --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ + --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ + --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ + --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ + --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ + --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ + --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ + --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ + --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ + --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ + --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ + --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ + --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ + --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ + --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ + --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ + --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ + --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ + --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ + --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ + --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ + --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ + --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ + --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f + # via yq +tomlkit==0.12.3 \ + --hash=sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4 \ + --hash=sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba + # via yq +xmltodict==0.13.0 \ + --hash=sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56 \ + --hash=sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852 + # via yq +yq==3.2.3 \ + --hash=sha256:29c8fe1d36b4f64163f4d01314c6ae217539870f610216dee6025dfb5eafafb1 \ + --hash=sha256:b50c91894dad9894d1d36ea77d5722d5495cac9482d2351e55089360a90709ae + # via -r verify-requirements.in diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml deleted file mode 100644 index a2b32c02756e..000000000000 --- a/.github/workflows/workflow-complete.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Workflow complete -# This workflow is only required for externally triggered jobs that have manually -# set the check status for a commit/PR - -permissions: - contents: read - -on: - # Do not run untrusted code here - workflow_run: - workflows: - - Publish & verify - types: - - completed - -jobs: - complete: - if: ${{ github.actor == 'trigger-workflow-envoy[bot]' }} - runs-on: ubuntu-22.04 - permissions: - statuses: write - steps: - - name: 'Download artifact' - uses: actions/github-script@v6 - with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "state_sha" - })[0]; - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/state_sha.zip`, Buffer.from(download.data)); - - - run: | - set -e - unzip state_sha.zip - STATE_SHA="$(cat state_sha)" - echo "state_sha=$STATE_SHA" >> "$GITHUB_OUTPUT" - STATE="${{ github.event.workflow_run.conclusion }}" - if [[ ${STATE} != "success" ]]; then - STATE=failure - fi - echo "state=${STATE}" >> "$GITHUB_OUTPUT" - id: job - - name: Complete status check - uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.18 - with: - authToken: ${{ secrets.GITHUB_TOKEN }} - context: Verify/examples - state: ${{ steps.job.outputs.state }} - sha: ${{ steps.job.outputs.state_sha }} - target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 39267d2a1ca0..bb9133367ce8 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -32,6 +32,12 @@ "type": "shell", "command": "EXCLUDE_CONTRIB=true tools/vscode/refresh_compdb.sh", "problemMatcher": [] + }, + { + "label": "Local Fix Format", + "type": "shell", + "command": "tools/local_fix_format.sh -all", + "problemMatcher": [] } ] } diff --git a/BUILD b/BUILD index 34c8e7a4b633..e7f90fcd9ea7 100644 --- a/BUILD +++ b/BUILD @@ -1,5 +1,12 @@ +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//tools/python:namespace.bzl", "envoy_py_namespace") + licenses(["notice"]) # Apache 2 +envoy_package() + +envoy_py_namespace() + exports_files([ "VERSION.txt", "API_VERSION.txt", @@ -8,6 +15,7 @@ exports_files([ ".coveragerc", "CODEOWNERS", "OWNERS.md", + ".github/config.yml", ]) alias( @@ -20,6 +28,12 @@ alias( actual = "//source/exe:envoy-static.stripped", ) +filegroup( + name = "clang_tidy_config", + srcs = [".clang-tidy"], + visibility = ["//visibility:public"], +) + # These two definitions exist to help reduce Envoy upstream core code depending on extensions. # To avoid visibility problems, see notes in source/extensions/extensions_build_config.bzl # diff --git a/CODEOWNERS b/CODEOWNERS index b20d4198f765..dd0f36e455fe 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -26,13 +26,13 @@ extensions/filters/common/original_src @klarose @mattklein123 /*/extensions/filters/http/ext_proc @gbrail @stevenzzzz @tyxia @mattklein123 @htuch @yanavlasov /*/extensions/filters/common/mutation_rules @gbrail @tyxia @mattklein123 @htuch @yanavlasov # jwt_authn http filter extension -/*/extensions/filters/http/jwt_authn @qiwzhang @lizan +/*/extensions/filters/http/jwt_authn @taoxuy @lizan # grpc_field_extraction http filter extension /*/extensions/filters/http/grpc_field_extraction @taoxuy @nareddyt @yanavlasov # grpc_http1_reverse_bridge http filter extension /*/extensions/filters/http/grpc_http1_reverse_bridge @zuercher @mattklein123 # alts transport socket extension -/*/extensions/transport_sockets/alts @adisuissa @yangminzhu +/*/extensions/transport_sockets/alts @adisuissa @matthewstevenson88 # tcp_stats transport socket extension /*/extensions/transport_sockets/tcp_stats @ggreenway @mattklein123 # tls transport socket extension @@ -54,9 +54,9 @@ extensions/filters/common/original_src @klarose @mattklein123 # sni_dynamic_forward_proxy extension /*/extensions/filters/network/sni_dynamic_forward_proxy @rshriram @lizan # tracers.datadog extension -/*/extensions/tracers/datadog @cgilmour @palazzem @mattklein123 +/*/extensions/tracers/datadog @cgilmour @dgoffredo @dmehala @mattklein123 # tracers.xray extension -/*/extensions/tracers/xray @abaptiste @suniltheta @mattklein123 +/*/extensions/tracers/xray @suniltheta @mattklein123 # tracers.skywalking extension /*/extensions/tracers/skywalking @wbpcode @lizan @Shikugawa # tracers.opentelemetry extension @@ -84,8 +84,8 @@ extensions/filters/common/original_src @klarose @mattklein123 /*/extensions/filters/http/cache @toddmgreer @jmarantz @penguingao @mpwarres @capoferro /*/extensions/http/cache/simple_http_cache @toddmgreer @jmarantz @penguingao @mpwarres @capoferro # aws_iam grpc credentials -/*/extensions/grpc_credentials/aws_iam @lavignes @mattklein123 @tonya11en -/*/extensions/common/aws @lavignes @mattklein123 @tonya11en +/*/extensions/grpc_credentials/aws_iam @suniltheta @lavignes @mattklein123 +/*/extensions/common/aws @suniltheta @lavignes @mattklein123 # adaptive concurrency limit extension. /*/extensions/filters/http/adaptive_concurrency @tonya11en @mattklein123 # admission control extension. @@ -112,12 +112,12 @@ extensions/filters/common/original_src @klarose @mattklein123 /*/extensions/filters/http/grpc_http1_bridge @jose @mattklein123 /*/extensions/filters/http/fault @rshriram @alyssawilk /*/extensions/filters/common/fault @rshriram @alyssawilk -/*/extensions/filters/http/grpc_json_transcoder @qiwzhang @lizan +/*/extensions/filters/http/grpc_json_transcoder @nareddyt @lizan /*/extensions/filters/http/router @alyssawilk @mattklein123 /*/extensions/filters/common/rbac/matchers @conqerAtapple @ggreenway @alyssawilk /*/extensions/filters/http/grpc_web @fengli79 @lizan /*/extensions/filters/http/grpc_stats @kyessenov @lizan -/*/extensions/filters/http/connect_grpc_bridge @UNOWNED @UNOWNED +/*/extensions/filters/http/connect_grpc_bridge @jchadwick-buf @mattklein123 /*/extensions/filters/common/original_src @klarose @mattklein123 /*/extensions/filters/listener/tls_inspector @ggreenway @KBaichoo /*/extensions/grpc_credentials/example @wozz @htuch @@ -140,15 +140,15 @@ extensions/filters/common/original_src @klarose @mattklein123 /*/extensions/filters/network/http_connection_manager @alyssawilk @mattklein123 /*/extensions/filters/network/tcp_proxy @alyssawilk @zuercher @ggreenway /*/extensions/filters/network/echo @htuch @alyssawilk -/*/extensions/filters/udp/dns_filter @abaptiste @mattklein123 @yanjunxiang-google +/*/extensions/filters/udp/dns_filter @mattklein123 @yanjunxiang-google /*/extensions/filters/network/direct_response @kyessenov @zuercher /*/extensions/filters/udp/udp_proxy @mattklein123 @danzh2010 /*/extensions/clusters/aggregate @yxue @mattklein123 # support for on-demand VHDS requests /*/extensions/filters/http/on_demand @dmitri-d @htuch @kyessenov /*/extensions/filters/network/connection_limit @mattklein123 @alyssawilk @delong-coder -/*/extensions/filters/http/aws_request_signing @derekargueta @mattklein123 @marcomagdy @tonya11en -/*/extensions/filters/http/aws_lambda @mattklein123 @marcomagdy @lavignes @tonya11en +/*/extensions/filters/http/aws_request_signing @derekargueta @suniltheta @mattklein123 @marcomagdy +/*/extensions/filters/http/aws_lambda @suniltheta @mattklein123 @marcomagdy @lavignes /*/extensions/filters/http/buffer @alyssawilk @mattklein123 /*/extensions/transport_sockets/raw_buffer @alyssawilk @mattklein123 # Watchdog Extensions @@ -192,9 +192,15 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123 /*/extensions/filters/http/rate_limit_quota @tyxia @yanavlasov # HTTP Bandwidth Limit /*/extensions/filters/http/bandwidth_limit @nitgoy @mattklein123 @yanavlasov @tonya11en +# HTTP Basic Auth +/*/extensions/filters/http/basic_auth @zhaohuabing @wbpcode # Original IP detection /*/extensions/http/original_ip_detection/custom_header @alyssawilk @mattklein123 /*/extensions/http/original_ip_detection/xff @alyssawilk @mattklein123 +# set_filter_state extension +/*/extensions/filters/common/set_filter_state @kyessenov @wbpcode +/*/extensions/filters/http/set_filter_state @kyessenov @wbpcode +/*/extensions/filters/network/set_filter_state @kyessenov @wbpcode # set_metadata extension /*/extensions/filters/http/set_metadata @aguinet @mattklein123 # Formatters @@ -258,7 +264,7 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123 /*/extensions/http/stateful_session/header @ramaraochavali @wbpcode @cpakulski /*/extensions/filters/http/stateful_session @wbpcode @cpakulski # tracers -/*/extensions/tracers/zipkin @wbpcode @Shikugawa @suniltheta @basvanbeek +/*/extensions/tracers/zipkin @wbpcode @Shikugawa @basvanbeek /*/extensions/tracers/dynamic_ot @wbpcode @Shikugawa @basvanbeek /*/extensions/tracers/opencensus @wbpcode @Shikugawa @basvanbeek /*/extensions/tracers/common @wbpcode @Shikugawa @basvanbeek @@ -299,10 +305,10 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123 # Dubbo codec /*/extensions/common/dubbo @wbpcode @lizan # upstream load balancing policies -/*/extensions/load_balancing_policies/common @wbpcode @UNOWNED -/*/extensions/load_balancing_policies/least_request @wbpcode @UNOWNED -/*/extensions/load_balancing_policies/random @wbpcode @UNOWNED -/*/extensions/load_balancing_policies/round_robin @wbpcode @UNOWNED +/*/extensions/load_balancing_policies/common @wbpcode @tonya11en +/*/extensions/load_balancing_policies/least_request @wbpcode @tonya11en +/*/extensions/load_balancing_policies/random @wbpcode @tonya11en +/*/extensions/load_balancing_policies/round_robin @wbpcode @tonya11en /*/extensions/load_balancing_policies/ring_hash @wbpcode @UNOWNED /*/extensions/load_balancing_policies/maglev @wbpcode @UNOWNED /*/extensions/load_balancing_policies/subset @wbpcode @zuercher @@ -321,9 +327,15 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123 /*/extensions/health_check/event_sinks/file @botengyao @yanavlasov # IP Geolocation /*/extensions/filters/http/geoip @nezdolik @ravenblackx +/*/extensions/geoip_providers/common @nezdolik @ravenblackx +# Maxmind geolocation provider +/*/extensions/geoip_providers/maxmind @nezdolik @ravenblackx /*/extensions/health_checkers/common @zuercher @botengyao +# Lua cluster specifier +/*/extensions/router/cluster_specifiers/lua @StarryVae @wbpcode + # Intentionally exempt (treated as core code) /*/extensions/filters/common @UNOWNED @UNOWNED /*/extensions/filters/http/common @UNOWNED @UNOWNED diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ff4a8d508cc..a1f77ded5e73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -237,18 +237,19 @@ Please note that if adding a runtime guarded feature, your [release notes](chang * Typically we try to turn around reviews within one business day. * See [OWNERS.md](OWNERS.md) for the current list of maintainers. * It is generally expected that a senior maintainer should review every PR to - core code. Test-only or extension-only changes need only be reviewed by a - maintainer, or senior extension maintainer. + core code. Changes which only touch tests, extensions, tools, docs or comments + need only be reviewed by a maintainer, or senior extension maintainer. * It is also generally expected that a "domain expert" for the code the PR touches should review the PR. This person does not necessarily need to have commit access. -* The previous two points generally mean that every PR should have two approvals. (Exceptions can - be made by the senior maintainers). -* The above rules may be waived for PRs which only update docs or comments, or trivial changes to - tests and tools (where trivial is decided by the maintainer in question). -* In general, we should also attempt to make sure that at least one of the approvals is *from an +* For new extensions (contrib or otherwise) and features, at least one of the approvals should be *from an organization different from the PR author.* E.g., if Lyft authors a PR, at least one approver should be from an organization other than Lyft. This helps us make sure that we aren't putting organization specific shortcuts into the code. + new HTTP/3 features are largely exempt from cross-company approvals as all of the + area experts work at a single company, but HTTP/3 changes which impact general + functionality still merit a cross-company check. +* contrib extensions do not need senior maintainer or maintainer review only contrib owner review and + a maintainer stamp to merge. * If there is a question on who should review a PR please discuss in Slack. * Anyone is welcome to review any PR that they want, whether they are a maintainer or not. * Please make sure that the PR title, commit message, and description are updated if the PR changes diff --git a/OWNERS.md b/OWNERS.md index bf0a1a65fb05..51a50bab403d 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -29,6 +29,8 @@ routing PRs, questions, etc. to the right place. * Docs, tooling, CI, containers and sandbox examples * Ryan Hamilton ([RyanTheOptimist](https://github.com/ryantheoptimist)) (rch@google.com) * HTTP/3, upstream connection management, Envoy Mobile. +* Baiping Wang ([wbpcode](https://github.com/wbpcode)) (wbphub@live.com) + * Upstream, LB, tracing, logging, performance, and generic/dubbo proxy. # Maintainers @@ -38,8 +40,6 @@ routing PRs, questions, etc. to the right place. * xDS APIs, configuration, control plane, fuzzing. * Kevin Baichoo ([KBaichoo](https://github.com/KBaichoo)) (kbaichoo@google.com) * Data plane, overload management, flow control. -* Baiping Wang ([wbpcode](https://github.com/wbpcode)) (wbphub@live.com) - * Upstream, LB, tracing, logging, performance, and generic/dubbo proxy. * Keith Smiley ([keith](https://github.com/keith)) (keithbsmiley@gmail.com) * Bazel, CI, compilers, linkers, general build issues, etc. * Kuat Yessenov ([kyessenov](https://github.com/kyessenov)) (kuat@google.com) @@ -48,6 +48,8 @@ routing PRs, questions, etc. to the right place. * Caches, file filters, and file I/O. * Alex Xu ([soulxu](https://github.com/soulxu)) (hejie.xu@intel.com) * Listeners, iouring, data plane. +* Kateryna Nezdolii ([nezdolik](https://github.com/nezdolik)) (kateryna.nezdolii@gmail.com) + * Load balancing, GeoIP, overload manager, security. # Envoy mobile maintainers diff --git a/README.md b/README.md index b3325ef8bdb1..5d292c80d5dd 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ involved and how Envoy plays a role, read the CNCF [announcement](https://www.cncf.io/blog/2017/09/13/cncf-hosts-envoy/). [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1266/badge)](https://bestpractices.coreinfrastructure.org/projects/1266) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/envoyproxy/envoy/badge)](https://api.securityscorecards.dev/projects/github.com/envoyproxy/envoy) [![Azure Pipelines](https://dev.azure.com/cncf/envoy/_apis/build/status/11?branchName=main)](https://dev.azure.com/cncf/envoy/_build/latest?definitionId=11&branchName=main) [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/envoy.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:envoy) [![Jenkins](https://powerci.osuosl.org/buildStatus/icon?job=build-envoy-static-master&subject=ppc64le%20build)](https://powerci.osuosl.org/job/build-envoy-static-master/) diff --git a/RELEASES.md b/RELEASES.md index bdd6ca20bdd9..564a0bfb1d79 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -20,13 +20,6 @@ Major releases happen quartely and follow the schedule below. Security fixes typ quarterly as well, but this depends on the number and severity of security bugs. Other releases are ad-hoc and best-effort. -### Hand-off - -Hand-off to the maintainers of stable releases happens after Envoy maintainers release a new -version from the `main` branch by creating a `vX.Y.0` tag and a corresponding `release/vX.Y` -branch, with merge permissions given to the release manager of stable releases, and CI configured -to execute tests on it. - ### Security releases Critical security fixes are owned by the Envoy security team, which provides fixes for the @@ -97,7 +90,8 @@ deadline of 3 weeks. | 1.25.0 | 2023/01/15 | 2023/01/18 | +3 days | 2024/01/18 | | 1.26.0 | 2023/04/15 | 2023/04/18 | +3 days | 2024/04/18 | | 1.27.0 | 2023/07/14 | 2023/07/27 | +13 days | 2024/07/27 | -| 1.28.0 | 2023/10/16 | | | | +| 1.28.0 | 2023/10/16 | 2023/10/19 | +3 days | 2024/10/19 | +| 1.29.0 | 2024/01/16 | | | | ### Cutting a major release @@ -121,15 +115,7 @@ deadline of 3 weeks. * Get a review and merge. * Create a pull request with the commit created by the project tool and **wait for tests to pass**. -* Once the tests have passed, and the PR has landed, CI will automatically create the tagged release. -* From the envoy [landing page](https://github.com/envoyproxy/envoy) use the branch drop-down to create a branch - using the minor version from the tagged release, e.g. `1.6.0` -> `release/v1.6`. It will be used for the - [stable releases](RELEASES.md#stable-releases). -* Tagging will kick off another run of [AZP postsubmit](https://dev.azure.com/cncf/envoy/_build?definitionId=11). Monitor that - tag build to make sure that the final docker images get pushed along with - the final docs and [release assets](https://github.com/envoyproxy/envoy/releases). The final documentation will end up in the - [envoy-website repository](https://github.com/envoyproxy/envoy-website/tree/main/docs/envoy). -* Update the website ([example PR](https://github.com/envoyproxy/envoy-website/pull/148)) with the new release version. +* Once the tests have passed, and the PR has landed, CI will automatically create the tagged release and corresponding release branch. * Craft a witty/uplifting email and send it to all the email aliases: envoy-announce@ envoy-users@ envoy-dev@ envoy-maintainers * Make sure we tweet the new release: either have Matt do it or email social@cncf.io and ask them to do an Envoy account post. diff --git a/SECURITY.md b/SECURITY.md index dceab5e0a447..a472a952f661 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -465,11 +465,11 @@ and security team to ensure they still qualify for inclusion on the list. | Organization | End User | Last Review | |:-------------:|:--------:|:-----------:| -| Aspen Mesh | No | 06/21 | | AWS | No | 06/21 | | Cilium | No | 06/21 | | Cloud Foundry | No | 06/21 | | Datawire | No | 06/21 | +| F5 | No | 06/21 | | Google | No | 06/21 | | IBM | No | 06/21 | | Istio | No | 06/21 | diff --git a/STYLE.md b/STYLE.md index 69de7b6e5cb3..42d255be31a1 100644 --- a/STYLE.md +++ b/STYLE.md @@ -18,7 +18,7 @@ # Documentation -* If you are modifying the data plane structually, please keep the [Life of a +* If you are modifying the data plane structurally, please keep the [Life of a Request](https://www.envoyproxy.io/docs/envoy/latest/intro/life_of_a_request) documentation up-to-date. # Deviations from Google C++ style guidelines diff --git a/VERSION.txt b/VERSION.txt index 364d77fabb84..ac3f6e1415bf 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.28.0-dev +1.29.0-dev diff --git a/api/API_VERSIONING.md b/api/API_VERSIONING.md index 8add2227291a..726d1d036632 100644 --- a/api/API_VERSIONING.md +++ b/api/API_VERSIONING.md @@ -87,7 +87,7 @@ no field will ever be removed nor will Envoy ever remove the implementation for field. **NOTE**: Client implementations are free to output additional warnings about field usage beyond -deprecation, if for example, the use of the continued use of the field is deemed a substantial +deprecation, if for example, the continued use of the field is deemed a substantial security risk. Individual client versions are also free to stop supporting fields if they want to, though Envoy Proxy (as an xDS client) commits to never doing so. diff --git a/api/BUILD b/api/BUILD index 37d46fa349dd..16a0037cf8e8 100644 --- a/api/BUILD +++ b/api/BUILD @@ -72,6 +72,7 @@ proto_library( name = "v3_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg", "//contrib/envoy/extensions/filters/http/checksum/v3alpha:pkg", "//contrib/envoy/extensions/filters/http/dynamo/v3:pkg", "//contrib/envoy/extensions/filters/http/golang/v3alpha:pkg", @@ -81,6 +82,7 @@ proto_library( "//contrib/envoy/extensions/filters/network/client_ssl_auth/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/action/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3:pkg", + "//contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/router/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg", @@ -156,12 +158,14 @@ proto_library( "//envoy/extensions/filters/common/dependency/v3:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/extensions/filters/common/matcher/action/v3:pkg", + "//envoy/extensions/filters/common/set_filter_state/v3:pkg", "//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg", "//envoy/extensions/filters/http/admission_control/v3:pkg", "//envoy/extensions/filters/http/alternate_protocols_cache/v3:pkg", "//envoy/extensions/filters/http/aws_lambda/v3:pkg", "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", "//envoy/extensions/filters/http/bandwidth_limit/v3:pkg", + "//envoy/extensions/filters/http/basic_auth/v3:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3:pkg", "//envoy/extensions/filters/http/cdn_loop/v3:pkg", @@ -169,6 +173,7 @@ proto_library( "//envoy/extensions/filters/http/compressor/v3:pkg", "//envoy/extensions/filters/http/connect_grpc_bridge/v3:pkg", "//envoy/extensions/filters/http/cors/v3:pkg", + "//envoy/extensions/filters/http/credential_injector/v3:pkg", "//envoy/extensions/filters/http/csrf/v3:pkg", "//envoy/extensions/filters/http/custom_response/v3:pkg", "//envoy/extensions/filters/http/decompressor/v3:pkg", @@ -202,6 +207,7 @@ proto_library( "//envoy/extensions/filters/http/ratelimit/v3:pkg", "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", + "//envoy/extensions/filters/http/set_filter_state/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", "//envoy/extensions/filters/http/stateful_session/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", @@ -225,6 +231,7 @@ proto_library( "//envoy/extensions/filters/network/ratelimit/v3:pkg", "//envoy/extensions/filters/network/rbac/v3:pkg", "//envoy/extensions/filters/network/redis_proxy/v3:pkg", + "//envoy/extensions/filters/network/set_filter_state/v3:pkg", "//envoy/extensions/filters/network/sni_cluster/v3:pkg", "//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3:pkg", "//envoy/extensions/filters/network/tcp_proxy/v3:pkg", @@ -236,10 +243,14 @@ proto_library( "//envoy/extensions/filters/network/wasm/v3:pkg", "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg", "//envoy/extensions/filters/udp/dns_filter/v3:pkg", + "//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg", + "//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg", "//envoy/extensions/filters/udp/udp_proxy/v3:pkg", "//envoy/extensions/formatter/cel/v3:pkg", "//envoy/extensions/formatter/metadata/v3:pkg", "//envoy/extensions/formatter/req_without_query/v3:pkg", + "//envoy/extensions/geoip_providers/common/v3:pkg", + "//envoy/extensions/geoip_providers/maxmind/v3:pkg", "//envoy/extensions/health_check/event_sinks/file/v3:pkg", "//envoy/extensions/health_checkers/redis/v3:pkg", "//envoy/extensions/health_checkers/thrift/v3:pkg", @@ -254,6 +265,8 @@ proto_library( "//envoy/extensions/http/original_ip_detection/xff/v3:pkg", "//envoy/extensions/http/stateful_session/cookie/v3:pkg", "//envoy/extensions/http/stateful_session/header/v3:pkg", + "//envoy/extensions/injected_credentials/generic/v3:pkg", + "//envoy/extensions/injected_credentials/oauth2/v3:pkg", "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg", "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", @@ -296,9 +309,12 @@ proto_library( "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg", "//envoy/extensions/retry/host/previous_hosts/v3:pkg", "//envoy/extensions/retry/priority/previous_priorities/v3:pkg", + "//envoy/extensions/router/cluster_specifiers/lua/v3:pkg", "//envoy/extensions/stat_sinks/graphite_statsd/v3:pkg", "//envoy/extensions/stat_sinks/open_telemetry/v3:pkg", "//envoy/extensions/stat_sinks/wasm/v3:pkg", + "//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg", + "//envoy/extensions/tracers/opentelemetry/samplers/v3:pkg", "//envoy/extensions/transport_sockets/alts/v3:pkg", "//envoy/extensions/transport_sockets/http_11_proxy/v3:pkg", "//envoy/extensions/transport_sockets/internal_upstream/v3:pkg", @@ -353,9 +369,9 @@ proto_library( name = "xds_protos", visibility = ["//visibility:public"], deps = [ - "@com_github_cncf_udpa//xds/core/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", - "@com_github_cncf_udpa//xds/type/v3:pkg", + "@com_github_cncf_xds//xds/core/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//xds/type/v3:pkg", ], ) diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index a1e61a7072c4..0ff244623984 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -23,19 +23,19 @@ documentation. The documentation can be built locally in the root of https://github.com/envoyproxy/envoy via: ``` -docs/build.sh +ci/do_ci.sh docs ``` To skip configuration examples validation: ``` -SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +SPHINX_SKIP_CONFIG_VALIDATION=true ci/do_ci.sh docs ``` Or to use a hermetic Docker container: ``` -./ci/run_envoy_docker.sh './ci/do_ci.sh docs' +./ci/run_envoy_docker.sh 'ci/do_ci.sh docs' ``` This process builds RST documentation directly from the proto files, merges it with the static RST diff --git a/api/STYLE.md b/api/STYLE.md index b0821c927be2..8d042e0e2224 100644 --- a/api/STYLE.md +++ b/api/STYLE.md @@ -60,13 +60,14 @@ In addition, the following conventions should be followed: * Always use upper camel case names for message types and enum types without embedded acronyms, such as `HttpRequest`. -* Prefer `oneof` selections to boolean overloads of fields, for example, prefer: +* Prefer multiple fields with defined precedence over boolean overloads of fields or + `oneof`. For example, prefer: ```proto - oneof path_specifier { - string simple_path = 1; - string regex_path = 2; - } + // Simple path matcher. If regex_path is set, this field is not used. + string simple_path = 1; + // Regex path matcher. If set, takes precedence over simple_path. + string regex_path = 2; ``` to @@ -76,7 +77,17 @@ In addition, the following conventions should be followed: bool path_is_regex = 2; ``` - This is more efficient, extendable and self-describing. + or + + ``` + oneof path_specifier { + string simple_path = 1; + string regex_path = 2; + } + ``` + + This is more efficient on the wire. It also allows new alternatives to be + added later in a way that allows control planes to be backward-compatible. * The API includes two types for representing [percents](envoy/type/percent.proto). `Percent` is effectively a double value in the range 0.0-100.0. `FractionalPercent` is an integral fraction @@ -126,7 +137,7 @@ To add an extension config to the API, the steps below should be followed: licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) ``` 1. If this is still WiP and subject to breaking changes, please tag it @@ -162,7 +173,7 @@ To add an extension config to the API, the steps below should be followed: [source/extensions/extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl) or [contrib/contrib_build_config.bzl](../contrib/contrib_build_config.bzl) to include the new extension. 1. If the extension is not hidden, find or create a docs file with a toctree - and to reference your proto to make sure users can navigate to it from the API docs + and reference your proto to make sure users can navigate to it from the API docs (and to not break the docs build), like [docs/root/api-v3/admin/admin.rst](../docs/root/api-v3/admin/admin.rst). 1. Run `./tools/proto_format/proto_format.sh fix`. **Before running the script**, you will need to **commit your local changes**. By adding the commit, the tool will recognize the change, and will regenerate the `BUILD` file and reformat `foobar.proto` as needed. If you have not followed any of the above steps correctly `proto_format.sh` may remove some of the files that you added. If that is the case you can revert to the committed state, and try again once any issues are resolved. 1. See the [key-value-store PR](https://github.com/envoyproxy/envoy/pull/17745/files) for an example of adding a new extension point to common. @@ -180,8 +191,6 @@ metadata. We describe these annotations below by category. been disallowed by default as per the [breaking change policy](../CONTRIBUTING.md#breaking-change-policy). * `[(udpa.annotations.field_migrate).rename = ""]` to denote that the field will be renamed to a given name in the next API major version. -* `[(udpa.annotations.field_migrate).oneof_promotion = ""]` to denote that - the field will be promoted to a given `oneof` in the next API major version. * `[(udpa.annotations.sensitive) = true]` to denote sensitive fields that should be redacted in output such as logging or configuration dumps. * [PGV annotations](https://github.com/bufbuild/protoc-gen-validate) to denote field @@ -258,8 +267,7 @@ xDS APIs: breaking changes where there is no substantial gain in functionality, performance, security or implementation simplification. We will tolerate technical debt in the API itself, e.g. in the form of vestigial deprecated - fields or reduced ergonomics (such as not using `oneof` when we would prefer - to), in order to meet this principle. + fields or reduced ergonomics in order to meet this principle. * Namespaces for extensions, metadata, etc. use a reverse DNS naming scheme, e.g. `com.google.widget`, `com.lyft.widget`. Client built-ins may be prefixed diff --git a/api/bazel/BUILD b/api/bazel/BUILD index 63651c1e5a48..c4116598f74c 100644 --- a/api/bazel/BUILD +++ b/api/bazel/BUILD @@ -1,5 +1,5 @@ +load("@envoy_toolshed//:macros.bzl", "json_data") load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler") -load(":utils.bzl", "json_data") load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") load(":repository_locations_utils.bzl", "load_repository_locations_spec") load( @@ -7,7 +7,6 @@ load( "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP", "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP", "EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP", - "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP", ) licenses(["notice"]) # Apache 2 @@ -38,6 +37,5 @@ json_data( cc = EXTERNAL_PROTO_CC_BAZEL_DEP_MAP, go = EXTERNAL_PROTO_GO_BAZEL_DEP_MAP, imports = EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP, - py = EXTERNAL_PROTO_PY_BAZEL_DEP_MAP, ), ) diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 0266f4d03f2d..1f24149b9f92 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -1,6 +1,6 @@ load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library") load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") -load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", _py_proto_library = "py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") load("@io_bazel_rules_go//go:def.bzl", "go_test") load("@rules_proto//proto:defs.bzl", "proto_library") @@ -8,10 +8,9 @@ load( "//bazel:external_proto_deps.bzl", "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP", "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP", - "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP", ) load( - "@envoy//bazel/cc_proto_descriptor_library:builddefs.bzl", + "//bazel/cc_proto_descriptor_library:builddefs.bzl", "cc_proto_descriptor_library", ) @@ -52,63 +51,6 @@ def _go_proto_mapping(dep): def _cc_proto_mapping(dep): return _proto_mapping(dep, EXTERNAL_PROTO_CC_BAZEL_DEP_MAP, _CC_PROTO_SUFFIX) -def _py_proto_mapping(dep): - return _proto_mapping(dep, EXTERNAL_PROTO_PY_BAZEL_DEP_MAP, _PY_PROTO_SUFFIX) - -# TODO(htuch): Convert this to native py_proto_library once -# https://github.com/bazelbuild/bazel/issues/3935 and/or -# https://github.com/bazelbuild/bazel/issues/2626 are resolved. -def _api_py_proto_library(name, srcs = [], deps = []): - mapped_deps = [_py_proto_mapping(dep) for dep in deps] - mapped_unique_deps = {k: True for k in mapped_deps}.keys() - _py_proto_library( - name = name + _PY_PROTO_SUFFIX, - srcs = srcs, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - deps = mapped_unique_deps + [ - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@com_google_googleapis//google/rpc:status_py_proto", - "@com_google_googleapis//google/api:annotations_py_proto", - "@com_google_googleapis//google/api:http_py_proto", - "@com_google_googleapis//google/api:httpbody_py_proto", - ], - visibility = ["//visibility:public"], - ) - -# This defines googleapis py_proto_library. The repository does not provide its definition and requires -# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details). -def py_proto_library(name, deps = [], plugin = None): - srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps] - proto_deps = [] - - # py_proto_library in googleapis specifies *_proto rules in dependencies. - # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved. - # As a workaround, manually specify the proto dependencies for the imported python rules. - if name == "annotations_py_proto": - proto_deps = proto_deps + [":http_py_proto"] - - # checked.proto depends on syntax.proto, we have to add this dependency manually as well. - if name == "checked_py_proto": - proto_deps = proto_deps + [":syntax_py_proto"] - - # Special handling for expr_proto target - if srcs[0] == ":expr_moved.proto": - srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"] - proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"] - - # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0: - # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72. - # plugin should also be passed in here when gRPC version is greater than v1.25.x. - _py_proto_library( - name = name, - srcs = srcs, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - deps = proto_deps + ["@com_google_protobuf//:protobuf_python"], - visibility = ["//visibility:public"], - ) - def _api_cc_grpc_library(name, proto, deps = []): cc_grpc_library( name = name, @@ -157,7 +99,15 @@ def api_cc_py_proto_library( deps = [relative_name], visibility = ["//visibility:public"], ) - _api_py_proto_library(name, srcs, deps) + + # Uses gRPC implementation of py_proto_library. + # https://github.com/grpc/grpc/blob/v1.59.1/bazel/python_rules.bzl#L160 + _py_proto_library( + name = name + _PY_PROTO_SUFFIX, + # Actual dependencies are resolved automatically from the proto_library dep tree. + deps = [relative_name], + visibility = ["//visibility:public"], + ) # Optionally define gRPC services if has_services: diff --git a/bazel/cc_proto_descriptor_library/BUILD b/api/bazel/cc_proto_descriptor_library/BUILD similarity index 100% rename from bazel/cc_proto_descriptor_library/BUILD rename to api/bazel/cc_proto_descriptor_library/BUILD diff --git a/bazel/cc_proto_descriptor_library/builddefs.bzl b/api/bazel/cc_proto_descriptor_library/builddefs.bzl similarity index 99% rename from bazel/cc_proto_descriptor_library/builddefs.bzl rename to api/bazel/cc_proto_descriptor_library/builddefs.bzl index 53e312002646..2da95d00a063 100644 --- a/bazel/cc_proto_descriptor_library/builddefs.bzl +++ b/api/bazel/cc_proto_descriptor_library/builddefs.bzl @@ -332,7 +332,6 @@ cc_proto_descriptor_library_aspect = aspect( attr_aspects = ["deps"], fragments = ["cpp"], toolchains = use_cpp_toolchain(), - incompatible_use_toolchain_transition = True, ) cc_proto_descriptor_library = rule( diff --git a/bazel/cc_proto_descriptor_library/create_dynamic_message.cc b/api/bazel/cc_proto_descriptor_library/create_dynamic_message.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/create_dynamic_message.cc rename to api/bazel/cc_proto_descriptor_library/create_dynamic_message.cc diff --git a/bazel/cc_proto_descriptor_library/create_dynamic_message.h b/api/bazel/cc_proto_descriptor_library/create_dynamic_message.h similarity index 100% rename from bazel/cc_proto_descriptor_library/create_dynamic_message.h rename to api/bazel/cc_proto_descriptor_library/create_dynamic_message.h diff --git a/bazel/cc_proto_descriptor_library/file_descriptor_generator.cc b/api/bazel/cc_proto_descriptor_library/file_descriptor_generator.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/file_descriptor_generator.cc rename to api/bazel/cc_proto_descriptor_library/file_descriptor_generator.cc diff --git a/bazel/cc_proto_descriptor_library/file_descriptor_generator.h b/api/bazel/cc_proto_descriptor_library/file_descriptor_generator.h similarity index 100% rename from bazel/cc_proto_descriptor_library/file_descriptor_generator.h rename to api/bazel/cc_proto_descriptor_library/file_descriptor_generator.h diff --git a/bazel/cc_proto_descriptor_library/file_descriptor_generator_main.cc b/api/bazel/cc_proto_descriptor_library/file_descriptor_generator_main.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/file_descriptor_generator_main.cc rename to api/bazel/cc_proto_descriptor_library/file_descriptor_generator_main.cc diff --git a/bazel/cc_proto_descriptor_library/file_descriptor_info.cc b/api/bazel/cc_proto_descriptor_library/file_descriptor_info.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/file_descriptor_info.cc rename to api/bazel/cc_proto_descriptor_library/file_descriptor_info.cc diff --git a/bazel/cc_proto_descriptor_library/file_descriptor_info.h b/api/bazel/cc_proto_descriptor_library/file_descriptor_info.h similarity index 100% rename from bazel/cc_proto_descriptor_library/file_descriptor_info.h rename to api/bazel/cc_proto_descriptor_library/file_descriptor_info.h diff --git a/bazel/cc_proto_descriptor_library/testdata/BUILD b/api/bazel/cc_proto_descriptor_library/testdata/BUILD similarity index 100% rename from bazel/cc_proto_descriptor_library/testdata/BUILD rename to api/bazel/cc_proto_descriptor_library/testdata/BUILD diff --git a/bazel/cc_proto_descriptor_library/testdata/create_dynamic_message_test.cc b/api/bazel/cc_proto_descriptor_library/testdata/create_dynamic_message_test.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/testdata/create_dynamic_message_test.cc rename to api/bazel/cc_proto_descriptor_library/testdata/create_dynamic_message_test.cc diff --git a/bazel/cc_proto_descriptor_library/testdata/global_fallback_test.cc b/api/bazel/cc_proto_descriptor_library/testdata/global_fallback_test.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/testdata/global_fallback_test.cc rename to api/bazel/cc_proto_descriptor_library/testdata/global_fallback_test.cc diff --git a/bazel/cc_proto_descriptor_library/testdata/test-extension.proto b/api/bazel/cc_proto_descriptor_library/testdata/test-extension.proto similarity index 100% rename from bazel/cc_proto_descriptor_library/testdata/test-extension.proto rename to api/bazel/cc_proto_descriptor_library/testdata/test-extension.proto diff --git a/bazel/cc_proto_descriptor_library/testdata/test.proto b/api/bazel/cc_proto_descriptor_library/testdata/test.proto similarity index 100% rename from bazel/cc_proto_descriptor_library/testdata/test.proto rename to api/bazel/cc_proto_descriptor_library/testdata/test.proto diff --git a/bazel/cc_proto_descriptor_library/testdata/test1.proto b/api/bazel/cc_proto_descriptor_library/testdata/test1.proto similarity index 66% rename from bazel/cc_proto_descriptor_library/testdata/test1.proto rename to api/bazel/cc_proto_descriptor_library/testdata/test1.proto index d8fcdd5155a9..bb0ad106c317 100644 --- a/bazel/cc_proto_descriptor_library/testdata/test1.proto +++ b/api/bazel/cc_proto_descriptor_library/testdata/test1.proto @@ -2,8 +2,6 @@ syntax = "proto2"; package testdata.dynamic_descriptors; -import "bazel/cc_proto_descriptor_library/testdata/test.proto"; - message FooCopy { optional string bar = 1; diff --git a/bazel/cc_proto_descriptor_library/testdata/text_format_transcoder_test.cc b/api/bazel/cc_proto_descriptor_library/testdata/text_format_transcoder_test.cc similarity index 100% rename from bazel/cc_proto_descriptor_library/testdata/text_format_transcoder_test.cc rename to api/bazel/cc_proto_descriptor_library/testdata/text_format_transcoder_test.cc diff --git a/bazel/cc_proto_descriptor_library/text_format_transcoder.cc b/api/bazel/cc_proto_descriptor_library/text_format_transcoder.cc similarity index 98% rename from bazel/cc_proto_descriptor_library/text_format_transcoder.cc rename to api/bazel/cc_proto_descriptor_library/text_format_transcoder.cc index 059ab94935c2..70b116d33b14 100644 --- a/bazel/cc_proto_descriptor_library/text_format_transcoder.cc +++ b/api/bazel/cc_proto_descriptor_library/text_format_transcoder.cc @@ -93,7 +93,7 @@ std::unique_ptr TextFormatTranscoder::createEmptyDyna absl::string_view type_name, google::protobuf::io::ErrorCollector* error_collector) const { const google::protobuf::Descriptor* descriptor = internals_->descriptor_pool.FindMessageTypeByName(std::string(type_name)); - // If you're built with the full runtime then embeddng the descriptors and + // If you're built with the full runtime then embedding the descriptors and // loading them would be information duplicated by the global descriptor // pool which hurts builds like superroot that are near all the blaze/forge // size limits. Teams that care about not silently falling into this fallback diff --git a/bazel/cc_proto_descriptor_library/text_format_transcoder.h b/api/bazel/cc_proto_descriptor_library/text_format_transcoder.h similarity index 100% rename from bazel/cc_proto_descriptor_library/text_format_transcoder.h rename to api/bazel/cc_proto_descriptor_library/text_format_transcoder.h diff --git a/api/bazel/envoy_http_archive.bzl b/api/bazel/envoy_http_archive.bzl index 15fd65b2af27..849e2500678f 100644 --- a/api/bazel/envoy_http_archive.bzl +++ b/api/bazel/envoy_http_archive.bzl @@ -1,6 +1,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -def envoy_http_archive(name, locations, **kwargs): +def envoy_http_archive(name, locations, location_name = None, **kwargs): # `existing_rule_keys` contains the names of repositories that have already # been defined in the Bazel workspace. By skipping repos with existing keys, # users can override dependency versions by using standard Bazel repository @@ -10,7 +10,7 @@ def envoy_http_archive(name, locations, **kwargs): # This repository has already been defined, probably because the user # wants to override the version. Do nothing. return - location = locations[name] + location = locations[location_name or name] # HTTP tarball at a given URL. Add a BUILD file if requested. http_archive( diff --git a/api/bazel/external_proto_deps.bzl b/api/bazel/external_proto_deps.bzl index bea4868dc580..331934132ab6 100644 --- a/api/bazel/external_proto_deps.bzl +++ b/api/bazel/external_proto_deps.bzl @@ -22,7 +22,7 @@ EXTERNAL_PROTO_GO_BAZEL_DEP_MAP = { # Note @com_google_googleapis are point to @go_googleapis. # # It is aligned to xDS dependency to suppress the conflicting package heights error between - # @com_github_cncf_udpa//xds/type/matcher/v3:pkg_go_proto + # @com_github_cncf_xds//xds/type/matcher/v3:pkg_go_proto # @envoy_api//envoy/config/rbac/v3:pkg_go_proto # # TODO(https://github.com/bazelbuild/rules_go/issues/1986): update to @@ -49,15 +49,3 @@ EXTERNAL_PROTO_CC_BAZEL_DEP_MAP = { "@opentelemetry_proto//:metrics": "@opentelemetry_proto//:metrics_cc_proto", "@opentelemetry_proto//:common": "@opentelemetry_proto//:common_cc_proto", } - -# This maps from the Bazel proto_library target to the Python language binding target for external dependencies. -EXTERNAL_PROTO_PY_BAZEL_DEP_MAP = { - "@com_google_googleapis//google/api/expr/v1alpha1:checked_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto", - "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto", - "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_py", - "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto_py", - "@opentelemetry_proto//:trace": "@opentelemetry_proto//:trace_py_proto", - "@opentelemetry_proto//:logs": "@opentelemetry_proto//:logs_py_proto", - "@opentelemetry_proto//:metrics": "@opentelemetry_proto//:metrics_py_proto", - "@opentelemetry_proto//:common": "@opentelemetry_proto//:common_py_proto", -} diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index e789fe67fd2b..33aaa220fb6d 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -25,8 +25,15 @@ def api_dependencies(): name = "com_google_googleapis", ) + external_http_archive( + name = "com_github_cncf_xds", + ) + + # Needed until @com_github_grpc_grpc renames @com_github_cncf_udpa + # to @com_github_cncf_xds as well. external_http_archive( name = "com_github_cncf_udpa", + location_name = "com_github_cncf_xds", ) external_http_archive( @@ -56,6 +63,10 @@ def api_dependencies(): name = "com_github_chrusty_protoc_gen_jsonschema", ) + external_http_archive( + name = "envoy_toolshed", + ) + PROMETHEUSMETRICS_BUILD_CONTENT = """ load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index c1f59e7a77d4..202a5f8fb07b 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -4,9 +4,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "bazel-skylib", project_desc = "Common useful functions and rules for Bazel", project_url = "https://github.com/bazelbuild/bazel-skylib", - version = "1.4.2", - sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", - release_date = "2023-05-31", + version = "1.5.0", + sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", + release_date = "2023-11-06", urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz"], use_category = ["api"], license = "Apache-2.0", @@ -34,14 +34,14 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Apache-2.0", license_url = "https://github.com/bufbuild/protoc-gen-validate/blob/v{version}/LICENSE", ), - com_github_cncf_udpa = dict( + com_github_cncf_xds = dict( project_name = "xDS API", project_desc = "xDS API Working Group (xDS-WG)", project_url = "https://github.com/cncf/xds", # During the UDPA -> xDS migration, we aren't working with releases. - version = "e9ce68804cb4e64cab5a52e3c8baf840d4ff87b7", - sha256 = "0d33b83f8c6368954e72e7785539f0d272a8aba2f6e2e336ed15fd1514bc9899", - release_date = "2023-06-07", + version = "3a472e524827f72d1ad621c4983dd5af54c46776", + sha256 = "dc305e20c9fa80822322271b50aa2ffa917bf4fd3973bcec52bfc28dc32c5927", + release_date = "2023-11-16", strip_prefix = "xds-{version}", urls = ["https://github.com/cncf/xds/archive/{version}.tar.gz"], use_category = ["api"], @@ -92,9 +92,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Prometheus client model", project_desc = "Data model artifacts for Prometheus", project_url = "https://github.com/prometheus/client_model", - version = "0.4.0", - sha256 = "82fc41d9481476a778b120d4553e9e4edf06cc4efd52ee09ba000933d3a2a53d", - release_date = "2023-05-03", + version = "0.5.0", + sha256 = "170873e0b91cab5da6634af1498b88876842ff3e01212e2dabf6b4e6512c948d", + release_date = "2023-10-03", strip_prefix = "client_model-{version}", urls = ["https://github.com/prometheus/client_model/archive/v{version}.tar.gz"], use_category = ["api"], @@ -131,11 +131,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "buf", project_desc = "A new way of working with Protocol Buffers.", # Used for breaking change detection in API protobufs project_url = "https://buf.build", - version = "1.26.1", - sha256 = "7286b1c6c84392f327991fd4c7b2e7f1bcff141cd1249e797a93d094c2f662ba", + version = "1.28.1", + sha256 = "870cf492d381a967d36636fdee9da44b524ea62aad163659b8dbf16a7da56987", strip_prefix = "buf", urls = ["https://github.com/bufbuild/buf/releases/download/v{version}/buf-Linux-x86_64.tar.gz"], - release_date = "2023-08-09", + release_date = "2023-11-15", use_category = ["api"], license = "Apache-2.0", license_url = "https://github.com/bufbuild/buf/blob/v{version}/LICENSE", @@ -151,4 +151,18 @@ REPOSITORY_LOCATIONS_SPEC = dict( use_category = ["build"], release_date = "2023-05-30", ), + envoy_toolshed = dict( + project_name = "envoy_toolshed", + project_desc = "Tooling, libraries, runners and checkers for Envoy proxy's CI", + project_url = "https://github.com/envoyproxy/toolshed", + version = "0.1.1", + sha256 = "ee759b57270a2747f3f2a3d6ecaad63b834dd9887505a9f1c919d72429dbeffd", + strip_prefix = "toolshed-bazel-v{version}/bazel", + urls = ["https://github.com/envoyproxy/toolshed/archive/bazel-v{version}.tar.gz"], + use_category = ["build"], + release_date = "2023-10-21", + cpe = "N/A", + license = "Apache-2.0", + license_url = "https://github.com/envoyproxy/envoy/blob/bazel-v{version}/LICENSE", + ), ) diff --git a/api/bazel/utils.bzl b/api/bazel/utils.bzl deleted file mode 100644 index 0961f00eb446..000000000000 --- a/api/bazel/utils.bzl +++ /dev/null @@ -1,18 +0,0 @@ -load("@bazel_skylib//rules:write_file.bzl", "write_file") - -def json_data( - name, - data, - visibility = ["//visibility:public"], - **kwargs): - """Write a bazel object to a file - - The provided `data` object should be json serializable. - """ - write_file( - name = name, - out = "%s.json" % name, - content = json.encode(data).split("\n"), - visibility = visibility, - **kwargs - ) diff --git a/api/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/BUILD b/api/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/BUILD new file mode 100644 index 000000000000..29ebf0741406 --- /dev/null +++ b/api/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], +) diff --git a/api/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/qatzip.proto b/api/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/qatzip.proto new file mode 100644 index 000000000000..5c8a3df2ab3d --- /dev/null +++ b/api/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/qatzip.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package envoy.extensions.compression.qatzip.compressor.v3alpha; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.compression.qatzip.compressor.v3alpha"; +option java_outer_classname = "QatzipProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/compression/qatzip/compressor/v3alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Qatzip Compressor] +// Qatzip :ref:`configuration overview `. +// [#extension: envoy.compression.qatzip.compressor] + +// [#next-free-field: 6] +message Qatzip { + enum HardwareBufferSize { + DEFAULT = 0; + SZ_4K = 1; + SZ_8K = 2; + SZ_32K = 3; + SZ_64K = 4; + SZ_128K = 5; + SZ_512K = 6; + } + + // Value from 1 to 9 that controls the main compression speed-density lever. + // The higher quality, the slower compression. The default value is 1. + google.protobuf.UInt32Value compression_level = 1 [(validate.rules).uint32 = {lte: 9 gte: 1}]; + + // A size of qat hardware buffer. This field will be set to "DEFAULT" if not specified. + HardwareBufferSize hardware_buffer_size = 2 [(validate.rules).enum = {defined_only: true}]; + + // Threshold of compression service’s input size for software failover. + // If the size of input request less than the threshold, qatzip will route the request to software + // compressor. The default value is 1024. The maximum value is 512*1024. + google.protobuf.UInt32Value input_size_threshold = 3 + [(validate.rules).uint32 = {lte: 524288 gte: 128}]; + + // A size of stream buffer. The default value is 128 * 1024. The maximum value is 2*1024*1024 - + // 5*1024 + google.protobuf.UInt32Value stream_buffer_size = 4 + [(validate.rules).uint32 = {lte: 2092032 gte: 1024}]; + + // Value for compressor's next output buffer. If not set, defaults to 4096. + google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}]; +} diff --git a/api/contrib/envoy/extensions/config/v3alpha/BUILD b/api/contrib/envoy/extensions/config/v3alpha/BUILD index cfd406e0852a..edd8d2b1f4a8 100644 --- a/api/contrib/envoy/extensions/config/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/config/v3alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/common/key_value/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/config/v3alpha/kv_store_xds_delegate_config.proto b/api/contrib/envoy/extensions/config/v3alpha/kv_store_xds_delegate_config.proto index 639264240d76..bc982743eaec 100644 --- a/api/contrib/envoy/extensions/config/v3alpha/kv_store_xds_delegate_config.proto +++ b/api/contrib/envoy/extensions/config/v3alpha/kv_store_xds_delegate_config.proto @@ -9,7 +9,7 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.config.v3alpha"; option java_outer_classname = "KvStoreXdsDelegateConfigProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/config/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/config/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.xds_delegates.kv_store] diff --git a/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/BUILD index 081219249cbd..b1f16574954e 100644 --- a/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/checksum.proto b/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/checksum.proto index ba4c2fe75795..4255437ec520 100644 --- a/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/checksum.proto +++ b/api/contrib/envoy/extensions/filters/http/checksum/v3alpha/checksum.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.checksum.v3alpha"; option java_outer_classname = "ChecksumProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/checksum/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/http/checksum/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/http/dynamo/v3/BUILD b/api/contrib/envoy/extensions/filters/http/dynamo/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/http/dynamo/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/http/dynamo/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/http/dynamo/v3/dynamo.proto b/api/contrib/envoy/extensions/filters/http/dynamo/v3/dynamo.proto index 9d6feb4ec9b4..d0d4c8c13fa7 100644 --- a/api/contrib/envoy/extensions/filters/http/dynamo/v3/dynamo.proto +++ b/api/contrib/envoy/extensions/filters/http/dynamo/v3/dynamo.proto @@ -8,7 +8,7 @@ import "udpa/annotations/versioning.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.dynamo.v3"; option java_outer_classname = "DynamoProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/dynamo/v3;dynamov3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/http/dynamo/v3;dynamov3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Dynamo] diff --git a/api/contrib/envoy/extensions/filters/http/golang/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/http/golang/v3alpha/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/contrib/envoy/extensions/filters/http/golang/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/http/golang/v3alpha/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto b/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto index ee1268a25683..2d05509b51f8 100644 --- a/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto +++ b/api/contrib/envoy/extensions/filters/http/golang/v3alpha/golang.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.golang.v3alpha"; option java_outer_classname = "GolangProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/golang/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/http/golang/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/http/language/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/http/language/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/http/language/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/http/language/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto b/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto index 47f296707b21..74d32828fd09 100644 --- a/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto +++ b/api/contrib/envoy/extensions/filters/http/language/v3alpha/language.proto @@ -8,7 +8,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.language.v3alpha"; option java_outer_classname = "LanguageProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/language/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/http/language/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Language] diff --git a/api/contrib/envoy/extensions/filters/http/squash/v3/BUILD b/api/contrib/envoy/extensions/filters/http/squash/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/http/squash/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/http/squash/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto b/api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto index 6f178e1a9f8d..d78263cf83e8 100644 --- a/api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto +++ b/api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.squash.v3"; option java_outer_classname = "SquashProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/squash/v3;squashv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/http/squash/v3;squashv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Squash] diff --git a/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/BUILD index 3ca8242f7780..63fb3642c4b5 100644 --- a/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/transport_sockets/tls/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/sxg.proto b/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/sxg.proto index 5d1deb4eb9ff..7f8dd1434edf 100644 --- a/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/sxg.proto +++ b/api/contrib/envoy/extensions/filters/http/sxg/v3alpha/sxg.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.sxg.v3alpha"; option java_outer_classname = "SxgProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/sxg/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/http/sxg/v3alpha"; option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; diff --git a/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/BUILD b/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/client_ssl_auth.proto b/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/client_ssl_auth.proto index 6b45a59e530e..80fd093755f7 100644 --- a/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/client_ssl_auth.proto +++ b/api/contrib/envoy/extensions/filters/network/client_ssl_auth/v3/client_ssl_auth.proto @@ -14,7 +14,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.client_ssl_auth.v3"; option java_outer_classname = "ClientSslAuthProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/client_ssl_auth/v3;client_ssl_authv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/client_ssl_auth/v3;client_ssl_authv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Client TLS authentication] diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/BUILD index d0a7c688bf76..b6c098a23b3a 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/BUILD @@ -8,7 +8,7 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto index 119b8e7f85ef..d60a6f760bc4 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/action/v3/action.proto @@ -15,7 +15,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.action.v3"; option java_outer_classname = "ActionProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/action/v3;actionv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/action/v3;actionv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/dubbo.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/dubbo.proto index d5b6782e6662..47a2af159394 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/dubbo.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3/dubbo.proto @@ -9,7 +9,7 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.codecs.dubbo.v3"; option java_outer_classname = "DubboProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3;dubbov3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3;dubbov3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/BUILD new file mode 100644 index 000000000000..d49202b74ab4 --- /dev/null +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + ], +) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/kafka.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/kafka.proto new file mode 100644 index 000000000000..0e6998f8ec3d --- /dev/null +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/kafka.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package envoy.extensions.filters.network.generic_proxy.codecs.kafka.v3; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.codecs.kafka.v3"; +option java_outer_classname = "KafkaProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3;kafkav3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: Kafka codec configuration for Generic Proxy] +// [#extension: envoy.generic_proxy.codecs.kafka] + +// Configuration for Kafka codec. This codec gives the generic proxy the ability to proxy +// Kafka traffic. But note any route configuration for Kafka traffic is not supported yet. +// The generic proxy can only used to generate logs or metrics for Kafka traffic but cannot +// do matching or routing. +// +// .. note:: +// The codec can currently only be used in the sidecar mode. And to ensure the codec works +// properly, please make sure the following conditions are met: +// +// 1. The generic proxy must be configured with a wildcard route that matches all traffic. +// 2. The target cluster must be configured as a original destination cluster. +// 3. The :ref:`bind_upstream_connection +// ` +// of generic proxy router must be set to true to ensure same upstream connection is used +// for all traffic from same downstream connection. +message KafkaCodecConfig { +} diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD index 081219249cbd..b1f16574954e 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto index 13990866e0fd..2490a0b9bffd 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3/matcher.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.matcher.v3"; option java_outer_classname = "MatcherProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/matcher/v3;matcherv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3;matcherv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/router.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/router.proto index 308943dc123f..a7064152354e 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/router.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/router/v3/router.proto @@ -9,7 +9,7 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.router.v3"; option java_outer_classname = "RouterProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/router/v3;routerv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/router/v3;routerv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; @@ -17,4 +17,27 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // [#extension: envoy.filters.generic.router] message Router { + // Set to true if the upstream connection should be bound to the downstream connection, false + // otherwise. + // + // By default, one random upstream connection will be selected from the upstream connection pool + // and used for every request. And after the request is finished, the upstream connection will be + // released back to the upstream connection pool. + // + // If this option is true, the upstream connection will be bound to the downstream connection and + // have same lifetime as the downstream connection. The same upstream connection will be used for + // all requests from the same downstream connection. + // + // And if this options is true, one of the following requirements must be met: + // + // 1. The request must be handled one by one. That is, the next request can not be sent to the + // upstream until the previous request is finished. + // 2. Unique request id must be provided for each request and response. The request id must be + // unique for each request and response pair in same connection pair. And the request id must + // be the same for the corresponding request and response. + // + // This could be useful for some protocols that require the same upstream connection to be used + // for all requests from the same downstream connection. For example, the protocol using stateful + // connection. + bool bind_upstream_connection = 1; } diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/BUILD b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/BUILD index 71c5730a78f0..75f8163e3f2c 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/BUILD @@ -9,8 +9,8 @@ api_proto_package( "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/extensions/filters/network/http_connection_manager/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto index 84a34265f736..7e35eabdb54c 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/generic_proxy.proto @@ -16,7 +16,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.v3"; option java_outer_classname = "GenericProxyProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/v3;generic_proxyv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/v3;generic_proxyv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto index 01442fa85d6d..2ac8b0f93377 100644 --- a/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto +++ b/api/contrib/envoy/extensions/filters/network/generic_proxy/v3/route.proto @@ -11,7 +11,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.generic_proxy.v3"; option java_outer_classname = "RouteProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/v3;generic_proxyv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/generic_proxy/v3;generic_proxyv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; @@ -43,7 +43,7 @@ message VirtualHost { xds.type.matcher.v3.Matcher routes = 3 [(validate.rules).message = {required: true}]; } -// The generic proxy makes use of the `xds matching API` for routing configurations. +// The generic proxy makes use of the xDS matching API for routing configurations. // // In the below example, we combine a top level tree matcher with a linear matcher to match // the incoming requests, and send the matching requests to v1 of the upstream service. diff --git a/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto index f1051bc1aff3..e4ef538b3439 100644 --- a/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto +++ b/api/contrib/envoy/extensions/filters/network/golang/v3alpha/golang.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.golang.v3alpha"; option java_outer_classname = "GolangProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/golang/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/golang/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/BUILD b/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/kafka_broker.proto b/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/kafka_broker.proto index 83fdd27b378c..b8ab1d7e86b8 100644 --- a/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/kafka_broker.proto +++ b/api/contrib/envoy/extensions/filters/network/kafka_broker/v3/kafka_broker.proto @@ -9,7 +9,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.kafka_broker.v3"; option java_outer_classname = "KafkaBrokerProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/kafka_broker/v3;kafka_brokerv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/kafka_broker/v3;kafka_brokerv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Kafka Broker] @@ -22,4 +22,42 @@ message KafkaBroker { // The prefix to use when emitting :ref:`statistics `. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; + + // Set to true if broker filter should attempt to serialize the received responses from the + // upstream broker instead of passing received bytes as is. + // Disabled by default. + bool force_response_rewrite = 2; + + // Optional broker address rewrite specification. + // Allows the broker filter to rewrite Kafka responses so that all connections established by + // the Kafka clients point to Envoy. + // This allows Kafka cluster not to configure its 'advertised.listeners' property + // (as the necessary re-pointing will be done by this filter). + // This collection of rules should cover all brokers in the cluster that is being proxied, + // otherwise some nodes' addresses might leak to the downstream clients. + oneof broker_address_rewrite_spec { + // Broker address rewrite rules that match by broker ID. + IdBasedBrokerRewriteSpec id_based_broker_address_rewrite_spec = 3; + } +} + +// Collection of rules matching by broker ID. +message IdBasedBrokerRewriteSpec { + repeated IdBasedBrokerRewriteRule rules = 1; +} + +// Defines a rule to rewrite broker address data. +message IdBasedBrokerRewriteRule { + // Broker ID to match. + uint32 id = 1 [(validate.rules).uint32 = {gte: 0}]; + + // The host value to use (resembling the host part of Kafka's advertised.listeners). + // The value should point to the Envoy (not Kafka) listener, so that all client traffic goes + // through Envoy. + string host = 2 [(validate.rules).string = {min_len: 1}]; + + // The port value to use (resembling the port part of Kafka's advertised.listeners). + // The value should point to the Envoy (not Kafka) listener, so that all client traffic goes + // through Envoy. + uint32 port = 3 [(validate.rules).uint32 = {lte: 65535}]; } diff --git a/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/kafka_mesh.proto b/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/kafka_mesh.proto index 68c71f296ee3..26336dc575f2 100644 --- a/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/kafka_mesh.proto +++ b/api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/kafka_mesh.proto @@ -10,7 +10,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.kafka_mesh.v3alpha"; option java_outer_classname = "KafkaMeshProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/kafka_mesh/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/BUILD b/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/mysql_proxy.proto b/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/mysql_proxy.proto index b1bce3bc931c..f3f2cefdc372 100644 --- a/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/mysql_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/mysql_proxy/v3/mysql_proxy.proto @@ -9,7 +9,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.mysql_proxy.v3"; option java_outer_classname = "MysqlProxyProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/mysql_proxy/v3;mysql_proxyv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/mysql_proxy/v3;mysql_proxyv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: MySQL proxy] diff --git a/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto b/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto index afded237e8da..21a3049a1cce 100644 --- a/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto @@ -10,7 +10,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.postgres_proxy.v3alpha"; option java_outer_classname = "PostgresProxyProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/postgres_proxy/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha"; option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; diff --git a/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/BUILD b/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/BUILD index 2f90ace882d9..fd0f6d5f15c4 100644 --- a/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/BUILD +++ b/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/rocketmq_proxy.proto b/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/rocketmq_proxy.proto index dc8de3aaeec7..cca6abbd7e4d 100644 --- a/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/rocketmq_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/rocketmq_proxy.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.rocketmq_proxy.v3"; option java_outer_classname = "RocketmqProxyProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/rocketmq_proxy/v3;rocketmq_proxyv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3;rocketmq_proxyv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: RocketMQ Proxy] diff --git a/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/route.proto b/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/route.proto index ff76d9d344d6..2810aaba7b1b 100644 --- a/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/route.proto +++ b/api/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3/route.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.rocketmq_proxy.v3"; option java_outer_classname = "RouteProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/rocketmq_proxy/v3;rocketmq_proxyv3"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/rocketmq_proxy/v3;rocketmq_proxyv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Rocketmq Proxy Route Configuration] diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/router.proto b/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/router.proto index 15a3137499a8..b4bb27938331 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/router.proto +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha/router.proto @@ -7,7 +7,7 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.sip_proxy.router.v3alpha"; option java_outer_classname = "RouterProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sip_proxy/router/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/sip_proxy/router/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Router] diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/BUILD index 7753cfeb3d6e..79668d20fb02 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( has_services = True, deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/tra.proto b/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/tra.proto index 8599a3817942..daeff268d51c 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/tra.proto +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha/tra.proto @@ -13,7 +13,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.sip_proxy.tra.v3alpha"; option java_outer_classname = "TraProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sip_proxy/tra/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: TRA] diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/BUILD b/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/BUILD index 808cd297266e..4b04e5d2092d 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//contrib/envoy/extensions/filters/network/sip_proxy/tra/v3alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/route.proto b/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/route.proto index 5f906b531335..614f1c510516 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/route.proto +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/route.proto @@ -8,7 +8,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.sip_proxy.v3alpha"; option java_outer_classname = "RouteProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sip_proxy/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Sip Proxy Route Configuration] diff --git a/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/sip_proxy.proto b/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/sip_proxy.proto index 6843f7d3b8a8..99396f3665f6 100644 --- a/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/sip_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha/sip_proxy.proto @@ -14,7 +14,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.network.sip_proxy.v3alpha"; option java_outer_classname = "SipProxyProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sip_proxy/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/filters/network/sip_proxy/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Sip Proxy] diff --git a/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/BUILD b/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/hyperscan.proto b/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/hyperscan.proto index 96fc8d7cec69..8336fdf717a7 100644 --- a/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/hyperscan.proto +++ b/api/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha/hyperscan.proto @@ -8,10 +8,11 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.matching.input_matchers.hyperscan.v3alpha"; option java_outer_classname = "HyperscanProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/input_matchers/hyperscan/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Hyperscan matcher] +// Hyperscan :ref:`configuration overview `. // [#extension: envoy.matching.input_matchers.hyperscan] // `Hyperscan `_ regex matcher. The matcher uses the Hyperscan diff --git a/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/BUILD b/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/dlb.proto b/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/dlb.proto index 57eac9b6a7b2..234a9f9a3975 100644 --- a/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/dlb.proto +++ b/api/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/dlb.proto @@ -7,7 +7,7 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.network.connection_balance.dlb.v3alpha"; option java_outer_classname = "DlbProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/network/connection_balance/dlb/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/network/connection_balance/dlb/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Dlb connection balancer configuration] diff --git a/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/BUILD b/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/cryptomb.proto b/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/cryptomb.proto index d3407beab3ee..20290d8882c0 100644 --- a/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/cryptomb.proto +++ b/api/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha/cryptomb.proto @@ -13,7 +13,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.private_key_providers.cryptomb.v3alpha"; option java_outer_classname = "CryptombProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/private_key_providers/cryptomb/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/private_key_providers/cryptomb/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: CryptoMb private key provider] diff --git a/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/BUILD b/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/qat.proto b/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/qat.proto index e5c58e11f78a..d90bb7e330c7 100644 --- a/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/qat.proto +++ b/api/contrib/envoy/extensions/private_key_providers/qat/v3alpha/qat.proto @@ -13,7 +13,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.private_key_providers.qat.v3alpha"; option java_outer_classname = "QatProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/private_key_providers/qat/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/private_key_providers/qat/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: `QAT` private key provider] diff --git a/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/BUILD b/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/hyperscan.proto b/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/hyperscan.proto index 242694934307..7353e2f27299 100644 --- a/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/hyperscan.proto +++ b/api/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha/hyperscan.proto @@ -7,10 +7,11 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.regex_engines.hyperscan.v3alpha"; option java_outer_classname = "HyperscanProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/regex_engines/hyperscan/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/regex_engines/hyperscan/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Hyperscan] +// Hyperscan :ref:`configuration overview `. // [#extension: envoy.regex_engines.hyperscan] // `Hyperscan `_ regex engine. The engine uses hybrid automata diff --git a/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/BUILD b/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/golang.proto b/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/golang.proto index f22685aff563..884f88e29d81 100644 --- a/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/golang.proto +++ b/api/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha/golang.proto @@ -12,7 +12,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.router.cluster_specifier.golang.v3alpha"; option java_outer_classname = "GolangProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/router/cluster_specifier/golang/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/router/cluster_specifier/golang/v3alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; diff --git a/api/contrib/envoy/extensions/vcl/v3alpha/BUILD b/api/contrib/envoy/extensions/vcl/v3alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/contrib/envoy/extensions/vcl/v3alpha/BUILD +++ b/api/contrib/envoy/extensions/vcl/v3alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/contrib/envoy/extensions/vcl/v3alpha/vcl_socket_interface.proto b/api/contrib/envoy/extensions/vcl/v3alpha/vcl_socket_interface.proto index fc9d2b88535a..48155ecf9c1a 100644 --- a/api/contrib/envoy/extensions/vcl/v3alpha/vcl_socket_interface.proto +++ b/api/contrib/envoy/extensions/vcl/v3alpha/vcl_socket_interface.proto @@ -7,7 +7,7 @@ import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.vcl.v3alpha"; option java_outer_classname = "VclSocketInterfaceProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/vcl/v3alpha"; +option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/vcl/v3alpha"; option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; diff --git a/api/envoy/admin/v2alpha/BUILD b/api/envoy/admin/v2alpha/BUILD index 6fe8cb995d34..0d0be4ad7d9f 100644 --- a/api/envoy/admin/v2alpha/BUILD +++ b/api/envoy/admin/v2alpha/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/config/bootstrap/v2:pkg", "//envoy/service/tap/v2alpha:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/admin/v3/BUILD b/api/envoy/admin/v3/BUILD index 507cffde4ade..d33f4e0b06cf 100644 --- a/api/envoy/admin/v3/BUILD +++ b/api/envoy/admin/v3/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/config/core/v3:pkg", "//envoy/config/tap/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD index 0aded6e51b71..b90e220bc8d6 100644 --- a/api/envoy/api/v2/BUILD +++ b/api/envoy/api/v2/BUILD @@ -17,6 +17,6 @@ api_proto_package( "//envoy/config/filter/accesslog/v2:pkg", "//envoy/config/listener/v2:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/auth/BUILD b/api/envoy/api/v2/auth/BUILD index aaab1df15547..ce0d681bc294 100644 --- a/api/envoy/api/v2/auth/BUILD +++ b/api/envoy/api/v2/auth/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/cluster/BUILD b/api/envoy/api/v2/cluster/BUILD index 2ffbc958786b..4810773b6086 100644 --- a/api/envoy/api/v2/cluster/BUILD +++ b/api/envoy/api/v2/cluster/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/core/BUILD b/api/envoy/api/v2/core/BUILD index 8475a4ba8376..fb33c9b2d291 100644 --- a/api/envoy/api/v2/core/BUILD +++ b/api/envoy/api/v2/core/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/type:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/endpoint/BUILD b/api/envoy/api/v2/endpoint/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/api/v2/endpoint/BUILD +++ b/api/envoy/api/v2/endpoint/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/listener/BUILD b/api/envoy/api/v2/listener/BUILD index ea23dff77c22..220a49100a6c 100644 --- a/api/envoy/api/v2/listener/BUILD +++ b/api/envoy/api/v2/listener/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/api/v2/auth:pkg", "//envoy/api/v2/core:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/api/v2/ratelimit/BUILD b/api/envoy/api/v2/ratelimit/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/api/v2/ratelimit/BUILD +++ b/api/envoy/api/v2/ratelimit/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/api/v2/route/BUILD b/api/envoy/api/v2/route/BUILD index 3d4e6acfeac1..a8df3ab5a31a 100644 --- a/api/envoy/api/v2/route/BUILD +++ b/api/envoy/api/v2/route/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/type:pkg", "//envoy/type/matcher:pkg", "//envoy/type/tracing/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/accesslog/v2/BUILD b/api/envoy/config/accesslog/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/accesslog/v2/BUILD +++ b/api/envoy/config/accesslog/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/accesslog/v3/BUILD b/api/envoy/config/accesslog/v3/BUILD index e657889317c5..17fdbdc97dfd 100644 --- a/api/envoy/config/accesslog/v3/BUILD +++ b/api/envoy/config/accesslog/v3/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/data/accesslog/v3:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/accesslog/v3/accesslog.proto b/api/envoy/config/accesslog/v3/accesslog.proto index 0904b5d055a7..fe3ba2bc97ca 100644 --- a/api/envoy/config/accesslog/v3/accesslog.proto +++ b/api/envoy/config/accesslog/v3/accesslog.proto @@ -254,6 +254,8 @@ message ResponseFlagFilter { in: "UPE" in: "NC" in: "OM" + in: "DF" + in: "DO" } } }]; diff --git a/api/envoy/config/bootstrap/v2/BUILD b/api/envoy/config/bootstrap/v2/BUILD index 0c656d1a9c5a..f5623b97232f 100644 --- a/api/envoy/config/bootstrap/v2/BUILD +++ b/api/envoy/config/bootstrap/v2/BUILD @@ -13,6 +13,6 @@ api_proto_package( "//envoy/config/metrics/v2:pkg", "//envoy/config/overload/v2alpha:pkg", "//envoy/config/trace/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/bootstrap/v3/BUILD b/api/envoy/config/bootstrap/v3/BUILD index 23067e57ca46..b402807628e0 100644 --- a/api/envoy/config/bootstrap/v3/BUILD +++ b/api/envoy/config/bootstrap/v3/BUILD @@ -16,6 +16,6 @@ api_proto_package( "//envoy/config/trace/v3:pkg", "//envoy/extensions/transport_sockets/tls/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index f12f9819dcf8..b5f36f273bcc 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -41,7 +41,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // ` for more detail. // Bootstrap :ref:`configuration overview `. -// [#next-free-field: 40] +// [#next-free-field: 41] message Bootstrap { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v2.Bootstrap"; @@ -136,6 +136,13 @@ message Bootstrap { bool enable_deferred_creation_stats = 1; } + message GrpcAsyncClientManagerConfig { + // Optional field to set the expiration time for the cached gRPC client object. + // The minimal value is 5s and the default is 50s. + google.protobuf.Duration max_cached_entry_idle_duration = 1 + [(validate.rules).duration = {gte {seconds: 5}}]; + } + reserved 10, 11; reserved "runtime"; @@ -401,6 +408,9 @@ message Bootstrap { // Optional application log configuration. ApplicationLogConfig application_log_config = 38; + + // Optional gRPC async manager config. + GrpcAsyncClientManagerConfig grpc_async_client_manager_config = 40; } // Administration interface :ref:`operations documentation diff --git a/api/envoy/config/cluster/aggregate/v2alpha/BUILD b/api/envoy/config/cluster/aggregate/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/cluster/aggregate/v2alpha/BUILD +++ b/api/envoy/config/cluster/aggregate/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD index 25c228fd5609..4f912f7ac49c 100644 --- a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/cluster/redis/BUILD b/api/envoy/config/cluster/redis/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/cluster/redis/BUILD +++ b/api/envoy/config/cluster/redis/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/cluster/v3/BUILD b/api/envoy/config/cluster/v3/BUILD index a98acddff0e4..80d74b61cd6a 100644 --- a/api/envoy/config/cluster/v3/BUILD +++ b/api/envoy/config/cluster/v3/BUILD @@ -11,7 +11,7 @@ api_proto_package( "//envoy/config/endpoint/v3:pkg", "//envoy/type/metadata/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/core/v3:pkg", ], ) diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index 91535b9ee1aa..9b847a33126b 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -1257,4 +1257,19 @@ message TrackClusterStats { // ` tracking header and body sizes // of requests and responses will be published. bool request_response_sizes = 2; + + // If true, some stats will be emitted per-endpoint, similar to the stats in admin ``/clusters`` + // output. + // + // This does not currently output correct stats during a hot-restart. + // + // This is not currently implemented by all stat sinks. + // + // These stats do not honor filtering or tag extraction rules in :ref:`StatsConfig + // ` (but fixed-value tags are supported). Admin + // endpoint filtering is supported. + // + // This may not be used at the same time as + // :ref:`load_stats_config `. + bool per_endpoint_stats = 3; } diff --git a/api/envoy/config/cluster/v3/filter.proto b/api/envoy/config/cluster/v3/filter.proto index 304179976198..54611edbf167 100644 --- a/api/envoy/config/cluster/v3/filter.proto +++ b/api/envoy/config/cluster/v3/filter.proto @@ -16,8 +16,8 @@ option java_multiple_files = true; option go_package = "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3;clusterv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; -// [#protodoc-title: Upstream filters] -// Upstream filters apply to the connections to the upstream cluster hosts. +// [#protodoc-title: Upstream network filters] +// Upstream network filters apply to the connections to the upstream cluster hosts. message Filter { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.cluster.Filter"; @@ -28,7 +28,7 @@ message Filter { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. // Note that Envoy's :ref:`downstream network - // filters ` are not valid upstream filters. + // filters ` are not valid upstream network filters. // Only one of typed_config or config_discovery can be used. google.protobuf.Any typed_config = 2; diff --git a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD index 631cd93a3964..37595060971d 100644 --- a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD +++ b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/common/key_value/v3/BUILD b/api/envoy/config/common/key_value/v3/BUILD index e9b556d681cf..628f71321fba 100644 --- a/api/envoy/config/common/key_value/v3/BUILD +++ b/api/envoy/config/common/key_value/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/config/common/matcher/v3/BUILD b/api/envoy/config/common/matcher/v3/BUILD index 221350b756d4..fd0f6d5f15c4 100644 --- a/api/envoy/config/common/matcher/v3/BUILD +++ b/api/envoy/config/common/matcher/v3/BUILD @@ -9,7 +9,6 @@ api_proto_package( "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/common/matcher/v3/matcher.proto b/api/envoy/config/common/matcher/v3/matcher.proto index 5b9da519687a..49a146d73bd1 100644 --- a/api/envoy/config/common/matcher/v3/matcher.proto +++ b/api/envoy/config/common/matcher/v3/matcher.proto @@ -6,8 +6,6 @@ import "envoy/config/core/v3/extension.proto"; import "envoy/config/route/v3/route_components.proto"; import "envoy/type/matcher/v3/string.proto"; -import "xds/annotations/v3/status.proto"; - import "udpa/annotations/status.proto"; import "validate/validate.proto"; @@ -24,9 +22,10 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // is found the action specified by the most specific on_no_match will be evaluated. // As an on_no_match might result in another matching tree being evaluated, this process // might repeat several times until the final OnMatch (or no match) is decided. +// +// .. note:: +// Please use the syntactically equivalent :ref:`matching API ` message Matcher { - option (xds.annotations.v3.message_status).work_in_progress = true; - // What to do if a match is successful. message OnMatch { oneof on_match { diff --git a/api/envoy/config/common/mutation_rules/v3/BUILD b/api/envoy/config/common/mutation_rules/v3/BUILD index 3f3a5395d2aa..e3bfc4e175f4 100644 --- a/api/envoy/config/common/mutation_rules/v3/BUILD +++ b/api/envoy/config/common/mutation_rules/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/common/tap/v2alpha/BUILD b/api/envoy/config/common/tap/v2alpha/BUILD index 3aed5a34a400..88cd9b521ebb 100644 --- a/api/envoy/config/common/tap/v2alpha/BUILD +++ b/api/envoy/config/common/tap/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/service/tap/v2alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/core/v3/BUILD b/api/envoy/config/core/v3/BUILD index 812e942fc3b0..15185f766497 100644 --- a/api/envoy/config/core/v3/BUILD +++ b/api/envoy/config/core/v3/BUILD @@ -9,8 +9,8 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/core/v3:pkg", ], ) diff --git a/api/envoy/config/core/v3/address.proto b/api/envoy/config/core/v3/address.proto index 3bd9b4cd3dc1..d8d47882655b 100644 --- a/api/envoy/config/core/v3/address.proto +++ b/api/envoy/config/core/v3/address.proto @@ -151,7 +151,7 @@ message BindConfig { // precompiled binaries. repeated SocketOption socket_options = 3; - // Extra source addresses appended to the address specified in the `source_address` + // Extra source addresses appended to the address specified in the ``source_address`` // field. This enables to specify multiple source addresses. // The source address selection is determined by :ref:`local_address_selector // `. diff --git a/api/envoy/config/core/v3/http_service.proto b/api/envoy/config/core/v3/http_service.proto new file mode 100644 index 000000000000..426994c033ca --- /dev/null +++ b/api/envoy/config/core/v3/http_service.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package envoy.config.core.v3; + +import "envoy/config/core/v3/base.proto"; +import "envoy/config/core/v3/http_uri.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.core.v3"; +option java_outer_classname = "HttpServiceProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/config/core/v3;corev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: HTTP services] + +// HTTP service configuration. +message HttpService { + // The service's HTTP URI. For example: + // + // .. code-block:: yaml + // + // http_uri: + // uri: https://www.myserviceapi.com/v1/data + // cluster: www.myserviceapi.com|443 + // + HttpUri http_uri = 1; + + // Specifies a list of HTTP headers that should be added to each request + // handled by this virtual host. + repeated HeaderValueOption request_headers_to_add = 2 + [(validate.rules).repeated = {max_items: 1000}]; +} diff --git a/api/envoy/config/core/v3/http_uri.proto b/api/envoy/config/core/v3/http_uri.proto index 93b63093e377..bac37c0d5839 100644 --- a/api/envoy/config/core/v3/http_uri.proto +++ b/api/envoy/config/core/v3/http_uri.proto @@ -52,6 +52,7 @@ message HttpUri { // Sets the maximum duration in milliseconds that a response can take to arrive upon request. google.protobuf.Duration timeout = 3 [(validate.rules).duration = { required: true + lt {seconds: 4294967296} gte {} }]; } diff --git a/api/envoy/config/core/v3/protocol.proto b/api/envoy/config/core/v3/protocol.proto index 71b12f7247e0..d128dc6d93d7 100644 --- a/api/envoy/config/core/v3/protocol.proto +++ b/api/envoy/config/core/v3/protocol.proto @@ -85,7 +85,7 @@ message QuicProtocolOptions { [(validate.rules).uint32 = {lte: 25165824 gte: 1}]; // The number of timeouts that can occur before port migration is triggered for QUIC clients. - // This defaults to 1. If set to 0, port migration will not occur on path degrading. + // This defaults to 4. If set to 0, port migration will not occur on path degrading. // Timeout here refers to QUIC internal path degrading timeout mechanism, such as PTO. // This has no effect on server sessions. google.protobuf.UInt32Value num_timeouts_to_trigger_port_migration = 4 @@ -96,11 +96,11 @@ message QuicProtocolOptions { QuicKeepAliveSettings connection_keepalive = 5; // A comma-separated list of strings representing QUIC connection options defined in - // `QUICHE ` and to be sent by upstream connections. + // `QUICHE `_ and to be sent by upstream connections. string connection_options = 6; // A comma-separated list of strings representing QUIC client connection options defined in - // `QUICHE ` and to be sent by upstream connections. + // `QUICHE `_ and to be sent by upstream connections. string client_connection_options = 7; } diff --git a/api/envoy/config/endpoint/v3/BUILD b/api/envoy/config/endpoint/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/config/endpoint/v3/BUILD +++ b/api/envoy/config/endpoint/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/endpoint/v3/endpoint.proto b/api/envoy/config/endpoint/v3/endpoint.proto index 6cdc179d8931..20939526eb5f 100644 --- a/api/envoy/config/endpoint/v3/endpoint.proto +++ b/api/envoy/config/endpoint/v3/endpoint.proto @@ -40,7 +40,6 @@ message ClusterLoadAssignment { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.ClusterLoadAssignment.Policy"; - // [#not-implemented-hide:] message DropOverload { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.ClusterLoadAssignment.Policy.DropOverload"; @@ -75,7 +74,9 @@ message ClusterLoadAssignment { // "throttle"_drop = 60% // "lb"_drop = 20% // 50% of the remaining 'actual' load, which is 40%. // actual_outgoing_load = 20% // remaining after applying all categories. - // [#not-implemented-hide:] + // + // Envoy supports only one element and will NACK if more than one element is present. + // Other xDS-capable data planes will not necessarily have this limitation. repeated DropOverload drop_overloads = 2; // Priority levels and localities are considered overprovisioned with this diff --git a/api/envoy/config/endpoint/v3/endpoint_components.proto b/api/envoy/config/endpoint/v3/endpoint_components.proto index c9572fd8a11d..ebd2bb4c3324 100644 --- a/api/envoy/config/endpoint/v3/endpoint_components.proto +++ b/api/envoy/config/endpoint/v3/endpoint_components.proto @@ -88,8 +88,8 @@ message Endpoint { // :ref:`auto_host_rewrite `. string hostname = 3; - // An ordered list of addresses that together with `address` comprise the - // list of addresses for an endpoint. The address given in the `address` is + // An ordered list of addresses that together with ``address`` comprise the + // list of addresses for an endpoint. The address given in the ``address`` is // prepended to this list. It is assumed that the list must already be // sorted by preference order of the addresses. This will only be supported // for STATIC and EDS clusters. diff --git a/api/envoy/config/filter/accesslog/v2/BUILD b/api/envoy/config/filter/accesslog/v2/BUILD index f7c626ac0e5a..8b7956534cbe 100644 --- a/api/envoy/config/filter/accesslog/v2/BUILD +++ b/api/envoy/config/filter/accesslog/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/api/v2/core:pkg", "//envoy/api/v2/route:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD b/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD +++ b/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/fault/v2/BUILD b/api/envoy/config/filter/fault/v2/BUILD index 29613b4c3487..ad7d3cbadf20 100644 --- a/api/envoy/config/filter/fault/v2/BUILD +++ b/api/envoy/config/filter/fault/v2/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD index 2ffbc958786b..4810773b6086 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/aws_lambda/v2alpha/BUILD b/api/envoy/config/filter/http/aws_lambda/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/aws_lambda/v2alpha/BUILD +++ b/api/envoy/config/filter/http/aws_lambda/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/aws_request_signing/v2alpha/BUILD b/api/envoy/config/filter/http/aws_request_signing/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/aws_request_signing/v2alpha/BUILD +++ b/api/envoy/config/filter/http/aws_request_signing/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/buffer/v2/BUILD b/api/envoy/config/filter/http/buffer/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/buffer/v2/BUILD +++ b/api/envoy/config/filter/http/buffer/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/cache/v2alpha/BUILD b/api/envoy/config/filter/http/cache/v2alpha/BUILD index 5cbf4e821fc8..a4882c58634e 100644 --- a/api/envoy/config/filter/http/cache/v2alpha/BUILD +++ b/api/envoy/config/filter/http/cache/v2alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/route:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/compressor/v2/BUILD b/api/envoy/config/filter/http/compressor/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/filter/http/compressor/v2/BUILD +++ b/api/envoy/config/filter/http/compressor/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/cors/v2/BUILD b/api/envoy/config/filter/http/cors/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/cors/v2/BUILD +++ b/api/envoy/config/filter/http/cors/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/csrf/v2/BUILD b/api/envoy/config/filter/http/csrf/v2/BUILD index aaab1df15547..ce0d681bc294 100644 --- a/api/envoy/config/filter/http/csrf/v2/BUILD +++ b/api/envoy/config/filter/http/csrf/v2/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD index 25c228fd5609..4f912f7ac49c 100644 --- a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/dynamo/v2/BUILD b/api/envoy/config/filter/http/dynamo/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/dynamo/v2/BUILD +++ b/api/envoy/config/filter/http/dynamo/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/ext_authz/v2/BUILD b/api/envoy/config/filter/http/ext_authz/v2/BUILD index 74e703c963cb..5dc4abc38cb8 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/BUILD +++ b/api/envoy/config/filter/http/ext_authz/v2/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/api/v2/core:pkg", "//envoy/type:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/fault/v2/BUILD b/api/envoy/config/filter/http/fault/v2/BUILD index df4feab714ff..568e1dad4019 100644 --- a/api/envoy/config/filter/http/fault/v2/BUILD +++ b/api/envoy/config/filter/http/fault/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/api/v2/route:pkg", "//envoy/config/filter/fault/v2:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/grpc_http1_bridge/v2/BUILD b/api/envoy/config/filter/http/grpc_http1_bridge/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/grpc_http1_bridge/v2/BUILD +++ b/api/envoy/config/filter/http/grpc_http1_bridge/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD +++ b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/grpc_stats/v2alpha/BUILD b/api/envoy/config/filter/http/grpc_stats/v2alpha/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/filter/http/grpc_stats/v2alpha/BUILD +++ b/api/envoy/config/filter/http/grpc_stats/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/grpc_web/v2/BUILD b/api/envoy/config/filter/http/grpc_web/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/grpc_web/v2/BUILD +++ b/api/envoy/config/filter/http/grpc_web/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/gzip/v2/BUILD b/api/envoy/config/filter/http/gzip/v2/BUILD index 9cb0d1293421..4089809e5f7b 100644 --- a/api/envoy/config/filter/http/gzip/v2/BUILD +++ b/api/envoy/config/filter/http/gzip/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/filter/http/compressor/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/header_to_metadata/v2/BUILD b/api/envoy/config/filter/http/header_to_metadata/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v2/BUILD +++ b/api/envoy/config/filter/http/header_to_metadata/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/health_check/v2/BUILD b/api/envoy/config/filter/http/health_check/v2/BUILD index 22fc8fd458e6..5cc84b4af88d 100644 --- a/api/envoy/config/filter/http/health_check/v2/BUILD +++ b/api/envoy/config/filter/http/health_check/v2/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/route:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/ip_tagging/v2/BUILD b/api/envoy/config/filter/http/ip_tagging/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/filter/http/ip_tagging/v2/BUILD +++ b/api/envoy/config/filter/http/ip_tagging/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD b/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD index 1e485f4e158a..ef28c91bc4be 100644 --- a/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD +++ b/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/api/v2/route:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/lua/v2/BUILD b/api/envoy/config/filter/http/lua/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/lua/v2/BUILD +++ b/api/envoy/config/filter/http/lua/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/on_demand/v2/BUILD b/api/envoy/config/filter/http/on_demand/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/on_demand/v2/BUILD +++ b/api/envoy/config/filter/http/on_demand/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/original_src/v2alpha1/BUILD b/api/envoy/config/filter/http/original_src/v2alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/original_src/v2alpha1/BUILD +++ b/api/envoy/config/filter/http/original_src/v2alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/rate_limit/v2/BUILD b/api/envoy/config/filter/http/rate_limit/v2/BUILD index 5b66057a82cd..e5e3cac0561b 100644 --- a/api/envoy/config/filter/http/rate_limit/v2/BUILD +++ b/api/envoy/config/filter/http/rate_limit/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/ratelimit/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/rbac/v2/BUILD b/api/envoy/config/filter/http/rbac/v2/BUILD index 90082d083a3f..a7b74db08b49 100644 --- a/api/envoy/config/filter/http/rbac/v2/BUILD +++ b/api/envoy/config/filter/http/rbac/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/rbac/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/router/v2/BUILD b/api/envoy/config/filter/http/router/v2/BUILD index 4b7ccc42a6ca..3e6564b8fdd4 100644 --- a/api/envoy/config/filter/http/router/v2/BUILD +++ b/api/envoy/config/filter/http/router/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/filter/accesslog/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/squash/v2/BUILD b/api/envoy/config/filter/http/squash/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/squash/v2/BUILD +++ b/api/envoy/config/filter/http/squash/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/http/tap/v2alpha/BUILD b/api/envoy/config/filter/http/tap/v2alpha/BUILD index cf02fc6c0b1f..2e7c51a2a621 100644 --- a/api/envoy/config/filter/http/tap/v2alpha/BUILD +++ b/api/envoy/config/filter/http/tap/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/common/tap/v2alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/http/transcoder/v2/BUILD b/api/envoy/config/filter/http/transcoder/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/http/transcoder/v2/BUILD +++ b/api/envoy/config/filter/http/transcoder/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/listener/http_inspector/v2/BUILD b/api/envoy/config/filter/listener/http_inspector/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/listener/http_inspector/v2/BUILD +++ b/api/envoy/config/filter/listener/http_inspector/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/listener/original_dst/v2/BUILD b/api/envoy/config/filter/listener/original_dst/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/listener/original_dst/v2/BUILD +++ b/api/envoy/config/filter/listener/original_dst/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD b/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD +++ b/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/listener/proxy_protocol/v2/BUILD b/api/envoy/config/filter/listener/proxy_protocol/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/listener/proxy_protocol/v2/BUILD +++ b/api/envoy/config/filter/listener/proxy_protocol/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/listener/tls_inspector/v2/BUILD b/api/envoy/config/filter/listener/tls_inspector/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/listener/tls_inspector/v2/BUILD +++ b/api/envoy/config/filter/listener/tls_inspector/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD b/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD +++ b/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/direct_response/v2/BUILD b/api/envoy/config/filter/network/direct_response/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/filter/network/direct_response/v2/BUILD +++ b/api/envoy/config/filter/network/direct_response/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD index 5fe475a5dcf8..90d0f23bdb20 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/api/v2/route:pkg", "//envoy/type:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/echo/v2/BUILD b/api/envoy/config/filter/network/echo/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/network/echo/v2/BUILD +++ b/api/envoy/config/filter/network/echo/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/network/ext_authz/v2/BUILD b/api/envoy/config/filter/network/ext_authz/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/filter/network/ext_authz/v2/BUILD +++ b/api/envoy/config/filter/network/ext_authz/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD index b03bcd437c3d..d88a165dc606 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD +++ b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD @@ -13,6 +13,6 @@ api_proto_package( "//envoy/config/trace/v2:pkg", "//envoy/type:pkg", "//envoy/type/tracing/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/kafka_broker/v2alpha1/BUILD b/api/envoy/config/filter/network/kafka_broker/v2alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/network/kafka_broker/v2alpha1/BUILD +++ b/api/envoy/config/filter/network/kafka_broker/v2alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/network/kafka_broker/v2alpha1/kafka_broker.proto b/api/envoy/config/filter/network/kafka_broker/v2alpha1/kafka_broker.proto index 3611c1d6759f..829726a49892 100644 --- a/api/envoy/config/filter/network/kafka_broker/v2alpha1/kafka_broker.proto +++ b/api/envoy/config/filter/network/kafka_broker/v2alpha1/kafka_broker.proto @@ -21,4 +21,42 @@ option (udpa.annotations.file_status).package_version_status = FROZEN; message KafkaBroker { // The prefix to use when emitting :ref:`statistics `. string stat_prefix = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Set to true if broker filter should attempt to serialize the received responses from the + // upstream broker instead of passing received bytes as is. + // Disabled by default. + bool force_response_rewrite = 2; + + // Optional broker address rewrite specification. + // Allows the broker filter to rewrite Kafka responses so that all connections established by + // the Kafka clients point to Envoy. + // This allows Kafka cluster not to configure its 'advertised.listeners' property + // (as the necessary re-pointing will be done by this filter). + // This collection of rules should cover all brokers in the cluster that is being proxied, + // otherwise some nodes' addresses might leak to the downstream clients. + oneof broker_address_rewrite_spec { + // Broker address rewrite rules that match by broker ID. + IdBasedBrokerRewriteSpec id_based_broker_address_rewrite_spec = 3; + } +} + +// Collection of rules matching by broker ID. +message IdBasedBrokerRewriteSpec { + repeated IdBasedBrokerRewriteRule rules = 1; +} + +// Defines a rule to rewrite broker address data. +message IdBasedBrokerRewriteRule { + // Broker ID to match. + uint32 id = 1 [(validate.rules).uint32 = {gte: 0}]; + + // The host value to use (resembling the host part of Kafka's advertised.listeners). + // The value should point to the Envoy (not Kafka) listener, so that all client traffic goes + // through Envoy. + string host = 2 [(validate.rules).string = {min_len: 1}]; + + // The port value to use (resembling the port part of Kafka's advertised.listeners). + // The value should point to the Envoy (not Kafka) listener, so that all client traffic goes + // through Envoy. + uint32 port = 3 [(validate.rules).uint32 = {lte: 65535}]; } diff --git a/api/envoy/config/filter/network/local_rate_limit/v2alpha/BUILD b/api/envoy/config/filter/network/local_rate_limit/v2alpha/BUILD index 2ffbc958786b..4810773b6086 100644 --- a/api/envoy/config/filter/network/local_rate_limit/v2alpha/BUILD +++ b/api/envoy/config/filter/network/local_rate_limit/v2alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/mongo_proxy/v2/BUILD b/api/envoy/config/filter/network/mongo_proxy/v2/BUILD index b4f275ad5f87..d301445b93b5 100644 --- a/api/envoy/config/filter/network/mongo_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/mongo_proxy/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/filter/fault/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD +++ b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/network/rate_limit/v2/BUILD b/api/envoy/config/filter/network/rate_limit/v2/BUILD index 6d29e84c421c..1bf86ec50318 100644 --- a/api/envoy/config/filter/network/rate_limit/v2/BUILD +++ b/api/envoy/config/filter/network/rate_limit/v2/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/ratelimit:pkg", "//envoy/config/ratelimit/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/rbac/v2/BUILD b/api/envoy/config/filter/network/rbac/v2/BUILD index 90082d083a3f..a7b74db08b49 100644 --- a/api/envoy/config/filter/network/rbac/v2/BUILD +++ b/api/envoy/config/filter/network/rbac/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/rbac/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/redis_proxy/v2/BUILD b/api/envoy/config/filter/network/redis_proxy/v2/BUILD index f91701518907..d9a6fd81b488 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/redis_proxy/v2/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/sni_cluster/v2/BUILD b/api/envoy/config/filter/network/sni_cluster/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/network/sni_cluster/v2/BUILD +++ b/api/envoy/config/filter/network/sni_cluster/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/network/tcp_proxy/v2/BUILD b/api/envoy/config/filter/network/tcp_proxy/v2/BUILD index c02167a174de..59e433512707 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/tcp_proxy/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/api/v2/core:pkg", "//envoy/config/filter/accesslog/v2:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD index 1e485f4e158a..ef28c91bc4be 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/api/v2/route:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD index 5b66057a82cd..e5e3cac0561b 100644 --- a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD +++ b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/ratelimit/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/filter/thrift/router/v2alpha1/BUILD b/api/envoy/config/filter/thrift/router/v2alpha1/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/thrift/router/v2alpha1/BUILD +++ b/api/envoy/config/filter/thrift/router/v2alpha1/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/filter/udp/udp_proxy/v2alpha/BUILD b/api/envoy/config/filter/udp/udp_proxy/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/filter/udp/udp_proxy/v2alpha/BUILD +++ b/api/envoy/config/filter/udp/udp_proxy/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/grpc_credential/v2alpha/BUILD b/api/envoy/config/grpc_credential/v2alpha/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/grpc_credential/v2alpha/BUILD +++ b/api/envoy/config/grpc_credential/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/grpc_credential/v3/BUILD b/api/envoy/config/grpc_credential/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/config/grpc_credential/v3/BUILD +++ b/api/envoy/config/grpc_credential/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/health_checker/redis/v2/BUILD b/api/envoy/config/health_checker/redis/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/health_checker/redis/v2/BUILD +++ b/api/envoy/config/health_checker/redis/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/listener/v2/BUILD b/api/envoy/config/listener/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/listener/v2/BUILD +++ b/api/envoy/config/listener/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/listener/v3/BUILD b/api/envoy/config/listener/v3/BUILD index f4f900c4db89..712a0d83856e 100644 --- a/api/envoy/config/listener/v3/BUILD +++ b/api/envoy/config/listener/v3/BUILD @@ -10,9 +10,9 @@ api_proto_package( "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/core/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/core/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto index 084b6f3e4e37..a1a3d82c1c86 100644 --- a/api/envoy/config/listener/v3/listener.proto +++ b/api/envoy/config/listener/v3/listener.proto @@ -249,7 +249,7 @@ message Listener { // Additional socket options that may not be present in Envoy source code or // precompiled binaries. The socket options can be updated for a listener when // :ref:`enable_reuse_port ` - // is `true`. Otherwise, if socket options change during a listener update the update will be rejected + // is ``true``. Otherwise, if socket options change during a listener update the update will be rejected // to make it clear that the options were not updated. repeated core.v3.SocketOption socket_options = 13; diff --git a/api/envoy/config/metrics/v2/BUILD b/api/envoy/config/metrics/v2/BUILD index aaab1df15547..ce0d681bc294 100644 --- a/api/envoy/config/metrics/v2/BUILD +++ b/api/envoy/config/metrics/v2/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/metrics/v3/BUILD b/api/envoy/config/metrics/v3/BUILD index 3f3a5395d2aa..e3bfc4e175f4 100644 --- a/api/envoy/config/metrics/v3/BUILD +++ b/api/envoy/config/metrics/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/metrics/v3/stats.proto b/api/envoy/config/metrics/v3/stats.proto index fb73e91f8f99..e7d7f80d648a 100644 --- a/api/envoy/config/metrics/v3/stats.proto +++ b/api/envoy/config/metrics/v3/stats.proto @@ -121,8 +121,8 @@ message StatsMatcher { // limited by either an exclusion or an inclusion list of :ref:`StringMatcher // ` protos: // - // * If ``reject_all`` is set to `true`, no stats will be instantiated. If ``reject_all`` is set to - // `false`, all stats will be instantiated. + // * If ``reject_all`` is set to ``true``, no stats will be instantiated. If ``reject_all`` is set to + // ``false``, all stats will be instantiated. // // * If an exclusion list is supplied, any stat name matching *any* of the StringMatchers in the // list will not instantiate. diff --git a/api/envoy/config/overload/v2alpha/BUILD b/api/envoy/config/overload/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/overload/v2alpha/BUILD +++ b/api/envoy/config/overload/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/overload/v3/BUILD b/api/envoy/config/overload/v3/BUILD index 9a76b7e148e0..ef19132f9180 100644 --- a/api/envoy/config/overload/v3/BUILD +++ b/api/envoy/config/overload/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/ratelimit/v2/BUILD b/api/envoy/config/ratelimit/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/ratelimit/v2/BUILD +++ b/api/envoy/config/ratelimit/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/ratelimit/v3/BUILD b/api/envoy/config/ratelimit/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/config/ratelimit/v3/BUILD +++ b/api/envoy/config/ratelimit/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/rbac/v2/BUILD b/api/envoy/config/rbac/v2/BUILD index 4bce7466dddf..58b321a9a5b4 100644 --- a/api/envoy/config/rbac/v2/BUILD +++ b/api/envoy/config/rbac/v2/BUILD @@ -9,7 +9,7 @@ api_proto_package( "//envoy/api/v2/core:pkg", "//envoy/api/v2/route:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", ], ) diff --git a/api/envoy/config/rbac/v3/BUILD b/api/envoy/config/rbac/v3/BUILD index c289def1f11d..a8efea386273 100644 --- a/api/envoy/config/rbac/v3/BUILD +++ b/api/envoy/config/rbac/v3/BUILD @@ -11,7 +11,7 @@ api_proto_package( "//envoy/config/route/v3:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@com_google_googleapis//google/api/expr/v1alpha1:checked_proto", "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", ], diff --git a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD +++ b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD b/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD +++ b/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/retry/omit_canary_hosts/v2/BUILD b/api/envoy/config/retry/omit_canary_hosts/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/retry/omit_canary_hosts/v2/BUILD +++ b/api/envoy/config/retry/omit_canary_hosts/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/retry/omit_host_metadata/v2/BUILD b/api/envoy/config/retry/omit_host_metadata/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/retry/omit_host_metadata/v2/BUILD +++ b/api/envoy/config/retry/omit_host_metadata/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/retry/previous_hosts/v2/BUILD b/api/envoy/config/retry/previous_hosts/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/retry/previous_hosts/v2/BUILD +++ b/api/envoy/config/retry/previous_hosts/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/retry/previous_priorities/BUILD b/api/envoy/config/retry/previous_priorities/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/retry/previous_priorities/BUILD +++ b/api/envoy/config/retry/previous_priorities/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/route/v3/BUILD b/api/envoy/config/route/v3/BUILD index 385c2c8c40e0..8c0e78d334ed 100644 --- a/api/envoy/config/route/v3/BUILD +++ b/api/envoy/config/route/v3/BUILD @@ -12,8 +12,8 @@ api_proto_package( "//envoy/type/metadata/v3:pkg", "//envoy/type/tracing/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/config/route/v3/route.proto b/api/envoy/config/route/v3/route.proto index 237bddebdef6..0d5867d00714 100644 --- a/api/envoy/config/route/v3/route.proto +++ b/api/envoy/config/route/v3/route.proto @@ -23,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // * Routing :ref:`architecture overview ` // * HTTP :ref:`router filter ` -// [#next-free-field: 17] +// [#next-free-field: 18] message RouteConfiguration { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.RouteConfiguration"; @@ -151,6 +151,13 @@ message RouteConfiguration { // :ref:`FilterConfig` // message to specify additional options.] map typed_per_filter_config = 16; + + // The metadata field can be used to provide additional information + // about the route configuration. It can be used for configuration, stats, and logging. + // The metadata should go under the filter namespace that will need it. + // For instance, if the metadata is intended for the Router filter, + // the filter name should be specified as ``envoy.filters.http.router``. + core.v3.Metadata metadata = 17; } message Vhds { diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 1800ee91b5bf..1e2b486d288b 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -41,7 +41,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // host header. This allows a single listener to service multiple top level domain path trees. Once // a virtual host is selected based on the domain, the routes are processed in order to see which // upstream cluster to route to or whether to perform a redirect. -// [#next-free-field: 24] +// [#next-free-field: 25] message VirtualHost { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.VirtualHost"; @@ -215,6 +215,13 @@ message VirtualHost { // It takes precedence over the route config mirror policy entirely. // That is, policies are not merged, the most specific non-empty one becomes the mirror policies. repeated RouteAction.RequestMirrorPolicy request_mirror_policies = 22; + + // The metadata field can be used to provide additional information + // about the virtual host. It can be used for configuration, stats, and logging. + // The metadata should go under the filter namespace that will need it. + // For instance, if the metadata is intended for the Router filter, + // the filter name should be specified as ``envoy.filters.http.router``. + core.v3.Metadata metadata = 24; } // A filter-defined action type. @@ -524,10 +531,20 @@ message RouteMatch { // If specified, the route will match against whether or not a certificate is validated. // If not specified, certificate validation status (true or false) will not be considered when route matching. + // + // .. warning:: + // + // Client certificate validation is not currently performed upon TLS session resumption. For + // a resumed TLS session the route will match only when ``validated`` is false, regardless of + // whether the client TLS certificate is valid. + // + // The only known workaround for this issue is to disable TLS session resumption entirely, by + // setting both :ref:`disable_stateless_session_resumption ` + // and :ref:`disable_stateful_session_resumption ` on the DownstreamTlsContext. google.protobuf.BoolValue validated = 2; } - // An extensible message for matching CONNECT requests. + // An extensible message for matching CONNECT or CONNECT-UDP requests. message ConnectMatcher { } @@ -560,11 +577,10 @@ message RouteMatch { // stripping. This needs more thought.] type.matcher.v3.RegexMatcher safe_regex = 10 [(validate.rules).message = {required: true}]; - // If this is used as the matcher, the matcher will only match CONNECT requests. - // Note that this will not match HTTP/2 upgrade-style CONNECT requests - // (WebSocket and the like) as they are normalized in Envoy as HTTP/1.1 style - // upgrades. - // This is the only way to match CONNECT requests for HTTP/1.1. For HTTP/2, + // If this is used as the matcher, the matcher will only match CONNECT or CONNECT-UDP requests. + // Note that this will not match other Extended CONNECT requests (WebSocket and the like) as + // they are normalized in Envoy as HTTP/1.1 style upgrades. + // This is the only way to match CONNECT requests for HTTP/1.1. For HTTP/2 and HTTP/3, // where Extended CONNECT requests may have a path, the path matchers will work if // there is a path present. // Note that CONNECT support is currently considered alpha in Envoy. @@ -619,7 +635,8 @@ message RouteMatch { // match. The router will check the query string from the ``path`` header // against all the specified query parameters. If the number of specified // query parameters is nonzero, they all must match the ``path`` header's - // query string for a match to occur. + // query string for a match to occur. In the event query parameters are + // repeated, only the first value for each key will be considered. // // .. note:: // @@ -882,7 +899,8 @@ message RouteAction { // The name of the URL query parameter that will be used to obtain the hash // key. If the parameter is not present, no hash will be produced. Query - // parameter names are case-sensitive. + // parameter names are case-sensitive. If query parameters are repeated, only + // the first value will be considered. string name = 1 [(validate.rules).string = {min_len: 1}]; } @@ -1137,7 +1155,9 @@ message RouteAction { // Indicates that during forwarding, the host header will be swapped with // the hostname of the upstream host chosen by the cluster manager. This // option is applicable only when the destination cluster for a route is of - // type ``strict_dns`` or ``logical_dns``. Setting this to true with other cluster types + // type ``strict_dns`` or ``logical_dns``, + // or when :ref:`hostname ` + // field is not empty. Setting this to true with other cluster types // has no effect. Using this option will append the // :ref:`config_http_conn_man_headers_x-forwarded-host` header if // :ref:`append_x_forwarded_host ` @@ -1191,7 +1211,7 @@ message RouteAction { // :ref:`host_rewrite_path_regex `) // causes the original value of the host header, if any, to be appended to the // :ref:`config_http_conn_man_headers_x-forwarded-host` HTTP header if it is different to the last value appended. - // This can be disabled by setting the runtime guard `envoy_reloadable_features_append_xfh_idempotent` to false. + // This can be disabled by setting the runtime guard ``envoy_reloadable_features_append_xfh_idempotent`` to false. bool append_x_forwarded_host = 38; // Specifies the upstream timeout for the route. If not specified, the default is 15s. This @@ -2345,6 +2365,7 @@ message QueryParameterMatcher { } // HTTP Internal Redirect :ref:`architecture overview `. +// [#next-free-field: 6] message InternalRedirectPolicy { // An internal redirect is not handled, unless the number of previous internal redirects that a // downstream request has encountered is lower than this value. @@ -2370,6 +2391,14 @@ message InternalRedirectPolicy { // Allow internal redirect to follow a target URI with a different scheme than the value of // x-forwarded-proto. The default is false. bool allow_cross_scheme_redirect = 4; + + // Specifies a list of headers, by name, to copy from the internal redirect into the subsequent + // request. If a header is specified here but not present in the redirect, it will be cleared in + // the subsequent request. + repeated string response_headers_to_copy = 5 [(validate.rules).repeated = { + unique: true + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; } // A simple wrapper for an HTTP filter config. This is intended to be used as a wrapper for the @@ -2388,6 +2417,8 @@ message FilterConfig { bool is_optional = 2; // If true, the filter is disabled in the route or virtual host and the ``config`` field is ignored. + // See :ref:`route based filter chain ` + // for more details. // // .. note:: // diff --git a/api/envoy/config/tap/v3/BUILD b/api/envoy/config/tap/v3/BUILD index a457820fce67..ccd4d1a08aea 100644 --- a/api/envoy/config/tap/v3/BUILD +++ b/api/envoy/config/tap/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/common/matcher/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/trace/v2/BUILD b/api/envoy/config/trace/v2/BUILD index e6505e4f15d0..a207a53c6c19 100644 --- a/api/envoy/config/trace/v2/BUILD +++ b/api/envoy/config/trace/v2/BUILD @@ -8,7 +8,7 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", ], ) diff --git a/api/envoy/config/trace/v2alpha/BUILD b/api/envoy/config/trace/v2alpha/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/config/trace/v2alpha/BUILD +++ b/api/envoy/config/trace/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/trace/v3/BUILD b/api/envoy/config/trace/v3/BUILD index 94596540dfc4..4d265c471e1c 100644 --- a/api/envoy/config/trace/v3/BUILD +++ b/api/envoy/config/trace/v3/BUILD @@ -8,7 +8,7 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", ], ) diff --git a/api/envoy/config/trace/v3/opentelemetry.proto b/api/envoy/config/trace/v3/opentelemetry.proto index e9c7430dcfdd..59028326f220 100644 --- a/api/envoy/config/trace/v3/opentelemetry.proto +++ b/api/envoy/config/trace/v3/opentelemetry.proto @@ -2,8 +2,11 @@ syntax = "proto3"; package envoy.config.trace.v3; +import "envoy/config/core/v3/extension.proto"; import "envoy/config/core/v3/grpc_service.proto"; +import "envoy/config/core/v3/http_service.proto"; +import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v3"; @@ -16,13 +19,42 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Configuration for the OpenTelemetry tracer. // [#extension: envoy.tracers.opentelemetry] +// [#next-free-field: 6] message OpenTelemetryConfig { // The upstream gRPC cluster that will receive OTLP traces. // Note that the tracer drops traces if the server does not read data fast enough. - // This field can be left empty to disable reporting traces to the collector. - core.v3.GrpcService grpc_service = 1; + // This field can be left empty to disable reporting traces to the gRPC service. + // Only one of ``grpc_service``, ``http_service`` may be used. + core.v3.GrpcService grpc_service = 1 + [(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"]; + + // The upstream HTTP cluster that will receive OTLP traces. + // This field can be left empty to disable reporting traces to the HTTP service. + // Only one of ``grpc_service``, ``http_service`` may be used. + // + // .. note:: + // + // Note: The ``request_headers_to_add`` property in the OTLP HTTP exporter service + // does not support the :ref:`format specifier ` as used for + // :ref:`HTTP access logging `. + // The values configured are added as HTTP headers on the OTLP export request + // without any formatting applied. + core.v3.HttpService http_service = 3 + [(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"]; // The name for the service. This will be populated in the ResourceSpan Resource attributes. // If it is not provided, it will default to "unknown_service:envoy". string service_name = 2; + + // An ordered list of resource detectors + // [#extension-category: envoy.tracers.opentelemetry.resource_detectors] + repeated core.v3.TypedExtensionConfig resource_detectors = 4; + + // Specifies the sampler to be used by the OpenTelemetry tracer. + // The configured sampler implements the Sampler interface defined by the OpenTelemetry specification. + // This field can be left empty. In this case, the default Envoy sampling decision is used. + // + // See: `OpenTelemetry sampler specification `_ + // [#extension-category: envoy.tracers.opentelemetry.samplers] + core.v3.TypedExtensionConfig sampler = 5; } diff --git a/api/envoy/config/trace/v3/skywalking.proto b/api/envoy/config/trace/v3/skywalking.proto index 327defe9ba69..57872a0fdb9d 100644 --- a/api/envoy/config/trace/v3/skywalking.proto +++ b/api/envoy/config/trace/v3/skywalking.proto @@ -22,9 +22,9 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: SkyWalking tracer] // Configuration for the SkyWalking tracer. Please note that if SkyWalking tracer is used as the -// provider of http tracer, then -// :ref:`start_child_span ` -// in the router must be set to true to get the correct topology and tracing data. Moreover, SkyWalking +// provider of tracing, then +// :ref:`spawn_upstream_span ` +// in the tracing config must be set to true to get the correct topology and tracing data. Moreover, SkyWalking // Tracer does not support SkyWalking extension header (``sw8-x``) temporarily. // [#extension: envoy.tracers.skywalking] message SkyWalkingConfig { diff --git a/api/envoy/config/trace/v3/zipkin.proto b/api/envoy/config/trace/v3/zipkin.proto index 96556c7b29b1..a9aefef0c6df 100644 --- a/api/envoy/config/trace/v3/zipkin.proto +++ b/api/envoy/config/trace/v3/zipkin.proto @@ -75,7 +75,7 @@ message ZipkinConfig { // // * The Envoy Proxy is used as gateway or ingress. // * The Envoy Proxy is used as sidecar but inbound traffic capturing or outbound traffic capturing is disabled. - // * Any case that the `start_child_span of router ` is set to true. + // * Any case that the :ref:`start_child_span of router ` is set to true. // // .. attention:: // diff --git a/api/envoy/config/transport_socket/alts/v2alpha/BUILD b/api/envoy/config/transport_socket/alts/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/transport_socket/alts/v2alpha/BUILD +++ b/api/envoy/config/transport_socket/alts/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/transport_socket/raw_buffer/v2/BUILD b/api/envoy/config/transport_socket/raw_buffer/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/transport_socket/raw_buffer/v2/BUILD +++ b/api/envoy/config/transport_socket/raw_buffer/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/transport_socket/tap/v2alpha/BUILD b/api/envoy/config/transport_socket/tap/v2alpha/BUILD index 52ca9859536e..34f67e3be13b 100644 --- a/api/envoy/config/transport_socket/tap/v2alpha/BUILD +++ b/api/envoy/config/transport_socket/tap/v2alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/config/common/tap/v2alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/config/upstream/local_address_selector/v3/BUILD b/api/envoy/config/upstream/local_address_selector/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/config/upstream/local_address_selector/v3/BUILD +++ b/api/envoy/config/upstream/local_address_selector/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/config/upstream/local_address_selector/v3/default_local_address_selector.proto b/api/envoy/config/upstream/local_address_selector/v3/default_local_address_selector.proto index 4ecd27d1fe09..852689dd859b 100644 --- a/api/envoy/config/upstream/local_address_selector/v3/default_local_address_selector.proto +++ b/api/envoy/config/upstream/local_address_selector/v3/default_local_address_selector.proto @@ -23,9 +23,9 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // is appended to the address specified in the // :ref:`source_address ` // field. The extra address should have a different IP version than the address in the -// `source_address` field. The address which has the same IP +// ``source_address`` field. The address which has the same IP // version with the target host's address IP version will be used as bind address. -// If there is no same IP version address found, the address in the `source_address` field will +// If there is no same IP version address found, the address in the ``source_address`` field will // be returned. message DefaultLocalAddressSelector { } diff --git a/api/envoy/data/accesslog/v2/BUILD b/api/envoy/data/accesslog/v2/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/data/accesslog/v2/BUILD +++ b/api/envoy/data/accesslog/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/accesslog/v3/BUILD b/api/envoy/data/accesslog/v3/BUILD index a1775bbe6f51..e74acc660850 100644 --- a/api/envoy/data/accesslog/v3/BUILD +++ b/api/envoy/data/accesslog/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/accesslog/v3/accesslog.proto b/api/envoy/data/accesslog/v3/accesslog.proto index 56d9e7b8c4ca..25e7a80bb2f4 100644 --- a/api/envoy/data/accesslog/v3/accesslog.proto +++ b/api/envoy/data/accesslog/v3/accesslog.proto @@ -44,6 +44,9 @@ enum AccessLogType { UpstreamPeriodic = 8; UpstreamEnd = 9; DownstreamTunnelSuccessfullyEstablished = 10; + UdpTunnelUpstreamConnected = 11; + UdpPeriodic = 12; + UdpSessionEnd = 13; } message TCPAccessLogEntry { diff --git a/api/envoy/data/cluster/v2alpha/BUILD b/api/envoy/data/cluster/v2alpha/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/data/cluster/v2alpha/BUILD +++ b/api/envoy/data/cluster/v2alpha/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/data/cluster/v3/BUILD b/api/envoy/data/cluster/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/data/cluster/v3/BUILD +++ b/api/envoy/data/cluster/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/data/core/v2alpha/BUILD b/api/envoy/data/core/v2alpha/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/data/core/v2alpha/BUILD +++ b/api/envoy/data/core/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/core/v3/BUILD b/api/envoy/data/core/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/data/core/v3/BUILD +++ b/api/envoy/data/core/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/dns/v2alpha/BUILD b/api/envoy/data/dns/v2alpha/BUILD index e305003238a5..22b193151178 100644 --- a/api/envoy/data/dns/v2alpha/BUILD +++ b/api/envoy/data/dns/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/dns/v3/BUILD b/api/envoy/data/dns/v3/BUILD index 516369f09675..30302a7baf53 100644 --- a/api/envoy/data/dns/v3/BUILD +++ b/api/envoy/data/dns/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/tap/v2alpha/BUILD b/api/envoy/data/tap/v2alpha/BUILD index 83bc0ab960e7..10580ab4a7aa 100644 --- a/api/envoy/data/tap/v2alpha/BUILD +++ b/api/envoy/data/tap/v2alpha/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/tap/v3/BUILD b/api/envoy/data/tap/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/data/tap/v3/BUILD +++ b/api/envoy/data/tap/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/data/tap/v3/common.proto b/api/envoy/data/tap/v3/common.proto index 741f7d73d1ae..7be656aee046 100644 --- a/api/envoy/data/tap/v3/common.proto +++ b/api/envoy/data/tap/v3/common.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.data.tap.v3; +import "envoy/config/core/v3/address.proto"; + import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; @@ -36,3 +38,14 @@ message Body { // ` settings. bool truncated = 3; } + +// Connection properties. +message Connection { + option (udpa.annotations.versioning).previous_message_type = "envoy.data.tap.v2alpha.Connection"; + + // Local address. + config.core.v3.Address local_address = 1; + + // Remote address. + config.core.v3.Address remote_address = 2; +} diff --git a/api/envoy/data/tap/v3/http.proto b/api/envoy/data/tap/v3/http.proto index aa991dd53b22..2e5c566e59ed 100644 --- a/api/envoy/data/tap/v3/http.proto +++ b/api/envoy/data/tap/v3/http.proto @@ -46,6 +46,9 @@ message HttpBufferedTrace { // Response message. Message response = 2; + + // downstream connection + Connection downstream_connection = 3; } // A streamed HTTP trace segment. Multiple segments make up a full trace. diff --git a/api/envoy/data/tap/v3/transport.proto b/api/envoy/data/tap/v3/transport.proto index efd2d4168e73..9338165058a2 100644 --- a/api/envoy/data/tap/v3/transport.proto +++ b/api/envoy/data/tap/v3/transport.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package envoy.data.tap.v3; -import "envoy/config/core/v3/address.proto"; import "envoy/data/tap/v3/common.proto"; import "google/protobuf/timestamp.proto"; @@ -20,17 +19,6 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Trace format for the tap transport socket extension. This dumps plain text read/write // sequences on a socket. -// Connection properties. -message Connection { - option (udpa.annotations.versioning).previous_message_type = "envoy.data.tap.v2alpha.Connection"; - - // Local address. - config.core.v3.Address local_address = 2; - - // Remote address. - config.core.v3.Address remote_address = 3; -} - // Event in a socket trace. message SocketEvent { option (udpa.annotations.versioning).previous_message_type = "envoy.data.tap.v2alpha.SocketEvent"; diff --git a/api/envoy/extensions/access_loggers/file/v3/BUILD b/api/envoy/extensions/access_loggers/file/v3/BUILD index a1775bbe6f51..e74acc660850 100644 --- a/api/envoy/extensions/access_loggers/file/v3/BUILD +++ b/api/envoy/extensions/access_loggers/file/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD b/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD +++ b/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto b/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto index d8ef21a1cdf1..750ffd30d251 100644 --- a/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto +++ b/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto @@ -21,7 +21,8 @@ message ExpressionFilter { // Expressions are based on the set of Envoy :ref:`attributes `. // The provided expression must evaluate to true for logging (expression errors are considered false). // Examples: - // - ``response.code >= 400`` - // - ``(connection.mtls && request.headers['x-log-mtls'] == 'true') || request.url_path.contains('v1beta3')`` + // + // * ``response.code >= 400`` + // * ``(connection.mtls && request.headers['x-log-mtls'] == 'true') || request.url_path.contains('v1beta3')`` string expression = 1; } diff --git a/api/envoy/extensions/access_loggers/grpc/v3/BUILD b/api/envoy/extensions/access_loggers/grpc/v3/BUILD index bde8902a86c8..ee825af0503a 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/BUILD +++ b/api/envoy/extensions/access_loggers/grpc/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/tracing/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD b/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD index 37737510d8ea..95eff6986b7f 100644 --- a/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD +++ b/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/access_loggers/grpc/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@opentelemetry_proto//:common", ], ) diff --git a/api/envoy/extensions/access_loggers/stream/v3/BUILD b/api/envoy/extensions/access_loggers/stream/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/access_loggers/stream/v3/BUILD +++ b/api/envoy/extensions/access_loggers/stream/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/access_loggers/wasm/v3/BUILD b/api/envoy/extensions/access_loggers/wasm/v3/BUILD index c37174bdefc4..ed3c664aedd7 100644 --- a/api/envoy/extensions/access_loggers/wasm/v3/BUILD +++ b/api/envoy/extensions/access_loggers/wasm/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/wasm/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD b/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD +++ b/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/clusters/aggregate/v3/BUILD b/api/envoy/extensions/clusters/aggregate/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/clusters/aggregate/v3/BUILD +++ b/api/envoy/extensions/clusters/aggregate/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD index 00ebd40c5d65..ef2bec727fa1 100644 --- a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD +++ b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/config/cluster/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto index 8cb7183fc4d9..6ad6b9eb0ba3 100644 --- a/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto +++ b/api/envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.proto @@ -60,7 +60,7 @@ message ClusterConfig { // resolved address for the new connection matches the peer address of the connection and // the TLS certificate is also valid for the new hostname. For example, if a connection // has previously been established to foo.example.com at IP 1.2.3.4 with a certificate - // that is valid for `*.example.com`, then this connection could be used for requests to + // that is valid for ``*.example.com``, then this connection could be used for requests to // bar.example.com if that also resolved to 1.2.3.4. // // .. note:: diff --git a/api/envoy/extensions/clusters/redis/v3/BUILD b/api/envoy/extensions/clusters/redis/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/clusters/redis/v3/BUILD +++ b/api/envoy/extensions/clusters/redis/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/common/async_files/v3/BUILD b/api/envoy/extensions/common/async_files/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/envoy/extensions/common/async_files/v3/BUILD +++ b/api/envoy/extensions/common/async_files/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD index b9cc22c7ee67..a220c748ba7f 100644 --- a/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD +++ b/api/envoy/extensions/common/dynamic_forward_proxy/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/cluster/v3:pkg", "//envoy/config/common/key_value/v3:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/common/matching/v3/BUILD b/api/envoy/extensions/common/matching/v3/BUILD index de9e120297ac..2afc0bdde334 100644 --- a/api/envoy/extensions/common/matching/v3/BUILD +++ b/api/envoy/extensions/common/matching/v3/BUILD @@ -9,8 +9,7 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/common/matcher/v3:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/extensions/common/matching/v3/extension_matcher.proto b/api/envoy/extensions/common/matching/v3/extension_matcher.proto index bef7c712eb2e..817cd27a37a2 100644 --- a/api/envoy/extensions/common/matching/v3/extension_matcher.proto +++ b/api/envoy/extensions/common/matching/v3/extension_matcher.proto @@ -5,7 +5,6 @@ package envoy.extensions.common.matching.v3; import "envoy/config/common/matcher/v3/matcher.proto"; import "envoy/config/core/v3/extension.proto"; -import "xds/annotations/v3/status.proto"; import "xds/type/matcher/v3/matcher.proto"; import "envoy/annotations/deprecation.proto"; @@ -24,8 +23,6 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // decorating an existing extension with a matcher, which can be used to match against // relevant protocol data. message ExtensionWithMatcher { - option (xds.annotations.v3.message_status).work_in_progress = true; - // The associated matcher. This is deprecated in favor of xds_matcher. config.common.matcher.v3.Matcher matcher = 1 [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; diff --git a/api/envoy/extensions/common/ratelimit/v3/BUILD b/api/envoy/extensions/common/ratelimit/v3/BUILD index 9a76b7e148e0..ef19132f9180 100644 --- a/api/envoy/extensions/common/ratelimit/v3/BUILD +++ b/api/envoy/extensions/common/ratelimit/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/common/tap/v3/BUILD b/api/envoy/extensions/common/tap/v3/BUILD index a99fa811f859..9e898366c9bb 100644 --- a/api/envoy/extensions/common/tap/v3/BUILD +++ b/api/envoy/extensions/common/tap/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/tap/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/compression/brotli/compressor/v3/BUILD b/api/envoy/extensions/compression/brotli/compressor/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/compression/brotli/compressor/v3/BUILD +++ b/api/envoy/extensions/compression/brotli/compressor/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD b/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD +++ b/api/envoy/extensions/compression/brotli/decompressor/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/compression/gzip/compressor/v3/BUILD b/api/envoy/extensions/compression/gzip/compressor/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/compression/gzip/compressor/v3/BUILD +++ b/api/envoy/extensions/compression/gzip/compressor/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/compression/gzip/decompressor/v3/BUILD b/api/envoy/extensions/compression/gzip/decompressor/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/compression/gzip/decompressor/v3/BUILD +++ b/api/envoy/extensions/compression/gzip/decompressor/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/compression/zstd/compressor/v3/BUILD b/api/envoy/extensions/compression/zstd/compressor/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/compression/zstd/compressor/v3/BUILD +++ b/api/envoy/extensions/compression/zstd/compressor/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/compression/zstd/decompressor/v3/BUILD b/api/envoy/extensions/compression/zstd/decompressor/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/compression/zstd/decompressor/v3/BUILD +++ b/api/envoy/extensions/compression/zstd/decompressor/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/config/validators/minimum_clusters/v3/BUILD b/api/envoy/extensions/config/validators/minimum_clusters/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/config/validators/minimum_clusters/v3/BUILD +++ b/api/envoy/extensions/config/validators/minimum_clusters/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/early_data/v3/BUILD b/api/envoy/extensions/early_data/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/early_data/v3/BUILD +++ b/api/envoy/extensions/early_data/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/common/dependency/v3/BUILD b/api/envoy/extensions/filters/common/dependency/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/common/dependency/v3/BUILD +++ b/api/envoy/extensions/filters/common/dependency/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/common/fault/v3/BUILD b/api/envoy/extensions/filters/common/fault/v3/BUILD index 9a76b7e148e0..ef19132f9180 100644 --- a/api/envoy/extensions/filters/common/fault/v3/BUILD +++ b/api/envoy/extensions/filters/common/fault/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/common/matcher/action/v3/BUILD b/api/envoy/extensions/filters/common/matcher/action/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/common/matcher/action/v3/BUILD +++ b/api/envoy/extensions/filters/common/matcher/action/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/common/set_filter_state/v3/BUILD b/api/envoy/extensions/filters/common/set_filter_state/v3/BUILD new file mode 100644 index 000000000000..09a37ad16b83 --- /dev/null +++ b/api/envoy/extensions/filters/common/set_filter_state/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/common/set_filter_state/v3/value.proto b/api/envoy/extensions/filters/common/set_filter_state/v3/value.proto new file mode 100644 index 000000000000..81d501033226 --- /dev/null +++ b/api/envoy/extensions/filters/common/set_filter_state/v3/value.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package envoy.extensions.filters.common.set_filter_state.v3; + +import "envoy/config/core/v3/substitution_format_string.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.common.set_filter_state.v3"; +option java_outer_classname = "ValueProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/set_filter_state/v3;set_filter_statev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Set-Filter-State filter state value] + +// A filter state key and value pair. +// [#next-free-field: 6] +message FilterStateValue { + enum SharedWithUpstream { + // Object is not shared with the upstream internal connections. + NONE = 0; + + // Object is shared with the upstream internal connection. + ONCE = 1; + + // Object is shared with the upstream internal connection and any internal connection upstream from it. + TRANSITIVE = 2; + } + + oneof key { + option (validate.required) = true; + + // Filter state object key. The key is expected to be registered via an object factory, see + // :ref:`the well-known filter state keys `. + string object_key = 1 [(validate.rules).string = {min_len: 1}]; + } + + oneof value { + option (validate.required) = true; + + // Uses the :ref:`format string ` to + // instantiate the filter state object value. + config.core.v3.SubstitutionFormatString format_string = 2; + } + + // If marked as read-only, the filter state key value is locked, and cannot + // be overridden by any filter, including this filter. + bool read_only = 3; + + // Configures the object to be shared with the upstream internal connections. See :ref:`internal upstream + // transport ` for more details on the filter state sharing with + // the internal connections. + SharedWithUpstream shared_with_upstream = 4; + + // Skip the update if the value evaluates to an empty string. + // This option can be used to supply multiple alternatives for the same filter state object key. + bool skip_if_empty = 5; +} diff --git a/api/envoy/extensions/filters/http/adaptive_concurrency/v3/BUILD b/api/envoy/extensions/filters/http/adaptive_concurrency/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/extensions/filters/http/adaptive_concurrency/v3/BUILD +++ b/api/envoy/extensions/filters/http/adaptive_concurrency/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/admission_control/v3/BUILD b/api/envoy/extensions/filters/http/admission_control/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/extensions/filters/http/admission_control/v3/BUILD +++ b/api/envoy/extensions/filters/http/admission_control/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/alternate_protocols_cache/v3/BUILD b/api/envoy/extensions/filters/http/alternate_protocols_cache/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/alternate_protocols_cache/v3/BUILD +++ b/api/envoy/extensions/filters/http/alternate_protocols_cache/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/aws_lambda/v3/BUILD b/api/envoy/extensions/filters/http/aws_lambda/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/aws_lambda/v3/BUILD +++ b/api/envoy/extensions/filters/http/aws_lambda/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/aws_request_signing/v3/BUILD b/api/envoy/extensions/filters/http/aws_request_signing/v3/BUILD index 693f0b92ff34..bfc486330911 100644 --- a/api/envoy/extensions/filters/http/aws_request_signing/v3/BUILD +++ b/api/envoy/extensions/filters/http/aws_request_signing/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3/BUILD b/api/envoy/extensions/filters/http/bandwidth_limit/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3/BUILD +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/basic_auth/v3/BUILD b/api/envoy/extensions/filters/http/basic_auth/v3/BUILD new file mode 100644 index 000000000000..09a37ad16b83 --- /dev/null +++ b/api/envoy/extensions/filters/http/basic_auth/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto b/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto new file mode 100644 index 000000000000..df23868a4260 --- /dev/null +++ b/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.basic_auth.v3; + +import "envoy/config/core/v3/base.proto"; + +import "udpa/annotations/sensitive.proto"; +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.basic_auth.v3"; +option java_outer_classname = "BasicAuthProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/basic_auth/v3;basic_authv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Basic Auth] +// Basic Auth :ref:`configuration overview `. +// [#extension: envoy.filters.http.basic_auth] + +// Basic HTTP authentication. +// +// Example: +// +// .. code-block:: yaml +// +// users: +// inline_string: |- +// user1:{SHA}hashed_user1_password +// user2:{SHA}hashed_user2_password +// +message BasicAuth { + // Username-password pairs used to verify user credentials in the "Authorization" header. + // The value needs to be the htpasswd format. + // Reference to https://httpd.apache.org/docs/2.4/programs/htpasswd.html + config.core.v3.DataSource users = 1 [(udpa.annotations.sensitive) = true]; +} diff --git a/api/envoy/extensions/filters/http/buffer/v3/BUILD b/api/envoy/extensions/filters/http/buffer/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/buffer/v3/BUILD +++ b/api/envoy/extensions/filters/http/buffer/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/cache/v3/BUILD b/api/envoy/extensions/filters/http/cache/v3/BUILD index c0ffdf28daaf..eb0224e4187c 100644 --- a/api/envoy/extensions/filters/http/cache/v3/BUILD +++ b/api/envoy/extensions/filters/http/cache/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/route/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/cdn_loop/v3/BUILD b/api/envoy/extensions/filters/http/cdn_loop/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/cdn_loop/v3/BUILD +++ b/api/envoy/extensions/filters/http/cdn_loop/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/composite/v3/BUILD b/api/envoy/extensions/filters/http/composite/v3/BUILD index e9b556d681cf..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/composite/v3/BUILD +++ b/api/envoy/extensions/filters/http/composite/v3/BUILD @@ -7,7 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/composite/v3/composite.proto b/api/envoy/extensions/filters/http/composite/v3/composite.proto index 08a72e411b9f..a6132b7043dd 100644 --- a/api/envoy/extensions/filters/http/composite/v3/composite.proto +++ b/api/envoy/extensions/filters/http/composite/v3/composite.proto @@ -2,11 +2,12 @@ syntax = "proto3"; package envoy.extensions.filters.http.composite.v3; +import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/extension.proto"; -import "xds/annotations/v3/status.proto"; - +import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; +import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.filters.http.composite.v3"; option java_outer_classname = "CompositeProto"; @@ -29,11 +30,32 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // :ref:`ExecuteFilterAction `) // which filter configuration to create and delegate to. message Composite { - option (xds.annotations.v3.message_status).work_in_progress = true; +} + +// Configuration for an extension configuration discovery service with name. +message DynamicConfig { + // The name of the extension configuration. It also serves as a resource name in ExtensionConfigDS. + string name = 1 [(validate.rules).string = {min_len: 1}]; + + // Configuration source specifier for an extension configuration discovery + // service. In case of a failure and without the default configuration, + // 500(Internal Server Error) will be returned. + config.core.v3.ExtensionConfigSource config_discovery = 2; } // Composite match action (see :ref:`matching docs ` for more info on match actions). // This specifies the filter configuration of the filter that the composite filter should delegate filter interactions to. message ExecuteFilterAction { - config.core.v3.TypedExtensionConfig typed_config = 1; + // Filter specific configuration which depends on the filter being + // instantiated. See the supported filters for further documentation. + // Only one of ``typed_config`` or ``dynamic_config`` can be set. + // [#extension-category: envoy.filters.http] + config.core.v3.TypedExtensionConfig typed_config = 1 + [(udpa.annotations.field_migrate).oneof_promotion = "config_type"]; + + // Dynamic configuration of filter obtained via extension configuration discovery + // service. + // Only one of ``typed_config`` or ``dynamic_config`` can be set. + DynamicConfig dynamic_config = 2 + [(udpa.annotations.field_migrate).oneof_promotion = "config_type"]; } diff --git a/api/envoy/extensions/filters/http/compressor/v3/BUILD b/api/envoy/extensions/filters/http/compressor/v3/BUILD index a1775bbe6f51..e74acc660850 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/BUILD +++ b/api/envoy/extensions/filters/http/compressor/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto index a106f8ee5c60..6fe4b137da6e 100644 --- a/api/envoy/extensions/filters/http/compressor/v3/compressor.proto +++ b/api/envoy/extensions/filters/http/compressor/v3/compressor.proto @@ -126,18 +126,21 @@ message Compressor { // ``.compressor...*``. ResponseDirectionConfig response_direction_config = 8; - // If true, chooses this compressor first to do compression when the q-values in `Accept-Encoding` are same. + // If true, chooses this compressor first to do compression when the q-values in ``Accept-Encoding`` are same. // The last compressor which enables choose_first will be chosen if multiple compressor filters in the chain have choose_first as true. bool choose_first = 9; } -// Per-route overrides of `ResponseDirectionConfig`. Anything added here should be optional, -// to allow overriding arbitrary subsets of configuration. Omitted fields must have no affect. +// Per-route overrides of ``ResponseDirectionConfig``. Anything added here should be optional, +// to allow overriding arbitrary subsets of configuration. Omitted fields must have no effect. message ResponseDirectionOverrides { + // If set, overrides the filter-level + // :ref:`remove_accept_encoding_header`. + google.protobuf.BoolValue remove_accept_encoding_header = 1; } // Per-route overrides. As per-route overrides are needed, they should be -// added here, mirroring the structure of `Compressor`. All fields should be +// added here, mirroring the structure of ``Compressor``. All fields should be // optional, to allow overriding arbitrary subsets of configuration. message CompressorOverrides { // If present, response compression is enabled. @@ -152,7 +155,7 @@ message CompressorPerRoute { // Overrides Compressor.runtime_enabled and CommonDirectionConfig.enabled. bool disabled = 1 [(validate.rules).bool = {const: true}]; - // Per-route overrides. Fields set here will override corresponding fields in `Compressor`. + // Per-route overrides. Fields set here will override corresponding fields in ``Compressor``. CompressorOverrides overrides = 2; } } diff --git a/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/BUILD b/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/BUILD +++ b/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/config.proto b/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/config.proto index 9b98d057aa9b..7fc956260353 100644 --- a/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/config.proto +++ b/api/envoy/extensions/filters/http/connect_grpc_bridge/v3/config.proto @@ -13,10 +13,10 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/fil option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; -// [#protodoc-title: Buf Connect to gRPC] Buf Connect to gRPC bridge +// [#protodoc-title: Connect RPC to gRPC] Connect RPC to gRPC bridge // :ref:`configuration overview `. // [#extension: envoy.filters.http.connect_grpc_bridge] -// Buf Connect gRPC bridge filter configuration +// Connect RPC to gRPC bridge filter configuration message FilterConfig { } diff --git a/api/envoy/extensions/filters/http/cors/v3/BUILD b/api/envoy/extensions/filters/http/cors/v3/BUILD index 3f3a5395d2aa..e3bfc4e175f4 100644 --- a/api/envoy/extensions/filters/http/cors/v3/BUILD +++ b/api/envoy/extensions/filters/http/cors/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/credential_injector/v3/BUILD b/api/envoy/extensions/filters/http/credential_injector/v3/BUILD new file mode 100644 index 000000000000..628f71321fba --- /dev/null +++ b/api/envoy/extensions/filters/http/credential_injector/v3/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + ], +) diff --git a/api/envoy/extensions/filters/http/credential_injector/v3/credential_injector.proto b/api/envoy/extensions/filters/http/credential_injector/v3/credential_injector.proto new file mode 100644 index 000000000000..efa16d3aca9a --- /dev/null +++ b/api/envoy/extensions/filters/http/credential_injector/v3/credential_injector.proto @@ -0,0 +1,85 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.credential_injector.v3; + +import "envoy/config/core/v3/extension.proto"; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.credential_injector.v3"; +option java_outer_classname = "CredentialInjectorProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/credential_injector/v3;credential_injectorv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: Credential Injector] +// [#not-implemented-hide:] +// Credential Injector :ref:`configuration overview `. +// [#extension: envoy.filters.http.credential_injector] + +// Credential Injector injects credentials into outgoing HTTP requests. The filter configuration is used to retrieve the credentials, or +// they can be requested through the OAuth2 client credential grant. The credentials obtained are then injected into the Authorization header +// of the proxied HTTP requests, utilizing either the Basic or Bearer scheme. +// +// If the credential is not present, the request will fail with 401 Unauthorized if fail_if_not_present is set to true. +// +// Notice: This filter is intended to be used for workload authentication, which means that the identity associated with the inserted credential +// is considered as the identity of the workload behind the envoy proxy(in this case, envoy is typically deployed as a sidecar alongside that +// workload). Please note that this filter does not handle end user authentication. Its purpose is solely to authenticate the workload itself. +// +// Here is an example of CredentialInjector configuration with Generic credential, which injects an HTTP Basic Auth credential into the proxied requests. +// +// .. code-block:: yaml +// +// overwrite: true +// fail_if_not_present: true +// credential: +// name: generic_credential +// typed_config: +// "@type": type.googleapis.com/envoy.extensions.injected_credentials.generic.v3.Generic +// credential: +// name: credential +// sds_config: +// path_config_source: +// path: credential.yaml +// header: Authorization +// +// credential.yaml for Basic Auth: +// .. code-block:: yaml +// +// resources: +// - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" +// name: credential +// generic_secret: +// secret: +// inline_string: "Basic base64EncodedUsernamePassword" +// +// It can also be configured to inject a Bearer token into the proxied requests. +// credential.yaml for Bearer Token: +// .. code-block:: yaml +// +// resources: +// - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" +// name: credential +// generic_secret: +// secret: +// inline_string: "Bearer myToken" +// +message CredentialInjector { + // Whether to overwrite the value or not if the injected headers already exist. + // Value defaults to false. + bool overwrite = 1; + + // Whether to fail the request if the credential is not present. + // Value defaults to false. + // If set to true, the request will fail with 401 Unauthorized if the credential is not present. + bool fail_if_not_present = 2; + + // The credential to inject into the proxied requests + // TODO add extension-category + config.core.v3.TypedExtensionConfig credential = 3 [(validate.rules).message = {required: true}]; +} diff --git a/api/envoy/extensions/filters/http/csrf/v3/BUILD b/api/envoy/extensions/filters/http/csrf/v3/BUILD index 3f3a5395d2aa..e3bfc4e175f4 100644 --- a/api/envoy/extensions/filters/http/csrf/v3/BUILD +++ b/api/envoy/extensions/filters/http/csrf/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/custom_response/v3/BUILD b/api/envoy/extensions/filters/http/custom_response/v3/BUILD index 4e7598f926bd..720cd87d94c8 100644 --- a/api/envoy/extensions/filters/http/custom_response/v3/BUILD +++ b/api/envoy/extensions/filters/http/custom_response/v3/BUILD @@ -6,8 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/decompressor/v3/BUILD b/api/envoy/extensions/filters/http/decompressor/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/decompressor/v3/BUILD +++ b/api/envoy/extensions/filters/http/decompressor/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/BUILD index 05f25a2fe5d9..73e98d4d40b2 100644 --- a/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/http/dynamic_forward_proxy/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ext_authz/v3/BUILD b/api/envoy/extensions/filters/http/ext_authz/v3/BUILD index d1ae5c00f93a..cabe849e71d1 100644 --- a/api/envoy/extensions/filters/http/ext_authz/v3/BUILD +++ b/api/envoy/extensions/filters/http/ext_authz/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto b/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto index cd2d1f6f4e21..ea40112a980e 100644 --- a/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto +++ b/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto @@ -10,6 +10,8 @@ import "envoy/type/matcher/v3/metadata.proto"; import "envoy/type/matcher/v3/string.proto"; import "envoy/type/v3/http_status.proto"; +import "google/protobuf/wrappers.proto"; + import "envoy/annotations/deprecation.proto"; import "udpa/annotations/sensitive.proto"; import "udpa/annotations/status.proto"; @@ -26,7 +28,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // External Authorization :ref:`configuration overview `. // [#extension: envoy.filters.http.ext_authz] -// [#next-free-field: 19] +// [#next-free-field: 23] message ExtAuthz { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.ext_authz.v2.ExtAuthz"; @@ -63,6 +65,12 @@ message ExtAuthz { // `. bool failure_mode_allow = 2; + // When ``failure_mode_allow`` and ``failure_mode_allow_header_add`` are both set to true, + // ``x-envoy-auth-failure-mode-allowed: true`` will be added to request headers if the communication + // with the authorization service has failed, or if the authorization service has returned a + // HTTP 5xx error. + bool failure_mode_allow_header_add = 19; + // Enables filter to buffer the client request body and send it within the authorization request. // A ``x-envoy-auth-partial-body: false|true`` metadata header will be added to the authorization // request message indicating if the body data is partial. @@ -85,7 +93,10 @@ message ExtAuthz { type.v3.HttpStatus status_on_error = 7; // Specifies a list of metadata namespaces whose values, if present, will be passed to the - // ext_authz service. :ref:`filter_metadata ` is passed as an opaque ``protobuf::Struct``. + // ext_authz service. The :ref:`filter_metadata ` + // is passed as an opaque ``protobuf::Struct``. + // + // Please note that this field exclusively applies to the gRPC ext_authz service and has no effect on the HTTP service. // // For example, if the ``jwt_authn`` filter is used and :ref:`payload_in_metadata // ` is set, @@ -99,13 +110,28 @@ message ExtAuthz { repeated string metadata_context_namespaces = 8; // Specifies a list of metadata namespaces whose values, if present, will be passed to the - // ext_authz service. :ref:`typed_filter_metadata ` is passed as an ``protobuf::Any``. + // ext_authz service. :ref:`typed_filter_metadata ` + // is passed as a ``protobuf::Any``. // - // It works in a way similar to ``metadata_context_namespaces`` but allows envoy and external authz server to share the protobuf message definition - // in order to do a safe parsing. + // Please note that this field exclusively applies to the gRPC ext_authz service and has no effect on the HTTP service. + // + // It works in a way similar to ``metadata_context_namespaces`` but allows Envoy and ext_authz server to share + // the protobuf message definition in order to do a safe parsing. // repeated string typed_metadata_context_namespaces = 16; + // Specifies a list of route metadata namespaces whose values, if present, will be passed to the + // ext_authz service at :ref:`route_metadata_context ` in + // :ref:`CheckRequest `. + // :ref:`filter_metadata ` is passed as an opaque ``protobuf::Struct``. + repeated string route_metadata_context_namespaces = 21; + + // Specifies a list of route metadata namespaces whose values, if present, will be passed to the + // ext_authz service at :ref:`route_metadata_context ` in + // :ref:`CheckRequest `. + // :ref:`typed_filter_metadata ` is passed as an ``protobuf::Any``. + repeated string route_typed_metadata_context_namespaces = 22; + // Specifies if the filter is enabled. // // If :ref:`runtime_key ` is specified, @@ -185,6 +211,10 @@ message ExtAuthz { // When this field is true, Envoy will include the SNI name used for TLSClientHello, if available, in the // :ref:`tls_session`. bool include_tls_session = 18; + + // Whether to increment cluster statistics (e.g. cluster..upstream_rq_*) on authorization failure. + // Defaults to true. + google.protobuf.BoolValue charge_cluster_response_stats = 20; } // Configuration for buffering the request data. @@ -304,8 +334,8 @@ message AuthorizationResponse { type.matcher.v3.ListStringMatcher allowed_upstream_headers = 1; // When this :ref:`list ` is set, authorization - // response headers that have a correspondent match will be added to the client's response. Note - // that coexistent headers will be appended. + // response headers that have a correspondent match will be added to the original client request. + // Note that coexistent headers will be appended. type.matcher.v3.ListStringMatcher allowed_upstream_headers_to_append = 3; // When this :ref:`list ` is set, authorization @@ -371,6 +401,19 @@ message CheckSettings { map context_extensions = 1 [(udpa.annotations.sensitive) = true]; // When set to true, disable the configured :ref:`with_request_body - // ` for a route. + // ` for a specific route. + // + // Please note that only one of *disable_request_body_buffering* or + // :ref:`with_request_body ` + // may be specified. bool disable_request_body_buffering = 2; + + // Enable or override request body buffering, which is configured using the + // :ref:`with_request_body ` + // option for a specific route. + // + // Please note that only only one of *with_request_body* or + // :ref:`disable_request_body_buffering ` + // may be specified. + BufferSettings with_request_body = 3; } diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/BUILD b/api/envoy/extensions/filters/http/ext_proc/v3/BUILD index 18cc12771da3..8322f99fa7df 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/BUILD +++ b/api/envoy/extensions/filters/http/ext_proc/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/config/common/mutation_rules/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto b/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto index eafdb1eabc65..66c04acc6426 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3/processing_mode.proto @@ -35,6 +35,22 @@ message ProcessingMode { } // Control how the request and response bodies are handled + // When body mutation by external processor is enabled, ext_proc filter will always remove + // the content length header in three cases below because content length can not be guaranteed + // to be set correctly: + // 1) STREAMED BodySendMode: header processing completes before body mutation comes back. + // 2) BUFFERED_PARTIAL BodySendMode: body is buffered and could be injected in different phases. + // 3) BUFFERED BodySendMode + SKIP HeaderSendMode: header processing (e.g., update content-length) is skipped. + // + // In Envoy's http1 codec implementation, removing content length will enable chunked transfer + // encoding whenever feasible. The recipient (either client or server) must be able + // to parse and decode the chunked transfer coding. + // (see `details in RFC9112 `_). + // + // In BUFFERED BodySendMode + SEND HeaderSendMode, content length header is allowed but it is + // external processor's responsibility to set the content length correctly matched to the length + // of mutated body. If they don't match, the corresponding body mutation will be rejected and + // local reply will be sent with an error message. enum BodySendMode { // Do not send the body at all. This is the default. NONE = 0; diff --git a/api/envoy/extensions/filters/http/fault/v3/BUILD b/api/envoy/extensions/filters/http/fault/v3/BUILD index 53db91cad82c..1bbe0b04c3d6 100644 --- a/api/envoy/extensions/filters/http/fault/v3/BUILD +++ b/api/envoy/extensions/filters/http/fault/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/config/route/v3:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/file_system_buffer/v3/BUILD b/api/envoy/extensions/filters/http/file_system_buffer/v3/BUILD index 26baeccd9941..5b108dcfee6c 100644 --- a/api/envoy/extensions/filters/http/file_system_buffer/v3/BUILD +++ b/api/envoy/extensions/filters/http/file_system_buffer/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/common/async_files/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/gcp_authn/v3/BUILD b/api/envoy/extensions/filters/http/gcp_authn/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/gcp_authn/v3/BUILD +++ b/api/envoy/extensions/filters/http/gcp_authn/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/geoip/v3/BUILD b/api/envoy/extensions/filters/http/geoip/v3/BUILD index e9b556d681cf..628f71321fba 100644 --- a/api/envoy/extensions/filters/http/geoip/v3/BUILD +++ b/api/envoy/extensions/filters/http/geoip/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/geoip/v3/geoip.proto b/api/envoy/extensions/filters/http/geoip/v3/geoip.proto index a01356333524..4ef26a8245e2 100644 --- a/api/envoy/extensions/filters/http/geoip/v3/geoip.proto +++ b/api/envoy/extensions/filters/http/geoip/v3/geoip.proto @@ -21,52 +21,6 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // [#extension: envoy.filters.http.geoip] message Geoip { - // The set of geolocation headers to add to request. If any of the configured headers is present - // in the incoming request, it will be overridden by Geoip filter. - // [#next-free-field: 10] - message GeolocationHeadersToAdd { - // If set, the header will be used to populate the country ISO code associated with the IP address. - string country = 1 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the header will be used to populate the city associated with the IP address. - string city = 2 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the header will be used to populate the region ISO code associated with the IP address. - string region = 3 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the header will be used to populate the ASN associated with the IP address. - string asn = 4 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the IP address will be checked if it belongs to any type of anonymization network (e.g. VPN, public proxy etc) - // and header will be populated with the check result. Header value will be set to either "true" or "false" depending on the check result. - string is_anon = 5 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the IP address will be checked if it belongs to a VPN and header will be populated with the check result. - // Header value will be set to either "true" or "false" depending on the check result. - string anon_vpn = 6 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the IP address will be checked if it belongs to a hosting provider and header will be populated with the check result. - // Header value will be set to either "true" or "false" depending on the check result. - string anon_hosting = 7 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the IP address will be checked if it belongs to a TOR exit node and header will be populated with the check result. - // Header value will be set to either "true" or "false" depending on the check result. - string anon_tor = 8 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - - // If set, the IP address will be checked if it belongs to a public proxy and header will be populated with the check result. - // Header value will be set to either "true" or "false" depending on the check result. - string anon_proxy = 9 - [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; - } - message XffConfig { // The number of additional ingress proxy hops from the right side of the // :ref:`config_http_conn_man_headers_x-forwarded-for` HTTP header to trust when @@ -77,14 +31,15 @@ message Geoip { } // If set, the :ref:`xff_num_trusted_hops ` field will be used to determine - // trusted client address from `x-forwarded-for` header. + // trusted client address from ``x-forwarded-for`` header. // Otherwise, the immediate downstream connection source address will be used. // [#next-free-field: 2] XffConfig xff_config = 1; - // Configuration for geolocation headers to add to request. - GeolocationHeadersToAdd geo_headers_to_add = 2 [(validate.rules).message = {required: true}]; - - // Geolocation provider specific configuration. + // Geoip driver specific configuration which depends on the driver being instantiated. + // See the geoip drivers for examples: + // + // - :ref:`MaxMindConfig ` + // [#extension-category: envoy.geoip_providers] config.core.v3.TypedExtensionConfig provider = 3 [(validate.rules).message = {required: true}]; } diff --git a/api/envoy/extensions/filters/http/grpc_field_extraction/v3/BUILD b/api/envoy/extensions/filters/http/grpc_field_extraction/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/grpc_field_extraction/v3/BUILD +++ b/api/envoy/extensions/filters/http/grpc_field_extraction/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/grpc_field_extraction/v3/config.proto b/api/envoy/extensions/filters/http/grpc_field_extraction/v3/config.proto index dbd6ce43f165..3684f994d65f 100644 --- a/api/envoy/extensions/filters/http/grpc_field_extraction/v3/config.proto +++ b/api/envoy/extensions/filters/http/grpc_field_extraction/v3/config.proto @@ -140,14 +140,14 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; message GrpcFieldExtractionConfig { // The proto descriptor set binary for the gRPC services. // - // It could be passed by a local file through `Datasource.filename` or embedded in the - // `Datasource.inline_bytes`. + // It could be passed by a local file through ``Datasource.filename`` or embedded in the + // ``Datasource.inline_bytes``. config.core.v3.DataSource descriptor_set = 1 [(validate.rules).message = {required: true}]; // Specify the extraction info. // The key is the fully qualified gRPC method name. - // `${package}.${Service}.${Method}`, like - // `endpoints.examples.bookstore.BookStore.GetShelf` + // ``${package}.${Service}.${Method}``, like + // ``endpoints.examples.bookstore.BookStore.GetShelf`` // // The value is the field extractions for individual gRPC method. map extractions_by_method = 2; @@ -158,8 +158,8 @@ message GrpcFieldExtractionConfig { message FieldExtractions { // The field extractions for requests. // The key is the field path within the grpc request. - // For example, we can define `foo.bar.name` if we want to extract - // Request.foo.bar.name. + // For example, we can define ``foo.bar.name`` if we want to extract + // ``Request.foo.bar.name``. // // .. code-block:: proto // diff --git a/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/BUILD b/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/BUILD +++ b/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/config.proto b/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/config.proto index e66ad4031023..1ca5b05ea9bc 100644 --- a/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/config.proto +++ b/api/envoy/extensions/filters/http/grpc_http1_bridge/v3/config.proto @@ -26,4 +26,7 @@ message Config { // For the requests that went through this upgrade the filter will also strip the frame before forwarding the // response to the client. bool upgrade_protobuf_to_grpc = 1; + + // If true then query parameters in request's URL path will be removed. + bool ignore_query_parameters = 2; } diff --git a/api/envoy/extensions/filters/http/grpc_http1_reverse_bridge/v3/BUILD b/api/envoy/extensions/filters/http/grpc_http1_reverse_bridge/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/grpc_http1_reverse_bridge/v3/BUILD +++ b/api/envoy/extensions/filters/http/grpc_http1_reverse_bridge/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/grpc_json_transcoder/v3/BUILD b/api/envoy/extensions/filters/http/grpc_json_transcoder/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/grpc_json_transcoder/v3/BUILD +++ b/api/envoy/extensions/filters/http/grpc_json_transcoder/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/grpc_stats/v3/BUILD b/api/envoy/extensions/filters/http/grpc_stats/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/grpc_stats/v3/BUILD +++ b/api/envoy/extensions/filters/http/grpc_stats/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/grpc_web/v3/BUILD b/api/envoy/extensions/filters/http/grpc_web/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/grpc_web/v3/BUILD +++ b/api/envoy/extensions/filters/http/grpc_web/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/gzip/v3/BUILD b/api/envoy/extensions/filters/http/gzip/v3/BUILD index bfe5d198e612..dbd9ebc365d2 100644 --- a/api/envoy/extensions/filters/http/gzip/v3/BUILD +++ b/api/envoy/extensions/filters/http/gzip/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/filters/http/compressor/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/header_mutation/v3/BUILD b/api/envoy/extensions/filters/http/header_mutation/v3/BUILD index 7af7ae042311..876a007c83cf 100644 --- a/api/envoy/extensions/filters/http/header_mutation/v3/BUILD +++ b/api/envoy/extensions/filters/http/header_mutation/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/common/mutation_rules/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/header_to_metadata/v3/BUILD b/api/envoy/extensions/filters/http/header_to_metadata/v3/BUILD index 693f0b92ff34..bfc486330911 100644 --- a/api/envoy/extensions/filters/http/header_to_metadata/v3/BUILD +++ b/api/envoy/extensions/filters/http/header_to_metadata/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/health_check/v3/BUILD b/api/envoy/extensions/filters/http/health_check/v3/BUILD index c6ef74063aab..c5d802c5d29f 100644 --- a/api/envoy/extensions/filters/http/health_check/v3/BUILD +++ b/api/envoy/extensions/filters/http/health_check/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/route/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/json_to_metadata/v3/BUILD b/api/envoy/extensions/filters/http/json_to_metadata/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/json_to_metadata/v3/BUILD +++ b/api/envoy/extensions/filters/http/json_to_metadata/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/json_to_metadata/v3/json_to_metadata.proto b/api/envoy/extensions/filters/http/json_to_metadata/v3/json_to_metadata.proto index 3dfb87f97a7d..8d7b53d1c841 100644 --- a/api/envoy/extensions/filters/http/json_to_metadata/v3/json_to_metadata.proto +++ b/api/envoy/extensions/filters/http/json_to_metadata/v3/json_to_metadata.proto @@ -62,7 +62,6 @@ message JsonToMetadata { } message Selector { - // TODO(kuochunghsu): Explore matchers for array handling. oneof selector { // key to match string key = 1 [(validate.rules).string = {min_len: 1}]; @@ -99,9 +98,9 @@ message JsonToMetadata { repeated Rule rules = 1 [(validate.rules).repeated = {min_items: 1}]; // Allowed content-type for json to metadata transformation. - // Default to {"application/json"}. + // Default to ``{"application/json"}``. // - // Set `allow_empty_content_type` if empty/missing content-type header + // Set ``allow_empty_content_type`` if empty/missing content-type header // is allowed. repeated string allow_content_types = 2 [(validate.rules).repeated = {items {string {min_len: 1}}}]; @@ -111,6 +110,10 @@ message JsonToMetadata { bool allow_empty_content_type = 3; } - // Rules to match json body of requests - MatchRules request_rules = 1 [(validate.rules).message = {required: true}]; + // At least one of request_rules and response_rules must be provided. + // Rules to match json body of requests. + MatchRules request_rules = 1; + + // Rules to match json body of responses. + MatchRules response_rules = 2; } diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/BUILD b/api/envoy/extensions/filters/http/jwt_authn/v3/BUILD index 6eb33fe8151a..cea648f6d0ec 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v3/BUILD +++ b/api/envoy/extensions/filters/http/jwt_authn/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto index bf88896e7030..7d5ae25a027a 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -53,11 +53,23 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // cache_duration: // seconds: 300 // -// [#next-free-field: 17] +// [#next-free-field: 19] message JwtProvider { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.jwt_authn.v2alpha.JwtProvider"; + // Alters the payload representation in the request dynamic metadata to facilitate its use in matching. + message NormalizePayload { + // Each claim in this list will be interpreted as a space-delimited string + // and converted to a list of strings based on the delimited values. + // Example: a token with a claim ``scopes: "email profile"`` is translated + // to dynamic metadata ``scopes: ["email", "profile"]`` if this field is + // set value ``["scopes"]``. This special handling of ``scopes`` is + // recommended by `RFC8693 + // `_. + repeated string space_delimited_claims = 1; + } + // Specify the `principal `_ that issued // the JWT, usually a URL or an email address. // @@ -230,6 +242,9 @@ message JwtProvider { // string payload_in_metadata = 9; + // Normalizes the payload representation in the request metadata. + NormalizePayload normalize_payload_in_metadata = 18; + // If not empty, similar to :ref:`payload_in_metadata `, // a successfully verified JWT header will be written to :ref:`Dynamic State ` // as an entry (``protobuf::Struct``) in ``envoy.filters.http.jwt_authn`` ``namespace`` with the @@ -270,8 +285,8 @@ message JwtProvider { // string header_in_metadata = 14; - // If non empty, the failure status `::google::jwt_verify::Status` for a non verified JWT will be written to StreamInfo DynamicMetadata - // in the format as: ``namespace`` is the jwt_authn filter name as ````envoy.filters.http.jwt_authn```` + // If non empty, the failure status ``::google::jwt_verify::Status`` for a non verified JWT will be written to StreamInfo DynamicMetadata + // in the format as: ``namespace`` is the jwt_authn filter name as ``envoy.filters.http.jwt_authn`` // The value is the ``protobuf::Struct``. The values of this field will be ``code`` and ``message`` // and they will contain the JWT authentication failure status code and a message describing the failure. // @@ -302,8 +317,18 @@ message JwtProvider { // - name: x-jwt-claim-nested-claim // claim: claim.nested.key // - // This header is only reserved for jwt claim; any other value will be overwrite. + // This header is only reserved for jwt claim; any other value will be overwritten. repeated JwtClaimToHeader claim_to_headers = 15; + + // Clears route cache in order to allow JWT token to correctly affect + // routing decisions. Filter clears all cached routes when: + // + // 1. The field is set to ``true``. + // + // 2. At least one ``claim_to_headers`` header is added to the request OR + // if ``payload_in_metadata`` is set. + // + bool clear_route_cache = 17; } // This message specifies JWT Cache configuration. diff --git a/api/envoy/extensions/filters/http/kill_request/v3/BUILD b/api/envoy/extensions/filters/http/kill_request/v3/BUILD index 9a76b7e148e0..ef19132f9180 100644 --- a/api/envoy/extensions/filters/http/kill_request/v3/BUILD +++ b/api/envoy/extensions/filters/http/kill_request/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD b/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD index 6c58a43e4ff6..1ef2f0c9bf47 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/config/core/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto index 1adc75ec0b75..c253d049731c 100644 --- a/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto +++ b/api/envoy/extensions/filters/http/local_ratelimit/v3/local_rate_limit.proto @@ -7,6 +7,8 @@ import "envoy/extensions/common/ratelimit/v3/ratelimit.proto"; import "envoy/type/v3/http_status.proto"; import "envoy/type/v3/token_bucket.proto"; +import "google/protobuf/wrappers.proto"; + import "udpa/annotations/status.proto"; import "validate/validate.proto"; @@ -20,7 +22,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Local Rate limit :ref:`configuration overview `. // [#extension: envoy.filters.http.local_ratelimit] -// [#next-free-field: 14] +// [#next-free-field: 16] message LocalRateLimit { // The human readable prefix to use when emitting stats. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -117,4 +119,15 @@ message LocalRateLimit { // Specifies if the local rate limit filter should include the virtual host rate limits. common.ratelimit.v3.VhRateLimitsOptions vh_rate_limits = 13 [(validate.rules).enum = {defined_only: true}]; + + // Specifies if default token bucket should be always consumed. + // If set to false, default token bucket will only be consumed when there is + // no matching descriptor. If set to true, default token bucket will always + // be consumed. Default is true. + google.protobuf.BoolValue always_consume_default_token_bucket = 14; + + // Specifies whether a ``RESOURCE_EXHAUSTED`` gRPC code must be returned instead + // of the default ``UNAVAILABLE`` gRPC code for a rate limited gRPC call. The + // HTTP code will be 200 for a gRPC response. + bool rate_limited_as_resource_exhausted = 15; } diff --git a/api/envoy/extensions/filters/http/lua/v3/BUILD b/api/envoy/extensions/filters/http/lua/v3/BUILD index a1775bbe6f51..e74acc660850 100644 --- a/api/envoy/extensions/filters/http/lua/v3/BUILD +++ b/api/envoy/extensions/filters/http/lua/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/oauth2/v3/BUILD b/api/envoy/extensions/filters/http/oauth2/v3/BUILD index 75d36b709935..19dc4b83616f 100644 --- a/api/envoy/extensions/filters/http/oauth2/v3/BUILD +++ b/api/envoy/extensions/filters/http/oauth2/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/route/v3:pkg", "//envoy/extensions/transport_sockets/tls/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto b/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto index 7c933d8726fa..8e0574afe4b1 100644 --- a/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto +++ b/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto @@ -7,6 +7,8 @@ import "envoy/config/route/v3/route_components.proto"; import "envoy/extensions/transport_sockets/tls/v3/secret.proto"; import "envoy/type/matcher/v3/path.proto"; +import "google/protobuf/wrappers.proto"; + import "udpa/annotations/status.proto"; import "validate/validate.proto"; @@ -71,7 +73,7 @@ message OAuth2Credentials { // OAuth config // -// [#next-free-field: 12] +// [#next-free-field: 13] message OAuth2Config { enum AuthType { // The ``client_id`` and ``client_secret`` will be sent in the URL encoded request body. @@ -123,6 +125,11 @@ message OAuth2Config { // Defines how ``client_id`` and ``client_secret`` are sent in OAuth client to OAuth server requests. // RFC https://datatracker.ietf.org/doc/html/rfc6749#section-2.3.1 AuthType auth_type = 11 [(validate.rules).enum = {defined_only: true}]; + + // If set to true, allows automatic access token refresh using the associated refresh token (see + // `RFC 6749 section 6 `_), provided that the OAuth server supports that. + // Default value is false. + google.protobuf.BoolValue use_refresh_token = 12; } // Filter config. diff --git a/api/envoy/extensions/filters/http/on_demand/v3/BUILD b/api/envoy/extensions/filters/http/on_demand/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/on_demand/v3/BUILD +++ b/api/envoy/extensions/filters/http/on_demand/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/original_src/v3/BUILD b/api/envoy/extensions/filters/http/original_src/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/original_src/v3/BUILD +++ b/api/envoy/extensions/filters/http/original_src/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/rate_limit_quota/v3/BUILD b/api/envoy/extensions/filters/http/rate_limit_quota/v3/BUILD index 39b7d6bb45d1..5a6a4b7e9fcd 100644 --- a/api/envoy/extensions/filters/http/rate_limit_quota/v3/BUILD +++ b/api/envoy/extensions/filters/http/rate_limit_quota/v3/BUILD @@ -8,8 +8,8 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.proto b/api/envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.proto index 93dded4345b8..57b8bdecd782 100644 --- a/api/envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.proto +++ b/api/envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.proto @@ -202,8 +202,7 @@ message RateLimitQuotaBucketSettings { // ` // message. // - // If the field is not set, the ``ExpiredAssignmentBehavior`` time is **not limited**: - // it applies to the bucket until replaced by an ``active`` assignment. + // If not set, defaults to zero, and the bucket is abandoned immediately. google.protobuf.Duration expired_assignment_behavior_timeout = 1 [(validate.rules).duration = {gt {}}]; @@ -389,6 +388,12 @@ message RateLimitQuotaBucketSettings { // // After sending the initial report, the data plane is to continue reporting the bucket usage with // the internal specified in this field. + // + // If for any reason RLQS client doesn't receive the initial assignment for the reported bucket, + // the data plane will eventually consider the bucket abandoned and stop sending the usage + // reports. This is explained in more details at :ref:`Rate Limit Quota Service (RLQS) + // `. + // // [#comment: 100000000 nanoseconds = 0.1 seconds] google.protobuf.Duration reporting_interval = 2 [(validate.rules).duration = { required: true diff --git a/api/envoy/extensions/filters/http/ratelimit/v3/BUILD b/api/envoy/extensions/filters/http/ratelimit/v3/BUILD index 77ed9cc64947..90cebe0b0b07 100644 --- a/api/envoy/extensions/filters/http/ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/http/ratelimit/v3/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/config/route/v3:pkg", "//envoy/type/metadata/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ratelimit/v3/rate_limit.proto b/api/envoy/extensions/filters/http/ratelimit/v3/rate_limit.proto index f9510db579d2..3e33536b228a 100644 --- a/api/envoy/extensions/filters/http/ratelimit/v3/rate_limit.proto +++ b/api/envoy/extensions/filters/http/ratelimit/v3/rate_limit.proto @@ -25,7 +25,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Rate limit :ref:`configuration overview `. // [#extension: envoy.filters.http.ratelimit] -// [#next-free-field: 12] +// [#next-free-field: 14] message RateLimit { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.rate_limit.v2.RateLimit"; @@ -126,6 +126,14 @@ message RateLimit { // have been rate limited. repeated config.core.v3.HeaderValueOption response_headers_to_add = 11 [(validate.rules).repeated = {max_items: 10}]; + + // Sets the HTTP status that is returned to the client when the ratelimit server returns an error + // or cannot be reached. The default status is 500. + type.v3.HttpStatus status_on_error = 12; + + // Optional additional prefix to use when emitting statistics. This allows to distinguish + // emitted statistics between configured ``ratelimit`` filters in an HTTP filter chain. + string stat_prefix = 13; } // Global rate limiting :ref:`architecture overview `. diff --git a/api/envoy/extensions/filters/http/rbac/v3/BUILD b/api/envoy/extensions/filters/http/rbac/v3/BUILD index 49cb2ccac4f7..f4f91ded2a89 100644 --- a/api/envoy/extensions/filters/http/rbac/v3/BUILD +++ b/api/envoy/extensions/filters/http/rbac/v3/BUILD @@ -7,8 +7,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/rbac/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/router/v3/BUILD b/api/envoy/extensions/filters/http/router/v3/BUILD index 3e49a416a43f..76b034d46fa1 100644 --- a/api/envoy/extensions/filters/http/router/v3/BUILD +++ b/api/envoy/extensions/filters/http/router/v3/BUILD @@ -6,8 +6,9 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/annotations:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/extensions/filters/network/http_connection_manager/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/router/v3/router.proto b/api/envoy/extensions/filters/http/router/v3/router.proto index a1ede3ffe351..75bca960da1f 100644 --- a/api/envoy/extensions/filters/http/router/v3/router.proto +++ b/api/envoy/extensions/filters/http/router/v3/router.proto @@ -8,6 +8,7 @@ import "envoy/extensions/filters/network/http_connection_manager/v3/http_connect import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; +import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -52,7 +53,13 @@ message Router { // useful in scenarios where other filters (auth, ratelimit, etc.) make // outbound calls and have child spans rooted at the same ingress // parent. Defaults to false. - bool start_child_span = 2; + // + // .. attention:: + // This field is deprecated by the + // :ref:`spawn_upstream_span `. + // Please use that ``spawn_upstream_span`` field to control the span creation. + bool start_child_span = 2 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; // Configuration for HTTP upstream logs emitted by the router. Upstream logs // are configured in the same way as access logs, but each log entry represents @@ -115,16 +122,16 @@ message Router { // .. note:: // Upstream HTTP filters are currently in alpha. // - // Optional HTTP filters for the upstream filter chain. + // Optional HTTP filters for the upstream HTTP filter chain. // // These filters will be applied for all requests that pass through the router. // They will also be applied to shadowed requests. - // Upstream filters cannot change route or cluster. - // Upstream filters specified on the cluster will override these filters. + // Upstream HTTP filters cannot change route or cluster. + // Upstream HTTP filters specified on the cluster will override these filters. // - // If using upstream filters, please be aware that local errors sent by - // upstream filters will not trigger retries, and local errors sent by - // upstream filters will count as a final response if hedging is configured. + // If using upstream HTTP filters, please be aware that local errors sent by + // upstream HTTP filters will not trigger retries, and local errors sent by + // upstream HTTP filters will count as a final response if hedging is configured. // [#extension-category: envoy.filters.http.upstream] repeated network.http_connection_manager.v3.HttpFilter upstream_http_filters = 8; } diff --git a/api/envoy/extensions/filters/http/set_filter_state/v3/BUILD b/api/envoy/extensions/filters/http/set_filter_state/v3/BUILD new file mode 100644 index 000000000000..7d18ef132da3 --- /dev/null +++ b/api/envoy/extensions/filters/http/set_filter_state/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/filters/common/set_filter_state/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.proto b/api/envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.proto new file mode 100644 index 000000000000..54f1f4c334f2 --- /dev/null +++ b/api/envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.extensions.filters.http.set_filter_state.v3; + +import "envoy/extensions/filters/common/set_filter_state/v3/value.proto"; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.set_filter_state.v3"; +option java_outer_classname = "SetFilterStateProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/set_filter_state/v3;set_filter_statev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Set-Filter-State Filter] +// +// This filter sets or updates the dynamic filter state. See :ref:`the filter +// documentation ` for more information on +// how this filter should be used. +// +// [#extension: envoy.filters.http.set_filter_state] + +message Config { + // A sequence of the filter state values to apply in the specified order + // when a new request is received. + repeated common.set_filter_state.v3.FilterStateValue on_request_headers = 1; +} diff --git a/api/envoy/extensions/filters/http/set_metadata/v3/BUILD b/api/envoy/extensions/filters/http/set_metadata/v3/BUILD index ee92fb652582..cd8fcbbc5e0d 100644 --- a/api/envoy/extensions/filters/http/set_metadata/v3/BUILD +++ b/api/envoy/extensions/filters/http/set_metadata/v3/BUILD @@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = [ + "//envoy/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], ) diff --git a/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto b/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto index a50a1d4fad95..e3dc67392752 100644 --- a/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto +++ b/api/envoy/extensions/filters/http/set_metadata/v3/set_metadata.proto @@ -2,8 +2,10 @@ syntax = "proto3"; package envoy.extensions.filters.http.set_metadata.v3; +import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; +import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; import "validate/validate.proto"; @@ -19,13 +21,41 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // // [#extension: envoy.filters.http.set_metadata] -message Config { +message Metadata { // The metadata namespace. string metadata_namespace = 1 [(validate.rules).string = {min_len: 1}]; - // The value to update the namespace with. See + // Allow the filter to overwrite or merge with an existing value in the namespace. + bool allow_overwrite = 2; + + // The value to place at the namespace. If ``allow_overwrite``, this will + // overwrite or merge with any existing values in that namespace. See + // :ref:`the filter documentation ` for + // more information on how this value is merged with potentially existing + // ones if ``allow_overwrite`` is configured. Only one of ``value`` and + // ``typed_value`` may be set. + google.protobuf.Struct value = 3; + + // The value to place at the namespace. If ``allow_overwrite``, this will + // overwrite any existing values in that namespace. Only one of ``value`` and + // ``typed_value`` may be set. + google.protobuf.Any typed_value = 4; +} + +message Config { + // The metadata namespace. + // This field is deprecated; please use ``metadata`` as replacement. + string metadata_namespace = 1 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; + + // The untyped value to update the dynamic metadata namespace with. See // :ref:`the filter documentation ` for // more information on how this value is merged with potentially existing // ones. - google.protobuf.Struct value = 2; + // This field is deprecated; please use ``metadata`` as replacement. + google.protobuf.Struct value = 2 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; + + // Defines changes to be made to dynamic metadata. + repeated Metadata metadata = 3; } diff --git a/api/envoy/extensions/filters/http/stateful_session/v3/BUILD b/api/envoy/extensions/filters/http/stateful_session/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/http/stateful_session/v3/BUILD +++ b/api/envoy/extensions/filters/http/stateful_session/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/stateful_session/v3/stateful_session.proto b/api/envoy/extensions/filters/http/stateful_session/v3/stateful_session.proto index e3c612edaacf..aa07083f81a4 100644 --- a/api/envoy/extensions/filters/http/stateful_session/v3/stateful_session.proto +++ b/api/envoy/extensions/filters/http/stateful_session/v3/stateful_session.proto @@ -23,6 +23,10 @@ message StatefulSession { // // [#extension-category: envoy.http.stateful_session] config.core.v3.TypedExtensionConfig session_state = 1; + + // If set to True, the HTTP request must be routed to the requested destination. + // If the requested destination is not available, Envoy returns 503. Defaults to False. + bool strict = 2; } message StatefulSessionPerRoute { diff --git a/api/envoy/extensions/filters/http/tap/v3/BUILD b/api/envoy/extensions/filters/http/tap/v3/BUILD index 6b2b1215048c..31d61dcfa206 100644 --- a/api/envoy/extensions/filters/http/tap/v3/BUILD +++ b/api/envoy/extensions/filters/http/tap/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/common/tap/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/tap/v3/tap.proto b/api/envoy/extensions/filters/http/tap/v3/tap.proto index b82735b2f13f..0ed35de97095 100644 --- a/api/envoy/extensions/filters/http/tap/v3/tap.proto +++ b/api/envoy/extensions/filters/http/tap/v3/tap.proto @@ -31,4 +31,7 @@ message Tap { // Request headers time stamp is stored after receiving request headers. // Response headers time stamp is stored after receiving response headers. bool record_headers_received_time = 2; + + // Indicates whether report downstream connection info + bool record_downstream_connection = 3; } diff --git a/api/envoy/extensions/filters/http/upstream_codec/v3/BUILD b/api/envoy/extensions/filters/http/upstream_codec/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/http/upstream_codec/v3/BUILD +++ b/api/envoy/extensions/filters/http/upstream_codec/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/http/wasm/v3/BUILD b/api/envoy/extensions/filters/http/wasm/v3/BUILD index c37174bdefc4..ed3c664aedd7 100644 --- a/api/envoy/extensions/filters/http/wasm/v3/BUILD +++ b/api/envoy/extensions/filters/http/wasm/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/wasm/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/listener/http_inspector/v3/BUILD b/api/envoy/extensions/filters/listener/http_inspector/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/listener/http_inspector/v3/BUILD +++ b/api/envoy/extensions/filters/listener/http_inspector/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/listener/local_ratelimit/v3/BUILD b/api/envoy/extensions/filters/listener/local_ratelimit/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/extensions/filters/listener/local_ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/listener/local_ratelimit/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/listener/original_dst/v3/BUILD b/api/envoy/extensions/filters/listener/original_dst/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/listener/original_dst/v3/BUILD +++ b/api/envoy/extensions/filters/listener/original_dst/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/listener/original_src/v3/BUILD b/api/envoy/extensions/filters/listener/original_src/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/listener/original_src/v3/BUILD +++ b/api/envoy/extensions/filters/listener/original_src/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/listener/proxy_protocol/v3/BUILD b/api/envoy/extensions/filters/listener/proxy_protocol/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/listener/proxy_protocol/v3/BUILD +++ b/api/envoy/extensions/filters/listener/proxy_protocol/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/listener/tls_inspector/v3/BUILD b/api/envoy/extensions/filters/listener/tls_inspector/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/listener/tls_inspector/v3/BUILD +++ b/api/envoy/extensions/filters/listener/tls_inspector/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/network/connection_limit/v3/BUILD b/api/envoy/extensions/filters/network/connection_limit/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/network/connection_limit/v3/BUILD +++ b/api/envoy/extensions/filters/network/connection_limit/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/direct_response/v3/BUILD b/api/envoy/extensions/filters/network/direct_response/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/filters/network/direct_response/v3/BUILD +++ b/api/envoy/extensions/filters/network/direct_response/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/dubbo_proxy/router/v3/BUILD b/api/envoy/extensions/filters/network/dubbo_proxy/router/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/network/dubbo_proxy/router/v3/BUILD +++ b/api/envoy/extensions/filters/network/dubbo_proxy/router/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/network/dubbo_proxy/v3/BUILD b/api/envoy/extensions/filters/network/dubbo_proxy/v3/BUILD index c690bfe279cb..824cb7cd0ce5 100644 --- a/api/envoy/extensions/filters/network/dubbo_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/dubbo_proxy/v3/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/config/route/v3:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/echo/v3/BUILD b/api/envoy/extensions/filters/network/echo/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/network/echo/v3/BUILD +++ b/api/envoy/extensions/filters/network/echo/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/network/ext_authz/v3/BUILD b/api/envoy/extensions/filters/network/ext_authz/v3/BUILD index 3f3a5395d2aa..e3bfc4e175f4 100644 --- a/api/envoy/extensions/filters/network/ext_authz/v3/BUILD +++ b/api/envoy/extensions/filters/network/ext_authz/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/BUILD b/api/envoy/extensions/filters/network/http_connection_manager/v3/BUILD index b1f4b3ba5b51..39bbb5c3d280 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/BUILD +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/BUILD @@ -14,6 +14,6 @@ api_proto_package( "//envoy/type/http/v3:pkg", "//envoy/type/tracing/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index f86be41f0493..7a92259eb43b 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -130,7 +130,7 @@ message HttpConnectionManager { UNESCAPE_AND_FORWARD = 4; } - // [#next-free-field: 10] + // [#next-free-field: 11] message Tracing { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.Tracing"; @@ -195,6 +195,27 @@ message HttpConnectionManager { // Such a constraint is inherent to OpenCensus itself. It cannot be overcome without changes // on OpenCensus side. config.trace.v3.Tracing.Http provider = 9; + + // Create separate tracing span for each upstream request if true. And if this flag is set to true, + // the tracing provider will assume that Envoy will be independent hop in the trace chain and may + // set span type to client or server based on this flag. + // This will deprecate the + // :ref:`start_child_span ` + // in the router. + // + // Users should set appropriate value based on their tracing provider and actual scenario: + // + // * If Envoy is used as sidecar and users want to make the sidecar and its application as only one + // hop in the trace chain, this flag should be set to false. And please also make sure the + // :ref:`start_child_span ` + // in the router is not set to true. + // * If Envoy is used as gateway or independent proxy, or users want to make the sidecar and its + // application as different hops in the trace chain, this flag should be set to true. + // * If tracing provider that has explicit requirements on span creation (like SkyWalking), + // this flag should be set to true. + // + // The default value is false for now for backward compatibility. + google.protobuf.BoolValue spawn_upstream_span = 10; } message InternalAddressConfig { @@ -361,7 +382,7 @@ message HttpConnectionManager { // on stream close, when the HTTP request is complete. If this field is set, the HCM will flush access // logs periodically at the specified interval. This is especially useful in the case of long-lived // requests, such as CONNECT and Websockets. Final access logs can be detected via the - // `requestComplete()` method of `StreamInfo` in access log filters, or thru the `%DURATION%` substitution + // ``requestComplete()`` method of ``StreamInfo`` in access log filters, or through the ``%DURATION%`` substitution // string. // The interval must be at least 1 millisecond. google.protobuf.Duration access_log_flush_interval = 1 @@ -862,12 +883,12 @@ message HttpConnectionManager { // [#extension-category: envoy.http.header_validators] config.core.v3.TypedExtensionConfig typed_header_validation_config = 50; - // Append the `x-forwarded-port` header with the port value client used to connect to Envoy. It - // will be ignored if the `x-forwarded-port` header has been set by any trusted proxy in front of Envoy. + // Append the ``x-forwarded-port`` header with the port value client used to connect to Envoy. It + // will be ignored if the ``x-forwarded-port`` header has been set by any trusted proxy in front of Envoy. bool append_x_forwarded_port = 51; - // Whether the HCM will add ProxyProtocolFilterState to the Connection lifetime filter state. Defaults to `true`. - // This should be set to `false` in cases where Envoy's view of the downstream address may not correspond to the + // Whether the HCM will add ProxyProtocolFilterState to the Connection lifetime filter state. Defaults to ``true``. + // This should be set to ``false`` in cases where Envoy's view of the downstream address may not correspond to the // actual client address, for example, if there's another proxy in front of the Envoy. google.protobuf.BoolValue add_proxy_protocol_connection_state = 53; } @@ -1033,7 +1054,9 @@ message ScopedRoutes { // .. note:: // // If the header appears multiple times only the first value is used. - string name = 1 [(validate.rules).string = {min_len: 1}]; + string name = 1 [ + (validate.rules).string = {min_len: 1 well_known_regex: HTTP_HEADER_NAME strict: false} + ]; // The element separator (e.g., ';' separates 'a;b;c;d'). // Default: empty string. This causes the entirety of the header field to be extracted. @@ -1109,7 +1132,7 @@ message ScopedRds { string srds_resources_locator = 2; } -// [#next-free-field: 7] +// [#next-free-field: 8] message HttpFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.http_connection_manager.v2.HttpFilter"; @@ -1146,6 +1169,14 @@ message HttpFilter { // filter but otherwise accept the config. // Otherwise, clients that do not support this filter must reject the config. bool is_optional = 6; + + // If true, the filter is disabled by default and must be explicitly enabled by setting + // per filter configuration in the route configuration. + // See :ref:`route based filter chain ` + // for more details. + // + // Terminal filters (e.g. ``envoy.filters.http.router``) cannot be marked as disabled. + bool disabled = 7; } message RequestIDExtension { diff --git a/api/envoy/extensions/filters/network/local_ratelimit/v3/BUILD b/api/envoy/extensions/filters/network/local_ratelimit/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/extensions/filters/network/local_ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/network/local_ratelimit/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/mongo_proxy/v3/BUILD b/api/envoy/extensions/filters/network/mongo_proxy/v3/BUILD index d399b876a7f4..01b06eb3efe9 100644 --- a/api/envoy/extensions/filters/network/mongo_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/mongo_proxy/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/filters/common/fault/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/ratelimit/v3/BUILD b/api/envoy/extensions/filters/network/ratelimit/v3/BUILD index 9276f5ab3d2d..6bc991f8e8c8 100644 --- a/api/envoy/extensions/filters/network/ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/network/ratelimit/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/ratelimit/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/rbac/v3/BUILD b/api/envoy/extensions/filters/network/rbac/v3/BUILD index 49cb2ccac4f7..f4f91ded2a89 100644 --- a/api/envoy/extensions/filters/network/rbac/v3/BUILD +++ b/api/envoy/extensions/filters/network/rbac/v3/BUILD @@ -7,8 +7,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/rbac/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/redis_proxy/v3/BUILD b/api/envoy/extensions/filters/network/redis_proxy/v3/BUILD index 3a6953663d6c..f4f1453d8809 100644 --- a/api/envoy/extensions/filters/network/redis_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/redis_proxy/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto index fba1c786f7c3..28e351fb0412 100644 --- a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto +++ b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto @@ -138,7 +138,7 @@ message RedisProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.redis_proxy.v2.RedisProxy.PrefixRoutes"; - // [#next-free-field: 6] + // [#next-free-field: 7] message Route { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.redis_proxy.v2.RedisProxy.PrefixRoutes.Route"; @@ -168,6 +168,11 @@ message RedisProxy { bool exclude_read_commands = 3; } + // ReadCommandPolicy specifies that Envoy should route read commands to another cluster. + message ReadCommandPolicy { + string cluster = 1 [(validate.rules).string = {min_len: 1}]; + } + // String prefix that must match the beginning of the keys. Envoy will always favor the // longest match. string prefix = 1 [(validate.rules).string = {max_bytes: 1000}]; @@ -184,6 +189,9 @@ message RedisProxy { // Indicates how redis key should be formatted. To substitute redis key into the formatting // expression, use %KEY% as a string replacement command. string key_formatter = 5; + + // Indicates that the route has a read command policy + ReadCommandPolicy read_command_policy = 6; } reserved 3; diff --git a/api/envoy/extensions/filters/network/set_filter_state/v3/BUILD b/api/envoy/extensions/filters/network/set_filter_state/v3/BUILD new file mode 100644 index 000000000000..7d18ef132da3 --- /dev/null +++ b/api/envoy/extensions/filters/network/set_filter_state/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/filters/common/set_filter_state/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/network/set_filter_state/v3/set_filter_state.proto b/api/envoy/extensions/filters/network/set_filter_state/v3/set_filter_state.proto new file mode 100644 index 000000000000..084f516e72fe --- /dev/null +++ b/api/envoy/extensions/filters/network/set_filter_state/v3/set_filter_state.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.extensions.filters.network.set_filter_state.v3; + +import "envoy/extensions/filters/common/set_filter_state/v3/value.proto"; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.network.set_filter_state.v3"; +option java_outer_classname = "SetFilterStateProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/set_filter_state/v3;set_filter_statev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Set-Filter-State Filter] +// +// This filter sets or updates the dynamic filter state. See :ref:`the filter +// documentation ` for more +// information on how this filter should be used. +// +// [#extension: envoy.filters.network.set_filter_state] + +message Config { + // A sequence of the filter state values to apply in the specified order + // when a new connection is received. + repeated common.set_filter_state.v3.FilterStateValue on_new_connection = 1; +} diff --git a/api/envoy/extensions/filters/network/sni_cluster/v3/BUILD b/api/envoy/extensions/filters/network/sni_cluster/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/network/sni_cluster/v3/BUILD +++ b/api/envoy/extensions/filters/network/sni_cluster/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3/BUILD index 05f25a2fe5d9..73e98d4d40b2 100644 --- a/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/tcp_proxy/v3/BUILD b/api/envoy/extensions/filters/network/tcp_proxy/v3/BUILD index 495e9c79112c..c9c87b7395d5 100644 --- a/api/envoy/extensions/filters/network/tcp_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/tcp_proxy/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/thrift_proxy/filters/header_to_metadata/v3/BUILD b/api/envoy/extensions/filters/network/thrift_proxy/filters/header_to_metadata/v3/BUILD index 693f0b92ff34..bfc486330911 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/filters/header_to_metadata/v3/BUILD +++ b/api/envoy/extensions/filters/network/thrift_proxy/filters/header_to_metadata/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD b/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD index 693f0b92ff34..bfc486330911 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD +++ b/api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3/BUILD b/api/envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3/BUILD index 0bad14913d21..928d9a6b885a 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3/BUILD +++ b/api/envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/ratelimit/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/thrift_proxy/router/v3/BUILD b/api/envoy/extensions/filters/network/thrift_proxy/router/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/router/v3/BUILD +++ b/api/envoy/extensions/filters/network/thrift_proxy/router/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/network/thrift_proxy/v3/BUILD b/api/envoy/extensions/filters/network/thrift_proxy/v3/BUILD index eee482f0a8fd..8cc8bfccf7bd 100644 --- a/api/envoy/extensions/filters/network/thrift_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/thrift_proxy/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/wasm/v3/BUILD b/api/envoy/extensions/filters/network/wasm/v3/BUILD index c37174bdefc4..ed3c664aedd7 100644 --- a/api/envoy/extensions/filters/network/wasm/v3/BUILD +++ b/api/envoy/extensions/filters/network/wasm/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/wasm/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/BUILD b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto index 9fdcaaf9db2f..a3825f10c9ff 100644 --- a/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto +++ b/api/envoy/extensions/filters/network/zookeeper_proxy/v3/zookeeper_proxy.proto @@ -19,7 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // ZooKeeper Proxy :ref:`configuration overview `. // [#extension: envoy.filters.network.zookeeper_proxy] -// [#next-free-field: 7] +// [#next-free-field: 10] message ZooKeeperProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.zookeeper_proxy.v1alpha1.ZooKeeperProxy"; @@ -42,8 +42,8 @@ message ZooKeeperProxy { // if that is set. If it isn't, ZooKeeper's default is also 1Mb. google.protobuf.UInt32Value max_packet_bytes = 3; - // Whether to emit latency threshold metrics. If not set, defaults to false. - // If false, setting `default_latency_threshold` and `latency_threshold_overrides` will not have effect. + // Whether to emit latency threshold metrics. If not set, it defaults to false. + // If false, setting ``default_latency_threshold`` and ``latency_threshold_overrides`` will not have effect. bool enable_latency_threshold_metrics = 4; // The default latency threshold to decide the fast/slow responses and emit metrics (used for error budget calculation). @@ -59,6 +59,15 @@ message ZooKeeperProxy { // threshold. // Specifying latency threshold overrides multiple times for one opcode is not allowed. repeated LatencyThresholdOverride latency_threshold_overrides = 6; + + // Whether to emit per opcode request bytes metrics. If not set, it defaults to false. + bool enable_per_opcode_request_bytes_metrics = 7; + + // Whether to emit per opcode response bytes metrics. If not set, it defaults to false. + bool enable_per_opcode_response_bytes_metrics = 8; + + // [#not-implemented-hide:] Whether to emit per opcode decoder error metrics. If not set, it defaults to false. + bool enable_per_opcode_decoder_error_metrics = 9; } message LatencyThresholdOverride { diff --git a/api/envoy/extensions/filters/udp/dns_filter/v3/BUILD b/api/envoy/extensions/filters/udp/dns_filter/v3/BUILD index 1f8dbc5af561..c95410c79d19 100644 --- a/api/envoy/extensions/filters/udp/dns_filter/v3/BUILD +++ b/api/envoy/extensions/filters/udp/dns_filter/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", "//envoy/data/dns/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/BUILD b/api/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/BUILD new file mode 100644 index 000000000000..73e98d4d40b2 --- /dev/null +++ b/api/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto b/api/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto new file mode 100644 index 000000000000..a264f4e3c56f --- /dev/null +++ b/api/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; + +package envoy.extensions.filters.udp.udp_proxy.session.dynamic_forward_proxy.v3; + +import "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.udp.udp_proxy.session.dynamic_forward_proxy.v3"; +option java_outer_classname = "DynamicForwardProxyProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3;dynamic_forward_proxyv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Filter state dynamic forward proxy] + +// Configuration for the filter state based dynamic forward proxy filter. See the +// :ref:`architecture overview ` for +// more information. Note this filter must be used in conjunction to another filter that +// sets the 'envoy.upstream.dynamic_host' and the 'envoy.upstream.dynamic_port' filter +// state keys for the required upstream UDP session. +// [#extension: envoy.filters.udp.session.dynamic_forward_proxy] +message FilterConfig { + // Configuration for UDP datagrams buffering. + message BufferOptions { + // If set, the filter will only buffer datagrams up to the requested limit, and will drop + // new UDP datagrams if the buffer contains the max_buffered_datagrams value at the time + // of a new datagram arrival. If not set, the default value is 1024 datagrams. + google.protobuf.UInt32Value max_buffered_datagrams = 1; + + // If set, the filter will only buffer datagrams up to the requested total buffered bytes limit, + // and will drop new UDP datagrams if the buffer contains the max_buffered_datagrams value + // at the time of a new datagram arrival. If not set, the default value is 16,384 (16KB). + google.protobuf.UInt64Value max_buffered_bytes = 2; + } + + // The prefix to use when emitting :ref:`statistics `. + string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; + + oneof implementation_specifier { + // The DNS cache configuration that the filter will attach to. Note this + // configuration must match that of associated :ref:`dynamic forward proxy cluster configuration + // `. + common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 2 + [(validate.rules).message = {required: true}]; + } + + // If configured, the filter will buffer datagrams in case that it is waiting for a DNS response. + // If this field is not configured, there will be no buffering and downstream datagrams that arrive + // while the DNS resolution is in progress will be dropped. In case this field is set but the options + // are not configured, the default values will be applied as described in the ``BufferOptions``. + BufferOptions buffer_options = 3; +} diff --git a/api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/BUILD b/api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/BUILD new file mode 100644 index 000000000000..29ebf0741406 --- /dev/null +++ b/api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.proto b/api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.proto new file mode 100644 index 000000000000..e3455462b67f --- /dev/null +++ b/api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3"; +option java_outer_classname = "HttpCapsuleProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3;http_capsulev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: UDP HTTP Capsule filter] +// UDP to HTTP capsules :ref:`overview `. +// [#extension: envoy.filters.udp.session.http_capsule] + +message FilterConfig { +} diff --git a/api/envoy/extensions/filters/udp/udp_proxy/v3/BUILD b/api/envoy/extensions/filters/udp/udp_proxy/v3/BUILD index 375c78d299a2..501298f89985 100644 --- a/api/envoy/extensions/filters/udp/udp_proxy/v3/BUILD +++ b/api/envoy/extensions/filters/udp/udp_proxy/v3/BUILD @@ -9,8 +9,8 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], ) diff --git a/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto b/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto index 25560b5c8c08..1d07668acb77 100644 --- a/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto +++ b/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto @@ -3,10 +3,12 @@ syntax = "proto3"; package envoy.extensions.filters.udp.udp_proxy.v3; import "envoy/config/accesslog/v3/accesslog.proto"; +import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/udp_socket_config.proto"; import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "xds/annotations/v3/status.proto"; import "xds/type/matcher/v3/matcher.proto"; @@ -27,7 +29,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.filters.udp_listener.udp_proxy] // Configuration for the UDP proxy filter. -// [#next-free-field: 12] +// [#next-free-field: 14] message UdpProxyConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.udp.udp_proxy.v2alpha.UdpProxyConfig"; @@ -62,6 +64,126 @@ message UdpProxyConfig { } } + // Configuration for tunneling UDP over other transports or application layers. + // Tunneling is currently supported over HTTP/2. + // [#next-free-field: 12] + message UdpTunnelingConfig { + // Configuration for UDP datagrams buffering. + message BufferOptions { + // If set, the filter will only buffer datagrams up to the requested limit, and will drop + // new UDP datagrams if the buffer contains the max_buffered_datagrams value at the time + // of a new datagram arrival. If not set, the default value is 1024 datagrams. + google.protobuf.UInt32Value max_buffered_datagrams = 1; + + // If set, the filter will only buffer datagrams up to the requested total buffered bytes limit, + // and will drop new UDP datagrams if the buffer contains the max_buffered_datagrams value + // at the time of a new datagram arrival. If not set, the default value is 16,384 (16KB). + google.protobuf.UInt64Value max_buffered_bytes = 2; + } + + message RetryOptions { + // The maximum number of unsuccessful connection attempts that will be made before giving up. + // If the parameter is not specified, 1 connection attempt will be made. + google.protobuf.UInt32Value max_connect_attempts = 1; + } + + // The hostname to send in the synthesized CONNECT headers to the upstream proxy. + // This field evaluates command operators if set, otherwise returns hostname as is. + // + // Example: dynamically set hostname using filter state + // + // .. code-block:: yaml + // + // tunneling_config: + // proxy_host: "%FILTER_STATE(proxy.host.key:PLAIN)%" + // + string proxy_host = 1 [(validate.rules).string = {min_len: 1}]; + + // Optional port value to add to the HTTP request URI. + // This value can be overridden per-session by setting the required port value for + // the filter state key ``udp.connect.proxy_port``. + google.protobuf.UInt32Value proxy_port = 2; + + // The target host to send in the synthesized CONNECT headers to the upstream proxy. + // This field evaluates command operators if set, otherwise returns hostname as is. + // + // Example: dynamically set target host using filter state + // + // .. code-block:: yaml + // + // tunneling_config: + // target_host: "%FILTER_STATE(target.host.key:PLAIN)%" + // + string target_host = 3 [(validate.rules).string = {min_len: 1}]; + + // The default target port to send in the CONNECT headers to the upstream proxy. + // This value can be overridden per-session by setting the required port value for + // the filter state key ``udp.connect.target_port``. + uint32 default_target_port = 4 [(validate.rules).uint32 = {lte: 65535 gt: 0}]; + + // Use POST method instead of CONNECT method to tunnel the UDP stream. + // + // .. note:: + // If use_post is set, the upstream stream does not comply with the connect-udp RFC, and + // instead it will be a POST request. the path used in the headers will be set from the + // post_path field, and the headers will not contain the target host and target port, as + // required by the connect-udp protocol. This flag should be used carefully. + // + bool use_post = 5; + + // The path used with POST method. Default path is ``/``. If post path is specified and + // use_post field isn't true, it will be rejected. + string post_path = 6; + + // Optional retry options, in case connecting to the upstream failed. + RetryOptions retry_options = 7; + + // Additional request headers to upstream proxy. Neither ``:-prefixed`` pseudo-headers + // nor the Host: header can be overridden. Values of the added headers evaluates command + // operators if they are set in the value template. + // + // Example: dynamically set a header with the local port + // + // .. code-block:: yaml + // + // headers_to_add: + // - header: + // key: original_dst_port + // value: "%DOWNSTREAM_LOCAL_PORT%" + // + repeated config.core.v3.HeaderValueOption headers_to_add = 8 + [(validate.rules).repeated = {max_items: 1000}]; + + // If configured, the filter will buffer datagrams in case that it is waiting for the upstream to be + // ready, whether if it is during the connection process or due to upstream buffer watermarks. + // If this field is not configured, there will be no buffering and downstream datagrams that arrive + // while the upstream is not ready will be dropped. In case this field is set but the options + // are not configured, the default values will be applied as described in the ``BufferOptions``. + BufferOptions buffer_options = 9; + + // Save the response headers to the downstream info filter state for consumption + // by the session filters. The filter state key is ``envoy.udp_proxy.propagate_response_headers``. + bool propagate_response_headers = 10; + + // Save the response trailers to the downstream info filter state for consumption + // by the session filters. The filter state key is ``envoy.udp_proxy.propagate_response_trailers``. + bool propagate_response_trailers = 11; + } + + message UdpAccessLogOptions { + // The interval to flush access log. The UDP proxy will flush only one access log when the session + // is ended by default. If this field is set, the UDP proxy will flush access log periodically with + // the specified interval. + // This field does not require on-tunnel-connected access logging enabled, and the other way around. + // The interval must be at least 1ms. + google.protobuf.Duration access_log_flush_interval = 1 + [(validate.rules).duration = {gte {nanos: 1000000}}]; + + // If set to true and UDP tunneling is configured, access log will be flushed when the UDP proxy has successfully + // established a connection tunnel with the upstream. If the connection failed, the access log will not be flushed. + bool flush_access_log_on_tunnel_connected = 2; + } + // The stat prefix used when emitting UDP proxy filter stats. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -127,5 +249,13 @@ message UdpProxyConfig { // Optional session filters that will run for each UDP session. // Only one of use_per_packet_load_balancing or session_filters can be used. + // [#extension-category: envoy.filters.udp.session] repeated SessionFilter session_filters = 11; + + // If set, this configures UDP tunneling. See `Proxying UDP in HTTP `_. + // More information can be found in the UDP Proxy and HTTP upgrade documentation. + UdpTunnelingConfig tunneling_config = 12; + + // Additional access log options for UDP Proxy. + UdpAccessLogOptions access_log_options = 13; } diff --git a/api/envoy/extensions/formatter/cel/v3/BUILD b/api/envoy/extensions/formatter/cel/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/formatter/cel/v3/BUILD +++ b/api/envoy/extensions/formatter/cel/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/formatter/cel/v3/cel.proto b/api/envoy/extensions/formatter/cel/v3/cel.proto index ca9d01dedb4d..4e19fa5db954 100644 --- a/api/envoy/extensions/formatter/cel/v3/cel.proto +++ b/api/envoy/extensions/formatter/cel/v3/cel.proto @@ -24,10 +24,11 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // truncation up to Z characters long. // // Examples: -// - ``%CEL(response.code)%`` -// - ``%CEL(connection.mtls)%`` -// - ``%CEL(request.headers['x-envoy-original-path']):10%`` -// - ``%CEL(request.headers['x-log-mtls'] || request.url_path.contains('v1beta3'))%`` +// +// * ``%CEL(response.code)%`` +// * ``%CEL(connection.mtls)%`` +// * ``%CEL(request.headers['x-envoy-original-path']):10%`` +// * ``%CEL(request.headers['x-log-mtls'] || request.url_path.contains('v1beta3'))%`` // Configuration for the CEL formatter. message Cel { diff --git a/api/envoy/extensions/formatter/metadata/v3/BUILD b/api/envoy/extensions/formatter/metadata/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/formatter/metadata/v3/BUILD +++ b/api/envoy/extensions/formatter/metadata/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/formatter/metadata/v3/metadata.proto b/api/envoy/extensions/formatter/metadata/v3/metadata.proto index 90f6094b3c27..bf79f76741b5 100644 --- a/api/envoy/extensions/formatter/metadata/v3/metadata.proto +++ b/api/envoy/extensions/formatter/metadata/v3/metadata.proto @@ -21,6 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // * CLUSTER // * ROUTE // * UPSTREAM_HOST +// * LISTENER // // See :ref:`here ` for more information on access log configuration. diff --git a/api/envoy/extensions/formatter/req_without_query/v3/BUILD b/api/envoy/extensions/formatter/req_without_query/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/formatter/req_without_query/v3/BUILD +++ b/api/envoy/extensions/formatter/req_without_query/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/geoip_providers/common/v3/BUILD b/api/envoy/extensions/geoip_providers/common/v3/BUILD new file mode 100644 index 000000000000..29ebf0741406 --- /dev/null +++ b/api/envoy/extensions/geoip_providers/common/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/geoip_providers/common/v3/common.proto b/api/envoy/extensions/geoip_providers/common/v3/common.proto new file mode 100644 index 000000000000..91a9126cfef8 --- /dev/null +++ b/api/envoy/extensions/geoip_providers/common/v3/common.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package envoy.extensions.geoip_providers.common.v3; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.geoip_providers.common.v3"; +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/geoip_providers/common/v3;commonv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Common Geolocation Provider Configuration] +// Common geolocation provider :ref:`configuration overview `. +// Common configuration shared across geolocation providers. + +message CommonGeoipProviderConfig { + // The set of geolocation headers to add to request. If any of the configured headers is present + // in the incoming request, it will be overridden by the :ref:`Geoip filter `. + // [#next-free-field: 10] + message GeolocationHeadersToAdd { + // If set, the header will be used to populate the country ISO code associated with the IP address. + string country = 1 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the header will be used to populate the city associated with the IP address. + string city = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the header will be used to populate the region ISO code associated with the IP address. + // The least specific subdivision will be selected as region value. + string region = 3 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the header will be used to populate the ASN associated with the IP address. + string asn = 4 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the IP address will be checked if it belongs to any type of anonymization network (e.g. VPN, public proxy etc) + // and header will be populated with the check result. Header value will be set to either "true" or "false" depending on the check result. + string is_anon = 5 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the IP address will be checked if it belongs to a VPN and header will be populated with the check result. + // Header value will be set to either "true" or "false" depending on the check result. + string anon_vpn = 6 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the IP address will be checked if it belongs to a hosting provider and header will be populated with the check result. + // Header value will be set to either "true" or "false" depending on the check result. + string anon_hosting = 7 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the IP address will be checked if it belongs to a TOR exit node and header will be populated with the check result. + // Header value will be set to either "true" or "false" depending on the check result. + string anon_tor = 8 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + + // If set, the IP address will be checked if it belongs to a public proxy and header will be populated with the check result. + // Header value will be set to either "true" or "false" depending on the check result. + string anon_proxy = 9 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; + } + + // Configuration for geolocation headers to add to request. + GeolocationHeadersToAdd geo_headers_to_add = 1 [(validate.rules).message = {required: true}]; +} diff --git a/api/envoy/extensions/geoip_providers/maxmind/v3/BUILD b/api/envoy/extensions/geoip_providers/maxmind/v3/BUILD new file mode 100644 index 000000000000..06e26d5c8079 --- /dev/null +++ b/api/envoy/extensions/geoip_providers/maxmind/v3/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/geoip_providers/common/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + ], +) diff --git a/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto b/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto new file mode 100644 index 000000000000..3fc7f7c16082 --- /dev/null +++ b/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package envoy.extensions.geoip_providers.maxmind.v3; + +import "envoy/extensions/geoip_providers/common/v3/common.proto"; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.geoip_providers.maxmind.v3"; +option java_outer_classname = "MaxmindProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/geoip_providers/maxmind/v3;maxmindv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: MaxMind Geolocation Provider] +// MaxMind geolocation provider :ref:`configuration overview `. +// At least one geolocation database path :ref:`city_db_path `, +// :ref:`isp_db_path ` or +// :ref:`anon_db_path ` must be configured. +// [#extension: envoy.geoip_providers.maxmind] + +message MaxMindConfig { + // Full file path to the Maxmind city database, e.g. /etc/GeoLite2-City.mmdb. + // Database file is expected to have .mmdb extension. + string city_db_path = 1 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}]; + + // Full file path to the Maxmind ASN database, e.g. /etc/GeoLite2-ASN.mmdb. + // Database file is expected to have .mmdb extension. + string isp_db_path = 2 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}]; + + // Full file path to the Maxmind anonymous IP database, e.g. /etc/GeoIP2-Anonymous-IP.mmdb. + // Database file is expected to have .mmdb extension. + string anon_db_path = 3 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}]; + + // Common provider configuration that specifies which geolocation headers will be populated with geolocation data. + common.v3.CommonGeoipProviderConfig common_provider_config = 4 + [(validate.rules).message = {required: true}]; +} diff --git a/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD b/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD +++ b/api/envoy/extensions/health_check/event_sinks/file/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/health_checkers/redis/v3/BUILD b/api/envoy/extensions/health_checkers/redis/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/health_checkers/redis/v3/BUILD +++ b/api/envoy/extensions/health_checkers/redis/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/health_checkers/thrift/v3/BUILD b/api/envoy/extensions/health_checkers/thrift/v3/BUILD index 8e325386105a..993cf11f30e9 100644 --- a/api/envoy/extensions/health_checkers/thrift/v3/BUILD +++ b/api/envoy/extensions/health_checkers/thrift/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/filters/network/thrift_proxy/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/http/cache/file_system_http_cache/v3/BUILD b/api/envoy/extensions/http/cache/file_system_http_cache/v3/BUILD index 26baeccd9941..5b108dcfee6c 100644 --- a/api/envoy/extensions/http/cache/file_system_http_cache/v3/BUILD +++ b/api/envoy/extensions/http/cache/file_system_http_cache/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/common/async_files/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/http/cache/simple_http_cache/v3/BUILD b/api/envoy/extensions/http/cache/simple_http_cache/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/http/cache/simple_http_cache/v3/BUILD +++ b/api/envoy/extensions/http/cache/simple_http_cache/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/http/custom_response/local_response_policy/v3/BUILD b/api/envoy/extensions/http/custom_response/local_response_policy/v3/BUILD index e9b556d681cf..628f71321fba 100644 --- a/api/envoy/extensions/http/custom_response/local_response_policy/v3/BUILD +++ b/api/envoy/extensions/http/custom_response/local_response_policy/v3/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/http/custom_response/local_response_policy/v3/local_response_policy.proto b/api/envoy/extensions/http/custom_response/local_response_policy/v3/local_response_policy.proto index deb13b0b0221..b40800c01ae5 100644 --- a/api/envoy/extensions/http/custom_response/local_response_policy/v3/local_response_policy.proto +++ b/api/envoy/extensions/http/custom_response/local_response_policy/v3/local_response_policy.proto @@ -26,11 +26,11 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // downstream. message LocalResponsePolicy { // Optional new local reply body text. It will be used - // in the `%LOCAL_REPLY_BODY%` command operator in the `body_format`. + // in the ``%LOCAL_REPLY_BODY%`` command operator in the ``body_format``. config.core.v3.DataSource body = 1; - // Optional body format to be used for this response. If `body_format` is not - // provided, and `body` is, the contents of `body` will be used to populate + // Optional body format to be used for this response. If ``body_format`` is not + // provided, and ``body`` is, the contents of ``body`` will be used to populate // the body of the local reply without formatting. config.core.v3.SubstitutionFormatString body_format = 2; diff --git a/api/envoy/extensions/http/custom_response/redirect_policy/v3/BUILD b/api/envoy/extensions/http/custom_response/redirect_policy/v3/BUILD index d0a7c688bf76..b6c098a23b3a 100644 --- a/api/envoy/extensions/http/custom_response/redirect_policy/v3/BUILD +++ b/api/envoy/extensions/http/custom_response/redirect_policy/v3/BUILD @@ -8,7 +8,7 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/http/custom_response/redirect_policy/v3/redirect_policy.proto b/api/envoy/extensions/http/custom_response/redirect_policy/v3/redirect_policy.proto index 73cf7ed7a864..ef8d050e2aa8 100644 --- a/api/envoy/extensions/http/custom_response/redirect_policy/v3/redirect_policy.proto +++ b/api/envoy/extensions/http/custom_response/redirect_policy/v3/redirect_policy.proto @@ -43,12 +43,12 @@ message RedirectPolicy { string uri = 1 [(validate.rules).string = {min_len: 1}]; // Specify elements of the redirect url individually. - // Note: Do not specify the `response_code` field in `redirect_action`, use - // `status_code` instead. - // The following fields in `redirect_action` are currently not supported, + // Note: Do not specify the ``response_code`` field in ``redirect_action``, use + // ``status_code`` instead. + // The following fields in ``redirect_action`` are currently not supported, // and specifying them will cause the config to be rejected: - // - `prefix_rewrite` - // - `regex_rewrite` + // - ``prefix_rewrite`` + // - ``regex_rewrite`` config.route.v3.RedirectAction redirect_action = 2; } diff --git a/api/envoy/extensions/http/early_header_mutation/header_mutation/v3/BUILD b/api/envoy/extensions/http/early_header_mutation/header_mutation/v3/BUILD index 7af7ae042311..876a007c83cf 100644 --- a/api/envoy/extensions/http/early_header_mutation/header_mutation/v3/BUILD +++ b/api/envoy/extensions/http/early_header_mutation/header_mutation/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/common/mutation_rules/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/http/header_formatters/preserve_case/v3/BUILD b/api/envoy/extensions/http/header_formatters/preserve_case/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/http/header_formatters/preserve_case/v3/BUILD +++ b/api/envoy/extensions/http/header_formatters/preserve_case/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/http/header_validators/envoy_default/v3/BUILD b/api/envoy/extensions/http/header_validators/envoy_default/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/http/header_validators/envoy_default/v3/BUILD +++ b/api/envoy/extensions/http/header_validators/envoy_default/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/http/original_ip_detection/custom_header/v3/BUILD b/api/envoy/extensions/http/original_ip_detection/custom_header/v3/BUILD index 9a76b7e148e0..ef19132f9180 100644 --- a/api/envoy/extensions/http/original_ip_detection/custom_header/v3/BUILD +++ b/api/envoy/extensions/http/original_ip_detection/custom_header/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/http/original_ip_detection/xff/v3/BUILD b/api/envoy/extensions/http/original_ip_detection/xff/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/http/original_ip_detection/xff/v3/BUILD +++ b/api/envoy/extensions/http/original_ip_detection/xff/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/http/stateful_session/cookie/v3/BUILD b/api/envoy/extensions/http/stateful_session/cookie/v3/BUILD index 7a3fc432b2f2..b6f8d3424c11 100644 --- a/api/envoy/extensions/http/stateful_session/cookie/v3/BUILD +++ b/api/envoy/extensions/http/stateful_session/cookie/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/http/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/http/stateful_session/header/v3/BUILD b/api/envoy/extensions/http/stateful_session/header/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/http/stateful_session/header/v3/BUILD +++ b/api/envoy/extensions/http/stateful_session/header/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/http/stateful_session/header/v3/header.proto b/api/envoy/extensions/http/stateful_session/header/v3/header.proto index 01b9381bd923..c5e7de1c3170 100644 --- a/api/envoy/extensions/http/stateful_session/header/v3/header.proto +++ b/api/envoy/extensions/http/stateful_session/header/v3/header.proto @@ -23,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // upstream host, this upstream host will be selected first. See :ref:`stateful session filter // `. // -// For example, if the header name is set to ``session-header``, envoy will prefer ``1.2.3.4:80`` +// For example, if the header name is set to ``session-header``, Envoy will prefer ``1.2.3.4:80`` // as the upstream host when the request contains the following header: // // .. code-block:: none diff --git a/api/envoy/extensions/injected_credentials/generic/v3/BUILD b/api/envoy/extensions/injected_credentials/generic/v3/BUILD new file mode 100644 index 000000000000..78e0ff699aae --- /dev/null +++ b/api/envoy/extensions/injected_credentials/generic/v3/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/transport_sockets/tls/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + ], +) diff --git a/api/envoy/extensions/injected_credentials/generic/v3/generic.proto b/api/envoy/extensions/injected_credentials/generic/v3/generic.proto new file mode 100644 index 000000000000..5519ec10f0db --- /dev/null +++ b/api/envoy/extensions/injected_credentials/generic/v3/generic.proto @@ -0,0 +1,76 @@ +syntax = "proto3"; + +package envoy.extensions.injected_credentials.generic.v3; + +import "envoy/extensions/transport_sockets/tls/v3/secret.proto"; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.injected_credentials.generic.v3"; +option java_outer_classname = "GenericProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/injected_credentials/generic/v3;genericv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: Generic Credential] +// [#not-implemented-hide:] +// [#extension: envoy.injected_credentials.generic] + +// Generic extension can be used to inject HTTP Basic Auth, Bearer Token, or any arbitrary credential +// into the proxied requests. +// The credential will be injected into the specified HTTP request header. +// Example: +// +// .. code-block:: yaml +// +// credential: +// name: generic_credential +// typed_config: +// "@type": type.googleapis.com/envoy.extensions.injected_credentials.generic.v3.Generic +// credential: +// name: credential +// sds_config: +// path_config_source: +// path: credential.yaml +// header: Authorization +// +// credential.yaml for Basic Auth: +// +// .. code-block:: yaml +// +// resources: +// - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" +// name: credential +// generic_secret: +// secret: +// inline_string: "Basic base64EncodedUsernamePassword" +// +// Refer to [RFC 7617: The 'Basic' HTTP Authentication Scheme](https://www.rfc-editor.org/rfc/rfc7617) for details. +// +// credential.yaml for Bearer Token: +// +// .. code-block:: yaml +// resources: +// - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" +// name: credential +// generic_secret: +// secret: +// inline_string: "Bearer myToken" +// +// Refer to [RFC 6750: The OAuth 2.0 Authorization Framework: Bearer Token Usage](https://www.rfc-editor.org/rfc/rfc6750) for details. +// +message Generic { + // The SDS configuration for the credential that will be injected to the specified HTTP request header. + // It must be a generic secret. + transport_sockets.tls.v3.SdsSecretConfig credential = 1 + [(validate.rules).message = {required: true}]; + + // The header that will be injected to the HTTP request with the provided credential. + // If not set, filter will default to: ``Authorization`` + string header = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME ignore_empty: true}]; +} diff --git a/api/envoy/extensions/injected_credentials/oauth2/v3/BUILD b/api/envoy/extensions/injected_credentials/oauth2/v3/BUILD new file mode 100644 index 000000000000..8cf427582df6 --- /dev/null +++ b/api/envoy/extensions/injected_credentials/oauth2/v3/BUILD @@ -0,0 +1,14 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "//envoy/extensions/transport_sockets/tls/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", + ], +) diff --git a/api/envoy/extensions/injected_credentials/oauth2/v3/oauth2.proto b/api/envoy/extensions/injected_credentials/oauth2/v3/oauth2.proto new file mode 100644 index 000000000000..bf89893361dd --- /dev/null +++ b/api/envoy/extensions/injected_credentials/oauth2/v3/oauth2.proto @@ -0,0 +1,70 @@ +syntax = "proto3"; + +package envoy.extensions.injected_credentials.oauth2.v3; + +import "envoy/config/core/v3/http_uri.proto"; +import "envoy/extensions/transport_sockets/tls/v3/secret.proto"; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.injected_credentials.oauth2.v3"; +option java_outer_classname = "Oauth2Proto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/injected_credentials/oauth2/v3;oauth2v3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: OAuth2 Credential] +// [#not-implemented-hide:] +// [#extension: envoy.injected_credentials.oauth2] + +// OAuth2 extension can be used to retrieve an OAuth2 access token from an authorization server and inject it into the +// proxied requests. +// Currently, only the Client Credentials Grant flow is supported. +// The access token will be injected into the request headers using the ``Authorization`` header as a bearer token. +message OAuth2 { + enum AuthType { + // The ``client_id`` and ``client_secret`` will be sent using HTTP Basic authentication scheme. + BASIC_AUTH = 0; + + // The ``client_id`` and ``client_secret`` will be sent in the URL encoded request body. + // This type should only be used when Auth server does not support Basic authentication. + URL_ENCODED_BODY = 1; + } + + // Credentials to authenticate client to the authorization server. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-2.3) for details. + message ClientCredentials { + // Client ID. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-2.3.1) for details. + string client_id = 1 [(validate.rules).string = {min_len: 1}]; + + // Client secret. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-2.3.1) for details. + transport_sockets.tls.v3.SdsSecretConfig client_secret = 2 + [(validate.rules).message = {required: true}]; + + // The method to use when sending credentials to the authorization server. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-2.3.1) for details. + AuthType auth_type = 3; + } + + // Endpoint on the authorization server to retrieve the access token from. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-3.2) for details. + config.core.v3.HttpUri token_endpoint = 1 [(validate.rules).message = {required: true}]; + + // Optional list of OAuth scopes to be claimed in the authorization request. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-4.4.2) for details. + repeated string scopes = 2; + + oneof flow_type { + option (validate.required) = true; + + // Client Credentials Grant. + // Refer to [RFC 6749: The OAuth 2.0 Authorization Framework](https://www.rfc-editor.org/rfc/rfc6749#section-4.4) for details. + ClientCredentials client_credentials = 3; + } +} diff --git a/api/envoy/extensions/internal_redirect/allow_listed_routes/v3/BUILD b/api/envoy/extensions/internal_redirect/allow_listed_routes/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/internal_redirect/allow_listed_routes/v3/BUILD +++ b/api/envoy/extensions/internal_redirect/allow_listed_routes/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/internal_redirect/previous_routes/v3/BUILD b/api/envoy/extensions/internal_redirect/previous_routes/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/internal_redirect/previous_routes/v3/BUILD +++ b/api/envoy/extensions/internal_redirect/previous_routes/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/internal_redirect/safe_cross_scheme/v3/BUILD b/api/envoy/extensions/internal_redirect/safe_cross_scheme/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/internal_redirect/safe_cross_scheme/v3/BUILD +++ b/api/envoy/extensions/internal_redirect/safe_cross_scheme/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/key_value/file_based/v3/BUILD b/api/envoy/extensions/key_value/file_based/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/envoy/extensions/key_value/file_based/v3/BUILD +++ b/api/envoy/extensions/key_value/file_based/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/BUILD b/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD b/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/cluster_provided/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/load_balancing_policies/common/v3/BUILD b/api/envoy/extensions/load_balancing_policies/common/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/extensions/load_balancing_policies/common/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/common/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD b/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD index 366a3c324b35..b45c78410e7d 100644 --- a/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/extensions/load_balancing_policies/common/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto index 87a379c66912..ebef61852e21 100644 --- a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto +++ b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto @@ -22,6 +22,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in LEAST_REQUEST LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. +// [#next-free-field: 6] message LeastRequest { // The number of random healthy hosts from which the host with the fewest active requests will // be chosen. Defaults to 2 so that we perform two-choice selection if the field is not set. @@ -30,18 +31,18 @@ message LeastRequest { // The following formula is used to calculate the dynamic weights when hosts have different load // balancing weights: // - // `weight = load_balancing_weight / (active_requests + 1)^active_request_bias` + // ``weight = load_balancing_weight / (active_requests + 1)^active_request_bias`` // // The larger the active request bias is, the more aggressively active requests will lower the // effective weight when all host weights are not equal. // - // `active_request_bias` must be greater than or equal to 0.0. + // ``active_request_bias`` must be greater than or equal to 0.0. // - // When `active_request_bias == 0.0` the Least Request Load Balancer doesn't consider the number + // When ``active_request_bias == 0.0`` the Least Request Load Balancer doesn't consider the number // of active requests at the time it picks a host and behaves like the Round Robin Load // Balancer. // - // When `active_request_bias > 0.0` the Least Request Load Balancer scales the load balancing + // When ``active_request_bias > 0.0`` the Least Request Load Balancer scales the load balancing // weight by the number of active requests at the time it does a pick. // // The value is cached for performance reasons and refreshed whenever one of the Load Balancer's @@ -58,4 +59,10 @@ message LeastRequest { // Configuration for local zone aware load balancing or locality weighted load balancing. common.v3.LocalityLbConfig locality_lb_config = 4; + + // [#not-implemented-hide:] + // Configuration for performing full scan on the list of hosts. + // If this configuration is set, when selecting the host a full scan on the list hosts will be + // used to select the one with least requests instead of using random choices. + google.protobuf.BoolValue enable_full_scan = 5; } diff --git a/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD b/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD index 6a0be4c9bf97..3ff3820af87e 100644 --- a/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/load_balancing_policies/common/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD b/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/pick_first/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/load_balancing_policies/random/v3/BUILD b/api/envoy/extensions/load_balancing_policies/random/v3/BUILD index 6a0be4c9bf97..3ff3820af87e 100644 --- a/api/envoy/extensions/load_balancing_policies/random/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/random/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/load_balancing_policies/common/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD index 9ec681aa9756..0698cc5b6360 100644 --- a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/extensions/load_balancing_policies/common/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto index c121bb05796d..b6583cc3a5ce 100644 --- a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto +++ b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto @@ -53,7 +53,7 @@ message RingHash { // :ref:`minimum_ring_size`. google.protobuf.UInt64Value maximum_ring_size = 3 [(validate.rules).uint64 = {lte: 8388608}]; - // If set to `true`, the cluster will use hostname instead of the resolved + // If set to ``true``, the cluster will use hostname instead of the resolved // address as the key to consistently hash to an upstream host. Only valid for StrictDNS clusters with hostnames which resolve to a single IP address. // // .. note:: @@ -68,7 +68,7 @@ message RingHash { // Minimum is 100. // // This is implemented based on the method described in the paper https://arxiv.org/abs/1608.01350. For the specified - // `hash_balance_factor`, requests to any upstream host are capped at `hash_balance_factor/100` times the average number of requests + // ``hash_balance_factor``, requests to any upstream host are capped at ``hash_balance_factor/100`` times the average number of requests // across the cluster. When a request arrives for an upstream host that is currently serving at its max capacity, linear probing // is used to identify an eligible host. Further, the linear probe is implemented using a random jump in hosts ring/table to identify // the eligible host (this technique is as described in the paper https://arxiv.org/abs/1908.08762 - the random jump avoids the @@ -76,7 +76,7 @@ message RingHash { // // If weights are specified on the hosts, they are respected. // - // This is an O(N) algorithm, unlike other load balancers. Using a lower `hash_balance_factor` results in more hosts + // This is an O(N) algorithm, unlike other load balancers. Using a lower ``hash_balance_factor`` results in more hosts // being probed, so use a higher value if you require better performance. // // .. note:: diff --git a/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD b/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD index 6a0be4c9bf97..3ff3820af87e 100644 --- a/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/load_balancing_policies/common/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD b/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD index b49ae9078cfc..9d41e8bdabcf 100644 --- a/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/subset/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/cluster/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/BUILD b/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/BUILD index b49ae9078cfc..9d41e8bdabcf 100644 --- a/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/cluster/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/matching/common_inputs/environment_variable/v3/BUILD b/api/envoy/extensions/matching/common_inputs/environment_variable/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/matching/common_inputs/environment_variable/v3/BUILD +++ b/api/envoy/extensions/matching/common_inputs/environment_variable/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/matching/common_inputs/network/v3/BUILD b/api/envoy/extensions/matching/common_inputs/network/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/matching/common_inputs/network/v3/BUILD +++ b/api/envoy/extensions/matching/common_inputs/network/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/matching/common_inputs/ssl/v3/BUILD b/api/envoy/extensions/matching/common_inputs/ssl/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/matching/common_inputs/ssl/v3/BUILD +++ b/api/envoy/extensions/matching/common_inputs/ssl/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/matching/input_matchers/consistent_hashing/v3/BUILD b/api/envoy/extensions/matching/input_matchers/consistent_hashing/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/matching/input_matchers/consistent_hashing/v3/BUILD +++ b/api/envoy/extensions/matching/input_matchers/consistent_hashing/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/matching/input_matchers/ip/v3/BUILD b/api/envoy/extensions/matching/input_matchers/ip/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/matching/input_matchers/ip/v3/BUILD +++ b/api/envoy/extensions/matching/input_matchers/ip/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD b/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD +++ b/api/envoy/extensions/matching/input_matchers/runtime_fraction/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/network/dns_resolver/apple/v3/BUILD b/api/envoy/extensions/network/dns_resolver/apple/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/network/dns_resolver/apple/v3/BUILD +++ b/api/envoy/extensions/network/dns_resolver/apple/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/network/dns_resolver/cares/v3/BUILD b/api/envoy/extensions/network/dns_resolver/cares/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/network/dns_resolver/cares/v3/BUILD +++ b/api/envoy/extensions/network/dns_resolver/cares/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/network/dns_resolver/getaddrinfo/v3/BUILD b/api/envoy/extensions/network/dns_resolver/getaddrinfo/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/network/dns_resolver/getaddrinfo/v3/BUILD +++ b/api/envoy/extensions/network/dns_resolver/getaddrinfo/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/network/socket_interface/v3/BUILD b/api/envoy/extensions/network/socket_interface/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/network/socket_interface/v3/BUILD +++ b/api/envoy/extensions/network/socket_interface/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/path/match/uri_template/v3/BUILD b/api/envoy/extensions/path/match/uri_template/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/path/match/uri_template/v3/BUILD +++ b/api/envoy/extensions/path/match/uri_template/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/path/rewrite/uri_template/v3/BUILD b/api/envoy/extensions/path/rewrite/uri_template/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/path/rewrite/uri_template/v3/BUILD +++ b/api/envoy/extensions/path/rewrite/uri_template/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/quic/connection_id_generator/v3/BUILD b/api/envoy/extensions/quic/connection_id_generator/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/quic/connection_id_generator/v3/BUILD +++ b/api/envoy/extensions/quic/connection_id_generator/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/quic/crypto_stream/v3/BUILD b/api/envoy/extensions/quic/crypto_stream/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/quic/crypto_stream/v3/BUILD +++ b/api/envoy/extensions/quic/crypto_stream/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/quic/proof_source/v3/BUILD b/api/envoy/extensions/quic/proof_source/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/quic/proof_source/v3/BUILD +++ b/api/envoy/extensions/quic/proof_source/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/quic/server_preferred_address/v3/BUILD b/api/envoy/extensions/quic/server_preferred_address/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/envoy/extensions/quic/server_preferred_address/v3/BUILD +++ b/api/envoy/extensions/quic/server_preferred_address/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD b/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD index facd82ce6de2..81b55729566c 100644 --- a/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD +++ b/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", ], ) diff --git a/api/envoy/extensions/rbac/audit_loggers/stream/v3/BUILD b/api/envoy/extensions/rbac/audit_loggers/stream/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/rbac/audit_loggers/stream/v3/BUILD +++ b/api/envoy/extensions/rbac/audit_loggers/stream/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/rbac/matchers/upstream_ip_port/v3/BUILD b/api/envoy/extensions/rbac/matchers/upstream_ip_port/v3/BUILD index ad2fc9a9a84f..eeae27ad54b4 100644 --- a/api/envoy/extensions/rbac/matchers/upstream_ip_port/v3/BUILD +++ b/api/envoy/extensions/rbac/matchers/upstream_ip_port/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/regex_engines/v3/BUILD b/api/envoy/extensions/regex_engines/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/regex_engines/v3/BUILD +++ b/api/envoy/extensions/regex_engines/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/request_id/uuid/v3/BUILD b/api/envoy/extensions/request_id/uuid/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/request_id/uuid/v3/BUILD +++ b/api/envoy/extensions/request_id/uuid/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD b/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD +++ b/api/envoy/extensions/resource_monitors/downstream_connections/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/extensions/resource_monitors/fixed_heap/v3/BUILD b/api/envoy/extensions/resource_monitors/fixed_heap/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/resource_monitors/fixed_heap/v3/BUILD +++ b/api/envoy/extensions/resource_monitors/fixed_heap/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/resource_monitors/injected_resource/v3/BUILD b/api/envoy/extensions/resource_monitors/injected_resource/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/resource_monitors/injected_resource/v3/BUILD +++ b/api/envoy/extensions/resource_monitors/injected_resource/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/retry/host/omit_canary_hosts/v3/BUILD b/api/envoy/extensions/retry/host/omit_canary_hosts/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/retry/host/omit_canary_hosts/v3/BUILD +++ b/api/envoy/extensions/retry/host/omit_canary_hosts/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/retry/host/omit_host_metadata/v3/BUILD b/api/envoy/extensions/retry/host/omit_host_metadata/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/retry/host/omit_host_metadata/v3/BUILD +++ b/api/envoy/extensions/retry/host/omit_host_metadata/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/retry/host/previous_hosts/v3/BUILD b/api/envoy/extensions/retry/host/previous_hosts/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/retry/host/previous_hosts/v3/BUILD +++ b/api/envoy/extensions/retry/host/previous_hosts/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/retry/priority/previous_priorities/v3/BUILD b/api/envoy/extensions/retry/priority/previous_priorities/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/retry/priority/previous_priorities/v3/BUILD +++ b/api/envoy/extensions/retry/priority/previous_priorities/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/router/cluster_specifiers/lua/v3/BUILD b/api/envoy/extensions/router/cluster_specifiers/lua/v3/BUILD new file mode 100644 index 000000000000..09a37ad16b83 --- /dev/null +++ b/api/envoy/extensions/router/cluster_specifiers/lua/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/router/cluster_specifiers/lua/v3/lua.proto b/api/envoy/extensions/router/cluster_specifiers/lua/v3/lua.proto new file mode 100644 index 000000000000..b8ea10a02df7 --- /dev/null +++ b/api/envoy/extensions/router/cluster_specifiers/lua/v3/lua.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.extensions.router.cluster_specifiers.lua.v3; + +import "envoy/config/core/v3/base.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.router.cluster_specifiers.lua.v3"; +option java_outer_classname = "LuaProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/router/cluster_specifiers/lua/v3;luav3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Lua] +// +// Lua cluster specifier :ref:`configuration reference documentation `. +// [#extension: envoy.router.cluster_specifier_plugin.lua] + +message LuaConfig { + // The lua code that Envoy will execute to select cluster. + config.core.v3.DataSource source_code = 1 [(validate.rules).message = {required: true}]; + + // Default cluster. It will be used when the lua code execute failure. + string default_cluster = 2; +} diff --git a/api/envoy/extensions/stat_sinks/graphite_statsd/v3/BUILD b/api/envoy/extensions/stat_sinks/graphite_statsd/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/stat_sinks/graphite_statsd/v3/BUILD +++ b/api/envoy/extensions/stat_sinks/graphite_statsd/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/stat_sinks/open_telemetry/v3/BUILD b/api/envoy/extensions/stat_sinks/open_telemetry/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/stat_sinks/open_telemetry/v3/BUILD +++ b/api/envoy/extensions/stat_sinks/open_telemetry/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/stat_sinks/wasm/v3/BUILD b/api/envoy/extensions/stat_sinks/wasm/v3/BUILD index c37174bdefc4..ed3c664aedd7 100644 --- a/api/envoy/extensions/stat_sinks/wasm/v3/BUILD +++ b/api/envoy/extensions/stat_sinks/wasm/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/wasm/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/BUILD b/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/BUILD new file mode 100644 index 000000000000..29ebf0741406 --- /dev/null +++ b/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.proto b/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.proto new file mode 100644 index 000000000000..adb072f75990 --- /dev/null +++ b/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.opentelemetry.resource_detectors.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.opentelemetry.resource_detectors.v3"; +option java_outer_classname = "DynatraceResourceDetectorProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3;resource_detectorsv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Dynatrace Resource Detector config] + +// Configuration for the Dynatrace Resource Detector extension. +// The resource detector reads from the Dynatrace enrichment files +// and adds host/process related attributes to the OpenTelemetry resource. +// +// See: +// +// `Enrich ingested data with Dynatrace-specific dimensions `_ +// +// [#extension: envoy.tracers.opentelemetry.resource_detectors.dynatrace] +message DynatraceResourceDetectorConfig { +} diff --git a/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.proto b/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.proto new file mode 100644 index 000000000000..df62fc2d9e42 --- /dev/null +++ b/api/envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.opentelemetry.resource_detectors.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.opentelemetry.resource_detectors.v3"; +option java_outer_classname = "EnvironmentResourceDetectorProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3;resource_detectorsv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Environment Resource Detector config] + +// Configuration for the Environment Resource detector extension. +// The resource detector reads from the ``OTEL_RESOURCE_ATTRIBUTES`` +// environment variable, as per the OpenTelemetry specification. +// +// See: +// +// `OpenTelemetry specification `_ +// +// [#extension: envoy.tracers.opentelemetry.resource_detectors.environment] +message EnvironmentResourceDetectorConfig { +} diff --git a/api/envoy/extensions/tracers/opentelemetry/samplers/v3/BUILD b/api/envoy/extensions/tracers/opentelemetry/samplers/v3/BUILD new file mode 100644 index 000000000000..29ebf0741406 --- /dev/null +++ b/api/envoy/extensions/tracers/opentelemetry/samplers/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/tracers/opentelemetry/samplers/v3/always_on_sampler.proto b/api/envoy/extensions/tracers/opentelemetry/samplers/v3/always_on_sampler.proto new file mode 100644 index 000000000000..241dc06eb1fc --- /dev/null +++ b/api/envoy/extensions/tracers/opentelemetry/samplers/v3/always_on_sampler.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.opentelemetry.samplers.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.opentelemetry.samplers.v3"; +option java_outer_classname = "AlwaysOnSamplerProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/samplers/v3;samplersv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Always On Sampler config] +// Configuration for the "AlwaysOn" Sampler extension. +// The sampler follows the "AlwaysOn" implementation from the OpenTelemetry +// SDK specification. +// +// See: +// `AlwaysOn sampler specification `_ +// [#extension: envoy.tracers.opentelemetry.samplers.always_on] + +message AlwaysOnSamplerConfig { +} diff --git a/api/envoy/extensions/transport_sockets/alts/v3/BUILD b/api/envoy/extensions/transport_sockets/alts/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/transport_sockets/alts/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/alts/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/transport_sockets/http_11_proxy/v3/BUILD b/api/envoy/extensions/transport_sockets/http_11_proxy/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/transport_sockets/http_11_proxy/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/http_11_proxy/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/internal_upstream/v3/BUILD b/api/envoy/extensions/transport_sockets/internal_upstream/v3/BUILD index 1bbba241ba61..450e5434d631 100644 --- a/api/envoy/extensions/transport_sockets/internal_upstream/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/internal_upstream/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/type/metadata/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/proxy_protocol/v3/BUILD b/api/envoy/extensions/transport_sockets/proxy_protocol/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/transport_sockets/proxy_protocol/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/proxy_protocol/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/quic/v3/BUILD b/api/envoy/extensions/transport_sockets/quic/v3/BUILD index 3ca8242f7780..63fb3642c4b5 100644 --- a/api/envoy/extensions/transport_sockets/quic/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/quic/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/extensions/transport_sockets/tls/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/raw_buffer/v3/BUILD b/api/envoy/extensions/transport_sockets/raw_buffer/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/transport_sockets/raw_buffer/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/raw_buffer/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/transport_sockets/s2a/v3/BUILD b/api/envoy/extensions/transport_sockets/s2a/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/transport_sockets/s2a/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/s2a/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/transport_sockets/starttls/v3/BUILD b/api/envoy/extensions/transport_sockets/starttls/v3/BUILD index 7ae3c01a9947..2addd072fbf8 100644 --- a/api/envoy/extensions/transport_sockets/starttls/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/starttls/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/extensions/transport_sockets/raw_buffer/v3:pkg", "//envoy/extensions/transport_sockets/tls/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/tap/v3/BUILD b/api/envoy/extensions/transport_sockets/tap/v3/BUILD index b97db3d63736..6f8c1c8f74ec 100644 --- a/api/envoy/extensions/transport_sockets/tap/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/tap/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/extensions/common/tap/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/tcp_stats/v3/BUILD b/api/envoy/extensions/transport_sockets/tcp_stats/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/transport_sockets/tcp_stats/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/tcp_stats/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/tls/v3/BUILD b/api/envoy/extensions/transport_sockets/tls/v3/BUILD index 75026a89c29b..8a81977d7bc3 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/BUILD +++ b/api/envoy/extensions/transport_sockets/tls/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/transport_sockets/tls/v3/common.proto b/api/envoy/extensions/transport_sockets/tls/v3/common.proto index 0d653050f5a3..d244adcdf549 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -180,8 +180,8 @@ message PrivateKeyProvider { } // If the private key provider isn't available (eg. the required hardware capability doesn't existed), - // Envoy will fallback to the BoringSSL default implementation when the `fallback` is true. - // The default value is `false`. + // Envoy will fallback to the BoringSSL default implementation when the ``fallback`` is true. + // The default value is ``false``. bool fallback = 4; } @@ -536,12 +536,10 @@ message CertificateValidationContext { bool only_verify_leaf_cert_crl = 14; // Defines maximum depth of a certificate chain accepted in verification, the default limit is 100, though this can be system-dependent. - // This number does not include the leaf, so a depth of 1 allows the leaf and one CA certificate. If a trusted issuer appears in the chain, - // but in a depth larger than configured, the certificate validation will fail. - // See `BoringSSL SSL_CTX_set_verify_depth ` - // If you use OpenSSL, its behavior is different from BoringSSL, this will define a limit on the number of certificates between the end-entity and trust-anchor certificates. - // Neither the end-entity nor the trust-anchor certificates count against depth. - // See `OpenSSL SSL set_verify_depth `_. + // This number does not include the leaf but includes the trust anchor, so a depth of 1 allows the leaf and one CA certificate. If a trusted issuer + // appears in the chain, but in a depth larger than configured, the certificate validation will fail. + // This matches the semantics of ``SSL_CTX_set_verify_depth`` in OpenSSL 1.0.x and older versions of BoringSSL. It differs from ``SSL_CTX_set_verify_depth`` + // in OpenSSL 1.1.x and newer versions of BoringSSL in that the trust anchor is included. // Trusted issues are specified by setting :ref:`trusted_ca ` google.protobuf.UInt32Value max_verify_depth = 16 [(validate.rules).uint32 = {lte: 100}]; } diff --git a/api/envoy/extensions/transport_sockets/tls/v3/tls.proto b/api/envoy/extensions/transport_sockets/tls/v3/tls.proto index ac3641ebd6f5..f94889cfad04 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/tls.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/tls.proto @@ -63,7 +63,7 @@ message UpstreamTlsContext { google.protobuf.BoolValue enforce_rsa_key_usage = 5; } -// [#next-free-field: 10] +// [#next-free-field: 11] message DownstreamTlsContext { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.auth.DownstreamTlsContext"; @@ -119,6 +119,10 @@ message DownstreamTlsContext { bool disable_stateless_session_resumption = 7; } + // If set to true, the TLS server will not maintain a session cache of TLS sessions. (This is + // relevant only for TLSv1.2 and earlier.) + bool disable_stateful_session_resumption = 10; + // If specified, ``session_timeout`` will change the maximum lifetime (in seconds) of the TLS session. // Currently this value is used as a hint for the `TLS session ticket lifetime (for TLSv1.2) `_. // Only seconds can be specified (fractional seconds are ignored). diff --git a/api/envoy/extensions/udp_packet_writer/v3/BUILD b/api/envoy/extensions/udp_packet_writer/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/udp_packet_writer/v3/BUILD +++ b/api/envoy/extensions/udp_packet_writer/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/upstreams/http/generic/v3/BUILD b/api/envoy/extensions/upstreams/http/generic/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/upstreams/http/generic/v3/BUILD +++ b/api/envoy/extensions/upstreams/http/generic/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/upstreams/http/generic/v3/generic_connection_pool.proto b/api/envoy/extensions/upstreams/http/generic/v3/generic_connection_pool.proto index e76dde8573e9..cb578db10cd0 100644 --- a/api/envoy/extensions/upstreams/http/generic/v3/generic_connection_pool.proto +++ b/api/envoy/extensions/upstreams/http/generic/v3/generic_connection_pool.proto @@ -12,7 +12,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Generic Connection Pool] -// A connection pool which forwards downstream HTTP as TCP or HTTP to upstream, +// A connection pool which forwards downstream HTTP as TCP, UDP or HTTP to upstream, // based on CONNECT configuration. // [#extension: envoy.upstreams.http.generic] message GenericConnectionPoolProto { diff --git a/api/envoy/extensions/upstreams/http/http/v3/BUILD b/api/envoy/extensions/upstreams/http/http/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/upstreams/http/http/v3/BUILD +++ b/api/envoy/extensions/upstreams/http/http/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/upstreams/http/tcp/v3/BUILD b/api/envoy/extensions/upstreams/http/tcp/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/upstreams/http/tcp/v3/BUILD +++ b/api/envoy/extensions/upstreams/http/tcp/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/upstreams/http/udp/v3/BUILD b/api/envoy/extensions/upstreams/http/udp/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/upstreams/http/udp/v3/BUILD +++ b/api/envoy/extensions/upstreams/http/udp/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/upstreams/http/v3/BUILD b/api/envoy/extensions/upstreams/http/v3/BUILD index a3fe9fa9e747..e6a0401f0645 100644 --- a/api/envoy/extensions/upstreams/http/v3/BUILD +++ b/api/envoy/extensions/upstreams/http/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/extensions/filters/network/http_connection_manager/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/upstreams/http/v3/http_protocol_options.proto b/api/envoy/extensions/upstreams/http/v3/http_protocol_options.proto index 1663fa4aad7a..ca4cb81fe033 100644 --- a/api/envoy/extensions/upstreams/http/v3/http_protocol_options.proto +++ b/api/envoy/extensions/upstreams/http/v3/http_protocol_options.proto @@ -154,14 +154,14 @@ message HttpProtocolOptions { // .. note:: // Upstream HTTP filters are currently in alpha. // - // Optional HTTP filters for the upstream filter chain. + // Optional HTTP filters for the upstream HTTP filter chain. // // These filters will be applied for all HTTP streams which flow through this - // cluster. Unlike downstream filters, they will *not* be applied to terminated CONNECT requests. + // cluster. Unlike downstream HTTP filters, they will *not* be applied to terminated CONNECT requests. // - // If using upstream filters, please be aware that local errors sent by - // upstream filters will not trigger retries, and local errors sent by - // upstream filters will count as a final response if hedging is configured. + // If using upstream HTTP filters, please be aware that local errors sent by + // upstream HTTP filters will not trigger retries, and local errors sent by + // upstream HTTP filters will count as a final response if hedging is configured. // [#extension-category: envoy.filters.http.upstream] repeated filters.network.http_connection_manager.v3.HttpFilter http_filters = 6; diff --git a/api/envoy/extensions/upstreams/tcp/generic/v3/BUILD b/api/envoy/extensions/upstreams/tcp/generic/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/upstreams/tcp/generic/v3/BUILD +++ b/api/envoy/extensions/upstreams/tcp/generic/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/upstreams/tcp/v3/BUILD b/api/envoy/extensions/upstreams/tcp/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/upstreams/tcp/v3/BUILD +++ b/api/envoy/extensions/upstreams/tcp/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/extensions/wasm/v3/BUILD b/api/envoy/extensions/wasm/v3/BUILD index 1c1a6f6b4423..09a37ad16b83 100644 --- a/api/envoy/extensions/wasm/v3/BUILD +++ b/api/envoy/extensions/wasm/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/watchdog/profile_action/v3/BUILD b/api/envoy/extensions/watchdog/profile_action/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/extensions/watchdog/profile_action/v3/BUILD +++ b/api/envoy/extensions/watchdog/profile_action/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/service/accesslog/v2/BUILD b/api/envoy/service/accesslog/v2/BUILD index 1253698c39d5..e05de7268986 100644 --- a/api/envoy/service/accesslog/v2/BUILD +++ b/api/envoy/service/accesslog/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/data/accesslog/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/accesslog/v3/BUILD b/api/envoy/service/accesslog/v3/BUILD index c913d31f62fe..10edf724f3bb 100644 --- a/api/envoy/service/accesslog/v3/BUILD +++ b/api/envoy/service/accesslog/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/data/accesslog/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/auth/v2/BUILD b/api/envoy/service/auth/v2/BUILD index fa00ca5127de..0fc0d204ca71 100644 --- a/api/envoy/service/auth/v2/BUILD +++ b/api/envoy/service/auth/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/auth/v3/BUILD b/api/envoy/service/auth/v3/BUILD index f39e4f85d88f..4f64fe2f9ee5 100644 --- a/api/envoy/service/auth/v3/BUILD +++ b/api/envoy/service/auth/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/auth/v3/attribute_context.proto b/api/envoy/service/auth/v3/attribute_context.proto index 77af84436de9..152672685bcc 100644 --- a/api/envoy/service/auth/v3/attribute_context.proto +++ b/api/envoy/service/auth/v3/attribute_context.proto @@ -38,7 +38,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // - field mask to send // - which return values from request_context are copied back // - which return values are copied into request_headers] -// [#next-free-field: 13] +// [#next-free-field: 14] message AttributeContext { option (udpa.annotations.versioning).previous_message_type = "envoy.service.auth.v2.AttributeContext"; @@ -183,6 +183,9 @@ message AttributeContext { // Dynamic metadata associated with the request. config.core.v3.Metadata metadata_context = 11; + // Metadata associated with the selected route. + config.core.v3.Metadata route_metadata_context = 13; + // TLS session details of the underlying connection. // This is not populated by default and will be populated if ext_authz filter's // :ref:`include_tls_session ` is set to true. diff --git a/api/envoy/service/cluster/v3/BUILD b/api/envoy/service/cluster/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/cluster/v3/BUILD +++ b/api/envoy/service/cluster/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/discovery/v2/BUILD b/api/envoy/service/discovery/v2/BUILD index ec687f753436..dc79641fe85b 100644 --- a/api/envoy/service/discovery/v2/BUILD +++ b/api/envoy/service/discovery/v2/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/api/v2:pkg", "//envoy/api/v2/core:pkg", "//envoy/api/v2/endpoint:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/discovery/v3/BUILD b/api/envoy/service/discovery/v3/BUILD index 7753cfeb3d6e..79668d20fb02 100644 --- a/api/envoy/service/discovery/v3/BUILD +++ b/api/envoy/service/discovery/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( has_services = True, deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/endpoint/v3/BUILD b/api/envoy/service/endpoint/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/endpoint/v3/BUILD +++ b/api/envoy/service/endpoint/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/event_reporting/v2alpha/BUILD b/api/envoy/service/event_reporting/v2alpha/BUILD index 4f58bd462f66..9b30e884abc7 100644 --- a/api/envoy/service/event_reporting/v2alpha/BUILD +++ b/api/envoy/service/event_reporting/v2alpha/BUILD @@ -8,6 +8,6 @@ api_proto_package( has_services = True, deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/event_reporting/v3/BUILD b/api/envoy/service/event_reporting/v3/BUILD index 7753cfeb3d6e..79668d20fb02 100644 --- a/api/envoy/service/event_reporting/v3/BUILD +++ b/api/envoy/service/event_reporting/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( has_services = True, deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/ext_proc/v3/BUILD b/api/envoy/service/ext_proc/v3/BUILD index 62a33c34631b..0e337d5c3ed1 100644 --- a/api/envoy/service/ext_proc/v3/BUILD +++ b/api/envoy/service/ext_proc/v3/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/config/core/v3:pkg", "//envoy/extensions/filters/http/ext_proc/v3:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/ext_proc/v3/external_processor.proto b/api/envoy/service/ext_proc/v3/external_processor.proto index 666e65296255..50fba503f846 100644 --- a/api/envoy/service/ext_proc/v3/external_processor.proto +++ b/api/envoy/service/ext_proc/v3/external_processor.proto @@ -167,7 +167,9 @@ message ProcessingResponse { // for the duration of this particular request/response only. Servers // may use this to intelligently control how requests are processed // based on the headers and other metadata that they see. - // This field is ignored by Envoy when the ext_proc filter config + // This field is only applicable when servers responding to the header requests. + // If it is set in the response to the body or trailer requests, it will be ignored by Envoy. + // It is also ignored by Envoy when the ext_proc filter config // :ref:`allow_mode_override // ` // is set to false. diff --git a/api/envoy/service/extension/v3/BUILD b/api/envoy/service/extension/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/extension/v3/BUILD +++ b/api/envoy/service/extension/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/health/v3/BUILD b/api/envoy/service/health/v3/BUILD index b28383997467..786d0d75d165 100644 --- a/api/envoy/service/health/v3/BUILD +++ b/api/envoy/service/health/v3/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/config/cluster/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/endpoint/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/listener/v3/BUILD b/api/envoy/service/listener/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/listener/v3/BUILD +++ b/api/envoy/service/listener/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/load_stats/v2/BUILD b/api/envoy/service/load_stats/v2/BUILD index 1263251505f6..55f6785dfcc9 100644 --- a/api/envoy/service/load_stats/v2/BUILD +++ b/api/envoy/service/load_stats/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/api/v2/endpoint:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/load_stats/v3/BUILD b/api/envoy/service/load_stats/v3/BUILD index 1ee733dc7d82..f3dcebe111fd 100644 --- a/api/envoy/service/load_stats/v3/BUILD +++ b/api/envoy/service/load_stats/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/config/endpoint/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/metrics/v2/BUILD b/api/envoy/service/metrics/v2/BUILD index 79fc6928c032..434723c8306a 100644 --- a/api/envoy/service/metrics/v2/BUILD +++ b/api/envoy/service/metrics/v2/BUILD @@ -8,7 +8,7 @@ api_proto_package( has_services = True, deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@prometheus_metrics_model//:client_model", ], ) diff --git a/api/envoy/service/metrics/v3/BUILD b/api/envoy/service/metrics/v3/BUILD index b9a1679e2cb8..ac56c2baa409 100644 --- a/api/envoy/service/metrics/v3/BUILD +++ b/api/envoy/service/metrics/v3/BUILD @@ -8,7 +8,7 @@ api_proto_package( has_services = True, deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@prometheus_metrics_model//:client_model", ], ) diff --git a/api/envoy/service/rate_limit_quota/v3/BUILD b/api/envoy/service/rate_limit_quota/v3/BUILD index 1ed447c6f271..8e1364681b5a 100644 --- a/api/envoy/service/rate_limit_quota/v3/BUILD +++ b/api/envoy/service/rate_limit_quota/v3/BUILD @@ -8,7 +8,7 @@ api_proto_package( has_services = True, deps = [ "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/service/rate_limit_quota/v3/rlqs.proto b/api/envoy/service/rate_limit_quota/v3/rlqs.proto index 84ba44976b98..b8fa2cd89820 100644 --- a/api/envoy/service/rate_limit_quota/v3/rlqs.proto +++ b/api/envoy/service/rate_limit_quota/v3/rlqs.proto @@ -43,6 +43,14 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // ` // it. // +// If for any reason the RLQS client doesn't receive the initial assignment for the reported bucket, +// in order to prevent memory exhaustion, the data plane will limit the time such bucket +// is retained. The exact time to wait for the initial assignment is chosen by the filter, +// and may vary based on the implementation. +// Once the duration ends, the data plane will stop reporting bucket usage, reject any enqueued +// requests, and purge the bucket from the memory. Subsequent requests matched into the bucket +// will re-initialize the bucket in the "no assignment" state, restarting the reports. +// // Refer to Rate Limit Quota :ref:`configuration overview ` // for further details. diff --git a/api/envoy/service/ratelimit/v2/BUILD b/api/envoy/service/ratelimit/v2/BUILD index eedc3e62b3b2..ff6dcdd6bfe1 100644 --- a/api/envoy/service/ratelimit/v2/BUILD +++ b/api/envoy/service/ratelimit/v2/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/api/v2/core:pkg", "//envoy/api/v2/ratelimit:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/ratelimit/v3/BUILD b/api/envoy/service/ratelimit/v3/BUILD index 1cec1e02cde9..1e1a8863cfbd 100644 --- a/api/envoy/service/ratelimit/v3/BUILD +++ b/api/envoy/service/ratelimit/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/extensions/common/ratelimit/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/route/v3/BUILD b/api/envoy/service/route/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/route/v3/BUILD +++ b/api/envoy/service/route/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/runtime/v3/BUILD b/api/envoy/service/runtime/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/runtime/v3/BUILD +++ b/api/envoy/service/runtime/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/secret/v3/BUILD b/api/envoy/service/secret/v3/BUILD index 9f2ae1e747c5..b0154480fed5 100644 --- a/api/envoy/service/secret/v3/BUILD +++ b/api/envoy/service/secret/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/service/discovery/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/status/v2/BUILD b/api/envoy/service/status/v2/BUILD index 39c38eb10a7c..87bca3035cf9 100644 --- a/api/envoy/service/status/v2/BUILD +++ b/api/envoy/service/status/v2/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/admin/v2alpha:pkg", "//envoy/api/v2/core:pkg", "//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/status/v3/BUILD b/api/envoy/service/status/v3/BUILD index 45ec162b9093..f3b9672c29e6 100644 --- a/api/envoy/service/status/v3/BUILD +++ b/api/envoy/service/status/v3/BUILD @@ -11,6 +11,6 @@ api_proto_package( "//envoy/annotations:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/tap/v2alpha/BUILD b/api/envoy/service/tap/v2alpha/BUILD index 8e0561a169c5..478272b81779 100644 --- a/api/envoy/service/tap/v2alpha/BUILD +++ b/api/envoy/service/tap/v2alpha/BUILD @@ -10,6 +10,6 @@ api_proto_package( "//envoy/api/v2/core:pkg", "//envoy/api/v2/route:pkg", "//envoy/data/tap/v2alpha:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/tap/v3/BUILD b/api/envoy/service/tap/v3/BUILD index 8948f580a51e..13564a427446 100644 --- a/api/envoy/service/tap/v3/BUILD +++ b/api/envoy/service/tap/v3/BUILD @@ -9,6 +9,6 @@ api_proto_package( deps = [ "//envoy/config/core/v3:pkg", "//envoy/data/tap/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/service/trace/v2/BUILD b/api/envoy/service/trace/v2/BUILD index 7e6d2b11bf16..f3ef22c3da6d 100644 --- a/api/envoy/service/trace/v2/BUILD +++ b/api/envoy/service/trace/v2/BUILD @@ -8,7 +8,7 @@ api_proto_package( has_services = True, deps = [ "//envoy/api/v2/core:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", ], ) diff --git a/api/envoy/service/trace/v3/BUILD b/api/envoy/service/trace/v3/BUILD index a00d454ff974..1e793010c62c 100644 --- a/api/envoy/service/trace/v3/BUILD +++ b/api/envoy/service/trace/v3/BUILD @@ -8,7 +8,7 @@ api_proto_package( has_services = True, deps = [ "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", ], ) diff --git a/api/envoy/type/BUILD b/api/envoy/type/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/type/BUILD +++ b/api/envoy/type/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/type/http/v3/BUILD b/api/envoy/type/http/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/type/http/v3/BUILD +++ b/api/envoy/type/http/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/type/matcher/BUILD b/api/envoy/type/matcher/BUILD index 29613b4c3487..ad7d3cbadf20 100644 --- a/api/envoy/type/matcher/BUILD +++ b/api/envoy/type/matcher/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/type:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/type/matcher/v3/BUILD b/api/envoy/type/matcher/v3/BUILD index fb28aa0e973d..320b988b1a53 100644 --- a/api/envoy/type/matcher/v3/BUILD +++ b/api/envoy/type/matcher/v3/BUILD @@ -8,6 +8,6 @@ api_proto_package( deps = [ "//envoy/annotations:pkg", "//envoy/type/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/type/matcher/v3/value.proto b/api/envoy/type/matcher/v3/value.proto index bd46acc0713c..d773c6057fcc 100644 --- a/api/envoy/type/matcher/v3/value.proto +++ b/api/envoy/type/matcher/v3/value.proto @@ -19,7 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Specifies the way to match a ProtobufWkt::Value. Primitive values and ListValue are supported. // StructValue is not supported and is always not matched. -// [#next-free-field: 7] +// [#next-free-field: 8] message ValueMatcher { option (udpa.annotations.versioning).previous_message_type = "envoy.type.matcher.ValueMatcher"; @@ -56,6 +56,9 @@ message ValueMatcher { // If specified, a match occurs if and only if the target value is a list value and // is matched to this field. ListMatcher list_match = 6; + + // If specified, a match occurs if and only if any of the alternatives in the match accept the value. + OrMatcher or_match = 7; } } @@ -70,3 +73,8 @@ message ListMatcher { ValueMatcher one_of = 1; } } + +// Specifies a list of alternatives for the match. +message OrMatcher { + repeated ValueMatcher value_matchers = 1 [(validate.rules).repeated = {min_items: 2}]; +} diff --git a/api/envoy/type/metadata/v2/BUILD b/api/envoy/type/metadata/v2/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/type/metadata/v2/BUILD +++ b/api/envoy/type/metadata/v2/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/type/metadata/v3/BUILD b/api/envoy/type/metadata/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/type/metadata/v3/BUILD +++ b/api/envoy/type/metadata/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/envoy/type/tracing/v2/BUILD b/api/envoy/type/tracing/v2/BUILD index aa64935f43d1..e0ccd69d66c1 100644 --- a/api/envoy/type/tracing/v2/BUILD +++ b/api/envoy/type/tracing/v2/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/metadata/v2:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/type/tracing/v3/BUILD b/api/envoy/type/tracing/v3/BUILD index c797ae66c28a..369c3541b913 100644 --- a/api/envoy/type/tracing/v3/BUILD +++ b/api/envoy/type/tracing/v3/BUILD @@ -7,6 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/type/metadata/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/type/v3/BUILD b/api/envoy/type/v3/BUILD index ec1e778e06e5..d49202b74ab4 100644 --- a/api/envoy/type/v3/BUILD +++ b/api/envoy/type/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) diff --git a/api/envoy/watchdog/v3/BUILD b/api/envoy/watchdog/v3/BUILD index ee92fb652582..29ebf0741406 100644 --- a/api/envoy/watchdog/v3/BUILD +++ b/api/envoy/watchdog/v3/BUILD @@ -5,5 +5,5 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/api/test/build/BUILD b/api/test/build/BUILD index 16956e8d5720..612cf86713a6 100644 --- a/api/test/build/BUILD +++ b/api/test/build/BUILD @@ -13,7 +13,7 @@ api_cc_test( "//envoy/service/discovery/v2:pkg_cc_proto", "//envoy/service/metrics/v2:pkg_cc_proto", "//envoy/service/ratelimit/v2:pkg_cc_proto", - "@com_github_cncf_udpa//udpa/service/orca/v1:pkg_cc_proto", + "@com_github_cncf_xds//udpa/service/orca/v1:pkg_cc_proto", ], ) diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 9b4dc169b54a..65ba4dc5c75f 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -9,6 +9,7 @@ proto_library( name = "active_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg", "//contrib/envoy/extensions/config/v3alpha:pkg", "//contrib/envoy/extensions/filters/http/checksum/v3alpha:pkg", "//contrib/envoy/extensions/filters/http/dynamo/v3:pkg", @@ -19,6 +20,7 @@ proto_library( "//contrib/envoy/extensions/filters/network/client_ssl_auth/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/action/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3:pkg", + "//contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/matcher/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/router/v3:pkg", "//contrib/envoy/extensions/filters/network/generic_proxy/v3:pkg", @@ -94,12 +96,14 @@ proto_library( "//envoy/extensions/filters/common/dependency/v3:pkg", "//envoy/extensions/filters/common/fault/v3:pkg", "//envoy/extensions/filters/common/matcher/action/v3:pkg", + "//envoy/extensions/filters/common/set_filter_state/v3:pkg", "//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg", "//envoy/extensions/filters/http/admission_control/v3:pkg", "//envoy/extensions/filters/http/alternate_protocols_cache/v3:pkg", "//envoy/extensions/filters/http/aws_lambda/v3:pkg", "//envoy/extensions/filters/http/aws_request_signing/v3:pkg", "//envoy/extensions/filters/http/bandwidth_limit/v3:pkg", + "//envoy/extensions/filters/http/basic_auth/v3:pkg", "//envoy/extensions/filters/http/buffer/v3:pkg", "//envoy/extensions/filters/http/cache/v3:pkg", "//envoy/extensions/filters/http/cdn_loop/v3:pkg", @@ -107,6 +111,7 @@ proto_library( "//envoy/extensions/filters/http/compressor/v3:pkg", "//envoy/extensions/filters/http/connect_grpc_bridge/v3:pkg", "//envoy/extensions/filters/http/cors/v3:pkg", + "//envoy/extensions/filters/http/credential_injector/v3:pkg", "//envoy/extensions/filters/http/csrf/v3:pkg", "//envoy/extensions/filters/http/custom_response/v3:pkg", "//envoy/extensions/filters/http/decompressor/v3:pkg", @@ -140,6 +145,7 @@ proto_library( "//envoy/extensions/filters/http/ratelimit/v3:pkg", "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", + "//envoy/extensions/filters/http/set_filter_state/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", "//envoy/extensions/filters/http/stateful_session/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", @@ -163,6 +169,7 @@ proto_library( "//envoy/extensions/filters/network/ratelimit/v3:pkg", "//envoy/extensions/filters/network/rbac/v3:pkg", "//envoy/extensions/filters/network/redis_proxy/v3:pkg", + "//envoy/extensions/filters/network/set_filter_state/v3:pkg", "//envoy/extensions/filters/network/sni_cluster/v3:pkg", "//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3:pkg", "//envoy/extensions/filters/network/tcp_proxy/v3:pkg", @@ -174,10 +181,14 @@ proto_library( "//envoy/extensions/filters/network/wasm/v3:pkg", "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg", "//envoy/extensions/filters/udp/dns_filter/v3:pkg", + "//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg", + "//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg", "//envoy/extensions/filters/udp/udp_proxy/v3:pkg", "//envoy/extensions/formatter/cel/v3:pkg", "//envoy/extensions/formatter/metadata/v3:pkg", "//envoy/extensions/formatter/req_without_query/v3:pkg", + "//envoy/extensions/geoip_providers/common/v3:pkg", + "//envoy/extensions/geoip_providers/maxmind/v3:pkg", "//envoy/extensions/health_check/event_sinks/file/v3:pkg", "//envoy/extensions/health_checkers/redis/v3:pkg", "//envoy/extensions/health_checkers/thrift/v3:pkg", @@ -192,6 +203,8 @@ proto_library( "//envoy/extensions/http/original_ip_detection/xff/v3:pkg", "//envoy/extensions/http/stateful_session/cookie/v3:pkg", "//envoy/extensions/http/stateful_session/header/v3:pkg", + "//envoy/extensions/injected_credentials/generic/v3:pkg", + "//envoy/extensions/injected_credentials/oauth2/v3:pkg", "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg", "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", @@ -235,9 +248,12 @@ proto_library( "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg", "//envoy/extensions/retry/host/previous_hosts/v3:pkg", "//envoy/extensions/retry/priority/previous_priorities/v3:pkg", + "//envoy/extensions/router/cluster_specifiers/lua/v3:pkg", "//envoy/extensions/stat_sinks/graphite_statsd/v3:pkg", "//envoy/extensions/stat_sinks/open_telemetry/v3:pkg", "//envoy/extensions/stat_sinks/wasm/v3:pkg", + "//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg", + "//envoy/extensions/tracers/opentelemetry/samplers/v3:pkg", "//envoy/extensions/transport_sockets/alts/v3:pkg", "//envoy/extensions/transport_sockets/http_11_proxy/v3:pkg", "//envoy/extensions/transport_sockets/internal_upstream/v3:pkg", diff --git a/bazel/BUILD b/bazel/BUILD index d081f142371f..37eafa57eda1 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -1,11 +1,12 @@ -load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//bazel:envoy_internal.bzl", "envoy_select_force_libcpp") -load("@envoy_api//bazel:utils.bzl", "json_data") load("@bazel_skylib//lib:selects.bzl", "selects") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") -load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") load("@envoy_api//bazel:repository_locations.bzl", API_REPOSITORY_LOCATIONS_SPEC = "REPOSITORY_LOCATIONS_SPEC") load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locations_spec", "merge_dicts") +load("@envoy_toolshed//:macros.bzl", "json_data") +load("@envoy_toolshed//dependency:macros.bzl", "updater") +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//bazel:envoy_internal.bzl", "envoy_select_force_libcpp") +load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") licenses(["notice"]) # Apache 2 @@ -41,8 +42,8 @@ genrule( outs = ["gnu_build_id.ldscript"], cmd = """ echo --build-id=0x$$( - grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\ - | sed 's/^BUILD_SCM_REVISION //') \\ + grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \ + | sed 's/^BUILD_SCM_REVISION //') \ > $@ """, # Undocumented attr to depend on workspace status files. @@ -55,8 +56,8 @@ genrule( name = "raw_build_id", outs = ["raw_build_id.ldscript"], cmd = """ - grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\ - | sed 's/^BUILD_SCM_REVISION //' \\ + grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \ + | sed 's/^BUILD_SCM_REVISION //' \ | tr -d '\\n' \\ > $@ """, @@ -361,13 +362,8 @@ config_setting( ) config_setting( - name = "disable_envoy_mobile_request_compression", - values = {"define": "envoy_mobile_request_compression=disabled"}, -) - -config_setting( - name = "disable_envoy_mobile_stats_reporting", - values = {"define": "envoy_mobile_stats_reporting=disabled"}, + name = "disable_envoy_mobile_xds", + values = {"define": "envoy_mobile_xds=disabled"}, ) config_setting( @@ -394,7 +390,7 @@ config_setting( config_setting( name = "disable_http_datagrams", - values = {"define": "enable_http_datagrams=disabled"}, + values = {"define": "envoy_enable_http_datagrams=disabled"}, ) config_setting( @@ -887,3 +883,28 @@ cc_library( name = "python_headers", visibility = ["//visibility:public"], ) + +# These can be run as follows: +# +# $ bazel run //bazel:update ENVOY_DEP NEW_VERSION +# $ bazel run //bazel:api-update API_DEP NEW_VERSION +updater( + name = "update", + data = ["//tools/dependency:check"], + dependencies = "//tools/dependency:filtered-dependencies", + post_script = ":version_update_post.sh", + pydict = True, + tags = ["skip_on_windows"], + version_file = ":repository_locations.bzl", +) + +updater( + name = "api-update", + data = ["//tools/dependency:check"], + dependencies = "@envoy_api//bazel:repository_locations", + post_script = ":version_update_post.sh", + pydict = True, + tags = ["skip_on_windows"], + version_file = "@envoy_api//bazel:repository_locations.bzl", + version_path_replace = "external/envoy_api:api", +) diff --git a/bazel/README.md b/bazel/README.md index 34b0a75239d2..03a8008bcf05 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -285,7 +285,7 @@ Envoy can also be built with the Docker image used for CI, by installing Docker On Linux, run: ``` -./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.dev' +./ci/run_envoy_docker.sh './ci/do_ci.sh dev' ``` From a Windows host with Docker installed, the Windows containers feature enabled, and bash (installed via diff --git a/bazel/api_binding.bzl b/bazel/api_binding.bzl index 65ed382836fc..8d46d4c1827b 100644 --- a/bazel/api_binding.bzl +++ b/bazel/api_binding.bzl @@ -13,7 +13,6 @@ def _default_envoy_api_impl(ctx): ] for d in api_dirs: ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child(ctx.attr.reldir).get_child(d), d) - ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child("api").get_child("bazel").get_child("utils.bzl"), "utils.bzl") _default_envoy_api = repository_rule( implementation = _default_envoy_api_impl, diff --git a/bazel/aspect.patch b/bazel/aspect.patch new file mode 100644 index 000000000000..b9047daa6df9 --- /dev/null +++ b/bazel/aspect.patch @@ -0,0 +1,20 @@ +diff --git a/lib/private/yq.bzl b/lib/private/yq.bzl +index 29ca3d7..c8cd5eb 100644 +--- a/lib/private/yq.bzl ++++ b/lib/private/yq.bzl +@@ -71,10 +71,13 @@ def _yq_impl(ctx): + + # For split operations, yq outputs files in the same directory so we + # must cd to the correct output dir before executing it +- bin_dir = "/".join([ctx.bin_dir.path, ctx.label.package]) if ctx.label.package else ctx.bin_dir.path ++ bin_dir = ctx.bin_dir.path ++ if ctx.label.workspace_name: ++ bin_dir = "%s/external/%s" % (bin_dir, ctx.label.workspace_name) ++ bin_dir = "/".join([bin_dir, ctx.label.package]) if ctx.label.package else bin_dir + escape_bin_dir = _escape_path(bin_dir) + cmd = "cd {bin_dir} && {yq} {args} {eval_cmd} {expression} {sources} {maybe_out}".format( +- bin_dir = ctx.bin_dir.path + "/" + ctx.label.package, ++ bin_dir = bin_dir, + yq = escape_bin_dir + yq_bin.path, + eval_cmd = "eval" if len(inputs) <= 1 else "eval-all", + args = " ".join(args), diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index b743a1936d0d..2bd87a371771 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -18,9 +18,9 @@ load("@com_google_cel_cpp//bazel:deps.bzl", "parser_deps") load("@com_github_chrusty_protoc_gen_jsonschema//:deps.bzl", protoc_gen_jsonschema_go_dependencies = "go_dependencies") # go version for rules_go -GO_VERSION = "1.18" +GO_VERSION = "1.20" -JQ_VERSION = "1.6" +JQ_VERSION = "1.7" YQ_VERSION = "4.24.4" def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, yq_version = YQ_VERSION): @@ -125,7 +125,7 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L60-L65" ) go_repository( - name = "com_github_lyft_protoc_gen_star", + name = "com_github_lyft_protoc_gen_star_v2", importpath = "github.com/lyft/protoc-gen-star/v2", sum = "h1:keaAo8hRuAT0O3DfJ/wM3rufbAjGeJ1lAtWZHDjKGB0=", version = "v2.0.1", diff --git a/bazel/engflow-bazel-credential-helper.sh b/bazel/engflow-bazel-credential-helper.sh new file mode 100755 index 000000000000..c6c1bd339b62 --- /dev/null +++ b/bazel/engflow-bazel-credential-helper.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Bazel expects the helper to read stdin. +# See https://github.com/bazelbuild/bazel/pull/17666 +cat /dev/stdin > /dev/null + +# `GITHUB_TOKEN` is provided as a secret. +echo "{\"headers\":{\"Authorization\":[\"Bearer ${GITHUB_TOKEN}\"]}}" diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index 9063c97b5b10..93910375b755 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -28,8 +28,7 @@ load( _envoy_select_enable_http_datagrams = "envoy_select_enable_http_datagrams", _envoy_select_enable_yaml = "envoy_select_enable_yaml", _envoy_select_envoy_mobile_listener = "envoy_select_envoy_mobile_listener", - _envoy_select_envoy_mobile_request_compression = "envoy_select_envoy_mobile_request_compression", - _envoy_select_envoy_mobile_stats_reporting = "envoy_select_envoy_mobile_stats_reporting", + _envoy_select_envoy_mobile_xds = "envoy_select_envoy_mobile_xds", _envoy_select_google_grpc = "envoy_select_google_grpc", _envoy_select_hot_restart = "envoy_select_hot_restart", _envoy_select_signal_trace = "envoy_select_signal_trace", @@ -62,7 +61,6 @@ load( "@envoy_build_config//:extensions_build_config.bzl", "CONTRIB_EXTENSION_PACKAGE_VISIBILITY", "EXTENSION_PACKAGE_VISIBILITY", - "MOBILE_PACKAGE_VISIBILITY", ) load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") @@ -82,10 +80,8 @@ def envoy_extension_package(enabled_default = True, default_visibility = EXTENSI flag_values = {":enabled": "True"}, ) -def envoy_mobile_package(): - # Mobile packages should only be visible to other mobile packages, not any other - # parts of the Envoy codebase. - envoy_extension_package(default_visibility = MOBILE_PACKAGE_VISIBILITY) +def envoy_mobile_package(default_visibility = ["//visibility:public"]): + envoy_extension_package(default_visibility = default_visibility) def envoy_contrib_package(): envoy_extension_package(default_visibility = CONTRIB_EXTENSION_PACKAGE_VISIBILITY) @@ -237,9 +233,8 @@ envoy_select_admin_html = _envoy_select_admin_html envoy_select_admin_no_html = _envoy_select_admin_no_html envoy_select_admin_functionality = _envoy_select_admin_functionality envoy_select_static_extension_registration = _envoy_select_static_extension_registration -envoy_select_envoy_mobile_request_compression = _envoy_select_envoy_mobile_request_compression -envoy_select_envoy_mobile_stats_reporting = _envoy_select_envoy_mobile_stats_reporting envoy_select_envoy_mobile_listener = _envoy_select_envoy_mobile_listener +envoy_select_envoy_mobile_xds = _envoy_select_envoy_mobile_xds envoy_select_boringssl = _envoy_select_boringssl envoy_select_disable_logging = _envoy_select_disable_logging envoy_select_google_grpc = _envoy_select_google_grpc diff --git a/bazel/envoy_mobile_defines.bzl b/bazel/envoy_mobile_defines.bzl index 9c2ef5671719..07f5be337178 100644 --- a/bazel/envoy_mobile_defines.bzl +++ b/bazel/envoy_mobile_defines.bzl @@ -8,8 +8,7 @@ load( "envoy_select_enable_http_datagrams", "envoy_select_enable_yaml", "envoy_select_envoy_mobile_listener", - "envoy_select_envoy_mobile_request_compression", - "envoy_select_envoy_mobile_stats_reporting", + "envoy_select_envoy_mobile_xds", "envoy_select_google_grpc", ) @@ -22,6 +21,5 @@ def envoy_mobile_defines(repository): envoy_select_disable_exceptions(["ENVOY_DISABLE_EXCEPTIONS"], repository) + \ envoy_select_enable_http_datagrams(["ENVOY_ENABLE_HTTP_DATAGRAMS"], repository) + \ envoy_select_envoy_mobile_listener(["ENVOY_MOBILE_ENABLE_LISTENER"], repository) + \ - envoy_select_envoy_mobile_stats_reporting(["ENVOY_MOBILE_STATS_REPORTING"], repository) + \ - envoy_select_envoy_mobile_request_compression(["ENVOY_MOBILE_REQUEST_COMPRESSION"], repository) + \ + envoy_select_envoy_mobile_xds(["ENVOY_MOBILE_XDS"], repository) + \ envoy_select_google_grpc(["ENVOY_GOOGLE_GRPC"], repository) diff --git a/bazel/envoy_select.bzl b/bazel/envoy_select.bzl index 6138ef9aa1dd..87d5c71eefa2 100644 --- a/bazel/envoy_select.bzl +++ b/bazel/envoy_select.bzl @@ -59,24 +59,17 @@ def envoy_select_static_extension_registration(xs, repository = ""): "//conditions:default": xs, }) -# Selects the given values if Envoy Mobile request compression is enabled in the current build. -def envoy_select_envoy_mobile_request_compression(xs, repository = ""): - return select({ - repository + "//bazel:disable_envoy_mobile_request_compression": [], - "//conditions:default": xs, - }) - -# Selects the given values if Envoy Mobile stats reporting is enabled in the current build. -def envoy_select_envoy_mobile_stats_reporting(xs, repository = ""): +# Selects the given values if the Envoy Mobile listener is enabled in the current build. +def envoy_select_envoy_mobile_listener(xs, repository = ""): return select({ - repository + "//bazel:disable_envoy_mobile_stats_reporting": [], + repository + "//bazel:disable_envoy_mobile_listener": [], "//conditions:default": xs, }) -# Selects the given values if the Envoy Mobile listener is enabled in the current build. -def envoy_select_envoy_mobile_listener(xs, repository = ""): +# Selects the given values if Envoy Mobile xDS is enabled in the current build. +def envoy_select_envoy_mobile_xds(xs, repository = ""): return select({ - repository + "//bazel:disable_envoy_mobile_listener": [], + repository + "//bazel:disable_envoy_mobile_xds": [], "//conditions:default": xs, }) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 95b2503b0f48..b377dc5ec42a 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -6,8 +6,11 @@ load( ) load( "@envoy//bazel/external:quiche.bzl", + "envoy_quic_cc_library", + "envoy_quic_cc_test_library", "envoy_quiche_platform_impl_cc_library", "envoy_quiche_platform_impl_cc_test_library", + "quiche_copts", ) load("@rules_proto//proto:defs.bzl", "proto_library") @@ -38,21 +41,6 @@ src_files = glob([ "**/*.proto", ]) -# These options are only used to suppress errors in brought-in QUICHE tests. -# Use #pragma GCC diagnostic ignored in integration code to suppress these errors. -quiche_common_copts = [ - # hpack_huffman_decoder.cc overloads operator<<. - "-Wno-unused-function", - "-Wno-old-style-cast", -] - -quiche_copts = select({ - # Ignore unguarded #pragma GCC statements in QUICHE sources - "@envoy//bazel:windows_x86_64": ["-wd4068"], - # Remove these after upstream fix. - "//conditions:default": quiche_common_copts, -}) - test_suite( name = "ci_tests", tests = [ @@ -75,7 +63,7 @@ test_suite( "quiche_balsa_header_properties_test", "quiche_balsa_simple_buffer_test", "quiche_common_test", - "spdy_core_http2_header_block_test", + "quiche_http_header_block_test", ], ) @@ -1330,17 +1318,6 @@ envoy_cc_library( deps = [":quiche_common_platform"], ) -envoy_cc_library( - name = "spdy_simple_arena_lib", - srcs = ["quiche/spdy/core/spdy_simple_arena.cc"], - hdrs = ["quiche/spdy/core/spdy_simple_arena.h"], - repository = "@envoy", - visibility = ["//visibility:public"], - deps = [ - ":quiche_common_platform", - ], -) - envoy_cc_library( name = "spdy_no_op_headers_handler_lib", hdrs = ["quiche/spdy/core/no_op_headers_handler.h"], @@ -1396,7 +1373,6 @@ envoy_cc_library( envoy_cc_library( name = "spdy_core_http2_header_block_lib", - srcs = ["quiche/spdy/core/http2_header_block.cc"], hdrs = ["quiche/spdy/core/http2_header_block.h"], copts = quiche_copts, repository = "@envoy", @@ -1405,19 +1381,7 @@ envoy_cc_library( ":quiche_common_lib", ":quiche_common_platform", ":quiche_common_text_utils_lib", - ":spdy_core_http2_header_storage_lib", - ], -) - -envoy_cc_library( - name = "spdy_core_http2_header_storage_lib", - srcs = ["quiche/spdy/core/http2_header_storage.cc"], - hdrs = ["quiche/spdy/core/http2_header_storage.h"], - copts = quiche_copts, - repository = "@envoy", - deps = [ - "spdy_simple_arena_lib", - ":quiche_common_platform", + ":quiche_http_header_block_lib", ], ) @@ -1894,13 +1858,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_alarm_lib", srcs = ["quiche/quic/core/quic_alarm.cc"], hdrs = ["quiche/quic/core/quic_alarm.h"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_arena_scoped_ptr_lib", ":quic_core_connection_context_lib", @@ -1908,12 +1869,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_alarm_factory_lib", hdrs = ["quiche/quic/core/quic_alarm_factory.h"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_lib", ":quic_core_one_block_arena_lib", @@ -2055,24 +2013,20 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_blocked_writer_interface_lib", hdrs = ["quiche/quic/core/quic_blocked_writer_interface.h"], - repository = "@envoy", tags = ["nofips"], deps = [":quic_platform_export"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_arena_scoped_ptr_lib", hdrs = ["quiche/quic/core/quic_arena_scoped_ptr.h"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_chaos_protector_lib", srcs = [ "quiche/quic/core/quic_chaos_protector.cc", @@ -2080,8 +2034,6 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/quic_chaos_protector.h", ], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_random_lib", ":quic_core_data_lib", @@ -2094,13 +2046,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_clock_lib", hdrs = ["quiche/quic/core/quic_clock.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_time_lib", ":quic_platform_base", @@ -2118,14 +2066,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_config_lib", srcs = ["quiche/quic/core/quic_config.cc"], hdrs = ["quiche/quic/core/quic_config.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", ":quic_core_crypto_crypto_handshake_lib", @@ -2138,13 +2082,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_bandwidth_sampler_lib", srcs = ["quiche/quic/core/congestion_control/bandwidth_sampler.cc"], hdrs = ["quiche/quic/core/congestion_control/bandwidth_sampler.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_congestion_control_congestion_control_interface_lib", @@ -2157,14 +2098,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_bbr_lib", srcs = ["quiche/quic/core/congestion_control/bbr_sender.cc"], hdrs = ["quiche/quic/core/congestion_control/bbr_sender.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_congestion_control_bandwidth_sampler_lib", @@ -2180,7 +2117,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_bbr2_lib", srcs = [ "quiche/quic/core/congestion_control/bbr2_drain.cc", @@ -2198,9 +2135,6 @@ envoy_cc_library( "quiche/quic/core/congestion_control/bbr2_sender.h", "quiche/quic/core/congestion_control/bbr2_startup.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_congestion_control_bandwidth_sampler_lib", @@ -2216,13 +2150,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_general_loss_algorithm_lib", srcs = ["quiche/quic/core/congestion_control/general_loss_algorithm.cc"], hdrs = ["quiche/quic/core/congestion_control/general_loss_algorithm.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_congestion_control_congestion_control_interface_lib", ":quic_core_congestion_control_rtt_stats_lib", @@ -2233,15 +2164,12 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_congestion_control_interface_lib", hdrs = [ "quiche/quic/core/congestion_control/loss_detection_interface.h", "quiche/quic/core/congestion_control/send_algorithm_interface.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_clock_lib", @@ -2256,7 +2184,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_congestion_control_lib", srcs = [ "quiche/quic/core/congestion_control/send_algorithm_interface.cc", @@ -2265,9 +2193,6 @@ envoy_cc_library( "quiche/quic/core/congestion_control/loss_detection_interface.h", "quiche/quic/core/congestion_control/send_algorithm_interface.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_config_lib", @@ -2284,13 +2209,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_pacing_sender_lib", srcs = ["quiche/quic/core/congestion_control/pacing_sender.cc"], hdrs = ["quiche/quic/core/congestion_control/pacing_sender.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_config_lib", @@ -2301,13 +2223,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_rtt_stats_lib", srcs = ["quiche/quic/core/congestion_control/rtt_stats.cc"], hdrs = ["quiche/quic/core/congestion_control/rtt_stats.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_core_time_lib", @@ -2315,7 +2234,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_tcp_cubic_helper", srcs = [ "quiche/quic/core/congestion_control/hybrid_slow_start.cc", @@ -2325,9 +2244,6 @@ envoy_cc_library( "quiche/quic/core/congestion_control/hybrid_slow_start.h", "quiche/quic/core/congestion_control/prr_sender.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_packets_lib", @@ -2337,7 +2253,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_tcp_cubic_bytes_lib", srcs = [ "quiche/quic/core/congestion_control/cubic_bytes.cc", @@ -2347,9 +2263,6 @@ envoy_cc_library( "quiche/quic/core/congestion_control/cubic_bytes.h", "quiche/quic/core/congestion_control/tcp_cubic_sender_bytes.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_congestion_control_congestion_control_interface_lib", @@ -2364,26 +2277,20 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_uber_loss_algorithm_lib", srcs = ["quiche/quic/core/congestion_control/uber_loss_algorithm.cc"], hdrs = ["quiche/quic/core/congestion_control/uber_loss_algorithm.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_core_congestion_control_general_loss_algorithm_lib"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_congestion_control_windowed_filter_lib", hdrs = ["quiche/quic/core/congestion_control/windowed_filter.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_core_time_lib"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_connection_context_lib", srcs = [ "quiche/quic/core/quic_connection_context.cc", @@ -2391,12 +2298,9 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/quic_connection_context.h", ], - copts = quiche_copts, external_deps = [ "abseil_str_format", ], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_export", ":quiche_common_platform", @@ -2404,13 +2308,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_connection_id_manager", srcs = ["quiche/quic/core/quic_connection_id_manager.cc"], hdrs = ["quiche/quic/core/quic_connection_id_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -2424,27 +2325,19 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_connection_id_generator_interface_lib", hdrs = ["quiche/quic/core/connection_id_generator.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_types_lib", ":quic_core_versions_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_deterministic_connection_id_generator_lib", srcs = ["quiche/quic/core/deterministic_connection_id_generator.cc"], hdrs = ["quiche/quic/core/deterministic_connection_id_generator.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_connection_id_generator_interface_lib", ":quic_core_utils_lib", @@ -2452,14 +2345,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_connection_lib", srcs = ["quiche/quic/core/quic_connection.cc"], hdrs = ["quiche/quic/core/quic_connection.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -2492,13 +2381,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_connection_stats_lib", srcs = ["quiche/quic/core/quic_connection_stats.cc"], hdrs = ["quiche/quic/core/quic_connection_stats.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_packets_lib", @@ -2542,7 +2428,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_crypto_handshake_lib", srcs = [ "quiche/quic/core/crypto/cert_compressor.cc", @@ -2574,17 +2460,13 @@ envoy_cc_library( "quiche/quic/core/crypto/quic_compressed_certs_cache.h", "quiche/quic/core/crypto/transport_parameters.h", ], - copts = quiche_copts, external_deps = [ "ssl", "zlib", ], - repository = "@envoy", tags = [ - "nofips", "pg3", ], - visibility = ["//visibility:public"], deps = [ ":quic_core_clock_lib", ":quic_core_connection_context_lib", @@ -2610,7 +2492,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_client_crypto_crypto_handshake_lib", srcs = [ "quiche/quic/core/crypto/quic_client_session_cache.cc", @@ -2620,17 +2502,12 @@ envoy_cc_library( "quiche/quic/core/crypto/quic_client_session_cache.h", "quiche/quic/core/crypto/quic_crypto_client_config.h", ], - copts = quiche_copts, external_deps = [ - "ssl", "zlib", ], - repository = "@envoy", tags = [ - "nofips", "pg3", ], - visibility = ["//visibility:public"], deps = [ ":quic_client_crypto_tls_handshake_lib", ":quic_core_crypto_client_proof_source_lib", @@ -2639,7 +2516,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_server_crypto_crypto_handshake_lib", srcs = [ "quiche/quic/core/crypto/quic_crypto_server_config.cc", @@ -2647,17 +2524,13 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/crypto/quic_crypto_server_config.h", ], - copts = quiche_copts, external_deps = [ "ssl", "zlib", ], - repository = "@envoy", tags = [ - "nofips", "pg3", ], - visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_crypto_handshake_lib", ":quic_core_proto_crypto_server_config_proto_header", @@ -2666,28 +2539,21 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_boring_utils_lib", hdrs = ["quiche/quic/core/crypto/boring_utils.h"], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_export", ":quiche_common_platform", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_certificate_view_lib", srcs = ["quiche/quic/core/crypto/certificate_view.cc"], hdrs = ["quiche/quic/core/crypto/certificate_view.h"], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_boring_utils_lib", ":quic_core_types_lib", @@ -2698,7 +2564,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_encryption_lib", srcs = [ "quiche/quic/core/crypto/aead_base_decrypter.cc", @@ -2747,11 +2613,7 @@ envoy_cc_library( "quiche/quic/core/crypto/quic_decrypter.h", "quiche/quic/core/crypto/quic_encrypter.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_hkdf_lib", ":quic_core_data_lib", @@ -2764,19 +2626,16 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_hkdf_lib", srcs = ["quiche/quic/core/crypto/quic_hkdf.cc"], hdrs = ["quiche/quic/core/crypto/quic_hkdf.h"], - external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_base", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_proof_source_lib", srcs = [ "quiche/quic/core/crypto/proof_source.cc", @@ -2786,10 +2645,6 @@ envoy_cc_library( "quiche/quic/core/crypto/proof_source.h", "quiche/quic/core/crypto/quic_crypto_proof.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_certificate_view_lib", ":quic_core_packets_lib", @@ -2799,18 +2654,14 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_proof_source_x509_lib", srcs = ["quiche/quic/core/crypto/proof_source_x509.cc"], hdrs = ["quiche/quic/core/crypto/proof_source_x509.h"], - copts = quiche_copts, external_deps = [ "ssl", "abseil_node_hash_map", ], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_certificate_view_lib", ":quic_core_crypto_crypto_handshake_lib", @@ -2823,7 +2674,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_client_proof_source_lib", srcs = [ "quiche/quic/core/crypto/client_proof_source.cc", @@ -2831,10 +2682,6 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/crypto/client_proof_source.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_proof_source_lib", ":quic_platform_base", @@ -2852,7 +2699,7 @@ envoy_cc_library( deps = [":quiche_common_random_lib"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_crypto_tls_handshake_lib", srcs = [ "quiche/quic/core/crypto/tls_connection.cc", @@ -2860,10 +2707,7 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/crypto/tls_connection.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_proof_source_lib", ":quic_core_types_lib", @@ -2871,7 +2715,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_server_crypto_tls_handshake_lib", srcs = [ "quiche/quic/core/crypto/tls_server_connection.cc", @@ -2879,10 +2723,7 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/crypto/tls_server_connection.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_proof_source_lib", ":quic_core_crypto_tls_handshake_lib", @@ -2891,7 +2732,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_client_crypto_tls_handshake_lib", srcs = [ "quiche/quic/core/crypto/tls_client_connection.cc", @@ -2899,10 +2740,7 @@ envoy_cc_library( hdrs = [ "quiche/quic/core/crypto/tls_client_connection.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_tls_handshake_lib", ], @@ -2926,7 +2764,6 @@ envoy_cc_library( ":quiche_common_platform_iovec", ":quiche_common_platform_logging", ":quiche_common_platform_prefetch", - "@envoy//source/common/quic/platform:quiche_logging_impl_lib", ], ) @@ -2988,6 +2825,7 @@ envoy_cc_library( copts = quiche_copts, repository = "@envoy", tags = ["nofips"], + visibility = ["//visibility:public"], deps = [ ":quiche_common_platform_export", "@com_google_absl//absl/functional:any_invocable", @@ -2995,7 +2833,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_data_lib", srcs = [ "quiche/quic/core/quic_data_reader.cc", @@ -3005,10 +2843,6 @@ envoy_cc_library( "quiche/quic/core/quic_data_reader.h", "quiche/quic/core/quic_data_writer.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", ":quic_core_crypto_random_lib", @@ -3051,13 +2885,10 @@ envoy_cc_library( visibility = ["//visibility:public"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_framer_lib", srcs = ["quiche/quic/core/quic_framer.cc"], hdrs = ["quiche/quic/core/quic_framer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_connection_id_generator_interface_lib", ":quic_core_constants_lib", @@ -3153,12 +2984,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_http_constants_lib", srcs = ["quiche/quic/core/http/http_constants.cc"], hdrs = ["quiche/quic/core/http/http_constants.h"], - copts = quiche_copts, - repository = "@envoy", deps = [":quic_core_types_lib"], ) @@ -3168,12 +2997,14 @@ envoy_cc_library( hdrs = ["quiche/common/capsule.h"], copts = quiche_copts, repository = "@envoy", + visibility = ["//visibility:public"], deps = [ ":quic_core_data_lib", ":quic_core_http_http_frames_lib", ":quic_core_types_lib", ":quic_platform_base", ":quiche_common_buffer_allocator_lib", + ":quiche_common_ip_address", ":quiche_common_wire_serialization", ":quiche_web_transport_web_transport_lib", ], @@ -3187,7 +3018,7 @@ envoy_cc_library( repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_core_data_lib", + ":quiche_common_lib", ":quiche_common_platform_bug_tracker", ":quiche_common_platform_logging", "@com_google_absl//absl/strings", @@ -3207,26 +3038,18 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_client_lib", srcs = [ - "quiche/quic/core/http/quic_client_promised_info.cc", - "quiche/quic/core/http/quic_client_push_promise_index.cc", "quiche/quic/core/http/quic_spdy_client_session.cc", "quiche/quic/core/http/quic_spdy_client_session_base.cc", "quiche/quic/core/http/quic_spdy_client_stream.cc", ], hdrs = [ - "quiche/quic/core/http/quic_client_promised_info.h", - "quiche/quic/core/http/quic_client_push_promise_index.h", "quiche/quic/core/http/quic_spdy_client_session.h", "quiche/quic/core/http/quic_spdy_client_session_base.h", "quiche/quic/core/http/quic_spdy_client_stream.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_client_session_lib", ":quic_core_alarm_lib", @@ -3241,18 +3064,13 @@ envoy_cc_library( ":quic_platform_base", ":spdy_core_framer_lib", ":spdy_core_protocol_lib", - "@envoy//source/common/quic:spdy_server_push_utils_for_envoy_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_header_list_lib", srcs = ["quiche/quic/core/http/quic_header_list.cc"], hdrs = ["quiche/quic/core/http/quic_header_list.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_packets_lib", ":quic_core_qpack_qpack_header_table_lib", @@ -3264,13 +3082,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_http_decoder_lib", srcs = ["quiche/quic/core/http/http_decoder.cc"], hdrs = ["quiche/quic/core/http/http_decoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":http2_constants_lib", ":quic_core_data_lib", @@ -3282,13 +3097,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_http_encoder_lib", srcs = ["quiche/quic/core/http/http_encoder.cc"], hdrs = ["quiche/quic/core/http/http_encoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_data_lib", ":quic_core_error_codes_lib", @@ -3298,12 +3110,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_http_frames_lib", hdrs = ["quiche/quic/core/http/http_frames.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_http_http_constants_lib", ":quic_core_types_lib", @@ -3312,33 +3121,17 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_server_initiated_spdy_stream_lib", srcs = ["quiche/quic/core/http/quic_server_initiated_spdy_stream.cc"], hdrs = ["quiche/quic/core/http/quic_server_initiated_spdy_stream.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_http_spdy_session_lib", ":quic_core_types_lib", ], ) -envoy_cc_library( - name = "quic_core_http_spdy_server_push_utils_header", - hdrs = ["quiche/quic/core/http/spdy_server_push_utils.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], - deps = [ - ":quic_platform_base", - ":spdy_core_http2_header_block_lib", - ], -) - -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_spdy_session_lib", srcs = [ "quiche/quic/core/http/quic_headers_stream.cc", @@ -3360,10 +3153,6 @@ envoy_cc_library( "quiche/quic/core/http/web_transport_stream_adapter.h", "quiche/quic/core/web_transport_stats.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_connection_lib", ":quic_core_crypto_crypto_handshake_lib", @@ -3395,7 +3184,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_server_http_spdy_session_lib", srcs = [ "quiche/quic/core/http/quic_server_session_base.cc", @@ -3405,23 +3194,16 @@ envoy_cc_library( "quiche/quic/core/http/quic_server_session_base.h", "quiche/quic/core/http/quic_spdy_server_stream_base.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_http_spdy_session_lib", ":quic_server_session_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_spdy_stream_body_manager_lib", srcs = ["quiche/quic/core/http/quic_spdy_stream_body_manager.cc"], hdrs = ["quiche/quic/core/http/quic_spdy_stream_body_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_http_http_decoder_lib", ":quic_core_session_lib", @@ -3429,13 +3211,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_http_spdy_utils_lib", srcs = ["quiche/quic/core/http/spdy_utils.cc"], hdrs = ["quiche/quic/core/http/spdy_utils.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_http_header_list_lib", ":quic_core_http_http_constants_lib", @@ -3446,13 +3225,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_idle_network_detector_lib", srcs = ["quiche/quic/core/quic_idle_network_detector.cc"], hdrs = ["quiche/quic/core/quic_idle_network_detector.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -3475,12 +3251,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_interval_deque_lib", hdrs = ["quiche/quic/core/quic_interval_deque.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_interval_lib", ":quic_core_types_lib", @@ -3507,17 +3280,22 @@ envoy_cc_library( name = "quic_core_io_event_loop", hdrs = select({ "@envoy//bazel:windows_x86_64": [], + "@envoy//bazel:disable_http3": [], "//conditions:default": ["quiche/quic/core/io/quic_event_loop.h"], }), copts = quiche_copts, repository = "@envoy", tags = ["nofips"], - deps = [ - ":quic_core_alarm_factory_lib", - ":quic_core_clock_lib", - ":quic_core_udp_socket_lib", - "@com_google_absl//absl/base:core_headers", - ], + deps = select({ + "@envoy//bazel:windows_x86_64": [], + "@envoy//bazel:disable_http3": [], + "//conditions:default": [ + ":quic_core_alarm_factory_lib", + ":quic_core_clock_lib", + ":quic_core_udp_socket_lib", + "@com_google_absl//absl/base:core_headers", + ], + }), ) envoy_cc_library( @@ -3554,6 +3332,7 @@ envoy_cc_library( name = "quic_core_io_event_loop_socket_factory_lib", srcs = select({ "@envoy//bazel:windows_x86_64": [], + "@envoy//bazel:disable_http3": [], "//conditions:default": [ "quiche/quic/core/io/event_loop_connecting_client_socket.cc", "quiche/quic/core/io/event_loop_socket_factory.cc", @@ -3561,6 +3340,7 @@ envoy_cc_library( }), hdrs = select({ "@envoy//bazel:windows_x86_64": [], + "@envoy//bazel:disable_http3": [], "//conditions:default": [ "quiche/quic/core/io/event_loop_connecting_client_socket.h", "quiche/quic/core/io/event_loop_socket_factory.h", @@ -3569,46 +3349,43 @@ envoy_cc_library( copts = quiche_copts, repository = "@envoy", tags = ["nofips"], - deps = [ - ":quic_core_io_event_loop", - ":quic_core_io_socket_lib", - ":quic_core_types_lib", - ":quic_platform_socket_address", - ":quiche_common_buffer_allocator_lib", - ":quiche_common_platform", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - "@com_google_absl//absl/types:variant", - ], + deps = select({ + "@envoy//bazel:windows_x86_64": [], + "@envoy//bazel:disable_http3": [], + "//conditions:default": [ + ":quic_core_io_event_loop", + ":quic_core_io_socket_lib", + ":quic_core_types_lib", + ":quic_platform_socket_address", + ":quiche_common_buffer_allocator_lib", + ":quiche_common_platform", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_absl//absl/types:variant", + ], + }), ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_lru_cache_lib", hdrs = ["quiche/quic/core/quic_lru_cache.h"], - repository = "@envoy", - tags = ["nofips"], deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_mtu_discovery_lib", srcs = ["quiche/quic/core/quic_mtu_discovery.cc"], hdrs = ["quiche/quic/core/quic_mtu_discovery.h"], - copts = quiche_copts, - repository = "@envoy", deps = [ ":quic_core_constants_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_one_block_arena_lib", srcs = ["quiche/quic/core/quic_one_block_arena.h"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_arena_scoped_ptr_lib", ":quic_core_types_lib", @@ -3656,13 +3433,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_network_blackhole_detector_lib", srcs = ["quiche/quic/core/quic_network_blackhole_detector.cc"], hdrs = ["quiche/quic/core/quic_network_blackhole_detector.h"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -3673,13 +3447,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_packet_creator_lib", srcs = ["quiche/quic/core/quic_packet_creator.cc"], hdrs = ["quiche/quic/core/quic_packet_creator.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_chaos_protector_lib", ":quic_core_coalesced_packet_lib", @@ -3697,11 +3468,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_packet_number_indexed_queue_lib", hdrs = ["quiche/quic/core/packet_number_indexed_queue.h"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_constants_lib", ":quic_core_types_lib", @@ -3758,13 +3527,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_path_validator_lib", srcs = ["quiche/quic/core/quic_path_validator.cc"], hdrs = ["quiche/quic/core/quic_path_validator.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -3779,13 +3545,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_ping_manager_lib", srcs = ["quiche/quic/core/quic_ping_manager.cc"], hdrs = ["quiche/quic/core/quic_ping_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -3796,12 +3559,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_process_packet_interface_lib", hdrs = ["quiche/quic/core/quic_process_packet_interface.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_platform_base", @@ -3817,25 +3577,20 @@ envoy_cc_library( visibility = ["//visibility:public"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_blocking_manager_lib", srcs = ["quiche/quic/core/qpack/qpack_blocking_manager.cc"], hdrs = ["quiche/quic/core/qpack/qpack_blocking_manager.h"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_types_lib", ":quic_platform_base", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_decoder_lib", srcs = ["quiche/quic/core/qpack/qpack_decoder.cc"], hdrs = ["quiche/quic/core/qpack/qpack_decoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_decoder_stream_sender_lib", ":quic_core_qpack_qpack_encoder_stream_receiver_lib", @@ -3846,13 +3601,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_encoder_lib", srcs = ["quiche/quic/core/qpack/qpack_encoder.cc"], hdrs = ["quiche/quic/core/qpack/qpack_encoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_blocking_manager_lib", ":quic_core_qpack_qpack_decoder_stream_receiver_lib", @@ -3868,13 +3620,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_header_table_lib", srcs = ["quiche/quic/core/qpack/qpack_header_table.cc"], hdrs = ["quiche/quic/core/qpack/qpack_header_table.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_static_table_lib", ":quic_platform_base", @@ -3883,13 +3632,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_instruction_decoder_lib", srcs = ["quiche/quic/core/qpack/qpack_instruction_decoder.cc"], hdrs = ["quiche/quic/core/qpack/qpack_instruction_decoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":http2_hpack_huffman_hpack_huffman_decoder_lib", ":http2_hpack_varint_hpack_varint_decoder_lib", @@ -3898,23 +3644,17 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_instructions_lib", srcs = ["quiche/quic/core/qpack/qpack_instructions.cc"], hdrs = ["quiche/quic/core/qpack/qpack_instructions.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_instruction_encoder_lib", srcs = ["quiche/quic/core/qpack/qpack_instruction_encoder.cc"], hdrs = ["quiche/quic/core/qpack/qpack_instruction_encoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":http2_hpack_huffman_hpack_huffman_encoder_lib", ":http2_hpack_varint_hpack_varint_encoder_lib", @@ -3923,13 +3663,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_progressive_decoder_lib", srcs = ["quiche/quic/core/qpack/qpack_progressive_decoder.cc"], hdrs = ["quiche/quic/core/qpack/qpack_progressive_decoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_decoder_stream_sender_lib", ":quic_core_qpack_qpack_encoder_stream_receiver_lib", @@ -3943,23 +3680,17 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_required_insert_count_lib", srcs = ["quiche/quic/core/qpack/qpack_required_insert_count.cc"], hdrs = ["quiche/quic/core/qpack/qpack_required_insert_count.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_encoder_stream_sender_lib", srcs = ["quiche/quic/core/qpack/qpack_encoder_stream_sender.cc"], hdrs = ["quiche/quic/core/qpack/qpack_encoder_stream_sender.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_instruction_encoder_lib", ":quic_core_qpack_qpack_instructions_lib", @@ -3969,13 +3700,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_encoder_stream_receiver_lib", srcs = ["quiche/quic/core/qpack/qpack_encoder_stream_receiver.cc"], hdrs = ["quiche/quic/core/qpack/qpack_encoder_stream_receiver.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":http2_decoder_decode_buffer_lib", ":http2_decoder_decode_status_lib", @@ -3987,13 +3715,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_decoder_stream_sender_lib", srcs = ["quiche/quic/core/qpack/qpack_decoder_stream_sender.cc"], hdrs = ["quiche/quic/core/qpack/qpack_decoder_stream_sender.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_instruction_encoder_lib", ":quic_core_qpack_qpack_instructions_lib", @@ -4003,13 +3728,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_decoder_stream_receiver_lib", srcs = ["quiche/quic/core/qpack/qpack_decoder_stream_receiver.cc"], hdrs = ["quiche/quic/core/qpack/qpack_decoder_stream_receiver.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":http2_decoder_decode_buffer_lib", ":http2_decoder_decode_status_lib", @@ -4021,41 +3743,33 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_index_conversions_lib", srcs = ["quiche/quic/core/qpack/qpack_index_conversions.cc"], hdrs = ["quiche/quic/core/qpack/qpack_index_conversions.h"], - copts = quiche_copts, - repository = "@envoy", deps = [ ":quic_platform_base", ":quic_platform_export", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_static_table_lib", srcs = ["quiche/quic/core/qpack/qpack_static_table.cc"], hdrs = ["quiche/quic/core/qpack/qpack_static_table.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_base", ":spdy_core_hpack_hpack_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_stream_receiver_lib", hdrs = ["quiche/quic/core/qpack/qpack_stream_receiver.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_streams_lib", srcs = [ "quiche/quic/core/qpack/qpack_receive_stream.cc", @@ -4065,8 +3779,6 @@ envoy_cc_library( "quiche/quic/core/qpack/qpack_receive_stream.h", "quiche/quic/core/qpack/qpack_send_stream.h", ], - copts = quiche_copts, - repository = "@envoy", deps = [ ":quic_core_qpack_qpack_stream_receiver_lib", ":quic_core_qpack_qpack_stream_sender_delegate_lib", @@ -4074,13 +3786,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_decoded_headers_accumulator_lib", srcs = ["quiche/quic/core/qpack/qpack_decoded_headers_accumulator.cc"], hdrs = ["quiche/quic/core/qpack/qpack_decoded_headers_accumulator.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_http_header_list_lib", ":quic_core_qpack_qpack_decoder_lib", @@ -4090,35 +3799,26 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_value_splitting_header_list_lib", srcs = ["quiche/quic/core/qpack/value_splitting_header_list.cc"], hdrs = ["quiche/quic/core/qpack/value_splitting_header_list.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_base", ":spdy_core_http2_header_block_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_qpack_qpack_stream_sender_delegate_lib", hdrs = ["quiche/quic/core/qpack/qpack_stream_sender_delegate.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_received_packet_manager_lib", srcs = ["quiche/quic/core/quic_received_packet_manager.cc"], hdrs = ["quiche/quic/core/quic_received_packet_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_config_lib", ":quic_core_congestion_control_rtt_stats_lib", @@ -4130,13 +3830,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_sent_packet_manager_lib", srcs = ["quiche/quic/core/quic_sent_packet_manager.cc"], hdrs = ["quiche/quic/core/quic_sent_packet_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_congestion_control_congestion_control_lib", ":quic_core_congestion_control_general_loss_algorithm_lib", @@ -4157,12 +3854,9 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_web_transport_interface_lib", hdrs = ["quiche/quic/core/web_transport_interface.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_session_lib", ":quic_core_types_lib", @@ -4171,19 +3865,17 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_server_id_lib", srcs = ["quiche/quic/core/quic_server_id.cc"], hdrs = ["quiche/quic/core/quic_server_id.h"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_base", - "@com_googlesource_googleurl//url", + ":quiche_common_platform_googleurl", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_server_lib", srcs = [ "quiche/quic/core/chlo_extractor.cc", @@ -4197,10 +3889,6 @@ envoy_cc_library( "quiche/quic/core/quic_dispatcher.h", "quiche/quic/core/tls_chlo_extractor.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_factory_lib", ":quic_core_alarm_lib", @@ -4247,7 +3935,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_session_lib", srcs = [ "quiche/quic/core/legacy_quic_stream_id_manager.cc", @@ -4281,11 +3969,7 @@ envoy_cc_library( "quiche/quic/core/tls_handshaker.h", "quiche/quic/core/uber_quic_stream_id_manager.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_client_crypto_crypto_handshake_lib", ":quic_core_config_lib", @@ -4318,7 +4002,7 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_client_session_lib", srcs = [ "quiche/quic/core/quic_crypto_client_handshaker.cc", @@ -4330,18 +4014,14 @@ envoy_cc_library( "quiche/quic/core/quic_crypto_client_stream.h", "quiche/quic/core/tls_client_handshaker.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_client_crypto_crypto_handshake_lib", ":quic_core_session_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_server_session_lib", srcs = [ "quiche/quic/core/quic_crypto_server_stream.cc", @@ -4353,11 +4033,7 @@ envoy_cc_library( "quiche/quic/core/quic_crypto_server_stream_base.h", "quiche/quic/core/tls_server_handshaker.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_session_lib", ":quic_server_crypto_crypto_handshake_lib", @@ -4365,45 +4041,35 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_session_notifier_interface_lib", hdrs = ["quiche/quic/core/session_notifier_interface.h"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_frames_frames_lib", ":quic_core_time_lib", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_socket_address_coder_lib", srcs = ["quiche/quic/core/quic_socket_address_coder.cc"], hdrs = ["quiche/quic/core/quic_socket_address_coder.h"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_base", ":quic_platform_socket_address", ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_stream_frame_data_producer_lib", hdrs = ["quiche/quic/core/quic_stream_frame_data_producer.h"], - repository = "@envoy", - tags = ["nofips"], deps = [":quic_core_types_lib"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_stream_send_buffer_lib", srcs = ["quiche/quic/core/quic_stream_send_buffer.cc"], hdrs = ["quiche/quic/core/quic_stream_send_buffer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_data_lib", ":quic_core_frames_frames_lib", @@ -4417,13 +4083,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_stream_sequencer_buffer_lib", srcs = ["quiche/quic/core/quic_stream_sequencer_buffer.cc"], hdrs = ["quiche/quic/core/quic_stream_sequencer_buffer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_constants_lib", ":quic_core_interval_lib", @@ -4434,13 +4097,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_sustained_bandwidth_recorder_lib", srcs = ["quiche/quic/core/quic_sustained_bandwidth_recorder.cc"], hdrs = ["quiche/quic/core/quic_sustained_bandwidth_recorder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_time_lib", @@ -4473,22 +4133,16 @@ envoy_cc_library( deps = [":quic_platform_base"], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_time_accumulator_lib", hdrs = ["quiche/quic/core/quic_time_accumulator.h"], - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_time_wait_list_manager_lib", srcs = ["quiche/quic/core/quic_time_wait_list_manager.cc"], hdrs = ["quiche/quic/core/quic_time_wait_list_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_blocked_writer_interface_lib", ":quic_core_crypto_encryption_lib", @@ -4503,13 +4157,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_transmission_info_lib", srcs = ["quiche/quic/core/quic_transmission_info.cc"], hdrs = ["quiche/quic/core/quic_transmission_info.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_ack_listener_interface_lib", ":quic_core_frames_frames_lib", @@ -4546,13 +4197,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_uber_received_packet_manager_lib", srcs = ["quiche/quic/core/uber_received_packet_manager.cc"], hdrs = ["quiche/quic/core/uber_received_packet_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_received_packet_manager_lib", ":quic_core_utils_lib", @@ -4590,13 +4238,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_unacked_packet_map_lib", srcs = ["quiche/quic/core/quic_unacked_packet_map.cc"], hdrs = ["quiche/quic/core/quic_unacked_packet_map.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_connection_stats_lib", ":quic_core_packets_lib", @@ -4629,13 +4274,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_core_version_manager_lib", srcs = ["quiche/quic/core/quic_version_manager.cc"], hdrs = ["quiche/quic/core/quic_version_manager.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_versions_lib", ":quic_platform_base", @@ -4660,13 +4302,10 @@ envoy_cc_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_config_peer_lib", srcs = ["quiche/quic/test_tools/quic_config_peer.cc"], hdrs = ["quiche/quic/test_tools/quic_config_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_config_lib", ":quic_core_packets_lib", @@ -4674,16 +4313,13 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_connection_id_manager_peer_lib", hdrs = ["quiche/quic/test_tools/quic_connection_id_manager_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_core_connection_id_manager"], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_crypto_server_config_peer_lib", srcs = [ "quiche/quic/test_tools/quic_crypto_server_config_peer.cc", @@ -4691,9 +4327,6 @@ envoy_cc_test_library( hdrs = [ "quiche/quic/test_tools/quic_crypto_server_config_peer.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_crypto_handshake_lib", ":quic_test_tools_mock_clock_lib", @@ -4703,7 +4336,7 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_first_flight_lib", srcs = [ "quiche/quic/test_tools/first_flight.cc", @@ -4711,9 +4344,6 @@ envoy_cc_test_library( hdrs = [ "quiche/quic/test_tools/first_flight.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_config_lib", ":quic_core_connection_lib", @@ -4728,7 +4358,7 @@ envoy_cc_test_library( ], ) -envoy_cc_library( +envoy_quic_cc_test_library( name = "quic_test_tools_flow_controller_peer_lib", srcs = [ "quiche/quic/test_tools/quic_flow_controller_peer.cc", @@ -4736,22 +4366,16 @@ envoy_cc_library( hdrs = [ "quiche/quic/test_tools/quic_flow_controller_peer.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_core_session_lib", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_framer_peer_lib", srcs = ["quiche/quic/test_tools/quic_framer_peer.cc"], hdrs = ["quiche/quic/test_tools/quic_framer_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_encryption_lib", ":quic_core_framer_lib", @@ -4760,51 +4384,39 @@ envoy_cc_test_library( ], ) -envoy_cc_library( +envoy_quic_cc_test_library( name = "quic_test_tools_interval_deque_peer_lib", hdrs = ["quiche/quic/test_tools/quic_interval_deque_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_interval_deque_lib", ":quic_core_interval_lib", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_mock_clock_lib", srcs = ["quiche/quic/test_tools/mock_clock.cc"], hdrs = ["quiche/quic/test_tools/mock_clock.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_clock_lib", ":quic_core_time_lib", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_mock_random_lib", srcs = ["quiche/quic/test_tools/mock_random.cc"], hdrs = ["quiche/quic/test_tools/mock_random.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_random_lib", ":quic_platform_test", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_mock_syscall_wrapper_lib", srcs = ["quiche/quic/test_tools/quic_mock_syscall_wrapper.cc"], hdrs = ["quiche/quic/test_tools/quic_mock_syscall_wrapper.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_syscall_wrapper_lib", ":quic_platform_base", @@ -4812,13 +4424,10 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_qpack_qpack_test_utils_lib", srcs = ["quiche/quic/test_tools/qpack/qpack_test_utils.cc"], hdrs = ["quiche/quic/test_tools/qpack/qpack_test_utils.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_encoder_lib", ":quic_core_qpack_qpack_stream_sender_delegate_lib", @@ -4826,13 +4435,10 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_sent_packet_manager_peer_lib", srcs = ["quiche/quic/test_tools/quic_sent_packet_manager_peer.cc"], hdrs = ["quiche/quic/test_tools/quic_sent_packet_manager_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_congestion_control_congestion_control_interface_lib", ":quic_core_packets_lib", @@ -4841,27 +4447,21 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_server_session_base_peer", hdrs = [ "quiche/quic/test_tools/quic_server_session_base_peer.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_utils_lib", ":quic_server_http_spdy_session_lib", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_simple_quic_framer_lib", srcs = ["quiche/quic/test_tools/simple_quic_framer.cc"], hdrs = ["quiche/quic/test_tools/simple_quic_framer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_crypto_encryption_lib", ":quic_core_framer_lib", @@ -4870,26 +4470,20 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_stream_send_buffer_peer_lib", srcs = ["quiche/quic/test_tools/quic_stream_send_buffer_peer.cc"], hdrs = ["quiche/quic/test_tools/quic_stream_send_buffer_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_stream_send_buffer_lib", ":quic_test_tools_interval_deque_peer_lib", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_stream_peer_lib", srcs = ["quiche/quic/test_tools/quic_stream_peer.cc"], hdrs = ["quiche/quic/test_tools/quic_stream_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_core_session_lib", @@ -4900,20 +4494,17 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_test_certificates_lib", srcs = ["quiche/quic/test_tools/test_certificates.cc"], hdrs = ["quiche/quic/test_tools/test_certificates.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_platform_base", ":quiche_common_platform", ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_test_utils_lib", srcs = [ "quiche/quic/test_tools/crypto_test_utils.cc", @@ -4934,10 +4525,7 @@ envoy_cc_test_library( "quiche/quic/test_tools/quic_dispatcher_peer.h", "quiche/quic/test_tools/quic_test_utils.h", ], - copts = quiche_copts, external_deps = ["ssl"], - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_client_session_lib", ":quic_core_congestion_control_congestion_control_interface_lib", @@ -4982,7 +4570,7 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_session_peer_lib", srcs = [ "quiche/quic/test_tools/quic_session_peer.cc", @@ -4990,9 +4578,6 @@ envoy_cc_test_library( hdrs = [ "quiche/quic/test_tools/quic_session_peer.h", ], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [ ":quic_client_session_lib", ":quic_core_packets_lib", @@ -5002,13 +4587,10 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( +envoy_quic_cc_test_library( name = "quic_test_tools_unacked_packet_map_peer_lib", srcs = ["quiche/quic/test_tools/quic_unacked_packet_map_peer.cc"], hdrs = ["quiche/quic/test_tools/quic_unacked_packet_map_peer.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], deps = [":quic_core_unacked_packet_map_lib"], ) @@ -5059,6 +4641,24 @@ envoy_quiche_platform_impl_cc_test_library( ], ) +envoy_cc_library( + name = "quiche_common_platform_googleurl", + hdrs = ["quiche/common/platform/api/quiche_googleurl.h"], + repository = "@envoy", + tags = ["nofips"], + deps = [":quiche_common_platform_default_quiche_platform_impl_googleurl_impl_lib"], +) + +envoy_quiche_platform_impl_cc_library( + name = "quiche_common_platform_default_quiche_platform_impl_googleurl_impl_lib", + hdrs = [ + "quiche/common/platform/default/quiche_platform_impl/quiche_googleurl_impl.h", + ], + deps = [ + "@com_googlesource_googleurl//url", + ], +) + envoy_cc_library( name = "quiche_common_platform_iovec", hdrs = [ @@ -5084,8 +4684,15 @@ envoy_cc_library( visibility = ["//visibility:public"], deps = [ ":quiche_common_platform_export", - "@envoy//source/common/quic/platform:quiche_logging_impl_lib", - ], + ] + select({ + "@platforms//os:android": [ + "@envoy//source/common/quic/platform/mobile_impl:mobile_quiche_bug_tracker_impl_lib", + ], + "@platforms//os:ios": [ + "@envoy//source/common/quic/platform/mobile_impl:mobile_quiche_bug_tracker_impl_lib", + ], + "//conditions:default": ["@envoy//source/common/quic/platform:quiche_logging_impl_lib"], + }), ) envoy_cc_library( @@ -5098,8 +4705,15 @@ envoy_cc_library( visibility = ["//visibility:public"], deps = [ ":quiche_common_platform_export", - "@envoy//source/common/quic/platform:quiche_logging_impl_lib", - ], + ] + select({ + "@platforms//os:android": [ + ":quiche_common_mobile_quiche_logging_lib", + ], + "@platforms//os:ios": [ + ":quiche_common_mobile_quiche_logging_lib", + ], + "//conditions:default": ["@envoy//source/common/quic/platform:quiche_logging_impl_lib"], + }), ) envoy_cc_library( @@ -5126,6 +4740,21 @@ envoy_quiche_platform_impl_cc_library( ], ) +envoy_quiche_platform_impl_cc_library( + name = "quiche_common_mobile_quiche_logging_lib", + srcs = [ + "quiche/common/platform/default/quiche_platform_impl/quiche_logging_impl.cc", + ], + hdrs = [ + "quiche/common/platform/default/quiche_platform_impl/quiche_logging_impl.h", + ], + deps = [ + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + ], +) + envoy_cc_library( name = "quiche_common_platform_mutex", srcs = [ @@ -5170,8 +4799,8 @@ envoy_cc_library( visibility = ["//visibility:public"], deps = [ ":quiche_common_platform_export", + ":quiche_common_platform_googleurl", ":quiche_common_platform_logging", - "@com_googlesource_googleurl//url", ], ) @@ -5340,7 +4969,7 @@ envoy_cc_library( ], ) -# Use the QUICHE default implmentation once the WIN32 compiler error is resolved. +# Use the QUICHE default implementation once the WIN32 compiler error is resolved. # envoy_quiche_platform_impl_cc_library( # name = "quiche_common_platform_default_quiche_platform_impl_export_impl_lib", # hdrs = [ @@ -5459,9 +5088,9 @@ envoy_cc_test_library( tags = ["nofips"], deps = [ ":quiche_common_platform", + ":quiche_common_platform_googleurl", ":quiche_common_platform_iovec", ":quiche_common_platform_test", - "@com_googlesource_googleurl//url", "@envoy//test/common/quic/platform:quiche_test_helpers_impl_lib", "@envoy//test/common/quic/platform:quiche_test_impl_lib", ], @@ -5502,6 +5131,58 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quiche_simple_arena_lib", + srcs = ["quiche/common/quiche_simple_arena.cc"], + hdrs = ["quiche/common/quiche_simple_arena.h"], + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quiche_common_platform_export", + ":quiche_common_platform_logging", + ], +) + +envoy_cc_library( + name = "quiche_http_header_storage_lib", + srcs = ["quiche/common/http/http_header_storage.cc"], + hdrs = ["quiche/common/http/http_header_storage.h"], + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quiche_common_platform_export", + ":quiche_common_platform_logging", + ":quiche_simple_arena_lib", + ], +) + +envoy_cc_library( + name = "quiche_http_header_block_lib", + srcs = ["quiche/common/http/http_header_block.cc"], + hdrs = ["quiche/common/http/http_header_block.h"], + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quiche_common_lib", + ":quiche_common_platform_export", + ":quiche_common_platform_logging", + ":quiche_common_text_utils_lib", + ":quiche_http_header_storage_lib", + ], +) + +envoy_cc_test( + name = "quiche_http_header_block_test", + srcs = ["quiche/common/http/http_header_block_test.cc"], + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quiche_common_platform_test", + ":quiche_http_header_block_lib", + ":spdy_test_tools_test_utils_lib", + ], +) + envoy_cc_library( name = "quiche_common_structured_headers_lib", srcs = ["quiche/common/structured_headers.cc"], @@ -5566,19 +5247,6 @@ envoy_cc_library( ], ) -envoy_cc_test( - name = "spdy_core_http2_header_block_test", - srcs = ["quiche/spdy/core/http2_header_block_test.cc"], - copts = quiche_copts, - coverage = False, - repository = "@envoy", - tags = ["nofips"], - deps = [ - ":spdy_core_http2_header_block_lib", - ":spdy_test_tools_test_utils_lib", - ], -) - envoy_cc_test( name = "quic_core_batch_writer_batch_writer_test", srcs = select({ @@ -5611,14 +5279,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_load_balancer_config_lib", srcs = ["quiche/quic/load_balancer/load_balancer_config.cc"], hdrs = ["quiche/quic/load_balancer/load_balancer_config.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_types_lib", ":quic_load_balancer_server_id_lib", @@ -5627,14 +5291,10 @@ envoy_cc_library( ], ) -envoy_cc_library( +envoy_quic_cc_library( name = "quic_load_balancer_encoder_lib", srcs = ["quiche/quic/load_balancer/load_balancer_encoder.cc"], hdrs = ["quiche/quic/load_balancer/load_balancer_encoder.h"], - copts = quiche_copts, - repository = "@envoy", - tags = ["nofips"], - visibility = ["//visibility:public"], deps = [ ":quic_core_connection_id_generator_interface_lib", ":quic_core_crypto_random_lib", diff --git a/bazel/external/quiche.bzl b/bazel/external/quiche.bzl index b3f18a8467d0..0868fb0eb887 100644 --- a/bazel/external/quiche.bzl +++ b/bazel/external/quiche.bzl @@ -3,6 +3,22 @@ load( "envoy_cc_library", "envoy_cc_test_library", ) +load("@envoy//bazel:envoy_select.bzl", "envoy_select_enable_http3") + +# These options are only used to suppress errors in brought-in QUICHE tests. +# Use #pragma GCC diagnostic ignored in integration code to suppress these errors. +quiche_common_copts = [ + # hpack_huffman_decoder.cc overloads operator<<. + "-Wno-unused-function", + "-Wno-old-style-cast", +] + +quiche_copts = select({ + # Ignore unguarded #pragma GCC statements in QUICHE sources + "@envoy//bazel:windows_x86_64": ["-wd4068"], + # Remove these after upstream fix. + "//conditions:default": quiche_common_copts, +}) def envoy_quiche_platform_impl_cc_library( name, @@ -34,3 +50,43 @@ def envoy_quiche_platform_impl_cc_test_library( strip_include_prefix = "quiche/common/platform/default/", tags = ["nofips"], ) + +# Used for QUIC libraries +def envoy_quic_cc_library( + name, + srcs = [], + hdrs = [], + deps = [], + defines = [], + external_deps = [], + tags = []): + envoy_cc_library( + name = name, + srcs = envoy_select_enable_http3(srcs, "@envoy"), + hdrs = envoy_select_enable_http3(hdrs, "@envoy"), + repository = "@envoy", + copts = quiche_copts, + tags = ["nofips"] + tags, + visibility = ["//visibility:public"], + defines = defines, + external_deps = external_deps, + deps = envoy_select_enable_http3(deps, "@envoy"), + ) + +def envoy_quic_cc_test_library( + name, + srcs = [], + hdrs = [], + tags = [], + external_deps = [], + deps = []): + envoy_cc_test_library( + name = name, + srcs = envoy_select_enable_http3(srcs, "@envoy"), + hdrs = envoy_select_enable_http3(hdrs, "@envoy"), + copts = quiche_copts, + repository = "@envoy", + tags = ["nofips"] + tags, + external_deps = external_deps, + deps = envoy_select_enable_http3(deps, "@envoy"), + ) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 4d2b98fd2eb0..eb9c9e81d2c4 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -1,4 +1,5 @@ load("@rules_foreign_cc//foreign_cc:configure.bzl", "configure_make") +load("@rules_foreign_cc//foreign_cc:make.bzl", "make") load("//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_cmake", "envoy_package") licenses(["notice"]) # Apache 2 @@ -59,13 +60,32 @@ cc_library( ], ) +make( + name = "lz4", + args = [ + "MOREFLAGS='-fPIC'", + "BUILD_SHARED=no", + ], + lib_source = "@com_github_lz4_lz4//:all", + out_static_libs = [ + "liblz4.a", + ], + tags = ["skip_on_windows"], + targets = [ + "lib", + "install", + ], + visibility = ["//visibility:public"], + alwayslink = False, +) + # Kafka client dependency used by Kafka-mesh filter. # librdkafka build generates extra headers that need to be copied into source to get it to compile. configure_make( name = "librdkafka_build", configure_in_place = True, - configure_options = ["--disable-ssl --disable-gssapi --disable-lz4-ext --disable-zstd --disable-curl && cp Makefile.config src/.. && cp config.h src/.."], - lib_source = "@edenhill_librdkafka//:all", + configure_options = ["--disable-ssl --disable-gssapi --disable-zstd --disable-curl && cp Makefile.config src/.. && cp config.h src/.."], + lib_source = "@confluentinc_librdkafka//:all", out_static_libs = [ "librdkafka.a", "librdkafka++.a", @@ -74,6 +94,7 @@ configure_make( targets = [ "ARFLAGS='' libs install-subdirs", ], + deps = [":lz4"], alwayslink = True, ) @@ -119,7 +140,7 @@ configure_make( # Workaround for the issue with statically linked libstdc++ # using -l:libstdc++.a. env = { - "CXXFLAGS": "-lstdc++ -Wno-unused-command-line-argument", + "CXXFLAGS": "--static -lstdc++ -Wno-unused-command-line-argument", }, lib_source = "@net_colm_open_source_colm//:all", out_binaries = ["colm"], @@ -141,7 +162,7 @@ configure_make( # Workaround for the issue with statically linked libstdc++ # using -l:libstdc++.a. env = { - "CXXFLAGS": "-lstdc++ -Wno-unused-command-line-argument", + "CXXFLAGS": "--static -lstdc++ -Wno-unused-command-line-argument", }, lib_source = "@net_colm_open_source_ragel//:all", out_binaries = ["ragel"], @@ -579,3 +600,27 @@ envoy_cmake( }), working_directory = "build/cmake", ) + +envoy_cmake( + name = "maxmind", + cache_entries = { + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_INSTALL_LIBDIR": "lib", + "CMAKE_CXX_COMPILER_FORCED": "on", + "BUILD_SHARED_LIBS": "off", + "BUILD_TESTING": "off", + }, + defines = ["MAXMIND_STATICLIB"], + lib_source = "@com_github_maxmind_libmaxminddb//:all", + out_static_libs = ["libmaxminddb.a"], + tags = ["skip_on_windows"], +) + +envoy_cc_library( + name = "maxmind_linux", + srcs = [], + deps = select({ + "//bazel:linux": [":maxmind"], + "//conditions:default": [], + }), +) diff --git a/bazel/foreign_cc/ipp-crypto-bn2lebinpad.patch b/bazel/foreign_cc/ipp-crypto-bn2lebinpad.patch new file mode 100644 index 000000000000..da1546a1afcd --- /dev/null +++ b/bazel/foreign_cc/ipp-crypto-bn2lebinpad.patch @@ -0,0 +1,17 @@ +diff --git a/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c b/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c +index 1099518..7526fdc 100644 +--- a/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c ++++ b/sources/ippcp/crypto_mb/src/common/ifma_cvt52.c +@@ -168,12 +168,6 @@ __INLINE void transform_8sb_to_mb8(U64 out_mb8[], int bitLen, int8u *inp[8], int + } + } + +-#ifdef OPENSSL_IS_BORINGSSL +-static int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen) { +- return BN_bn2le_padded(to, tolen, a); +-} +-#endif +- + #ifndef BN_OPENSSL_DISABLE + // Convert BIGNUM into MB8(Radix=2^52) format + // Returns bitmask of succesfully converted values diff --git a/bazel/foreign_cc/vectorscan.patch b/bazel/foreign_cc/vectorscan.patch new file mode 100644 index 000000000000..2ff5aab68bc1 --- /dev/null +++ b/bazel/foreign_cc/vectorscan.patch @@ -0,0 +1,33 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1db128b..ee3b4a9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -27,7 +27,7 @@ include (${CMAKE_MODULE_PATH}/platform.cmake) + include (${CMAKE_MODULE_PATH}/boost.cmake) + include (${CMAKE_MODULE_PATH}/ragel.cmake) + +-find_package(PkgConfig REQUIRED) ++find_package(PkgConfig QUIET) + + find_program(RAGEL ragel) + +diff --git a/cmake/sqlite3.cmake b/cmake/sqlite3.cmake +index 92b18ce..5291726 100644 +--- a/cmake/sqlite3.cmake ++++ b/cmake/sqlite3.cmake +@@ -2,6 +2,8 @@ + # a lot of noise to find sqlite + # + ++if(NOT SQLITE_SKIP_CHECK) ++ + option(SQLITE_PREFER_STATIC "Build sqlite3 statically instead of using an installed lib" OFF) + + if(NOT SQLITE_PREFER_STATIC) +@@ -33,4 +35,6 @@ else() + endif() + endif() + ++endif() ++ + # that's enough about sqlite diff --git a/bazel/get_workspace_status b/bazel/get_workspace_status index ca5159e6dea9..bc43475f01ac 100755 --- a/bazel/get_workspace_status +++ b/bazel/get_workspace_status @@ -23,6 +23,7 @@ if [ -f SOURCE_VERSION ] then echo "BUILD_SCM_REVISION $(cat SOURCE_VERSION)" + echo "ENVOY_BUILD_SCM_REVISION $(cat SOURCE_VERSION)" echo "STABLE_BUILD_SCM_REVISION $(cat SOURCE_VERSION)" echo "BUILD_SCM_STATUS Distribution" exit 0 @@ -30,11 +31,13 @@ fi if [[ -n "$BAZEL_FAKE_SCM_REVISION" ]]; then echo "BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION" + echo "ENVOY_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION" echo "STABLE_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION" else # The code below presents an implementation that works for git repository git_rev=$(git rev-parse HEAD) || exit 1 echo "BUILD_SCM_REVISION ${git_rev}" + echo "ENVOY_BUILD_SCM_REVISION ${git_rev}" echo "STABLE_BUILD_SCM_REVISION ${git_rev}" fi diff --git a/bazel/grpc.patch b/bazel/grpc.patch index c8872879824c..4608049f1bf8 100644 --- a/bazel/grpc.patch +++ b/bazel/grpc.patch @@ -23,4 +23,17 @@ index 1bb970e049..81265483e9 100644 + "-layering_check", ], ) - + +diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h +index 38bb070213..b53086e680 100644 +--- a/src/core/lib/channel/channel_args.h ++++ b/src/core/lib/channel/channel_args.h +@@ -284,7 +284,7 @@ class ChannelArgs { + + class Value { + public: +- explicit Value(int n) : rep_(reinterpret_cast(n), &int_vtable_) {} ++ explicit Value(int n) : rep_(reinterpret_cast(static_cast(n)), &int_vtable_) {} + explicit Value(std::string s) + : rep_(RefCountedString::Make(s).release(), &string_vtable_) {} + explicit Value(Pointer p) : rep_(std::move(p)) {} diff --git a/bazel/python_dependencies.bzl b/bazel/python_dependencies.bzl index 0033a5364547..ea50bf30ba38 100644 --- a/bazel/python_dependencies.bzl +++ b/bazel/python_dependencies.bzl @@ -1,7 +1,10 @@ load("@rules_python//python:pip.bzl", "pip_parse") load("@python3_11//:defs.bzl", "interpreter") +load("@envoy_toolshed//:packages.bzl", "load_packages") def envoy_python_dependencies(): + # TODO(phlax): rename base_pip3 -> pip3 and remove this + load_packages() pip_parse( name = "base_pip3", python_interpreter_target = interpreter, diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 17cb1180490e..4affb6bb757f 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,8 +1,8 @@ -load(":dev_binding.bzl", "envoy_dev_binding") +load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") load("@envoy_api//bazel:external_deps.bzl", "load_repository_locations") +load(":dev_binding.bzl", "envoy_dev_binding") load(":repository_locations.bzl", "PROTOC_VERSIONS", "REPOSITORY_LOCATIONS_SPEC") -load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") PPC_SKIP_TARGETS = ["envoy.filters.http.lua"] @@ -16,6 +16,13 @@ WINDOWS_SKIP_TARGETS = [ "envoy.tracers.opencensus", ] +NO_HTTP3_SKIP_TARGETS = [ + "envoy.quic.crypto_stream.server.quiche", + "envoy.quic.deterministic_connection_id_generator", + "envoy.quic.crypto_stream.server.quiche", + "envoy.quic.proof_source.filter_chain", +] + # Make all contents of an external repository accessible under a filegroup. Used for external HTTP # archives, e.g. cares. def _build_all_content(exclude = []): @@ -108,13 +115,14 @@ envoy_entry_point( name = "get_project_json", pkg = "envoy.base.utils", script = "envoy.project_data", + init_data = [":__init__.py"], ) genrule( name = "project", outs = ["project.json"], cmd = """ - $(location :get_project_json) . > $@ + $(location :get_project_json) $$(dirname $(location @envoy//:VERSION.txt)) > $@ """, tools = [ ":get_project_json", @@ -129,9 +137,12 @@ envoy_entry_point( args = [ "release", PATH, + "--release-message-path=$(location @envoy//changelogs:summary)", ], + data = ["@envoy//changelogs:summary"], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -142,6 +153,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -152,6 +164,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -162,6 +175,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -172,6 +186,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) ''') @@ -256,7 +271,7 @@ def envoy_dependencies(skip_targets = []): _com_github_axboe_liburing() _com_github_bazel_buildtools() _com_github_c_ares_c_ares() - _com_github_circonus_labs_libcircllhist() + _com_github_openhistogram_libcircllhist() _com_github_cyan4973_xxhash() _com_github_datadog_dd_trace_cpp() _com_github_mirror_tclap() @@ -270,9 +285,13 @@ def envoy_dependencies(skip_targets = []): _com_github_google_tcmalloc() _com_github_gperftools_gperftools() _com_github_grpc_grpc() + _com_github_rules_proto_grpc() _com_github_unicode_org_icu() _com_github_intel_ipp_crypto_crypto_mb() + _com_github_intel_ipp_crypto_crypto_mb_fips() _com_github_intel_qatlib() + _com_github_intel_qatzip() + _com_github_lz4_lz4() _com_github_jbeder_yaml_cpp() _com_github_libevent_libevent() _com_github_luajit_luajit() @@ -294,6 +313,7 @@ def envoy_dependencies(skip_targets = []): _com_github_google_quiche() _com_googlesource_googleurl() _io_hyperscan() + _io_vectorscan() _io_opentracing_cpp() _net_colm_open_source_colm() _net_colm_open_source_ragel() @@ -318,6 +338,7 @@ def envoy_dependencies(skip_targets = []): external_http_archive("bazel_toolchains") external_http_archive("bazel_compdb") external_http_archive("envoy_build_tools") + _com_github_maxmind_libmaxminddb() # TODO(keith): Remove patch when we update rules_pkg external_http_archive( @@ -325,7 +346,12 @@ def envoy_dependencies(skip_targets = []): patches = ["@envoy//bazel:rules_pkg.patch"], ) external_http_archive("com_github_aignas_rules_shellcheck") - external_http_archive("aspect_bazel_lib") + external_http_archive( + "aspect_bazel_lib", + patch_args = ["-p1"], + patches = ["@envoy//bazel:aspect.patch"], + ) + _com_github_fdio_vpp_vcl() # Unconditional, since we use this only for compiler-agnostic fuzzing utils. @@ -346,10 +372,8 @@ def envoy_dependencies(skip_targets = []): name = "com_google_googleapis_imports", cc = True, go = True, + python = True, grpc = True, - rules_override = { - "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""], - }, ) native.bind( name = "bazel_runfiles", @@ -371,14 +395,14 @@ def _boringssl_fips(): build_file = "@envoy//bazel/external:boringssl_fips.BUILD", ) -def _com_github_circonus_labs_libcircllhist(): +def _com_github_openhistogram_libcircllhist(): external_http_archive( - name = "com_github_circonus_labs_libcircllhist", + name = "com_github_openhistogram_libcircllhist", build_file = "@envoy//bazel/external:libcircllhist.BUILD", ) native.bind( name = "libcircllhist", - actual = "@com_github_circonus_labs_libcircllhist//:libcircllhist", + actual = "@com_github_openhistogram_libcircllhist//:libcircllhist", ) def _com_github_axboe_liburing(): @@ -463,6 +487,10 @@ def _com_github_google_benchmark(): external_http_archive( name = "com_github_google_benchmark", ) + external_http_archive( + name = "libpfm", + build_file = "@com_github_google_benchmark//tools:libpfm.BUILD.bazel", + ) native.bind( name = "benchmark", actual = "@com_github_google_benchmark//:benchmark", @@ -500,9 +528,29 @@ def _com_github_intel_ipp_crypto_crypto_mb(): # to create dynamic *.so library target. Linker fails when linking # with boringssl_fips library. Envoy uses only static library # anyways, so created dynamic library would not be used anyways. + patches = [ + "@envoy//bazel/foreign_cc:ipp-crypto-skip-dynamic-lib.patch", + "@envoy//bazel/foreign_cc:ipp-crypto-bn2lebinpad.patch", + ], + patch_args = ["-p1"], + build_file_content = BUILD_ALL_CONTENT, + ) + +def _com_github_intel_ipp_crypto_crypto_mb_fips(): + # Temporary fix for building ipp-crypto when boringssl-fips is used. + # Build will fail if bn2lebinpad patch is applied. Remove this archive + # when upstream dependency fixes this issue. + external_http_archive( + name = "com_github_intel_ipp_crypto_crypto_mb_fips", + # Patch removes from CMakeLists.txt instructions to + # to create dynamic *.so library target. Linker fails when linking + # with boringssl_fips library. Envoy uses only static library + # anyways, so created dynamic library would not be used anyways. patches = ["@envoy//bazel/foreign_cc:ipp-crypto-skip-dynamic-lib.patch"], patch_args = ["-p1"], build_file_content = BUILD_ALL_CONTENT, + # Use existing ipp-crypto repository location name to avoid redefinition. + location_name = "com_github_intel_ipp_crypto_crypto_mb", ) def _com_github_intel_qatlib(): @@ -511,6 +559,18 @@ def _com_github_intel_qatlib(): build_file_content = BUILD_ALL_CONTENT, ) +def _com_github_intel_qatzip(): + external_http_archive( + name = "com_github_intel_qatzip", + build_file_content = BUILD_ALL_CONTENT, + ) + +def _com_github_lz4_lz4(): + external_http_archive( + name = "com_github_lz4_lz4", + build_file_content = BUILD_ALL_CONTENT, + ) + def _com_github_jbeder_yaml_cpp(): external_http_archive( name = "com_github_jbeder_yaml_cpp", @@ -665,6 +725,15 @@ def _io_hyperscan(): patches = ["@envoy//bazel/foreign_cc:hyperscan.patch"], ) +def _io_vectorscan(): + external_http_archive( + name = "io_vectorscan", + build_file_content = BUILD_ALL_CONTENT, + type = "tar.gz", + patch_args = ["-p1"], + patches = ["@envoy//bazel/foreign_cc:vectorscan.patch"], + ) + def _io_opentracing_cpp(): external_http_archive( name = "io_opentracing_cpp", @@ -774,6 +843,10 @@ def _com_google_absl(): name = "absl-base", actual = "@com_google_absl//absl/base", ) + native.bind( + name = "abseil_btree", + actual = "@com_google_absl//absl/container:btree", + ) native.bind( name = "abseil_flat_hash_map", actual = "@com_google_absl//absl/container:flat_hash_map", @@ -834,6 +907,10 @@ def _com_google_absl(): name = "abseil_stacktrace", actual = "@com_google_absl//absl/debugging:stacktrace", ) + native.bind( + name = "abseil_statusor", + actual = "@com_google_absl//absl/status:statusor", + ) # Require abseil_time as an indirect dependency as it is needed by the # direct dependency jwt_verify_lib. @@ -952,7 +1029,9 @@ def _io_opencensus_cpp(): ) def _com_github_curl(): - # Used by OpenCensus Zipkin exporter. + # The usage by AWS extensions common utilities is deprecated and will be removed by Q3 2024 after + # the deprecation period of 2 releases. Please DO NOT USE curl dependency for any new or existing extensions. + # See https://github.com/envoyproxy/envoy/issues/11816 & https://github.com/envoyproxy/envoy/pull/30731. external_http_archive( name = "com_github_curl", build_file_content = BUILD_ALL_CONTENT + """ @@ -975,7 +1054,10 @@ cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy/ def _v8(): external_http_archive( name = "v8", - patches = ["@envoy//bazel:v8.patch"], + patches = [ + "@envoy//bazel:v8.patch", + "@envoy//bazel:v8_include.patch", + ], patch_args = ["-p1"], ) native.bind( @@ -1131,6 +1213,9 @@ def _com_github_grpc_grpc(): actual = "@upb//:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", ) +def _com_github_rules_proto_grpc(): + external_http_archive("com_github_rules_proto_grpc") + def _re2(): external_http_archive("com_googlesource_code_re2") @@ -1328,7 +1413,7 @@ filegroup( # This archive provides Kafka C/CPP client used by mesh filter to communicate with upstream # Kafka clusters. external_http_archive( - name = "edenhill_librdkafka", + name = "confluentinc_librdkafka", build_file_content = BUILD_ALL_CONTENT, # (adam.kotwasinski) librdkafka bundles in cJSON, which is also bundled in by libvppinfra. # For now, let's just drop this dependency from Kafka, as it's used only for monitoring. @@ -1370,18 +1455,12 @@ def _rules_ruby(): def _foreign_cc_dependencies(): external_http_archive("rules_foreign_cc") -def _is_linux(ctxt): - return ctxt.os.name == "linux" - -def _is_arch(ctxt, arch): - res = ctxt.execute(["uname", "-m"]) - return arch in res.stdout - -def _is_linux_ppc(ctxt): - return _is_linux(ctxt) and _is_arch(ctxt, "ppc") - -def _is_linux_s390x(ctxt): - return _is_linux(ctxt) and _is_arch(ctxt, "s390x") - -def _is_linux_x86_64(ctxt): - return _is_linux(ctxt) and _is_arch(ctxt, "x86_64") +def _com_github_maxmind_libmaxminddb(): + external_http_archive( + name = "com_github_maxmind_libmaxminddb", + build_file_content = BUILD_ALL_CONTENT, + ) + native.bind( + name = "maxmind", + actual = "@envoy//bazel/foreign_cc:maxmind_linux", + ) diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index 40d348073fa4..a5bc2d527769 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -1,8 +1,9 @@ +load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies") +load("@com_github_rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains") load("@emsdk//:deps.bzl", emsdk_deps = "deps") -load("@rules_python//python:repositories.bzl", "python_register_toolchains") load("@proxy_wasm_cpp_host//bazel/cargo/wasmtime:crates.bzl", "wasmtime_fetch_remote_crates") +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") load("//bazel/external/cargo:crates.bzl", "raze_fetch_remote_crates") -load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies") def _python_minor_version(python_version): return "_".join(python_version.split(".")[:-1]) @@ -18,6 +19,8 @@ def envoy_dependencies_extra( emsdk_deps() raze_fetch_remote_crates() wasmtime_fetch_remote_crates() + rules_proto_grpc_toolchains() + py_repositories() # Registers underscored Python minor version - eg `python3_10` python_register_toolchains( diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0302660b40ed..00c8b9678dc3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -60,10 +60,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Apple Rules for Bazel", project_desc = "Bazel rules for Apple platforms", project_url = "https://github.com/bazelbuild/rules_apple", - version = "3.0.0-rc1", - sha256 = "62b24b9c3c1eb5bdb6fe1a3f4bf541d6e61eac8997e87c25a50c821f85bf8ff2", + version = "3.1.1", + sha256 = "34c41bfb59cdaea29ac2df5a2fa79e5add609c71bb303b2ebb10985f93fa20e7", urls = ["https://github.com/bazelbuild/rules_apple/releases/download/{version}/rules_apple.{version}.tar.gz"], - release_date = "2023-08-28", + release_date = "2023-10-19", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_apple/blob/{version}/LICENSE", @@ -72,9 +72,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel build tools", project_desc = "Developer tools for working with Google's bazel buildtool.", project_url = "https://github.com/bazelbuild/buildtools", - version = "6.3.3", - sha256 = "42968f9134ba2c75c03bb271bd7bb062afb7da449f9b913c96e5be4ce890030a", - release_date = "2023-08-25", + version = "6.4.0", + sha256 = "05c3c3602d25aeda1e9dbc91d3b66e624c1f9fdadf273e5480b489e744ca7269", + release_date = "2023-11-15", strip_prefix = "buildtools-{version}", urls = ["https://github.com/bazelbuild/buildtools/archive/v{version}.tar.gz"], use_category = ["test_only"], @@ -84,11 +84,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_desc = "Bazel rules for fuzz tests", project_url = "https://github.com/bazelbuild/rules_fuzzing", # Patch contains workaround for https://github.com/bazelbuild/rules_python/issues/1221 - version = "0.3.2", - sha256 = "f85dc70bb9672af0e350686461fe6fdd0d61e10e75645f9e44fedf549b21e369", + version = "0.4.1", + sha256 = "f6f3f42c48576acd5653bf07637deee2ae4ebb77ccdb0dacc67c184508bedc8c", strip_prefix = "rules_fuzzing-{version}", urls = ["https://github.com/bazelbuild/rules_fuzzing/archive/v{version}.tar.gz"], - release_date = "2022-08-31", + release_date = "2023-10-19", use_category = ["test_only"], implied_untracked_deps = [ # This is a repository rule generated to define an OSS-Fuzz fuzzing @@ -102,11 +102,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "envoy-build-tools", project_desc = "Common build tools shared by the Envoy/UDPA ecosystem", project_url = "https://github.com/envoyproxy/envoy-build-tools", - version = "633f57439ba683c1370fb8b1025680f1ce678caf", - sha256 = "88e4b7d12429d488daff522b765f0f21a3204d2c4b262b4b9d67587230415454", + version = "157af2cabe71d054d4b9837a66f8458cb7bee2f0", + sha256 = "ab284f8b15f2b590d6fa3140d1f93d3fb96fbbea5ea0dfa01f48eb6d61d1c96b", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2023-09-15", + release_date = "2023-12-20", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/envoyproxy/envoy-build-tools/blob/{version}/LICENSE", @@ -120,13 +120,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( # 2. Open https://chromium.googlesource.com/chromium/src/+/refs/tags//DEPS and note . # 3. Find a commit in BoringSSL's "master-with-bazel" branch that merges . # - # chromium-112.0.5615.39 (linux/beta) - version = "88d7a40bd06a34da6ee0d985545755199d047258", - sha256 = "1e759891e168c5957f2f4d519929e2b4cef9303b7cf2049601081f4fca95bf21", + # chromium-118.0.5993.54 (linux/beta) + version = "45cf810dbdbd767f09f8cb0b0fcccd342c39041f", + sha256 = "f1f421738e9ba39dd88daf8cf3096ddba9c53e2b6b41b32fff5a3ff82f4cd162", strip_prefix = "boringssl-{version}", urls = ["https://github.com/google/boringssl/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2023-02-14", + release_date = "2023-08-28", cpe = "cpe:2.3:a:google:boringssl:*", license = "Mixed", license_url = "https://github.com/google/boringssl/blob/{version}/LICENSE", @@ -148,12 +148,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Aspect Bazel helpers", project_desc = "Base Starlark libraries and basic Bazel rules which are useful for constructing rulesets and BUILD files", project_url = "https://github.com/aspect-build/bazel-lib", - version = "1.34.1", - sha256 = "271d5f38c218a0c2fe2e94f94dfc0b497e931cbb335348bf1695015191be5367", + version = "2.1.0", + sha256 = "fc8bd670380eaba5314769abbe9fee21d641e3da06d9d26b8073a301f6d62332", strip_prefix = "bazel-lib-{version}", urls = ["https://github.com/aspect-build/bazel-lib/archive/v{version}.tar.gz"], use_category = ["build"], - release_date = "2023-08-30", + release_date = "2023-12-23", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/aspect-build/bazel-lib/blob/v{version}/LICENSE", @@ -176,11 +176,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Shellcheck rules for bazel", project_desc = "Now you do not need to depend on the system shellcheck version in your bazel-managed (mono)repos.", project_url = "https://github.com/aignas/rules_shellcheck", - version = "0.1.1", - sha256 = "4e7cc56d344d0adfd20283f7ad8cb4fba822c0b15ce122665b00dd87a27a74b6", + version = "0.2.4", + sha256 = "ce4d0e7a9beb1fb3f0d37424465060491a91dae68de1ef1c92ee57d94c773b46", strip_prefix = "rules_shellcheck-{version}", - urls = ["https://github.com/aignas/rules_shellcheck/archive/v{version}.tar.gz"], - release_date = "2022-05-30", + urls = ["https://github.com/aignas/rules_shellcheck/archive/{version}.tar.gz"], + release_date = "2023-10-27", use_category = ["build"], cpe = "N/A", license = "MIT", @@ -204,12 +204,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Perfetto", project_desc = "Perfetto Tracing SDK", project_url = "https://perfetto.dev/", - version = "36.1", - sha256 = "b46145b6009dd7367ab12ef1e36a1656ec004674d3df167184a0ba6ceb384283", + version = "40.0", + sha256 = "bd78f0165e66026c31c8c39221ed2863697a8bba5cd39b12e4b43d0b7f71626f", strip_prefix = "perfetto-{version}/sdk", urls = ["https://github.com/google/perfetto/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2023-07-06", + release_date = "2023-12-05", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/google/perfetto/blob/v{version}/LICENSE", @@ -228,19 +228,19 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "c-ares", license_url = "https://github.com/c-ares/c-ares/blob/cares-{underscore_version}/LICENSE.md", ), - com_github_circonus_labs_libcircllhist = dict( + com_github_openhistogram_libcircllhist = dict( project_name = "libcircllhist", - project_desc = "An implementation of Circonus log-linear histograms", - project_url = "https://github.com/circonus-labs/libcircllhist", + project_desc = "An implementation of OpenHistogram log-linear histograms", + project_url = "https://github.com/openhistogram/libcircllhist", version = "39f9db724a81ba78f5d037f1cae79c5a07107c8e", sha256 = "fd2492f6cc1f8734f8f57be8c2e7f2907e94ee2a4c02445ce59c4241fece144b", strip_prefix = "libcircllhist-{version}", - urls = ["https://github.com/circonus-labs/libcircllhist/archive/{version}.tar.gz"], + urls = ["https://github.com/openhistogram/libcircllhist/archive/{version}.tar.gz"], use_category = ["controlplane", "observability_core", "dataplane_core"], release_date = "2019-05-21", cpe = "N/A", license = "Apache-2.0", - license_url = "https://github.com/circonus-labs/libcircllhist/blob/{version}/LICENSE", + license_url = "https://github.com/openhistogram/libcircllhist/blob/{version}/LICENSE", ), com_github_cyan4973_xxhash = dict( project_name = "xxHash", @@ -376,16 +376,31 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "gRPC", project_desc = "gRPC C core library", project_url = "https://grpc.io", - version = "1.56.2", - sha256 = "931f07db9d48cff6a6007c1033ba6d691fe655bea2765444bc1ad974dfc840aa", + version = "1.59.1", + sha256 = "916f88a34f06b56432611aaa8c55befee96d0a7b7d7457733b9deeacbc016f99", strip_prefix = "grpc-{version}", urls = ["https://github.com/grpc/grpc/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2023-07-14", + release_date = "2023-10-06", cpe = "cpe:2.3:a:grpc:grpc:*", license = "Apache-2.0", license_url = "https://github.com/grpc/grpc/blob/v{version}/LICENSE", ), + com_github_rules_proto_grpc = dict( + project_name = "Protobuf and gRPC rules for Bazel", + project_desc = "Bazel rules for building Protobuf and gRPC code and libraries from proto_library targets", + project_url = "https://github.com/rules-proto-grpc/rules_proto_grpc", + version = "4.6.0", + sha256 = "2a0860a336ae836b54671cbbe0710eec17c64ef70c4c5a88ccfd47ea6e3739bd", + strip_prefix = "rules_proto_grpc-{version}", + urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/{version}/rules_proto_grpc-{version}.tar.gz"], + use_category = ["dataplane_ext"], + extensions = ["envoy.transport_sockets.alts"], + release_date = "2023-12-14", + cpe = "N/A", + license = "Apache-2.0", + license_url = "https://github.com/rules-proto-grpc/rules_proto_grpc/blob/{version}/LICENSE", + ), com_github_unicode_org_icu = dict( project_name = "ICU Library", project_desc = "Development files for International Components for Unicode", @@ -425,17 +440,32 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "qatlib", project_desc = "Intel QuickAssist Technology Library", project_url = "https://github.com/intel/qatlib", - version = "23.02.0", - sha256 = "075d900e44be30c117585054f0fd5eea853d3804106a401e537b1fff45297542", + version = "23.11.0", + sha256 = "f649613c243df98c2b005e58af7e0c9bb6d9638e0a12d2757d18d4930bf893cd", strip_prefix = "qatlib-{version}", urls = ["https://github.com/intel/qatlib/archive/refs/tags/{version}.tar.gz"], use_category = ["dataplane_ext"], - release_date = "2023-02-24", - extensions = ["envoy.tls.key_providers.qat"], + release_date = "2023-11-15", + extensions = ["envoy.tls.key_providers.qat", "envoy.compression.qatzip.compressor"], cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/intel/qatlib/blob/{version}/LICENSE", ), + com_github_intel_qatzip = dict( + project_name = "qatzip", + project_desc = "Intel QuickAssist Technology QATzip Library", + project_url = "https://github.com/intel/qatzip", + version = "1.1.2", + sha256 = "31419fa4b42d217b3e55a70a34545582cbf401a4f4d44738d21b4a3944b1e1ef", + strip_prefix = "QATzip-{version}", + urls = ["https://github.com/intel/QATzip/archive/v{version}.tar.gz"], + use_category = ["dataplane_ext"], + release_date = "2023-03-24", + extensions = ["envoy.compression.qatzip.compressor"], + cpe = "N/A", + license = "BSD-3-Clause", + license_url = "https://github.com/intel/QATzip/blob/{version}/LICENSE", + ), com_github_luajit_luajit = dict( project_name = "LuaJIT", project_desc = "Just-In-Time compiler for Lua", @@ -447,7 +477,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/LuaJIT/LuaJIT/archive/{version}.tar.gz"], release_date = "2023-04-16", use_category = ["dataplane_ext"], - extensions = ["envoy.filters.http.lua"], + extensions = [ + "envoy.filters.http.lua", + "envoy.router.cluster_specifier_plugin.lua", + ], cpe = "cpe:2.3:a:luajit:luajit:*", license = "MIT", license_url = "https://github.com/LuaJIT/LuaJIT/blob/{version}/COPYRIGHT", @@ -456,12 +489,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Nghttp2", project_desc = "Implementation of HTTP/2 and its header compression algorithm HPACK in C", project_url = "https://nghttp2.org", - version = "1.55.1", - sha256 = "e12fddb65ae3218b4edc083501519379928eba153e71a1673b185570f08beb96", + version = "1.58.0", + sha256 = "9ebdfbfbca164ef72bdf5fd2a94a4e6dfb54ec39d2ef249aeb750a91ae361dfb", strip_prefix = "nghttp2-{version}", urls = ["https://github.com/nghttp2/nghttp2/releases/download/v{version}/nghttp2-{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2023-07-14", + release_date = "2023-10-27", cpe = "cpe:2.3:a:nghttp2:nghttp2:*", license = "MIT", license_url = "https://github.com/nghttp2/nghttp2/blob/v{version}/LICENSE", @@ -484,6 +517,24 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "BSD-3-Clause", license_url = "https://github.com/intel/hyperscan/blob/v{version}/LICENSE", ), + io_vectorscan = dict( + project_name = "Vectorscan", + project_desc = "Hyperscan port for additional CPU architectures", + project_url = "https://www.vectorcamp.gr/vectorscan/", + version = "5.4.11", + sha256 = "905f76ad1fa9e4ae0eb28232cac98afdb96c479666202c5a4c27871fb30a2711", + strip_prefix = "vectorscan-vectorscan-{version}", + urls = ["https://codeload.github.com/VectorCamp/vectorscan/tar.gz/refs/tags/vectorscan/{version}"], + use_category = ["dataplane_ext"], + extensions = [ + "envoy.matching.input_matchers.hyperscan", + "envoy.regex_engines.hyperscan", + ], + release_date = "2023-11-20", + cpe = "N/A", + license = "BSD-3-Clause", + license_url = "https://github.com/VectorCamp/vectorscan/blob/vectorscan/{version}/LICENSE", + ), io_opentracing_cpp = dict( project_name = "OpenTracing", project_desc = "Vendor-neutral APIs and instrumentation for distributed tracing", @@ -506,7 +557,6 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "skywalking-data-collect-protocol", project_desc = "Data Collect Protocols of Apache SkyWalking", project_url = "https://github.com/apache/skywalking-data-collect-protocol", - name = "skywalking_data_collect_protocol", sha256 = "49bd689b9c1c0ea12064bd35581689cef7835e5ac15d335dc425fbfc2029aa90", urls = ["https://github.com/apache/skywalking-data-collect-protocol/archive/v{version}.tar.gz"], strip_prefix = "skywalking-data-collect-protocol-{version}", @@ -551,12 +601,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Benchmark", project_desc = "Library to benchmark code snippets", project_url = "https://github.com/google/benchmark", - version = "1.7.0", - sha256 = "3aff99169fa8bdee356eaa1f691e835a6e57b1efeadb8a0f9f228531158246ac", + version = "1.8.3", + sha256 = "6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce", strip_prefix = "benchmark-{version}", urls = ["https://github.com/google/benchmark/archive/v{version}.tar.gz"], use_category = ["test_only"], - release_date = "2022-07-25", + release_date = "2023-08-31", license = "Apache-2.0", license_url = "https://github.com/google/benchmark/blob/v{version}/LICENSE", ), @@ -650,16 +700,16 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Boost", project_desc = "Boost C++ source libraries", project_url = "https://www.boost.org/", - version = "1.78.0", - sha256 = "94ced8b72956591c4775ae2207a9763d3600b30d9d7446562c552f0a14a63be7", + version = "1.84.0", + sha256 = "a5800f405508f5df8114558ca9855d2640a2de8f0445f051fa1c7c3383045724", strip_prefix = "boost_{underscore_version}", - urls = ["https://boostorg.jfrog.io/artifactory/main/release/{version}/source/boost_{underscore_version}.tar.gz"], + urls = ["https://archives.boost.io/release/{version}/source/boost_{underscore_version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ "envoy.matching.input_matchers.hyperscan", "envoy.regex_engines.hyperscan", ], - release_date = "2021-12-08", + release_date = "2023-12-13", cpe = "cpe:2.3:a:boost:boost:*", license = "Boost", license_url = "https://github.com/boostorg/boost/blob/boost-{version}/LICENSE_1_0.txt", @@ -680,7 +730,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2023-08-31", cpe = "cpe:2.3:a:google:brotli:*", license = "MIT", - license_url = "https://github.com/google/brotli/blob/{version}/LICENSE", + license_url = "https://github.com/google/brotli/blob/v{version}/LICENSE", ), com_github_facebook_zstd = dict( project_name = "zstd", @@ -732,13 +782,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "jwt_verify_lib", project_desc = "JWT verification library for C++", project_url = "https://github.com/google/jwt_verify_lib", - version = "c29ba4bdab2cc9a7b4d80d1d3ebff3bf5b9bf6e2", - sha256 = "5851ab1857edf46b31dc298fba984e1b7638f80a58f88a84a83402540643a99f", + version = "b59e8075d4a4f975ba6f109e1916d6e60aeb5613", + sha256 = "637e4983506c4f26bbe2808ae4e1944e46cbb2277d34ff0b8a3b72bdac3c4b91", strip_prefix = "jwt_verify_lib-{version}", urls = ["https://github.com/google/jwt_verify_lib/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.http.jwt_authn", "envoy.filters.http.gcp_authn"], - release_date = "2022-11-04", + release_date = "2023-05-17", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/google/jwt_verify_lib/blob/{version}/LICENSE", @@ -778,14 +828,14 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "nlohmann JSON", project_desc = "Fast JSON parser/generator for C++", project_url = "https://nlohmann.github.io/json", - version = "3.11.2", - sha256 = "d69f9deb6a75e2580465c6c4c5111b89c4dc2fa94e3a85fcd2ffcd9a143d9273", + version = "3.11.3", + sha256 = "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406", strip_prefix = "json-{version}", urls = ["https://github.com/nlohmann/json/archive/v{version}.tar.gz"], # This will be a replacement for rapidJSON used in extensions and may also be a fast # replacement for protobuf JSON. use_category = ["controlplane", "dataplane_core"], - release_date = "2022-08-12", + release_date = "2023-11-28", cpe = "cpe:2.3:a:json-for-modern-cpp_project:json-for-modern-cpp:*", license = "MIT", license_url = "https://github.com/nlohmann/json/blob/v{version}/LICENSE.MIT", @@ -938,9 +988,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Python rules for Bazel", project_desc = "Bazel rules for the Python language", project_url = "https://github.com/bazelbuild/rules_python", - version = "0.25.0", - sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036", - release_date = "2023-08-22", + version = "0.27.1", + sha256 = "e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793", + release_date = "2023-12-05", strip_prefix = "rules_python-{version}", urls = ["https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz"], use_category = ["build"], @@ -1069,13 +1119,14 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Apache-2.0", license_url = "https://github.com/census-instrumentation/opencensus-cpp/blob/{version}/LICENSE", ), - # This should be removed, see https://github.com/envoyproxy/envoy/issues/11816. + # Curl usage is under deprecation and will be removed by Q3 2024 before v1.31 release in July-2024. + # See https://github.com/envoyproxy/envoy/issues/11816 & https://github.com/envoyproxy/envoy/pull/30731. com_github_curl = dict( project_name = "curl", project_desc = "Library for transferring data with URLs", project_url = "https://curl.haxx.se", - version = "8.2.1", - sha256 = "f98bdb06c0f52bdd19e63c4a77b5eb19b243bcbbd0f5b002b9f3cba7295a3a42", + version = "8.4.0", + sha256 = "816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427", strip_prefix = "curl-{version}", urls = ["https://github.com/curl/curl/releases/download/curl-{underscore_version}/curl-{version}.tar.gz"], use_category = ["dataplane_ext", "observability_ext"], @@ -1085,7 +1136,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.grpc_credentials.aws_iam", "envoy.tracers.opencensus", ], - release_date = "2023-07-26", + release_date = "2023-10-11", cpe = "cpe:2.3:a:haxx:libcurl:*", license = "curl", license_url = "https://github.com/curl/curl/blob/curl-{underscore_version}/COPYING", @@ -1124,12 +1175,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "667c58656fe58f27672c58c76023373f2d49ce2e", - sha256 = "b82e5e626a3f51dc3a57154f2ed92c5ecfe9f5bc7f390fa6b74a9f3854022b1e", + version = "0860476f3e59c023bc96e8cea1f18b11e241fda7", + sha256 = "05d1210e4a3e381cf682564a3d350adb702aacc60c098434b6470301df2613f0", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2023-09-13", + release_date = "2023-12-28", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", @@ -1211,12 +1262,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "RE2", project_desc = "RE2, a regular expression library", project_url = "https://github.com/google/re2", - version = "2023-07-01", - sha256 = "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4", + version = "2023-09-01", + sha256 = "5bb6875ae1cd1e9fedde98018c346db7260655f86fdb8837e3075103acd3649b", strip_prefix = "re2-{version}", urls = ["https://github.com/google/re2/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2023-06-30", + release_date = "2023-08-31", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/re2/blob/{version}/LICENSE", @@ -1272,20 +1323,20 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Apache-2.0", license_url = "https://github.com/apache/kafka/blob/{version}/LICENSE", ), - edenhill_librdkafka = dict( + confluentinc_librdkafka = dict( project_name = "Kafka (C/C++ client)", project_desc = "C/C++ client for Apache Kafka (open-source distributed event streaming platform)", - project_url = "https://github.com/edenhill/librdkafka", - version = "2.2.0", - sha256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226", + project_url = "https://github.com/confluentinc/librdkafka", + version = "2.3.0", + sha256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12", strip_prefix = "librdkafka-{version}", - urls = ["https://github.com/edenhill/librdkafka/archive/v{version}.tar.gz"], + urls = ["https://github.com/confluentinc/librdkafka/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_mesh"], - release_date = "2023-07-12", + release_date = "2023-10-25", cpe = "N/A", license = "librdkafka", - license_url = "https://github.com/edenhill/librdkafka/blob/v{version}/LICENSE", + license_url = "https://github.com/confluentinc/librdkafka/blob/v{version}/LICENSE", ), kafka_server_binary = dict( project_name = "Kafka (server binary)", @@ -1315,8 +1366,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ SDK)", project_desc = "WebAssembly for Proxies (C++ SDK)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-sdk", - version = "e30535b7c0cd3126e6401bc3769063a74bbb37be", - sha256 = "94e474ebea782225821224734ed5992fa749301e12e06b6520b8b4d4e1c05ffc", + version = "921039ae983ce053bf5cba78a85a3c08ff9791e5", + sha256 = "a11adfe4e6346d3318ff72643aa5569dc8439d7e8927ed148f93226fa255cc7a", strip_prefix = "proxy-wasm-cpp-sdk-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-sdk/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1332,7 +1383,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2022-03-15", + release_date = "2023-05-01", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-sdk/blob/{version}/LICENSE", @@ -1341,8 +1392,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - version = "5d76116c449d6892b298b7ae79a84ef1cf5752bf", - sha256 = "a5825a1a5bbd5b0178c6189b227d5cf4370ac713a883b41f6a54edd768a03cb7", + version = "e200fee8af40918c41f3275cff090993e3b26940", + sha256 = "9711411b3b8d48a3ee9278f44824ce569c1fdd491183255f568f2b938360e964", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1358,7 +1409,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2023-06-01", + release_date = "2023-12-19", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host/blob/{version}/LICENSE", @@ -1433,6 +1484,17 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2022-12-15", cpe = "N/A", ), + libpfm = dict( + project_name = "libpfm", + project_desc = "A helper library to develop monitoring tools", + project_url = "https://sourceforge.net/projects/perfmon2", + version = "4.11.0", + sha256 = "5da5f8872bde14b3634c9688d980f68bda28b510268723cc12973eedbab9fecc", + strip_prefix = "libpfm-{version}", + use_category = ["test_only"], + urls = ["https://downloads.sourceforge.net/project/perfmon2/libpfm4/libpfm-{version}.tar.gz"], + release_date = "2020-09-03", + ), rules_license = dict( project_name = "rules_license", project_desc = "Bazel rules for checking open source licenses", @@ -1460,6 +1522,34 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "MIT", license_url = "https://github.com/protocolbuffers/utf8_range/blob/{version}/LICENSE", ), + com_github_maxmind_libmaxminddb = dict( + project_name = "maxmind_libmaxminddb", + project_desc = "C library for reading MaxMind DB files", + project_url = "https://github.com/maxmind/libmaxminddb", + version = "1.8.0", + sha256 = "1107799f77be6aa3b9796ad0eed8ffcc334bf45f8bd18e6a984d8adf3e719c6d", + strip_prefix = "libmaxminddb-{version}", + urls = ["https://github.com/maxmind/libmaxminddb/releases/download/{version}/libmaxminddb-{version}.tar.gz"], + use_category = ["dataplane_ext"], + extensions = ["envoy.geoip_providers.maxmind"], + release_date = "2023-11-07", + cpe = "cpe:2.3:a:maxmind:libmaxminddb:*", + license = "Apache-2.0", + license_url = "https://github.com/maxmind/libmaxminddb/blob/{version}/LICENSE", + ), + com_github_lz4_lz4 = dict( + project_name = "LZ4", + project_desc = "Extremely Fast Compression algorithm", + project_url = "http://www.lz4.org/", + version = "1.9.4", + sha256 = "0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b", + strip_prefix = "lz4-{version}", + urls = ["https://github.com/lz4/lz4/archive/v{version}.tar.gz"], + use_category = ["dataplane_ext"], + release_date = "2022-08-15", + extensions = ["envoy.compression.qatzip.compressor"], + cpe = "N/A", + ), ) def _compiled_protoc_deps(locations, versions): diff --git a/bazel/v8_include.patch b/bazel/v8_include.patch new file mode 100644 index 000000000000..3e6b492bf05d --- /dev/null +++ b/bazel/v8_include.patch @@ -0,0 +1,41 @@ +# fix include types for late clang (15.0.7) / gcc (13.2.1) +# for Arch linux / Fedora, like in +# In file included from external/v8/src/torque/torque.cc:5: +# In file included from external/v8/src/torque/source-positions.h:10: +# In file included from external/v8/src/torque/contextual.h:10: +# In file included from external/v8/src/base/macros.h:12: +# external/v8/src/base/logging.h:154:26: error: use of undeclared identifier 'uint16_t' + +diff --git a/src/base/logging.h b/src/base/logging.h +--- a/src/base/logging.h ++++ b/src/base/logging.h +@@ -5,6 +5,7 @@ + #ifndef V8_BASE_LOGGING_H_ + #define V8_BASE_LOGGING_H_ + ++#include + #include + #include + #include +diff --git a/src/base/macros.h b/src/base/macros.h +--- a/src/base/macros.h ++++ b/src/base/macros.h +@@ -5,6 +5,7 @@ + #ifndef V8_BASE_MACROS_H_ + #define V8_BASE_MACROS_H_ + ++#include + #include + #include + +diff --git a/src/inspector/v8-string-conversions.h b/src/inspector/v8-string-conversions.h +--- a/src/inspector/v8-string-conversions.h ++++ b/src/inspector/v8-string-conversions.h +@@ -5,6 +5,7 @@ + #ifndef V8_INSPECTOR_V8_STRING_CONVERSIONS_H_ + #define V8_INSPECTOR_V8_STRING_CONVERSIONS_H_ + ++#include + #include + + // Conversion routines between UT8 and UTF16, used by string-16.{h,cc}. You may diff --git a/bazel/version_update_post.sh b/bazel/version_update_post.sh new file mode 100644 index 000000000000..ac877c1861f3 --- /dev/null +++ b/bazel/version_update_post.sh @@ -0,0 +1,78 @@ +#!/bin/bash -e + +set -o pipefail + + +EXISTING_DATE="$("${JQ}" -r ".${DEP}.release_date" "${DEP_DATA}")" +DATE_SEARCH="release_date = \"${EXISTING_DATE}\"," +DEP_CHECK="${DEP_CHECK:-tools/dependency/check}" + +find_date_line () { + local match match_ln date_match_ln + # This needs to find the correct date to replace + match="$(\ + grep -n "${DEP_SEARCH}" "${VERSION_FILE}" \ + | cut -d: -f-2)" + match_ln="$(\ + echo "${match}" \ + | cut -d: -f1)" + match_ln="$((match_ln + 1))" + date_match_ln="$(\ + tail -n "+${match_ln}" "${VERSION_FILE}" \ + | grep -n "${DATE_SEARCH}" \ + | head -n1 \ + | cut -d: -f1)" + date_match_ln="$((match_ln + date_match_ln - 1))" + printf '%s' "$date_match_ln" +} + +update_date () { + local match_ln search replace + match_ln="$1" + search="$2" + replace="$3" + echo "Updating date(${match_ln}): ${search} -> ${replace}" + sed -i "${match_ln}s/${search}/${replace}/" "$VERSION_FILE" +} + +get_new_date () { + # create a repository_locations with just the dep and with updated version + tmpfile="$(mktemp)" + # shellcheck disable=SC2016 + "$JQ" --arg new_version "$VERSION" \ + --arg existing_version "$EXISTING_VERSION" \ + --arg dep "$DEP" \ + 'if has($dep) then .[$dep].version = $new_version | .[$dep].urls |= map(gsub($existing_version; $new_version)) else . end' \ + "$DEP_DATA" > "$tmpfile" + output="$(\ + "$DEP_CHECK" \ + --repository_locations="$tmpfile" \ + --path "${BUILD_WORKSPACE_DIRECTORY}" \ + -c release_dates 2>&1)" + echo "$output" \ + | grep -E "^Mismatch" \ + | grep "$DEP" \ + | cut -d= -f2 \ + | xargs || { + cat "$tmpfile" >&2 + echo "$output" >&2 + rm "$tmpfile" + exit 1 + } + rm "$tmpfile" +} + +post_version_update () { + local date_ln new_date + if [[ "$EXISTING_VERSION" == "$VERSION" ]]; then + echo "Nothing to update" >&2 + exit 0 + fi + date_ln="$(find_date_line)" + new_date="$(get_new_date)" + if [[ -z "$new_date" ]]; then + echo "Unable to retrieve date" >&2 + exit 1 + fi + update_date "$date_ln" "$EXISTING_DATE" "$new_date" +} diff --git a/changelogs/1.24.0.yaml b/changelogs/1.24.0.yaml index fd5fc87991f5..52fad3a1659c 100644 --- a/changelogs/1.24.0.yaml +++ b/changelogs/1.24.0.yaml @@ -99,7 +99,7 @@ minor_behavior_changes: changed the filter callback interfaces to make sure that downstream-only functionality is explicit. - area: http change: | - the upstream remote address is now available to downstream filters via the ``upstreamRemoteAddress`` function. + the upstream remote address is now available to downstream HTTP filters via the ``upstreamRemoteAddress`` function. - area: stats change: | Default tag extraction rules were changed for ``worker_id`` extraction. Previously, ``worker_`` was removed from the @@ -193,13 +193,13 @@ new_features: - area: http change: | made the :ref:`admission control ` work - as an upstream filter. + as an upstream HTTP filter. - area: http change: | added default-false ``envoy.reloadable_features.http1_use_balsa_parser`` for experimental BalsaParser. - area: http change: | - added ``envoy.reloadable_features.allow_upstream_filters`` for experimental upstream filters. + added ``envoy.reloadable_features.allow_upstream_filters`` for experimental upstream HTTP filters. - area: dns_resolver change: | added DNS stats for c-ares DNS resolver. Detailed documentation is available :ref:`here `. diff --git a/changelogs/1.24.11.yaml b/changelogs/1.24.11.yaml new file mode 100644 index 000000000000..c5c5e55329bb --- /dev/null +++ b/changelogs/1.24.11.yaml @@ -0,0 +1,19 @@ +date: October 10, 2023 + +behavior_changes: +- area: http + change: | + Close HTTP/2 and HTTP/3 connections that prematurely reset streams. The runtime key + ``overload.premature_reset_min_stream_lifetime_seconds`` determines the interval where received stream + reset is considered premature (with 1 second default). The runtime key ``overload.premature_reset_total_stream_count``, + with the default value of 500, determines the number of requests received from a connection before the check for premature + resets is applied. The connection is disconnected if more than 50% of resets are premature. + Setting the runtime key ``envoy.restart_features.send_goaway_for_premature_rst_streams`` to ``false`` completely disables + this check. +- area: http + change: | + Add runtime flag ``http.max_requests_per_io_cycle`` for setting the limit on the number of HTTP requests processed + from a single connection in a single I/O cycle. Requests over this limit are processed in subsequent I/O cycles. This + mitigates CPU starvation by connections that simultaneously send high number of requests by allowing requests from other + connections to make progress. This runtime value can be set to 1 in the presence of abusive HTTP/2 or HTTP/3 connections. + By default this limit is disabled. diff --git a/changelogs/1.25.10.yaml b/changelogs/1.25.10.yaml new file mode 100644 index 000000000000..087ad323021d --- /dev/null +++ b/changelogs/1.25.10.yaml @@ -0,0 +1,34 @@ +date: October 10, 2023 + +behavior_changes: +- area: http + change: | + Close HTTP/2 and HTTP/3 connections that prematurely reset streams. The runtime key + ``overload.premature_reset_min_stream_lifetime_seconds`` determines the interval where received stream + reset is considered premature (with 1 second default). The runtime key ``overload.premature_reset_total_stream_count``, + with the default value of 500, determines the number of requests received from a connection before the check for premature + resets is applied. The connection is disconnected if more than 50% of resets are premature. + Setting the runtime key ``envoy.restart_features.send_goaway_for_premature_rst_streams`` to ``false`` completely disables + this check. +- area: http + change: | + Add runtime flag ``http.max_requests_per_io_cycle`` for setting the limit on the number of HTTP requests processed + from a single connection in a single I/O cycle. Requests over this limit are processed in subsequent I/O cycles. This + mitigates CPU starvation by connections that simultaneously send high number of requests by allowing requests from other + connections to make progress. This runtime value can be set to 1 in the presence of abusive HTTP/2 or HTTP/3 connections. + By default this limit is disabled. +- area: http + change: | + Add runtime flag ``http.max_requests_per_io_cycle`` for setting the limit on the number of HTTP requests processed + from a single connection in a single I/O cycle. Requests over this limit are processed in subsequent I/O cycles. This + mitigates CPU starvation by connections that simultaneously send high number of requests by allowing requests from other + connections to make progress. This runtime value can be set to 1 in the presence of abusive HTTP/2 or HTTP/3 connections. + By default this limit is disabled. + +bug_fixes: +- area: tls + change: | + fixed a bug where handshake may fail when both private key provider and cert validation are set. +- area: docker/publishing + change: | + Update base images to resolve various glibc vulnerabilities. diff --git a/changelogs/1.25.11.yaml b/changelogs/1.25.11.yaml new file mode 100644 index 000000000000..4beae10fad69 --- /dev/null +++ b/changelogs/1.25.11.yaml @@ -0,0 +1,7 @@ +date: October 16, 2023 + +bug_fixes: +- area: http + change: | + Fixed a bug where processing of deferred streams with the value of ``http.max_requests_per_io_cycle`` more than 1, + can cause a crash. diff --git a/changelogs/1.26.5.yaml b/changelogs/1.26.5.yaml new file mode 100644 index 000000000000..5f248d665be6 --- /dev/null +++ b/changelogs/1.26.5.yaml @@ -0,0 +1,24 @@ +date: October 10, 2023 + +behavior_changes: +- area: http + change: | + Close HTTP/2 and HTTP/3 connections that prematurely reset streams. The runtime key + ``overload.premature_reset_min_stream_lifetime_seconds`` determines the interval where received stream + reset is considered premature (with 1 second default). The runtime key ``overload.premature_reset_total_stream_count``, + with the default value of 500, determines the number of requests received from a connection before the check for premature + resets is applied. The connection is disconnected if more than 50% of resets are premature. + Setting the runtime key ``envoy.restart_features.send_goaway_for_premature_rst_streams`` to ``false`` completely disables + this check. +- area: http + change: | + Add runtime flag ``http.max_requests_per_io_cycle`` for setting the limit on the number of HTTP requests processed + from a single connection in a single I/O cycle. Requests over this limit are processed in subsequent I/O cycles. This + mitigates CPU starvation by connections that simultaneously send high number of requests by allowing requests from other + connections to make progress. This runtime value can be set to 1 in the presence of abusive HTTP/2 or HTTP/3 connections. + By default this limit is disabled. + +bug_fixes: +- area: tls + change: | + fixed a bug where handshake may fail when both private key provider and cert validation are set. diff --git a/changelogs/1.26.6.yaml b/changelogs/1.26.6.yaml new file mode 100644 index 000000000000..a5caeaa72fa5 --- /dev/null +++ b/changelogs/1.26.6.yaml @@ -0,0 +1,10 @@ +date: October 17, 2023 + +bug_fixes: +- area: tracing + change: | + Fixed a bug in the Datadog tracer where Datadog's "operation name" field would contain what should be in the "resource name" field. +- area: http + change: | + Fixed a bug where processing of deferred streams with the value of ``http.max_requests_per_io_cycle`` more than 1, + can cause a crash. diff --git a/changelogs/1.27.1.yaml b/changelogs/1.27.1.yaml new file mode 100644 index 000000000000..a6ce59291213 --- /dev/null +++ b/changelogs/1.27.1.yaml @@ -0,0 +1,30 @@ +date: October 11, 2023 + +behavior_changes: +- area: http + change: | + Close HTTP/2 and HTTP/3 connections that prematurely reset streams. The runtime key + ``overload.premature_reset_min_stream_lifetime_seconds`` determines the interval where received stream + reset is considered premature (with 1 second default). The runtime key ``overload.premature_reset_total_stream_count``, + with the default value of 500, determines the number of requests received from a connection before the check for premature + resets is applied. The connection is disconnected if more than 50% of resets are premature. + Setting the runtime key ``envoy.restart_features.send_goaway_for_premature_rst_streams`` to ``false`` completely disables + this check. +- area: http + change: | + Add runtime flag ``http.max_requests_per_io_cycle`` for setting the limit on the number of HTTP requests processed + from a single connection in a single I/O cycle. Requests over this limit are processed in subsequent I/O cycles. This + mitigates CPU starvation by connections that simultaneously send high number of requests by allowing requests from other + connections to make progress. This runtime value can be set to 1 in the presence of abusive HTTP/2 or HTTP/3 connections. + By default this limit is disabled. + +bug_fixes: +- area: connection limit + change: | + fixed a use-after-free bug in the connection limit filter. +- area: tls + change: | + fixed a bug where handshake may fail when both private key provider and cert validation are set. +- area: docker/publishing + change: | + Update base images to resolve various glibc vulnerabilities. diff --git a/changelogs/1.27.2.yaml b/changelogs/1.27.2.yaml new file mode 100644 index 000000000000..91d3633c0154 --- /dev/null +++ b/changelogs/1.27.2.yaml @@ -0,0 +1,10 @@ +date: October 16, 2023 + +bug_fixes: +- area: tracing + change: | + Fixed a bug in the Datadog tracer where Datadog's "operation name" field would contain what should be in the "resource name" field. +- area: http + change: | + Fixed a bug where processing of deferred streams with the value of ``http.max_requests_per_io_cycle`` more than 1, + can cause a crash. diff --git a/changelogs/1.28.0.yaml b/changelogs/1.28.0.yaml new file mode 100644 index 000000000000..72899ea63eb9 --- /dev/null +++ b/changelogs/1.28.0.yaml @@ -0,0 +1,445 @@ +date: October 18, 2023 + +behavior_changes: +- area: jwt + change: | + Changed behavior of the JWT extraction, passing entire token for validation, instead cut him in the non-Base64 character. + This change can be reverted temporarily by setting the runtime guard ``envoy.reloadable_features.token_passed_entirely`` to ``false``. +- area: eds + change: | + Introduced caching of EDS assignments when used with ADS. Prior to this change, Envoy required that EDS assignments were sent + after an EDS cluster was updated. If no EDS assignment was received for the cluster, it ended up with an empty assignment. + Following this change, after a cluster update, Envoy waits for an EDS assignment until + :ref:`initial_fetch_timeout ` times out, and will then apply + the cached assignment and finish updating the warmed cluster. This change is disabled by default, and can be enabled by setting + the runtime flag ``envoy.restart_features.use_eds_cache_for_ads`` to ``true``. +- area: http + change: | + Introduced a new runtime flag ``envoy.reloadable_features.no_downgrade_to_canonical_name`` to disable the name downgrading in the + per filter config searching. + See doc :ref:`Http filter route specific config ` or + issue https://github.com/envoyproxy/envoy/issues/29461 for more specific detail and examples. +- area: http + change: | + Switch from http_parser to BalsaParser for handling HTTP/1.1 traffic. See https://github.com/envoyproxy/envoy/issues/21245 for + details. This behavioral change can be reverted by setting runtime flag ``envoy.reloadable_features.http1_use_balsa_parser`` to + ``false``. +- area: udp_proxy + change: | + When the UDP proxy has session filters, choosing the upstream host and creating a socket only happens after iterating all + ``onNewSession()`` calls for all the filters in the chain. Upstream host health check for each downstream datagram does + not apply when there are session filters, and per-packet load balancing can't be used when there are session filters. +- area: zone-aware routing + change: | + Zone-aware routing is now enabled even when the originating and upstream cluster have different numbers of zones. + Previously, zone-aware routing was disabled in that case and the ``lb_zone_number_differs`` stat on the cluster was incremented. + This behavioral change can be reverted by setting runtime guard + ``envoy.reloadable_features.enable_zone_routing_different_zone_counts`` to ``false``. + Additionally, zone-aware routing now works correctly even when the originating and upstream cluster have different zone sets. + Previously, zone-aware routing would not route fairly in this case. + To revert the entire change, set the runtime flag ``envoy.reloadable_features.locality_routing_use_new_routing_logic`` + to ``false`` to get the old behavior and well-tested codepaths, undoing both changes. +- area: UHV + change: | + Introduced runtime flag ``envoy.reloadable_features.enable_universal_header_validator`` for toggling Universal Header Validator + (UHV) on and off. + The default value is off. This option is currently functional only when the ``ENVOY_ENABLE_UHV`` build flag is enabled. + See https://github.com/envoyproxy/envoy/issues/10646 for more information about UHV. +- area: http + change: | + Add runtime flag ``http.max_requests_per_io_cycle`` for setting the limit on the number of HTTP requests processed + from a single connection in a single I/O cycle. Requests over this limit are processed in subsequent I/O cycles. This + mitigates CPU starvation by connections that simultaneously send high number of requests by allowing requests from other + connections to make progress. This runtime value can be set to 1 in the presence of abusive HTTP/2 or HTTP/3 connections. + By default this limit is disabled. + +minor_behavior_changes: +- area: ext_authz + change: | + Removing any query parameter in the presence of repeated query parameter keys no longer drops the repeats. +- area: alternate_protocols_cache_filter + change: | + Changed the alternate protocols cache filter to get the cache from cluster config rather than filter config. + This allows one downstream HTTP filter to be used with multiple clusters with different caches. This change can be + reverted by setting runtime guard ``envoy.reloadable_features.use_cluster_cache_for_alt_protocols_filter`` to ``false``. +- area: ext_authz + change: | + Don't append the local address to ``x-forwarded-for`` header when sending an http (not gRPC) auth request. + This behavior can be reverted by setting runtime flag + ``envoy.reloadable_features.ext_authz_http_send_original_xff`` to ``false``. +- area: ext_proc + change: | + Envoy will only take + :ref:`mode_override ` + when waiting for the header responses. It will be ignored if it is in other processing states. +- area: outlier detection + change: | + Outlier detection will always respect ``max_ejection_percent`` now. + This behavioral change can be reverted by setting runtime guard + ``envoy.reloadable_features.check_mep_on_first_eject`` to ``false``. +- area: outlier detection + change: | + A node must stay in healthy state for at least one period of + :ref:`check interval ` before ejection time multiplier is decremented. +- area: quic + change: | + Enable QUICHE request and response headers validation. This behavior can be reverted by setting runtime flag + ``envoy.reloadable_features.FLAGS_envoy_quic_reloadable_flag_quic_act_upon_invalid_header`` to ``false``. +- area: http oauth2 filter + change: | + Change HMAC cookie encoding to base64-encoded only. This change can be reverted temporarily by + setting the runtime guard ``envoy.reloadable_features.hmac_base64_encoding_only`` to ``false``. +- area: router + change: | + Enable copying response_code from the upstream stream_info onto the downstream stream_info. + This behavior can be reverted by setting runtime guard + ``envoy.reloadable_features.copy_response_code_to_downstream_stream_info`` to ``false``. +- area: xds + change: | + Set the lower bound of :ref:`fill_rate ` + to once per year. Values lower than once per year will automatically be set to that value. +- area: redis + change: | + The redis network filter :ref:`connection_rate_limit_per_sec + ` + must be greater than 0. A config that sets this value to 0 will be rejected. +- area: upstream + change: | + Deprecate code path of legacy upstream load balancer. Ideally, this is implementation detail changes and should not + affect users. However, if there is any user who encounters issues, this behavior can be reverted by setting runtime flag + ``envoy_reloadable_features_convert_legacy_lb_config`` to false. +- area: http + change: | + Change the proxy status for ``UpstreamRequestTimeout`` to ``HttpResponseTimeout``. + It can be disabled by the runtime guard ``envoy.reloadable_features.proxy_status_upstream_request_timeout``. +- area: local_rate_limit + change: | + Added new configuration field :ref:`always_consume_default_token_bucket + ` + to allow for setting if default token bucket should be always consumed or only be consumed when there is no matching descriptor. +- area: tls + change: | + changed ssl failure reason format in ssl socket for a better handling. + It can be disabled by the runtime guard ``envoy.reloadable_features.ssl_transport_failure_reason_format``. +- area: tls_inspector + change: | + Updated the security posture of the :ref:`TLS inspector listener filter ` to + robust against untrusted downstreams and upstreams. +- area: router + change: | + Enable environment_variable in router direct response. +- area: access_log + change: | + When emitting grpc logs, only downstream HTTP filter state was used. Now, both downstream and upstream HTTP filter states + will be tried to find the keys configured in filter_state_objects_to_log. + +bug_fixes: +- area: connection limit + change: | + Fixed a use-after-free bug in the connection limit filter. +- area: subset load balancer + change: | + Fixed a bug where + :ref:`overprovisioning_factor` and + :ref:`weighted_priority_health ` + values were not respected when subset load balacing was enabled. The default values of ``140`` and ``false`` were always used. +- area: http1 + change: | + Fixed a bug where HTTP/1.1 requests with ``Connection: close`` header is handled differently if the requested is internally redirected. + Without internal redirect, the response will also have a ``Connection: close`` header and the connection will be closed after finishing + that request. Requests with internal redirect should be handled in the same way. This behavior can be reverted by setting runtime + ``envoy.reloadable_features.http1_connection_close_header_in_redirect`` to ``false``. +- area: redis + change: | + Fixed a bug where redis key formatter is using the closed stream because of life time issues. +- area: extension_discovery_service + change: | + Fixed a bug causing crash if ECDS is used with upstream HTTP filters. +- area: tls + change: | + Fixed a bug where handshake may fail when both private key provider and cert validation are set. +- area: dns + change: | + Fixed a bug where when ``respect_dns_ttl`` was set to ``true``, c-ares dns resolver only considered address record for TTL calculation + while ignoring CNAME records TTL. Now when ``respect_dns_ttl`` is set to ``true`` minimum of all TTL records is considered. +- area: dns + change: | + Fixed a bug where dns response was not always conforming `RFC 2181 `_ for TTL values. + Previously a malicious user could add a TTL greater than 2^31 - 1, and with c-ares library using 32 bit signed int data type + would overflow and send a negative TTL. +- area: healthcheck + change: | + The default behavior of unejecting outlier-detection-ejected host on successful active health checking can + be disabled by setting :ref:`outlier_detection.successful_active_health_check_uneject_host + ` + to ``false``. This new configuration flag is a substitute for the removed runtime option + ``envoy.reloadable_features_successful_active_health_check_uneject_host``. +- area: aws signer + change: | + Fixed a bug where expiration timestamp on task roles failed to validate. This causes failure of credential caching which + results in constant hits to the task role metadata URL. +- area: router check tool + change: | + Fixed a bug where the route coverage is not correctly calculated when a route has weighted clusters. +- area: unix domain sockets + change: | + Fixed a crash on some versions of macOS when using a listener on a unix-domain socket. +- area: redis + change: | + Fixed a bug where redis key with % in the key is failing with a validation error. +- area: tracing + change: | + Fixed a bug in the Datadog tracer where Datadog's "operation name" field would contain what should be in the "resource name" field. +- area: http + change: | + Close HTTP/2 and HTTP/3 connections that prematurely reset streams. The runtime key + ``overload.premature_reset_min_stream_lifetime_seconds`` determines the interval where received stream + reset is considered premature (with 1 second default). The runtime key ``overload.premature_reset_total_stream_count``, + with the default value of 500, determines the number of requests received from a connection before the check for premature + resets is applied. The connection is disconnected if more than 50% of resets are premature, or if + the number of suspect streams is already large enough to guarantee that more than 50% of the streams will be suspect + upon reaching the total stream threshold (even if all the remaining streams are considered benign). + Setting the runtime key ``envoy.restart_features.send_goaway_for_premature_rst_streams`` to ``false`` completely disables + this check. +- area: http + change: | + Fixed a bug that could cause metadata to be decoded after a local reply has been triggered. + Can be disabled by setting ``envoy.reloadable_features.stop_decode_metadata_on_local_reply`` to false. +- area: docker/publishing + change: | + Update base images to resolve various glibc vulnerabilities. +- area: xds + change: | + Fix a bug where the nonce was not reset after reconnecting to the xDS server, when using State-of-the-World. + +removed_config_or_runtime: +- area: listener + change: | + Removed ``envoy.reloadable_features.enable_update_listener_socket_options`` runtime flag and legacy code paths. +- area: tcp + change: | + Removed runtime key ``envoy.reloadable_features.tcp_pool_idle_timeout``. +- area: http filters + change: | + Removed ``envoy_reloadable_features_http_filter_avoid_reentrant_local_reply`` runtime flag and legacy code paths. +- area: tcp_proxy + change: | + Removed ``envoy_reloadable_features_finish_reading_on_decode_trailers`` runtime flag and legacy code paths. +- area: dns + change: | + Removed ``envoy.restart_features.use_apple_api_for_dns_lookups`` and legacy code paths. +- area: runtime + change: | + Removed ``envoy.restart_features.remove_runtime_singleton`` and legacy code paths. +- area: runtime + change: | + Removed ``envoy_reloadable_features_append_query_parameters_path_rewriter`` and legacy code paths. +- area: xDS + change: | + Removed ``envoy.restart_features.explicit_wildcard_resource`` and legacy code paths. +- area: quic + change: | + Removed ``envoy.reloadable_features.reject_require_client_certificate_with_quic`` and legacy code paths. +- area: healthcheck + change: | + Removed ``envoy.reloadable_features_successful_active_health_check_uneject_host`` runtime option and + substituted it with :ref:`outlier_detection.successful_active_health_check_uneject_host + ` + outlier detection configuration flag. +- area: ext_authz + change: | + Removed ``envoy.reloadable_features.http_ext_auth_failure_mode_allow_header_add`` + runtime option and substituted it with :ref:`failure_mode_allow_header_add + ` + configuration flag. + +new_features: +- area: access_log + change: | + Added ``%RESPONSE_FLAGS_LONG%`` substitution string, that will output a pascal case string representing the response flags. + The output response flags will correspond with ``%RESPONSE_FLAGS%``, only with a long textual string representation. +- area: compression + change: | + Added :ref:`remove_accept_encoding_header + ` + for per-route configuration of this value. +- area: config + change: | + Added the capability to defer broadcasting of certain cluster (CDS, EDS) to + worker threads from the main thread. This optimization can save significant + amount of memory in cases where there are (1) a large number of workers and + (2) a large amount of config, most of which is unused. This capability is + guarded by :ref:`enable_deferred_cluster_creation + `. +- area: extension_discovery_service + change: | + Added ECDS support for :ref:`downstream network filters `. +- area: ext_proc + change: | + Added + :ref:`disable_immediate_response ` + config API to ignore the + :ref:`immediate_response ` + message from the external processing server. +- area: access_log + change: | + Added a field lookup to ``%FILTER_STATE%`` for objects that have reflection enabled. +- area: http + change: | + Added :ref:`Json-To-Metadata filter `. +- area: listener + change: | + Added possibility to track global downstream connection limit via :ref:`downstream connections monitor + ` in overload manager. +- area: extension_discovery_service + change: | + Added metric ``listener.listener_stat.network_extension_config_missing`` to track closed connections due to missing config. +- area: lua + change: | + Added :ref:`downstreamRemoteAddress() ` + method to the Stream info object API. +- area: quic + change: | + Added support for QUIC listener filters with ECDS support reusing the same config API + :ref:`listener_filters ` as TCP does. +- area: oauth2 + change: | + Added :ref:`use_refresh_token ` + to support updating an access token via a refresh token if that is provided by authorization server. +- area: redis + change: | + Added support for ``time`` command (returns a local response). +- area: extension_discovery_service + change: | + Added ECDS support for :ref:`upstream network filters `. +- area: redis + change: | + Added support for ``lmove`` command. +- area: upstream + change: | + Added :ref:`allow_redundant_keys ` + to suppport redundant keys in request metadata for subset load balancing. +- area: access_logs + change: | + Added :ref:`json_format_options ` config option to + support JSON output formatting and the :ref:`sort_properties ` + option to print the JSON output with sorted properties. +- area: tap + change: | + Added :ref:`custom_sink ` type to enable writing tap data + out to a custom sink extension. +- area: tls + change: | + Added :ref:`disable_stateful_session_resumption + ` config option to + disable stateful TLS session resumption. +- area: udp_proxy + change: | + Added :ref:`session_filters ` config to + support optional filters that will run for each upstream UDP session. More information can be found in the UDP proxy documentation. +- area: udp_proxy + change: | + Added ``injectDatagramToFilterChain()`` callback to UDP session filters that allows session filters to inject datagrams downstream + or upstream the filter chain during a filter chain iteration. This can be used, for example, by session filters that are required + to buffer datagrams due to an asynchronous call. +- area: otlp_stats_sink + change: | + Added :ref:`stats prefix option ` + to OTLP stats sink that enables adding a static prefix to all stats flushed by this sink. +- area: udp_proxy + change: | + Added :ref:`http_capsule ` UDP session filter + that can be used to encapsule or decapsulate UDP datagrams in HTTP, when used for UDP tunneling. +- area: tap + change: | + Added :ref:`record_headers_received_time ` + to control writing request and response headers received time in trace output. +- area: udp_proxy + change: | + added :ref:`dynamic_forward_proxy ` + UDP session filter that can be used to have dynamic forward proxy UDP flows, when used in conjunction with another session filter + that sets required filter state values. +- area: zookeeper + change: | + Added support for emitting per opcode request bytes metrics via :ref:`enable_per_opcode_request_bytes_metrics + `. + added support for emitting per opcode response bytes metrics via :ref:`enable_per_opcode_response_bytes_metrics + `. +- area: tls + change: | + Added fallback :ref:`fallback + ` + to support private key provider to fallback to boringssl TLS handshake. + If the private key provider isn't available (eg. the required hardware capability doesn't existed), + Envoy will fallback to the BoringSSL default implementation when the fallback is ``true``. + The default value is ``false``. +- area: tcp + change: | + Added the support to detect and send TCP RST for raw buffer socket based connections. This is currently supported on Linux only. + It can be disabled by the runtime guard ``envoy_reloadable_features_detect_and_raise_rst_tcp_connection``. +- area: upstream + change: | + Added the ability to specify a custom upstream local address selector using + :ref:`local_address_selector `. +- area: redis + change: | + Added new configuration field :ref:`read_command_policy + ` + to specify Envoy should route read commands to another cluster. +- area: tap + change: | + Added :ref:`record_downstream_connection ` + to control writing downstream connection address info in trace output. +- area: tracing + change: | + Added :ref:`spawn_upstream_span + ` + to control whether to create separate upstream span for upstream request. +- area: original_dst + change: | + added support for the internal listener address recovery using the original destination listener filter. +- area: filters + change: | + Added filters to update the filter state for :ref:`the HTTP requests ` and + :ref:`the TCP connections `. +- area: http + change: | + Added :ref:`disabled + ` + flag to disable a filter in the filter chain by default. The filter can be enabled explicitly by setting + valid per filter config in the route configuration. +- area: admin_logging + change: | + added support for glob control of fine-grain loggers in admin /logging interface. +- area: geoip + change: | + Added support for :ref:`Maxmind geolocation provider `. +- area: admin + change: | + Added a new ``skip_exit`` query parameter to ``/drain_listeners`` to skip exiting after the drain period. +- area: router + change: | + Added ``metadata`` support for :ref:`virtual host ` and + :ref:`route configuration `. +- area: tracing + change: | + Added support for exporting spans via HTTP on the OpenTelemetry tracer. + +deprecated: +- area: tracing + change: | + OpenTracing is deprecated and will be removed at version 1.30, since the upstream project has been abandoned. +- area: tracing + change: | + Opencensus is deprecated and will be removed at version 1.30, since the upstream project has been abandoned. +- area: tracing + change: | + :ref:`start_child_span ` + is deprecated by + :ref:`spawn_upstream_span + `. + Please use the new field to control whether to create separate upstream span for upstream request. +- area: listener + change: | + deprecated runtime key ``overload.global_downstream_max_connections`` in favor of :ref:`downstream connections monitor + `. diff --git a/changelogs/BUILD b/changelogs/BUILD index 57ad9389b3b1..e36d6bfbb93a 100644 --- a/changelogs/BUILD +++ b/changelogs/BUILD @@ -12,3 +12,13 @@ filegroup( name = "changelogs", srcs = glob(["*.*.*.yaml"]) + ["current.yaml"], ) + +genrule( + name = "summary", + outs = ["summary.txt"], + cmd = """ + cat $(location :summary.md) > $@ + """, + tools = ["summary.md"], + visibility = ["//visibility:public"], +) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 964a4cd719cb..715baf916d7e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -2,281 +2,350 @@ date: Pending behavior_changes: # *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* -- area: jwt - change: | - Changed behavior of the jwt extraction, passing entire token for validation, instead cut him in the non-Base64 character. - This change can be reverted temporarily by setting the runtime guard ``envoy.reloadable_features.token_passed_entirely`` to false. -- area: eds +- area: http change: | - Introduced caching of EDS assignments when used with ADS. Prior to this change, Envoy required that EDS assignments were sent - after an EDS cluster was updated. If no EDS assignment was received for the cluster, it ended up with an empty assignment. - Following this change, after a cluster update, Envoy waits for an EDS assignment until - :ref:`initial_fetch_timeout ` times out, and will then apply - the cached assignment and finish updating the warmed cluster. This change is disabled by default, and can be enabled by setting - the runtime flag ``envoy.restart_features.use_eds_cache_for_ads`` to true. + Remove the hop by hop TE header from downstream request headers. This change can be temporarily reverted + by setting ``envoy.reloadable_features.sanitize_te`` to false. - area: http change: | - Introduced a new runtime flag ``envoy.reloadable_features.no_downgrade_to_canonical_name`` to disable the name downgrading in the - per filter config searching. + Flip runtime flag ``envoy.reloadable_features.no_downgrade_to_canonical_name`` to true. Name downgrading in the + per filter config searching will be disabled by default. This behavior can be temporarily reverted by setting + the flag to false explicitly. See doc :ref:`Http filter route specific config ` or issue https://github.com/envoyproxy/envoy/issues/29461 for more specific detail and examples. -- area: http +- area: listener change: | - Switch from http_parser to BalsaParser for handling HTTP/1.1 traffic. See https://github.com/envoyproxy/envoy/issues/21245 for - details. This behavioral change can be reverted by setting runtime flag ``envoy.reloadable_features.http1_use_balsa_parser`` to - false. -- area: udp_proxy + undeprecated runtime key ``overload.global_downstream_max_connections`` until :ref:`downstream connections monitor + ` extension becomes stable. +- area: stats dns_filter change: | - When the UDP proxy has session filters, choosing the upstream host and creating a socket only happens after iterating all - ``onNewSession()`` calls for all the filters in the chain. Upstream host health check for each downstream datagram does - not apply when there are session filters, and per-packet load balancing can't be used when there are session filters. -- area: zone-aware routing - change: | - Zone-aware routing is now enabled even when the originating and upstream cluster have different numbers of zones. - Previously, zone-aware routing was disabled in that case and the ``lb_zone_number_differs`` stat on the cluster was incremented. - This behavioral change can be reverted by setting runtime guard - ``envoy.reloadable_features.enable_zone_routing_different_zone_counts`` to false. - Additionally, zone-aware routing now works correctly even when the originating and upstream cluster have different zone sets. - Previously, zone-aware routing would not route fairly in this case. - To revert the entire change, set the runtime flag ``envoy.reloadable_features.locality_routing_use_new_routing_logic`` - to false to get the old behavior and well-tested codepaths, undoing both changes. - -- area: UHV + Fixed tag extraction so that :ref:`stat_prefix ` + is properly extracted. This changes the Prometheus name from + dns_filter_myprefix_local_a_record_answers{} to dns_filter_local_a_record_answers{envoy.dns_filter_prefix="myprefix"}. +- area: stats connection_limit + change: | + Fixed tag extraction so that :ref:`stat_prefix ` + is properly extracted. This changes the Prometheus name from ``envoy_connection_limit_myprefix_limited_connections{}`` to + ``envoy_connection_limit_limited_connections{envoy_connection_limit_prefix="myprefix"}``. +- area: http2 + change: | + Changes the default value of ``envoy.reloadable_features.http2_use_oghttp2`` to true. This changes the codec used for HTTP/2 + requests and responses. This behavior can be reverted by setting the feature to false. +- area: http2 + change: | + Discard the ``Host`` header if the ``:authority`` header was received to bring Envoy into compliance with + https://www.rfc-editor.org/rfc/rfc9113#section-8.3.1 This behavioral change can be reverted by setting runtime flag + ``envoy.reloadable_features.http2_discard_host_header`` to false. +- area: grpc reverse bridge change: | - Introduced runtime flag ``envoy.reloadable_features.enable_universal_header_validator`` for toggling Universal Header Validator - (UHV) on and off. - The default value is off. This option is currently functional only when the ``ENVOY_ENABLE_UHV`` build flag is enabled. - See https://github.com/envoyproxy/envoy/issues/10646 for more information about UHV. + Handle empty response bodies in grpc_http1_reverse_bridge. This may cause problems for clients expecting the filter to crash + for empty responses. This behavioral change can be temporarily reverted by setting runtime guard + ``envoy.reloadable_features.grpc_http1_reverse_bridge_handle_empty_response`` to false. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* -- area: ext_authz +- area: golang + change: | + Remove Protocol method from RequestHeaderMap. + To get the protocol, please use GetProperty("request.protocol") instead. +- area: aws + change: | + Added support to use http async client to fetch the credentials from EC2 instance metadata and ECS task metadata providers + instead of libcurl which is deprecated. By default this behavior is disabled. To enable set + ``envoy.reloadable_features.use_http_client_to_fetch_aws_credentials`` to true. +- area: local_rate_limit + change: | + Added new configuration field :ref:`rate_limited_as_resource_exhausted + ` + to allow for setting if rate limit grpc response should be RESOURCE_EXHAUSTED instead of the default UNAVAILABLE. +- area: config parsing, http cache filter + change: | + Replaces protobuf hashing by human-readable string with a dedicated deterministic hashing algorithm. + The performance of the hash operation is improved by 2-10x depending on the structure of the message, + which is expected to reduce config update time or startup time by 10-25%. The new algorithm is also + used for http_cache_filter hashing, which will effectively cause a one-time cache flush on update + for users with a persistent cache. To enable this behavior set ``envoy.restart_features.use_fast_protobuf_hash`` to true. +- area: filter state + change: | + Added config name of filter sending a local reply in filter state with key + ``envoy.filters.network.http_connection_manager.local_reply_owner``. + See :ref:`the well-known filter state keys ` for more detail. +- area: http2 + change: | + Flip the runtime guard ``envoy.reloadable_features.defer_processing_backedup_streams`` to be on by default. + This feature improves flow control within the proxy by deferring work on the receiving end if the other + end is backed up. +- area: admin + change: | + Switch no admin ``warning`` -> ``info``. +- area: generic_proxy + change: | + Update the stats prefix of generic proxy from ```` to ``generic_proxy.``. + +bug_fixes: +# *Changes expected to improve the state of the world and are unlikely to have negative effects* +- area: buffer change: | - removing any query parameter in the presence of repeated query parameter keys no longer drops the repeats. -- area: alternate_protocols_cache_filter + Fixed a bug (https://github.com/envoyproxy/envoy/issues/28760) that the internal listener causes an undefined + behavior due to the unintended release of the buffer memory. +- area: xds change: | - Changed the alternate protocols cache filter to get the cache from cluster config rather than filter config. - This allows one downstream filter to be used with multiple clusters with different caches. This change can be reverted by - setting runtime guard ``envoy.reloadable_features.use_cluster_cache_for_alt_protocols_filter`` to false. -- area: ext_authz + Fixed a bug (https://github.com/envoyproxy/envoy/issues/27702) that caused ADS initialization + to fail on the first attempt and set a back-off retry interval of up to 1 second, if ADS is + using an Envoy Cluster for the backend. The issue was fixed to ensure that ADS initialization + happens after the Envoy Cluster it depends upon has been properly initialized. ADS that does + not depend on an Envoy Cluster (i.e. GoogleGrpc) is not affected by this change. +- area: grpc change: | - Don't append the local address to ``x-forwarded-for`` header when sending an http (not gRPC) auth request. - This behavior can be reverted by setting runtime flag - ``envoy.reloadable_features.ext_authz_http_send_original_xff`` to false. -- area: outlier detection + Fixed a bug in gRPC async client cache which intermittently causes CPU spikes due to busy loop in timer expiration. +- area: tracing change: | - Outlier detection will always respect max_ejection_percent now. - This behavioral change can be reverted by setting runtime guard - ``envoy.reloadable_features.check_mep_on_first_eject`` to false. + Fixed a bug that caused the Datadog tracing extension to drop traces that + should be kept on account of an extracted sampling decision. - area: quic change: | - Enable QUICHE request and response headers validation. This behavior can be reverted by setting runtime flag - ``envoy.reloadable_features.FLAGS_envoy_quic_reloadable_flag_quic_act_upon_invalid_header`` to false. -- area: http oauth2 filter - change: | - Change HMAC cookie encoding to base64-encoded only. This change can be reverted temporarily by - setting the runtime guard ``envoy.reloadable_features.hmac_base64_encoding_only`` to false. -- area: router + Fixed a bug in QUIC and HCM interaction which could cause use-after-free during asynchronous certificates retrieval. + The fix is guarded by runtime ``envoy.reloadable_features.quic_fix_filter_manager_uaf``. +- area: redis change: | - Enable copying response_code from the upstream stream_info onto the downstream stream_info. - This behavior can be reverted by setting runtime guard - ``envoy.reloadable_features.copy_response_code_to_downstream_stream_info`` to false. -- area: xds + Fixed a bug causing crash if incoming redis key does not match against a prefix_route and catch_all_route is not defined. +- area: access log change: | - Set the lower bound of :ref:`fill_rate ` - to once per year. Values lower than once per year will automatically be set to that value. -- area: redis + Fixed a bug where the omit_empty_values field was not honored for access logs specifying formats via text_format_source. +- area: ext_proc change: | - The redis network filter :ref:`connection_rate_limit_per_sec - ` - must be greater than 0. A config that sets this value to 0 will be rejected. - -bug_fixes: -# *Changes expected to improve the state of the world and are unlikely to have negative effects* -- area: connection limit + Fixed content_length related issues when body mutation by external processor is enabled. ext_proc filter removes the content + length header in 1)STREAMED BodySendMode 2) BUFFERED_PARTIAL BodySendMode and 3) BUFFERED BodySendMode + SKIP HeaderSendMode. + This will enable chunked-encoding whenever feasible in HTTP1.1. Besides, ext_proc filter keep content length header + in BUFFERED BodySendMode + SEND HeaderSendMode. It is now external processor's responsibility to set the content length + correctly matched to the mutated body. if those two doesn't match, the mutation will be rejected and local reply with error + status will be returned. +- area: dynamic_forward_proxy + change: | + Fixed a bug where the preresolved hostnames specified in the Dynamic Forward Proxy cluster + config would not use the normalized hostname as the DNS cache key, which is the same key + used for retrieval. This caused cache misses on initial use, even though the host DNS entry + was pre-resolved. The fix is guarded by runtime guard ``envoy.reloadable_features.normalize_host_for_preresolve_dfp_dns``, + which defaults to true. +- area: otlp_stat_sink + change: | + Fixed a bug where a histogram bucket counts were wrong. Additionally, the number of buckets is fixed and is now + one element larger than the explicit bounds elements, as required by the specification. +- area: tracing change: | - fixed a use-after-free bug in the connection limit filter. -- area: subset load balancer + Fixed a bug where child spans produced by the Datadog tracer would have an incorrect operation name. +- area: DNS change: | - Fixed a bug where - :ref:`overprovisioning_factor` and - :ref:`weighted_priority_health ` - values were not respected when subset load balacing was enabled. The default values of 140 and false were always used. -- area: http1 + Fixed a race condition that when multiple requests with the same authority header are sent to Envoy, sometimes some requests + may receive 503 response with no_healthy_upstream from Envoy. The fix is guarded by runtime guard + ``envoy.reloadable_features.dns_cache_set_first_resolve_complete``, which defaults to true. +- area: upstream change: | - Fixed a bug where HTTP/1.1 requests with "Connection: close" header is handled differently if the requested is internally redirected. - Without internal redirect, the response will also have a "Connection: close" header and the connection will be closed after finishing - that request. Requests with internal redirect should be handled in the same way. This behavior can be reverted by setting runtime - ``envoy.reloadable_features.http1_connection_close_header_in_redirect`` to false. -- area: redis + Fixed a bug that the subset load balancer will always be used even if the subset load balancer config does not + contain any subset selector. +- area: docker change: | - fixed a bug where redis key formatter is using the closed stream because of life time issues. -- area: extension_discovery_service + Updated base image to ``ubuntu:22.04`` to fix Redis memory issue (https://github.com/envoyproxy/envoy/issues/31248). +- area: ext_authz change: | - Fixed a bug causing crash if ECDS is used with upstream HTTP filters. -- area: tls + Fixed a bug to ensure the proper functioning of the ``with_request_body`` feature within the per-route ExtAuthZ filter. + +removed_config_or_runtime: +# *Normally occurs at the end of the* :ref:`deprecation period ` +- area: http change: | - fixed a bug where handshake may fail when both private key provider and cert validation are set. -- area: dns + Removed ``envoy.reloadable_features.expand_agnostic_stream_lifetime`` and legacy code paths. +- area: http change: | - Fixed a bug where when respect_dns_ttl was set to true, c-ares dns resolver only considered address record for ttl calculation - while ignoring CNAME records TTL. Now when respect_dns_ttl is set to true minimum of all TTL records is considered. -- area: dns + removed ``envoy.reloadable_features.sanitize_original_path`` and legacy code paths. +- area: maglev change: | - Fixed a bug where dns response was not always conforming [RFC 2181](https://datatracker.ietf.org/doc/html/rfc2181) for TTL values. - Previously a malicious user could add a TTL greater than 2^31 - 1, and with c-ares library using 32 bit signed int data type - would overflow and send a negative TTL. -- area: healthcheck + Removed ``envoy.reloadable_features.allow_compact_maglev`` and legacy code paths. +- area: router change: | - The default behavior of unejecting outlier-detection-ejected host on successful active health checking can - be disabled by setting :ref:`outlier_detection.successful_active_health_check_uneject_host - ` - to ``false``. This new configuration flag is a substitute for the removed runtime option - ``envoy.reloadable_features_successful_active_health_check_uneject_host``. -- area: aws signer + Removed the deprecated ``envoy.reloadable_features.prohibit_route_refresh_after_response_headers_sent`` + runtime flag and legacy code path. +- area: upstream change: | - fixed a bug where expiration timestamp on task roles failed to validate. This causes failure of credential caching which - results in constant hits to the task role metadata URL. -- area: router check tool + Removed the deprecated ``envoy.reloadable_features.validate_detailed_override_host_statuses`` + runtime flag and legacy code path. +- area: grpc change: | - Fixed a bug where the route coverage is not correctly calculated when a route has weighted clusters. -- area: unix domain sockets + Removed the deprecated ``envoy.reloadable_features.service_sanitize_non_utf8_strings`` + runtime flag and legacy code path. +- area: access log change: | - Fixed a crash on some versions of macOS when using a listener on a unix-domain socket. -- area: redis + Removed the deprecated ``envoy.reloadable_features.format_ports_as_numbers`` + runtime flag and legacy code path. +- area: router change: | - Fixed a bug where redis key with % in the key is failing with a validation error. + Removed the deprecated ``envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config`` + runtime flag and legacy code path. -removed_config_or_runtime: -# *Normally occurs at the end of the* :ref:`deprecation period ` -- area: listener - change: | - Removed ``envoy.reloadable_features.enable_update_listener_socket_options`` runtime flag and legacy code paths. -- area: tcp +new_features: +- area: composite filter + change: | + added :ref:`ExtensionConfiguration discovery service` support for + :ref:`composite filter `. +- area: aws + change: | + Added support for AWS common utility to fetch metadata credentials from AWS STS by using ``WebIdentityToken``. To enable + you need to set ``envoy.reloadable_features.use_http_client_to_fetch_aws_credentials`` to ``true`` so that web identity + credentials provider can use http async client to fetch credentials. Web identity credentials provider cannot use current + default libcurl credentials fetcher which is under deprecation and will soon be removed. Web identity credentials provider + is not compatible with :ref:`Grpc Credentials AWS IAM ` + plugin which can only support deprecated libcurl credentials fetcher, see https://github.com/envoyproxy/envoy/pull/30626. +- area: filters + change: | + Added :ref:`the Basic Auth filter `, which can be used to + authenticate user credentials in the HTTP Authentication heaer defined in `RFC7617 `_. +- area: upstream change: | - Removed runtime key ``envoy.reloadable_features.tcp_pool_idle_timeout``. -- area: http filters + Implmented API :ref:`drop_overloads` + which can be used to drop certain percentage of traffic from Envoy. +- area: stats change: | - Removed ``envoy_reloadable_features_http_filter_avoid_reentrant_local_reply`` runtime flag and legacy code paths. -- area: tcp_proxy + added :ref:`per_endpoint_stats ` to get some metrics + for each endpoint in a cluster. +- area: jwt change: | - Removed ``envoy_reloadable_features_finish_reading_on_decode_trailers`` runtime flag and legacy code paths. -- area: dns + The jwt filter can now serialize non-primitive custom claims when maping claims to headers. + These claims will be serialized as JSON and encoded as Base64. +- area: jwt change: | - Removed ``envoy.restart_features.use_apple_api_for_dns_lookups`` and legacy code paths. -- area: runtime + The JWT authentication filter supports changing the routes when either the headers or the + dynamic metadata are modified. +- area: tcp_proxy change: | - Removed ``envoy.restart_features.remove_runtime_singleton`` and legacy code paths. -- area: xDS + added support to TCP Proxy for recording the latency in ``UpstreamTiming`` from when the first + initial connection to the upstream cluster was attempted to when either the + connection was successfully established or the filiter failed to initialize + any connection to the upstream. +- area: ratelimit change: | - Removed ``envoy.restart_features.explicit_wildcard_resource`` and legacy code paths. -- area: quic + Ratelimit supports setting the HTTP status that is returned to the client when the ratelimit server + returns an error or cannot be reached with :ref:`status_on_error + ` + configuration flag. +- area: tracing change: | - Removed ``envoy.reloadable_features.reject_require_client_certificate_with_quic`` and legacy code paths. -- area: healthcheck + Added support for configuring resource detectors on the OpenTelemetry tracer. +- area: tracing change: | - Removed ``envoy.reloadable_features_successful_active_health_check_uneject_host`` runtime option and - substituted it with :ref:`outlier_detection.successful_active_health_check_uneject_host - ` - outlier detection configuration flag. - -new_features: -- area: access_log + Added support to configure a sampler for the OpenTelemetry tracer. +- area: CEL-attributes change: | - added %RESPONSE_FLAGS_LONG% substitution string, that will output a pascal case string representing the resonse flags. - The output response flags will correspond with %RESPONSE_FLAGS%, only with a long textual string representation. -- area: config + Added :ref:`attribute ` ``connection.transport_failure_reason`` + for looking up connection transport failure reason. +- area: ext_authz change: | - Added the capability to defer broadcasting of certain cluster (CDS, EDS) to - worker threads from the main thread. This optimization can save significant - amount of memory in cases where there are (1) a large number of workers and - (2) a large amount of config, most of which is unused. This capability is - guarded by :ref:`enable_deferred_cluster_creation - `. -- area: extension_discovery_service + New config parameter :ref:`charge_cluster_response_stats + ` + for not incrementing cluster statistics on ext_authz response. Default true, no behavior change. +- area: ext_authz change: | - added ECDS support for :ref:` downstream network filters`. -- area: ext_proc + forward :ref:`filter_metadata ` selected by + :ref:`route_metadata_context_namespaces + ` + and :ref:`typed_filter_metadata ` selected by + :ref:`route_typed_metadata_context_namespaces + ` + from the metadata of the selected route to external auth service. + This metadata propagation is independent from the dynamic metadata from connection and request. +- area: ext_authz_filter + change: | + added :ref:`with_request_body + ` to optionally override + the default behavior of sending the request body to the authorization server from the per-route filter. +- area: grpc async client + change: | + added :ref:`max_cached_entry_idle_duration + ` + to control the cached grpc client eviction time in the cache. +- area: ratelimit + change: | + Ratelimit supports optional additional prefix to use when emitting statistics with :ref:`stat_prefix + ` + configuration flag. +- area: udp_proxy change: | - added - :ref:`disable_immediate_response ` - config API to ignore the - :ref:`immediate_response ` - message from the external processing server. -- area: access_log + added support for propagating the response headers in :ref:`UdpTunnelingConfig + ` and + response trailers in :ref:`UdpTunnelingConfig + ` to + the downstream info filter state. +- area: tracing change: | - added a field lookup to %FILTER_STATE% for objects that have reflection enabled. -- area: http + Provide initial span attributes to a sampler used in the OpenTelemetry tracer. +- area: tracing change: | - added :ref:`Json-To-Metadata filter `. -- area: extension_discovery_service + Added support to configure a Dynatrace resource detector for the OpenTelemetry tracer. +- area: compression change: | - added metric listener.listener_stat.network_extension_config_missing to track closed connections due to missing config. -- area: quic + Added qatzip :ref:`compressor `. +- area: udp_proxy change: | - added support for QUIC listener filters with ECDS support reusing the same config API - :ref:`listener_filters ` as TCP does. + add :ref:`access log options + ` + to allow recording an access log entry periodically for the UDP session, and allow recording an access + log entry on the connection tunnel created successfully to upstream when UDP tunneling is configured. +- area: internal_redirects + change: | + Added support to copy headers from the redirect response to the + triggered request. See + :ref:`response_headers_to_copy`. +- area: stateful_session + change: | + Added :ref:`strict mode ` + to cookie and header based stateful session. If a destination encoded in :ref:`cookie + ` + or in :ref:`specified header + ` respectively + is not available, Envoy will return ``503`` instead of selecting another destination from the cluster. +- area: stream info + change: | + Added time spent reading request headers to ``DownstreamTiming``. - area: redis change: | - added support for time command (returns a local response). -- area: extension_discovery_service + Added support for the watch command (aborts multi transactions if watched keys change). +- area: grpc_http_bridge change: | - added ECDS support for :ref:` upstream network filters`. -- area: redis + added :ref:`ignore_query_parameters + ` option for + automatically stripping query parameters in request URL path. +- area: access_log change: | - added support for lmove command. -- area: upstream + Added new access log command operator ``%EMIT_TIME%`` to get the time when the log entry is emitted. +- area: access_log change: | - added :ref:`allow_redundant_keys ` - to suppport redundant keys in request metadata for subset load balancing. -- area: access_logs + Added support for listener metadata in ``%METADATA%`` formatter. +- area: attributes change: | - added :ref:`json_format_options ` config option to - support JSON output formatting and the :ref:`sort_properties ` - option to print the JSON output with sorted properties. -- area: tap + Added support for listener metadata and listener direction in XDS attributes. +- area: attributes change: | - added :ref:`custom_sink ` type to enable writing tap data - out to a custom sink extension. -- area: udp_proxy + Added support for node data in ``%CEL%`` formatter. +- area: set_metadata change: | - added :ref:`session_filters ` config to - support optional filters that will run for each upstream UDP session. More information can be found in the UDP proxy documentation. -- area: udp_proxy + Added support for injecting typed and untyped dynamic metadata with this filter, also adds the ability + to add multiple namespaces with one filter and config to overwrite existing metadata is opt-in. + :ref:`untyped_metadata ` + may now be used to configure the ``set_metadata`` filter. +- area: lua change: | - added ``injectDatagramToFilterChain()`` callback to UDP session filters that allows session filters to inject datagrams downstream - or upstream the filter chain during a filter chain iteration. This can be used, for example, by session filters that are required - to buffer datagrams due to an asynchronous call. -- area: otlp_stats_sink - change: | - added :ref:` stats prefix option` - to OTLP stats sink that enables adding a static prefix to all stats flushed by this sink. -- area: tap - change: | - added :ref:`record_headers_received_time ` - to control writing request and response headers received time in trace output. -- area: tls - change: | - added fallback :ref:`fallback - ` - to support private key provider to fallback to boringssl tls handshake. - If the private key provider isn't available (eg. the required hardware capability doesn't existed), - Envoy will fallback to the BoringSSL default implementation when the fallback is true. - The default value is false. -- area: tcp - change: | - added the support to detect and send TCP RST for raw buffer socket based connections. This is currently supported on Linux only. - It can be disabled by the runtime guard ``envoy_reloadable_features_detect_and_raise_rst_tcp_connection``. - -- area: upstream + Added lua extension of router cluster specifier plugin to support selecting cluster dynamically by lua code. +- area: redis change: | - Added the ability to specify a custom upstream local address selector using - :ref:`local_address_selector:`. + Added support for the getdel command. deprecated: -- area: tracing +- area: wasm change: | - OpenTracing is deprecated and will be removed at version 1.30, since the upstream project has been abandoned. -- area: tracing + Wasm-specific configuration attributes are deprecated in favor of ``xds`` attributes. +- area: set_metadata change: | - Opencensus is deprecated and will be removed at version 1.30, since the upstream project has been abandoned. + :ref:`metadata_namespace ` + and :ref:`value ` + are deprecated. Please use the new field + :ref:`untyped_metadata ` + to configure static metadata to inject. diff --git a/changelogs/summary.md b/changelogs/summary.md new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/changelogs/summary.md @@ -0,0 +1 @@ + diff --git a/ci/Dockerfile-envoy b/ci/Dockerfile-envoy index 572a4f460124..9902045cbf4f 100644 --- a/ci/Dockerfile-envoy +++ b/ci/Dockerfile-envoy @@ -1,5 +1,5 @@ ARG BUILD_OS=ubuntu -ARG BUILD_TAG=20.04 +ARG BUILD_TAG=22.04@sha256:8eab65df33a6de2844c9aefd19efe8ddb87b7df5e9185a4ab73af936225685bb ARG ENVOY_VRP_BASE_IMAGE=envoy-base @@ -58,8 +58,7 @@ COPY --chown=0:0 --chmod=755 \ # STAGE: envoy-distroless -# gcr.io/distroless/base-nossl-debian11:nonroot -FROM gcr.io/distroless/base-nossl-debian11:nonroot@sha256:a156aae8df01d39f2390021016c672bee4fb34f3a90759f4d9aa74c116ec142a AS envoy-distroless +FROM gcr.io/distroless/base-nossl-debian12:nonroot@sha256:8c957f0c06030921ee439d028b5778dd1cee9e095092833fe8e4ee795d3a2298 AS envoy-distroless EXPOSE 10000 ENTRYPOINT ["/usr/local/bin/envoy"] CMD ["-c", "/etc/envoy/envoy.yaml"] diff --git a/ci/README.md b/ci/README.md index 143b8f323599..4305e52692c1 100644 --- a/ci/README.md +++ b/ci/README.md @@ -5,7 +5,7 @@ and an image based on Windows2019. ## Ubuntu Envoy image -The Ubuntu based Envoy Docker image at [`envoyproxy/envoy-build:`](https://hub.docker.com/r/envoyproxy/envoy-build/) is used for CI checks, +The Ubuntu based Envoy Docker image at [`envoyproxy/envoy-build-ubuntu:`](https://hub.docker.com/r/envoyproxy/envoy-build/) is used for CI checks, where `` is specified in [`envoy_build_sha.sh`](https://github.com/envoyproxy/envoy/blob/main/ci/envoy_build_sha.sh). Developers may work with the latest build image SHA in [envoy-build-tools](https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8) repo to provide a self-contained environment for building Envoy binaries and running tests that reflects the latest built Ubuntu Envoy image. @@ -13,9 +13,14 @@ Moreover, the Docker image at [`envoyproxy/envoy:dev-`](https://hub.docker The `` corresponds to the main commit at which the binary was compiled. Lastly, `envoyproxy/envoy:dev` contains an Envoy binary built from the latest tip of main that passed tests. -## Alpine Envoy image +## Distroless Envoy image + +Minimal images based on a [distroless](https://github.com/GoogleContainerTools/distroless) allow for quicker deployment of Envoy. + +The Distroless base image is only built with symbols stripped. + +## Debug Envoy image -Minimal images based on Alpine Linux allow for quicker deployment of Envoy. The Alpine base image is only built with symbols stripped. To get the binary with symbols, use the corresponding Ubuntu based debug image. The image is pushed with two different tags: `` and `latest`. Parallel to the Ubuntu images above, `` corresponds to the main commit at which the binary was compiled, and `latest` corresponds to a binary built from the latest tip of main that passed tests. @@ -81,7 +86,7 @@ ENVOY_DOCKER_PULL=true ./ci/run_envoy_docker.sh An example basic invocation to build a developer version of the Envoy static binary (using the Bazel `fastbuild` type) is: ```bash -./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.dev' +./ci/run_envoy_docker.sh './ci/do_ci.sh dev' ``` The Envoy binary can be found in `/tmp/envoy-docker-build/envoy/source/exe/envoy-fastbuild` on the Docker host. You @@ -89,22 +94,29 @@ can control this by setting `ENVOY_DOCKER_BUILD_DIR` in the environment, e.g. to generate the binary in `~/build/envoy/source/exe/envoy-fastbuild` you can run: ```bash -ENVOY_DOCKER_BUILD_DIR=~/build ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.dev' +ENVOY_DOCKER_BUILD_DIR=~/build ./ci/run_envoy_docker.sh './ci/do_ci.sh dev' ``` For a release version of the Envoy binary you can run: ```bash -./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release.server_only' +./ci/run_envoy_docker.sh './ci/do_ci.sh release.server_only' ``` The build artifact can be found in `/tmp/envoy-docker-build/envoy/source/exe/envoy` (or wherever `$ENVOY_DOCKER_BUILD_DIR` points). +To enable the previous behavior of the `release.server_only` target where the final binary was copied to a tar.gz file +(e.g. envoy-binary.tar.gz), you can run: + + ```bash + ./ci/run_envoy_docker.sh './ci/do_ci.sh release.server_only.binary + ``` + For a debug version of the Envoy binary you can run: ```bash -./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.debug.server_only' +./ci/run_envoy_docker.sh './ci/do_ci.sh debug.server_only' ``` The build artifact can be found in `/tmp/envoy-docker-build/envoy/source/exe/envoy-debug` (or wherever @@ -114,38 +126,39 @@ To leverage a [bazel remote cache](https://github.com/envoyproxy/envoy/tree/main the BAZEL_BUILD_EXTRA_OPTIONS environment variable ```bash -./ci/run_envoy_docker.sh "BAZEL_BUILD_EXTRA_OPTIONS='--remote_cache=http://127.0.0.1:28080' ./ci/do_ci.sh bazel.release" +./ci/run_envoy_docker.sh "BAZEL_BUILD_EXTRA_OPTIONS='--remote_cache=http://127.0.0.1:28080' ./ci/do_ci.sh release" ``` The `./ci/run_envoy_docker.sh './ci/do_ci.sh '` targets are: -* `bazel.api` — build and run API tests under `-c fastbuild` with clang. -* `bazel.asan` — build and run tests under `-c dbg --config=clang-asan` with clang. -* `bazel.asan ` — build and run a specified test or test dir under `-c dbg --config=clang-asan` with clang. -* `bazel.debug` — build Envoy static binary and run tests under `-c dbg`. -* `bazel.debug ` — build Envoy static binary and run a specified test or test dir under `-c dbg`. -* `bazel.debug.server_only` — build Envoy static binary under `-c dbg`. -* `bazel.dev` — build Envoy static binary and run tests under `-c fastbuild` with clang. -* `bazel.dev ` — build Envoy static binary and run a specified test or test dir under `-c fastbuild` with clang. -* `bazel.dev.contrib` — build Envoy static binary with contrib and run tests under `-c fastbuild` with clang. -* `bazel.dev.contrib ` — build Envoy static binary with contrib and run a specified test or test dir under `-c fastbuild` with clang. -* `bazel.release` — build Envoy static binary and run tests under `-c opt` with clang. -* `bazel.release ` — build Envoy static binary and run a specified test or test dir under `-c opt` with clang. -* `bazel.release.server_only` — build Envoy static binary under `-c opt` with clang. -* `bazel.sizeopt` — build Envoy static binary and run tests under `-c opt --config=sizeopt` with clang. -* `bazel.sizeopt ` — build Envoy static binary and run a specified test or test dir under `-c opt --config=sizeopt` with clang. -* `bazel.sizeopt.server_only` — build Envoy static binary under `-c opt --config=sizeopt` with clang. -* `bazel.coverage` — build and run tests under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`. -* `bazel.coverage ` — build and run a specified test or test dir under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`. Specify `//contrib/...` to get contrib coverage. -* `bazel.msan` — build and run tests under `-c dbg --config=clang-msan` with clang. -* `bazel.msan ` — build and run a specified test or test dir under `-c dbg --config=clang-msan` with clang. -* `bazel.tsan` — build and run tests under `-c dbg --config=clang-tsan` with clang. -* `bazel.tsan ` — build and run a specified test or test dir under `-c dbg --config=clang-tsan` with clang. -* `bazel.fuzz` — build and run fuzz tests under `-c dbg --config=asan-fuzzer` with clang. -* `bazel.fuzz ` — build and run a specified fuzz test or test dir under `-c dbg --config=asan-fuzzer` with clang. If specifying a single fuzz test, must use the full target name with "_with_libfuzzer" for ``. -* `bazel.compile_time_options` — build Envoy and run tests with various compile-time options toggled to their non-default state, to ensure they still build. -* `bazel.compile_time_options ` — build Envoy and run a specified test or test dir with various compile-time options toggled to their non-default state, to ensure they still build. -* `bazel.clang_tidy ` — build and run clang-tidy specified source files, if no files specified, runs against the diff with the last GitHub commit. +* `api` — build and run API tests under `-c fastbuild` with clang. +* `asan` — build and run tests under `-c dbg --config=clang-asan` with clang. +* `asan ` — build and run a specified test or test dir under `-c dbg --config=clang-asan` with clang. +* `debug` — build Envoy static binary and run tests under `-c dbg`. +* `debug ` — build Envoy static binary and run a specified test or test dir under `-c dbg`. +* `debug.server_only` — build Envoy static binary under `-c dbg`. +* `docker` — build Docker images, expects `release` or `release.server_only` to have been run furst. +* `dev` — build Envoy static binary and run tests under `-c fastbuild` with clang. +* `dev ` — build Envoy static binary and run a specified test or test dir under `-c fastbuild` with clang. +* `dev.contrib` — build Envoy static binary with contrib and run tests under `-c fastbuild` with clang. +* `dev.contrib ` — build Envoy static binary with contrib and run a specified test or test dir under `-c fastbuild` with clang. +* `release` — build Envoy static binary and run tests under `-c opt` with clang. +* `release ` — build Envoy static binaries and run a specified test or test dir under `-c opt` with clang. +* `release.server_only` — build Envoy static binaries under `-c opt` with clang. +* `sizeopt` — build Envoy static binary and run tests under `-c opt --config=sizeopt` with clang. +* `sizeopt ` — build Envoy static binary and run a specified test or test dir under `-c opt --config=sizeopt` with clang. +* `sizeopt.server_only` — build Envoy static binary under `-c opt --config=sizeopt` with clang. +* `coverage` — build and run tests under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`. +* `coverage ` — build and run a specified test or test dir under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`. Specify `//contrib/...` to get contrib coverage. +* `msan` — build and run tests under `-c dbg --config=clang-msan` with clang. +* `msan ` — build and run a specified test or test dir under `-c dbg --config=clang-msan` with clang. +* `tsan` — build and run tests under `-c dbg --config=clang-tsan` with clang. +* `tsan ` — build and run a specified test or test dir under `-c dbg --config=clang-tsan` with clang. +* `fuzz` — build and run fuzz tests under `-c dbg --config=asan-fuzzer` with clang. +* `fuzz ` — build and run a specified fuzz test or test dir under `-c dbg --config=asan-fuzzer` with clang. If specifying a single fuzz test, must use the full target name with "_with_libfuzzer" for ``. +* `compile_time_options` — build Envoy and run tests with various compile-time options toggled to their non-default state, to ensure they still build. +* `compile_time_options ` — build Envoy and run a specified test or test dir with various compile-time options toggled to their non-default state, to ensure they still build. +* `clang_tidy ` — build and run clang-tidy specified source files, if no files specified, runs against the diff with the last GitHub commit. * `check_proto_format`— check configuration, formatting and build issues in API proto files. * `fix_proto_format`— fix configuration, formatting and build issues in API proto files. * `format`— run validation, linting and formatting tools. diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 2d54fa423bc3..c56a6eb746a7 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -56,6 +56,10 @@ function setup_gcc_toolchain() { } function setup_clang_toolchain() { + if [[ -n "$CLANG_TOOLCHAIN_SETUP" ]]; then + return + fi + export CLANG_TOOLCHAIN_SETUP=1 ENVOY_STDLIB="${ENVOY_STDLIB:-libc++}" if [[ -z "${ENVOY_RBE}" ]]; then if [[ "${ENVOY_STDLIB}" == "libc++" ]]; then @@ -76,12 +80,15 @@ function setup_clang_toolchain() { echo "clang toolchain with ${ENVOY_STDLIB} configured" } -export BUILD_DIR=${BUILD_DIR:-/build} -if [[ ! -d "${BUILD_DIR}" ]] -then - echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}? Creating." - mkdir -p "${BUILD_DIR}" +if [[ -z "${BUILD_DIR}" ]]; then + echo "BUILD_DIR not set - defaulting to ~/.cache/envoy-bazel" >&2 + BUILD_DIR="${HOME}/.cache/envoy-bazel" +fi +if [[ ! -d "${BUILD_DIR}" ]]; then + echo "${BUILD_DIR} missing - Creating." >&2 + mkdir -p "${BUILD_DIR}" fi +export BUILD_DIR # Environment setup. export ENVOY_TEST_TMPDIR="${ENVOY_TEST_TMPDIR:-$BUILD_DIR/tmp}" @@ -119,14 +126,6 @@ bazel () { export _bazel export -f bazel -if [[ -n "$BAZEL_NO_CACHE_TEST_RESULTS" ]]; then - VERSION_DEV="$(cut -d- -f2 "${ENVOY_SRCDIR}/VERSION.txt")" - # Use uncached test results for non-release commits to a branch. - if [[ $VERSION_DEV == "dev" ]]; then - BAZEL_EXTRA_TEST_OPTIONS+=("--nocache_test_results") - fi -fi - # Use https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_repository_cache_hardlinks # to save disk space. BAZEL_GLOBAL_OPTIONS=( diff --git a/ci/create_release_branch.sh b/ci/create_release_branch.sh deleted file mode 100755 index 0996a493faeb..000000000000 --- a/ci/create_release_branch.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -e - -ENVOY_GIT_USERNAME="${ENVOY_GIT_USERNAME:-envoy-bot}" -ENVOY_GIT_EMAIL="${ENVOY_GIT_EMAIL:-envoy-bot@users.noreply.github.com}" -ENVOY_RELEASE_VERSION="${ENVOY_RELEASE_VERSION:-}" - -MAIN_BRANCH=refs/heads/main -MAIN_BRANCH_SHORTNAME=main -CURRENT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" - - -if [[ "$CURRENT_BRANCH" != "$MAIN_BRANCH" ]] && [[ "$CURRENT_BRANCH" != "$MAIN_BRANCH_SHORTNAME" ]]; then - echo "Current branch ($CURRENT_BRANCH) must be \`main\`, exiting" >&2 - exit 1 -else - # TODO(phlax): remove once its clear what this should be - echo "Current branch: $CURRENT_BRANCH" -fi - - -configure_git_user () { - if [[ -z "$ENVOY_GIT_USERNAME" || -z "$ENVOY_GIT_EMAIL" ]]; then - echo 'Unable to set git name/email, using existing git config' >&2 - return - fi - git config --global user.name "$ENVOY_GIT_USERNAME" - git config --global user.email "$ENVOY_GIT_EMAIL" -} - -create_dev_commit () { - bazel run @envoy_repo//:dev -- --patch -} - -get_release_name () { - local version - if [[ -z "$ENVOY_RELEASE_VERSION" ]]; then - version="$(cut -d- -f1 < VERSION.txt | cut -d. -f-2)" - else - version="$ENVOY_RELEASE_VERSION" - fi - echo -n "release/v${version}" -} - -create_branch () { - local release_name commit_sha - release_name="$(get_release_name)" - commit_sha="$(git rev-parse HEAD)" - - echo "Creating ${release_name} from ${commit_sha}" - git checkout -b "$release_name" - git push origin "$release_name" -} - -create_release_branch () { - configure_git_user - create_dev_commit - create_branch -} - -create_release_branch diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 79a11d10ba29..fc25e25c4068 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -8,8 +8,6 @@ set -e export SRCDIR="${SRCDIR:-$PWD}" export ENVOY_SRCDIR="${ENVOY_SRCDIR:-$PWD}" -# shellcheck source=ci/setup_cache.sh -. "$(dirname "$0")"/setup_cache.sh # shellcheck source=ci/build_setup.sh . "$(dirname "$0")"/build_setup.sh @@ -165,7 +163,7 @@ function bazel_binary_build() { # The COMPILE_TYPE variable is redundant in this case and is only here for # readability. It is already set in the .bazelrc config for sizeopt. COMPILE_TYPE="opt" - CONFIG_ARGS="--config=sizeopt" + CONFIG_ARGS=("--config=sizeopt") elif [[ "${BINARY_TYPE}" == "fastbuild" ]]; then COMPILE_TYPE="fastbuild" fi @@ -183,7 +181,7 @@ function bazel_binary_build() { # This is a workaround for https://github.com/bazelbuild/bazel/issues/11834 [[ -n "${ENVOY_RBE}" ]] && rm -rf bazel-bin/"${ENVOY_BIN}"* - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" "${BUILD_TARGET}" ${CONFIG_ARGS} + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" "${BUILD_TARGET}" "${CONFIG_ARGS[@]}" collect_build_profile "${BINARY_TYPE}"_build # Copy the built envoy binary somewhere that we can access outside of the @@ -193,14 +191,14 @@ function bazel_binary_build() { if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then # Generate dwp file for debugging since we used split DWARF to reduce binary # size - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" "${BUILD_DEBUG_INFORMATION}" ${CONFIG_ARGS} + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" "${BUILD_DEBUG_INFORMATION}" "${CONFIG_ARGS[@]}" # Copy the debug information cp -f bazel-bin/"${ENVOY_BIN}".dwp "${FINAL_DELIVERY_DIR}"/envoy.dwp fi # Validation tools for the tools image. bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" \ - //test/tools/schema_validator:schema_validator_tool ${CONFIG_ARGS} + //test/tools/schema_validator:schema_validator_tool "${CONFIG_ARGS[@]}" # Build su-exec utility bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" external:su-exec @@ -280,9 +278,7 @@ case $CI_TARGET in ;& api.go) - if [[ -z "$NO_BUILD_SETUP" ]]; then - setup_clang_toolchain - fi + setup_clang_toolchain GO_IMPORT_BASE="github.com/envoyproxy/go-control-plane" GO_TARGETS=(@envoy_api//...) read -r -a GO_PROTOS <<< "$(bazel query "${BAZEL_GLOBAL_OPTIONS[@]}" "kind('go_proto_library', ${GO_TARGETS[*]})" | tr '\n' ' ')" @@ -375,17 +371,15 @@ case $CI_TARGET in export FIX_YAML="${ENVOY_TEST_TMPDIR}/lint-fixes/clang-tidy-fixes.yaml" export CLANG_TIDY_APPLY_FIXES=1 mkdir -p "${ENVOY_TEST_TMPDIR}/lint-fixes" - BAZEL_BUILD_OPTIONS+=(--remote_download_minimal) - NUM_CPUS=$NUM_CPUS "${ENVOY_SRCDIR}"/ci/run_clang_tidy.sh "$@" || { - if [[ -s "$FIX_YAML" ]]; then - echo >&2 - echo "Diff/yaml files with (some) fixes will be uploaded. Please check the artefacts for this PR run in the azure pipeline." >&2 - echo >&2 - else - echo "Clang-tidy failed." >&2 - fi - exit 1 - } + CLANG_TIDY_TARGETS=( + //contrib/... + //source/... + //test/... + @envoy_api//...) + bazel build \ + "${BAZEL_BUILD_OPTIONS[@]}" \ + --config clang-tidy \ + "${CLANG_TIDY_TARGETS[@]}" ;; clean|expunge) @@ -520,6 +514,7 @@ case $CI_TARGET in TODAY_DATE=$(date -u -I"date") export TODAY_DATE bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check \ + --//tools/dependency:preload_cve_data \ --action_env=TODAY_DATE \ -- -v warn \ -c cves release_dates releases @@ -527,6 +522,11 @@ case $CI_TARGET in echo "Check dependabot ..." bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ //tools/dependency:dependatool + # Disable this pending resolution to https://github.com/envoyproxy/envoy/issues/30286 + # Run pip requirements tests + # echo "Check pip requirements ..." + # bazel test "${BAZEL_BUILD_OPTIONS[@]}" \ + # //tools/base:requirements_test ;; dev) @@ -567,11 +567,17 @@ case $CI_TARGET in -- --stdout \ -d "$ENVOY_RELEASE_TARBALL" \ | tar xfO - envoy > distribution/custom/envoy + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + //tools/zstd \ + -- --stdout \ + -d "$ENVOY_RELEASE_TARBALL" \ + | tar xfO - envoy-contrib > distribution/custom/envoy-contrib # Build the packages bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ --remote_download_toplevel \ -c opt \ --//distribution:envoy-binary=//distribution:custom/envoy \ + --//distribution:envoy-contrib-binary=//distribution:custom/envoy-contrib \ //distribution:packages.tar.gz if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" @@ -580,21 +586,51 @@ case $CI_TARGET in fi ;; - docs) - setup_clang_toolchain - echo "generating docs..." - # Build docs. - "${ENVOY_SRCDIR}/docs/build.sh" - ;; - - docs-upload) - setup_clang_toolchain - "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs - ;; - - docs-publish-latest) - BUILD_SHA=$(git rev-parse HEAD) - curl -X POST -d "$BUILD_SHA" "$NETLIFY_TRIGGER_URL" + docker) + # This is limited to linux x86/arm64 and expects `release` or `release.server_only` to have + # been run first. + if ! docker ps &> /dev/null; then + echo "Unable to build with Docker. If you are running with ci/run_envoy_docker.sh" \ + "you should set ENVOY_DOCKER_IN_DOCKER=1" + exit 1 + fi + if [[ -z "$CI_SHA1" ]]; then + CI_SHA1="$(git rev-parse HEAD~1)" + export CI_SHA1 + fi + ENVOY_ARCH_DIR="$(dirname "${ENVOY_BUILD_DIR}")" + ENVOY_TARBALL_DIR="${ENVOY_TARBALL_DIR:-${ENVOY_ARCH_DIR}}" + _PLATFORMS=() + PLATFORM_NAMES=( + x64:linux/amd64 + arm64:linux/arm64) + # TODO(phlax): avoid copying bins + for platform_name in "${PLATFORM_NAMES[@]}"; do + path="$(echo "${platform_name}" | cut -d: -f1)" + platform="$(echo "${platform_name}" | cut -d: -f2)" + bin_folder="${ENVOY_TARBALL_DIR}/${path}/bin" + if [[ ! -e "${bin_folder}/release.tar.zst" ]]; then + continue + fi + _PLATFORMS+=("$platform") + if [[ -e "$platform" ]]; then + rm -rf "$platform" + fi + mkdir -p "${platform}" + cp -a "${bin_folder}"/* "$platform" + done + if [[ -z "${_PLATFORMS[*]}" ]]; then + echo "No tarballs found in ${ENVOY_TARBALL_DIR}, did you run \`release\` first?" >&2 + exit 1 + fi + PLATFORMS="$(IFS=, ; echo "${_PLATFORMS[*]}")" + export DOCKER_PLATFORM="$PLATFORMS" + if [[ -z "${DOCKERHUB_PASSWORD}" && "${#_PLATFORMS[@]}" -eq 1 && -z $ENVOY_DOCKER_SAVE_IMAGE ]]; then + # if you are not pushing the images and there is only one platform + # then load to Docker (ie local build) + export DOCKER_LOAD_IMAGES=1 + fi + "${ENVOY_SRCDIR}/ci/docker_ci.sh" ;; docker-upload) @@ -616,6 +652,34 @@ case $CI_TARGET in cat bazel-bin/distribution/dockerhub/readme.md ;; + docs) + setup_clang_toolchain + echo "generating docs..." + # Build docs. + [[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs + rm -rf "${DOCS_OUTPUT_DIR}" + mkdir -p "${DOCS_OUTPUT_DIR}" + if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then + export SPHINX_RUNNER_ARGS="-v warn" + BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS") + fi + if [[ -n "${DOCS_BUILD_RST}" ]]; then + bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst + cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz + fi + DOCS_OUTPUT_DIR="$(realpath "$DOCS_OUTPUT_DIR")" + bazel "${BAZEL_STARTUP_OPTIONS[@]}" run \ + "${BAZEL_BUILD_OPTIONS[@]}" \ + --//tools/tarball:target=//docs:html \ + //tools/tarball:unpack \ + "$DOCS_OUTPUT_DIR" + ;; + + docs-upload) + setup_clang_toolchain + "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" /source/generated/docs docs + ;; + fetch|fetch-*) case $CI_TARGET in fetch) @@ -724,30 +788,40 @@ case $CI_TARGET in publish) setup_clang_toolchain BUILD_SHA="$(git rev-parse HEAD)" + ENVOY_COMMIT="${ENVOY_COMMIT:-${BUILD_SHA}}" + ENVOY_REPO="${ENVOY_REPO:-envoyproxy/envoy}" VERSION_DEV="$(cut -d- -f2 < VERSION.txt)" PUBLISH_ARGS=( - --publish-commitish="$BUILD_SHA" + --publish-commitish="$ENVOY_COMMIT" + --publish-commit-message --publish-assets=/build/release.signed/release.signed.tar.zst) if [[ "$VERSION_DEV" == "dev" ]] || [[ -n "$ENVOY_PUBLISH_DRY_RUN" ]]; then PUBLISH_ARGS+=(--dry-run) fi bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ @envoy_repo//:publish \ - -- "${PUBLISH_ARGS[@]}" - ;; - - release) - # When testing memory consumption, we want to test against exact byte-counts - # where possible. As these differ between platforms and compile options, we - # define the 'release' builds as canonical and test them only in CI, so the - # toolchain is kept consistent. This ifdef is checked in - # test/common/stats/stat_test_utility.cc when computing - # Stats::TestUtil::MemoryTest::mode(). - if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then - BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") + -- --repo="$ENVOY_REPO" \ + "${PUBLISH_ARGS[@]}" + ;; + + release|release.server_only) + if [[ "$CI_TARGET" == "release" ]]; then + # When testing memory consumption, we want to test against exact byte-counts + # where possible. As these differ between platforms and compile options, we + # define the 'release' builds as canonical and test them only in CI, so the + # toolchain is kept consistent. This ifdef is checked in + # test/common/stats/stat_test_utility.cc when computing + # Stats::TestUtil::MemoryTest::mode(). + if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + BAZEL_BUILD_OPTIONS+=("--test_env=ENVOY_MEMORY_TEST_EXACT=true") + fi fi setup_clang_toolchain ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" + if [[ -e "${ENVOY_BINARY_DIR}" ]]; then + echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." + rm -rf "${ENVOY_BINARY_DIR}" + fi mkdir -p "$ENVOY_BINARY_DIR" # As the binary build package enforces compiler options, adding here to ensure the tests and distribution build # reuse settings and any already compiled artefacts, the bundle itself will always be compiled @@ -755,16 +829,18 @@ case $CI_TARGET in BAZEL_RELEASE_OPTIONS=( --stripopt=--strip-all -c opt) - # Run release tests - echo "Testing with:" - echo " targets: ${TEST_TARGETS[*]}" - echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" - echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" - bazel_with_collection \ - test "${BAZEL_BUILD_OPTIONS[@]}" \ - --remote_download_minimal \ - "${BAZEL_RELEASE_OPTIONS[@]}" \ - "${TEST_TARGETS[@]}" + if [[ "$CI_TARGET" == "release" ]]; then + # Run release tests + echo "Testing with:" + echo " targets: ${TEST_TARGETS[*]}" + echo " build options: ${BAZEL_BUILD_OPTIONS[*]}" + echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" + bazel_with_collection \ + test "${BAZEL_BUILD_OPTIONS[@]}" \ + --remote_download_minimal \ + "${BAZEL_RELEASE_OPTIONS[@]}" \ + "${TEST_TARGETS[@]}" + fi # Build release binaries bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ "${BAZEL_RELEASE_OPTIONS[@]}" \ @@ -783,9 +859,10 @@ case $CI_TARGET in cp -a \ bazel-bin/test/tools/schema_validator/schema_validator_tool.stripped \ "${ENVOY_BINARY_DIR}/schema_validator_tool" + echo "Release files created in ${ENVOY_BINARY_DIR}" ;; - release.server_only) + release.server_only.binary) setup_clang_toolchain echo "bazel release build..." bazel_envoy_binary_build release @@ -794,9 +871,6 @@ case $CI_TARGET in release.signed) echo "Signing binary packages..." setup_clang_toolchain - # The default config expects these files - mkdir -p distribution/custom - cp -a /build/*/*64 distribution/custom/ bazel build "${BAZEL_BUILD_OPTIONS[@]}" //distribution:signed cp -a bazel-bin/distribution/release.signed.tar.zst "${BUILD_DIR}/envoy/" "${ENVOY_SRCDIR}/ci/upload_gcs_artifact.sh" "${BUILD_DIR}/envoy" release diff --git a/ci/docker_ci.sh b/ci/docker_ci.sh index 3845486acf07..fdf6cdaf74b5 100755 --- a/ci/docker_ci.sh +++ b/ci/docker_ci.sh @@ -14,22 +14,23 @@ set -e # DOCKERHUB_PASSWORD=mypassword # ## Set these to simulate types of CI run -# AZP_SHA1=MOCKSHA -# AZP_BRANCH=refs/heads/main -# AZP_BRANCH=refs/heads/release/v1.43 -# AZP_BRANCH=refs/tags/v1.77.3 +# CI_SHA1=MOCKSHA +# CI_BRANCH=refs/heads/main +# CI_BRANCH=refs/heads/release/v1.43 +# CI_BRANCH=refs/tags/v1.77.3 ## # Workaround for https://github.com/envoyproxy/envoy/issues/26634 DOCKER_BUILD_TIMEOUT="${DOCKER_BUILD_TIMEOUT:-400}" +DOCKER_PLATFORM="${DOCKER_PLATFORM:-linux/arm64,linux/amd64}" function is_windows() { [[ -n "$DOCKER_FAKE_WIN" ]] || [[ "$(uname -s)" == *NT* ]] } if [[ -n "$DOCKER_CI_DRYRUN" ]]; then - AZP_SHA1="${AZP_SHA1:-MOCKSHA}" + CI_SHA1="${CI_SHA1:-MOCKSHA}" if is_windows; then WINDOWS_IMAGE_BASE="${WINDOWS_IMAGE_BASE:-mcr.microsoft.com/windows/fakecore}" @@ -50,7 +51,7 @@ fi if [[ "$ENVOY_VERSION" =~ $DEV_VERSION_REGEX ]]; then # Dev version IMAGE_POSTFIX="-dev" - IMAGE_NAME="${AZP_SHA1}" + IMAGE_NAME="${CI_SHA1}" else # Non-dev version IMAGE_POSTFIX="" @@ -58,12 +59,14 @@ else fi # Only push images for main builds, and non-dev release branch builds -if [[ -n "$DOCKERHUB_USERNAME" ]] && [[ -n "$DOCKERHUB_PASSWORD" ]]; then - if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then +if [[ -n "$DOCKER_LOAD_IMAGES" ]]; then + LOAD_IMAGES=1 +elif [[ -n "$DOCKERHUB_USERNAME" ]] && [[ -n "$DOCKERHUB_PASSWORD" ]]; then + if [[ "${CI_BRANCH}" == "${MAIN_BRANCH}" ]]; then echo "Pushing images for main." PUSH_IMAGES_TO_REGISTRY=1 - elif [[ "${AZP_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]] && ! [[ "$ENVOY_VERSION" =~ $DEV_VERSION_REGEX ]]; then - echo "Pushing images for release branch ${AZP_BRANCH}." + elif [[ "${CI_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]] && ! [[ "$ENVOY_VERSION" =~ $DEV_VERSION_REGEX ]]; then + echo "Pushing images for release branch ${CI_BRANCH}." PUSH_IMAGES_TO_REGISTRY=1 else echo 'Ignoring non-release branch for docker push.' @@ -72,7 +75,7 @@ else echo 'No credentials for docker push.' fi -ENVOY_DOCKER_IMAGE_DIRECTORY="${ENVOY_DOCKER_IMAGE_DIRECTORY:-${BUILD_STAGINGDIRECTORY:-.}/build_images}" +ENVOY_DOCKER_IMAGE_DIRECTORY="${ENVOY_DOCKER_IMAGE_DIRECTORY:-${BUILD_DIR:-.}/build_images}" # This prefix is altered for the private security images on setec builds. DOCKER_IMAGE_PREFIX="${DOCKER_IMAGE_PREFIX:-envoyproxy/envoy}" if [[ -z "$DOCKER_CI_DRYRUN" ]]; then @@ -84,7 +87,7 @@ config_env() { echo ">> BUILDX: install" echo "> docker run --rm --privileged tonistiigi/binfmt --install all" echo "> docker buildx rm multi-builder 2> /dev/null || :" - echo "> docker buildx create --use --name multi-builder --platform linux/arm64,linux/amd64" + echo "> docker buildx create --use --name multi-builder --platform ${DOCKER_PLATFORM}" if [[ -n "$DOCKER_CI_DRYRUN" ]]; then return @@ -95,7 +98,7 @@ config_env() { # Remove older build instance docker buildx rm multi-builder 2> /dev/null || : - docker buildx create --use --name multi-builder --platform linux/arm64,linux/amd64 + docker buildx create --use --name multi-builder --platform "${DOCKER_PLATFORM}" } if is_windows; then @@ -152,7 +155,7 @@ build_platforms() { elif [[ "${build_type}" == *-google-vrp ]]; then echo -n "linux/amd64" else - echo -n "linux/arm64,linux/amd64" + echo -n "$DOCKER_PLATFORM" fi } @@ -210,7 +213,10 @@ build_and_maybe_push_image () { args+=( "--sbom=false" "--provenance=false") - if [[ "${image_type}" =~ debug ]]; then + if [[ -n "$LOAD_IMAGES" ]]; then + action="BUILD+LOAD" + args+=("--load") + elif [[ "${image_type}" =~ debug ]]; then # For linux if its the debug image then push immediately for release branches, # otherwise just test the build if [[ -n "$PUSH_IMAGES_TO_REGISTRY" ]]; then @@ -341,7 +347,7 @@ tag_variants () { # Only push latest on main/dev builds. if [[ "$ENVOY_VERSION" =~ $DEV_VERSION_REGEX ]]; then - if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then + if [[ "${CI_BRANCH}" == "${MAIN_BRANCH}" ]]; then variant_type="latest" fi else diff --git a/ci/format_pre.sh b/ci/format_pre.sh index f1a4c77fe079..f4627dea89d8 100755 --- a/ci/format_pre.sh +++ b/ci/format_pre.sh @@ -57,7 +57,7 @@ CURRENT=spelling "${ENVOY_SRCDIR}/tools/spelling/check_spelling_pedantic.py" --mark check # TODO(phlax): move clang/buildifier checks to bazel rules (/aspects) -if [[ -n "$AZP_BRANCH" ]]; then +if [[ -n "$CI_BRANCH" ]]; then CURRENT=check_format_test "${ENVOY_SRCDIR}/tools/code_format/check_format_test_helper.sh" --log=WARN fi diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh index 2ab857c72970..98fbf618d744 100755 --- a/ci/mac_ci_steps.sh +++ b/ci/mac_ci_steps.sh @@ -2,18 +2,6 @@ set -e -function finish { - echo "disk space at end of build:" - df -h -} -trap finish EXIT - -echo "disk space at beginning of build:" -df -h - -# shellcheck source=ci/setup_cache.sh -. "$(dirname "$0")"/setup_cache.sh - read -ra BAZEL_BUILD_EXTRA_OPTIONS <<< "${BAZEL_BUILD_EXTRA_OPTIONS:-}" read -ra BAZEL_EXTRA_TEST_OPTIONS <<< "${BAZEL_EXTRA_TEST_OPTIONS:-}" @@ -24,7 +12,6 @@ BUILD_CONFIG="$(dirname "$(realpath "$0")")"/osx-build-config BAZEL_BUILD_OPTIONS=( "--curses=no" --verbose_failures - "--test_output=all" "--flaky_test_attempts=integration@2" "--override_repository=envoy_build_config=${BUILD_CONFIG}" "${BAZEL_BUILD_EXTRA_OPTIONS[@]}" diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 1b82f96d72c4..094572f3b56d 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -11,18 +11,13 @@ function is_windows() { read -ra ENVOY_DOCKER_OPTIONS <<< "${ENVOY_DOCKER_OPTIONS:-}" -# TODO(phlax): uppercase these env vars -export HTTP_PROXY="${http_proxy:-}" -export HTTPS_PROXY="${https_proxy:-}" -export NO_PROXY="${no_proxy:-}" -export GOPROXY="${go_proxy:-}" +export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}" +export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}" +export NO_PROXY="${NO_PROXY:-${no_proxy:-}}" +export GOPROXY="${GOPROXY:-${go_proxy:-}}" if is_windows; then [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019" - # Container networking is unreliable in the most recently built images, pin Windows to a known - # good container. This can create a mismatch between the host environment, and the toolchain - # environment. - ENVOY_BUILD_SHA=41c5a05d708972d703661b702a63ef5060125c33 # TODO(sunjayBhatia): Currently ENVOY_DOCKER_OPTIONS is ignored on Windows because # CI sets it to a Linux-specific value. Undo this once https://github.com/envoyproxy/envoy/issues/13272 # is resolved. @@ -56,6 +51,7 @@ else BUILD_DIR_MOUNT_DEST=/build SOURCE_DIR="${PWD}" SOURCE_DIR_MOUNT_DEST=/source + ENVOY_DOCKER_SOURCE_DIR="${ENVOY_DOCKER_SOURCE_DIR:-${SOURCE_DIR_MOUNT_DEST}}" START_COMMAND=( "/bin/bash" "-lc" @@ -64,7 +60,7 @@ else && usermod -a -G pcap envoybuild \ && chown envoybuild:envoygroup /build \ && chown envoybuild /proc/self/fd/2 \ - && sudo -EHs -u envoybuild bash -c 'cd /source && $*'") + && sudo -EHs -u envoybuild bash -c 'cd ${ENVOY_DOCKER_SOURCE_DIR} && $*'") fi if [[ -n "$ENVOY_DOCKER_PLATFORM" ]]; then @@ -96,13 +92,17 @@ VOLUMES=( -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}" -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}") -if ! is_windows && [[ -n "$ENVOY_DOCKER_IN_DOCKER" ]]; then +if ! is_windows; then + export BUILD_DIR="${BUILD_DIR_MOUNT_DEST}" +fi + +if [[ -n "$ENVOY_DOCKER_IN_DOCKER" || -n "$ENVOY_SHARED_TMP_DIR" ]]; then # Create a "shared" directory that has the same path in/outside the container # This allows the host docker engine to see artefacts using a temporary path created inside the container, # at the same path. # For example, a directory created with `mktemp -d --tmpdir /tmp/bazel-shared` can be mounted as a volume # from within the build container. - SHARED_TMP_DIR=/tmp/bazel-shared + SHARED_TMP_DIR="${ENVOY_SHARED_TMP_DIR:-/tmp/bazel-shared}" mkdir -p "${SHARED_TMP_DIR}" chmod +rwx "${SHARED_TMP_DIR}" VOLUMES+=(-v "${SHARED_TMP_DIR}":"${SHARED_TMP_DIR}") @@ -112,13 +112,11 @@ if [[ -n "${ENVOY_DOCKER_PULL}" ]]; then time docker pull "${ENVOY_BUILD_IMAGE}" fi - # Since we specify an explicit hash, docker-run will pull from the remote repo if missing. docker run --rm \ "${ENVOY_DOCKER_OPTIONS[@]}" \ "${VOLUMES[@]}" \ - -e AZP_BRANCH \ - -e AZP_COMMIT_SHA \ + -e BUILD_DIR \ -e HTTP_PROXY \ -e HTTPS_PROXY \ -e NO_PROXY \ @@ -129,15 +127,17 @@ docker run --rm \ -e BAZEL_FAKE_SCM_REVISION \ -e BAZEL_REMOTE_CACHE \ -e BAZEL_STARTUP_EXTRA_OPTIONS \ + -e CI_BRANCH \ + -e CI_SHA1 \ -e CI_TARGET_BRANCH \ -e DOCKERHUB_USERNAME \ -e DOCKERHUB_PASSWORD \ + -e ENVOY_DOCKER_SAVE_IMAGE \ -e ENVOY_STDLIB \ -e BUILD_REASON \ - -e BAZEL_NO_CACHE_TEST_RESULTS \ -e BAZEL_REMOTE_INSTANCE \ - -e GOOGLE_BES_PROJECT_ID \ -e GCP_SERVICE_ACCOUNT_KEY \ + -e GCP_SERVICE_ACCOUNT_KEY_PATH \ -e NUM_CPUS \ -e ENVOY_BRANCH \ -e ENVOY_RBE \ @@ -150,6 +150,7 @@ docker run --rm \ -e ENVOY_HEAD_REF \ -e ENVOY_PUBLISH_DRY_RUN \ -e ENVOY_REPO \ + -e ENVOY_TARBALL_DIR \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ -e GITHUB_REF_NAME \ @@ -157,7 +158,7 @@ docker run --rm \ -e GITHUB_TOKEN \ -e GITHUB_APP_ID \ -e GITHUB_INSTALL_ID \ - -e NETLIFY_TRIGGER_URL \ + -e MOBILE_DOCS_CHECKOUT_DIR \ -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ -e ENVOY_BUILD_ARCH \ diff --git a/ci/setup_cache.sh b/ci/setup_cache.sh deleted file mode 100755 index da0b189dd4a8..000000000000 --- a/ci/setup_cache.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -n "${GCP_SERVICE_ACCOUNT_KEY:0:1}" ]]; then - # mktemp will create a tempfile with u+rw permission minus umask, it will not be readable by all - # users by default. - GCP_SERVICE_ACCOUNT_KEY_FILE=$(mktemp -t gcp_service_account.XXXXXX.json) - - gcp_service_account_cleanup() { - echo "Deleting service account key file..." - rm -rf "${GCP_SERVICE_ACCOUNT_KEY_FILE}" - } - - trap gcp_service_account_cleanup EXIT - - bash -c 'echo "${GCP_SERVICE_ACCOUNT_KEY}"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_FILE}" - - export BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_FILE}" - - if [[ -n "${GOOGLE_BES_PROJECT_ID}" ]]; then - export BAZEL_BUILD_EXTRA_OPTIONS+=" --config=rbe-google-bes --bes_instance_name=${GOOGLE_BES_PROJECT_ID}" - fi - -fi - -if [[ -n "${BAZEL_REMOTE_CACHE}" ]]; then - export BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}" - echo "Set up bazel remote read/write cache at ${BAZEL_REMOTE_CACHE}." - - if [[ -z "${ENVOY_RBE}" ]]; then - export BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_timeout=600" - echo "using local build cache." - # Normalize branches - `release/vX.xx`, `vX.xx`, `vX.xx.x` -> `vX.xx` - TARGET_BRANCH="${CI_TARGET_BRANCH}" - if [[ "$TARGET_BRANCH" =~ ^origin/ ]]; then - TARGET_BRANCH=$(echo "$TARGET_BRANCH" | cut -d/ -f2-) - fi - BRANCH_NAME="$(echo "${TARGET_BRANCH}" | cut -d/ -f2 | cut -d. -f-2)" - if [[ "$BRANCH_NAME" == "merge" ]]; then - # Manually run PR commit - there is no easy way of telling which branch - # it is, so just set it to `main` - otherwise it tries to cache as `branch/merge` - BRANCH_NAME=main - fi - BAZEL_REMOTE_INSTANCE="branch/${BRANCH_NAME}" - fi - - if [[ -n "${BAZEL_REMOTE_INSTANCE}" ]]; then - export BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_instance_name=${BAZEL_REMOTE_INSTANCE}" - echo "instance_name: ${BAZEL_REMOTE_INSTANCE}." - fi -fi diff --git a/ci/test_docker_ci.sh b/ci/test_docker_ci.sh index 6bfa4479aa4b..bd9748aa2b05 100755 --- a/ci/test_docker_ci.sh +++ b/ci/test_docker_ci.sh @@ -54,7 +54,7 @@ _test () { fi export ENVOY_VERSION="${version}" - export AZP_BRANCH="$branch" + export CI_BRANCH="$branch" # this should be ignored if the non-push export DOCKERHUB_USERNAME=DHUSER export DOCKERHUB_PASSWORD=DHPASSWORD @@ -68,13 +68,13 @@ _test () { if [[ "$DOCKER_CI_TEST_COMMIT" ]]; then echo "COMMIT(${name}): > ${testdata}" - echo " DOCKER_FAKE_WIN=${DOCKER_FAKE_WIN} ENVOY_VERSION=${version} ENVOY_DOCKER_IMAGE_DIRECTORY=/non/existent/test/path AZP_BRANCH=${branch} DOCKER_CI_DRYRUN=1 ./ci/docker_ci.sh | grep -E \"^>\"" + echo " DOCKER_FAKE_WIN=${DOCKER_FAKE_WIN} ENVOY_VERSION=${version} ENVOY_DOCKER_IMAGE_DIRECTORY=/non/existent/test/path CI_BRANCH=${branch} DOCKER_CI_DRYRUN=1 ./ci/docker_ci.sh | grep -E \"^>\"" ./ci/docker_ci.sh | grep -E "^>" > "$testdata" return fi echo "TEST(${name}): <> ${testdata}" - echo " DOCKER_FAKE_WIN=${DOCKER_FAKE_WIN} ENVOY_VERSION=${version} ENVOY_DOCKER_IMAGE_DIRECTORY=/non/existent/test/path AZP_BRANCH=${branch} DOCKER_CI_DRYRUN=1 ./ci/docker_ci.sh | grep -E \"^>\"" + echo " DOCKER_FAKE_WIN=${DOCKER_FAKE_WIN} ENVOY_VERSION=${version} ENVOY_DOCKER_IMAGE_DIRECTORY=/non/existent/test/path CI_BRANCH=${branch} DOCKER_CI_DRYRUN=1 ./ci/docker_ci.sh | grep -E \"^>\"" generated="$(mktemp)" ./ci/docker_ci.sh | grep -E "^>" > "$generated" diff --git a/ci/upload_gcs_artifact.sh b/ci/upload_gcs_artifact.sh index 6367184a408b..339a4e98dc4d 100755 --- a/ci/upload_gcs_artifact.sh +++ b/ci/upload_gcs_artifact.sh @@ -7,27 +7,17 @@ if [[ -z "${GCS_ARTIFACT_BUCKET}" ]]; then exit 1 fi -if [[ -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then - echo "GCP key is not set, not uploading artifacts." - exit 1 -fi - read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTION_LIST:-}" read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}" -remove_key () { - rm -rf "$KEYFILE" -} - -trap remove_key EXIT - -# Fail when service account key is not specified -KEYFILE="$(mktemp)" -bash -c 'echo ${GCP_SERVICE_ACCOUNT_KEY}' | base64 --decode > "$KEYFILE" +if [[ ! -s "${GCP_SERVICE_ACCOUNT_KEY_PATH}" ]]; then + echo "GCP key is not set, not uploading artifacts." + exit 1 +fi cat < ~/.boto [Credentials] -gs_service_key_file=${KEYFILE} +gs_service_key_file=${GCP_SERVICE_ACCOUNT_KEY_PATH} EOF SOURCE_DIRECTORY="$1" diff --git a/ci/verify_examples.sh b/ci/verify_examples.sh index 12de2fefe0a5..f267a4a464f7 100755 --- a/ci/verify_examples.sh +++ b/ci/verify_examples.sh @@ -11,6 +11,10 @@ WARNINGS=() FLAKY_SANDBOXES=( # https://github.com/envoyproxy/envoy/issues/28542 double-proxy + # https://github.com/envoyproxy/envoy/issues/31347 + local_ratelimit + # https://github.com/envoyproxy/envoy/issues/31333 + locality-load-balancing # https://github.com/envoyproxy/envoy/issues/28541 wasm-cc # https://github.com/envoyproxy/envoy/issues/28546 diff --git a/ci/windows_ci_steps.sh b/ci/windows_ci_steps.sh index 58fd0a9a81d5..8881be13dc99 100755 --- a/ci/windows_ci_steps.sh +++ b/ci/windows_ci_steps.sh @@ -11,9 +11,6 @@ trap finish EXIT echo "disk space at beginning of build:" df -h -# shellcheck source=ci/setup_cache.sh -. "$(dirname "$0")"/setup_cache.sh - [ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/c/source read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTIONS:-}" @@ -77,13 +74,13 @@ fi if [[ $1 == "//source/exe:envoy-static" ]]; then BUILD_ENVOY_STATIC=1 shift - TEST_TARGETS=$* + TEST_TARGETS=("${@}") elif [[ $# -gt 0 ]]; then BUILD_ENVOY_STATIC=0 - TEST_TARGETS=$* + TEST_TARGETS=("$@") else BUILD_ENVOY_STATIC=1 - TEST_TARGETS='//test/...' + TEST_TARGETS=('//test/...') fi # Complete envoy-static build @@ -100,8 +97,8 @@ if [[ $BUILD_ENVOY_STATIC -eq 1 ]]; then fi # Test invocations of known-working tests on Windows -if [[ $TEST_TARGETS == "//test/..." ]]; then - bazel "${BAZEL_STARTUP_OPTIONS[@]}" test "${BAZEL_BUILD_OPTIONS[@]}" $TEST_TARGETS --test_tag_filters=-skip_on_windows,-fails_on_${FAIL_GROUP} --build_tests_only +if [[ "${TEST_TARGETS[*]}" == "//test/..." ]]; then + bazel "${BAZEL_STARTUP_OPTIONS[@]}" test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}" --test_tag_filters=-skip_on_windows,-fails_on_${FAIL_GROUP} --build_tests_only # Build tests that are known flaky or failing to ensure no compilation regressions bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //test/... --test_tag_filters=fails_on_${FAIL_GROUP} --build_tests_only @@ -111,8 +108,8 @@ if [[ $TEST_TARGETS == "//test/..." ]]; then # not triggered by envoy-static or //test/... targets and not deliberately tagged skip_on_windows bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //bazel/... --build_tag_filters=-skip_on_windows fi -elif [[ -n "$TEST_TARGETS" ]]; then - bazel "${BAZEL_STARTUP_OPTIONS[@]}" test "${BAZEL_BUILD_OPTIONS[@]}" $TEST_TARGETS --build_tests_only +elif [[ -n "${TEST_TARGETS[*]}" ]]; then + bazel "${BAZEL_STARTUP_OPTIONS[@]}" test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}" --build_tests_only fi # Summarize known unbuildable or inapplicable tests (example) diff --git a/configs/proxy_connect_udp_http3_downstream.yaml b/configs/proxy_connect_udp_http3_downstream.yaml index 89ce62f156d9..a61cfc97ba43 100644 --- a/configs/proxy_connect_udp_http3_downstream.yaml +++ b/configs/proxy_connect_udp_http3_downstream.yaml @@ -46,8 +46,8 @@ static_resources: - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - http2_protocol_options: - allow_connect: true + http3_protocol_options: + allow_extended_connect: true upgrade_configs: - upgrade_type: CONNECT-UDP clusters: @@ -56,7 +56,8 @@ static_resources: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions explicit_http_config: - http2_protocol_options: {} + http3_protocol_options: + allow_extended_connect: true load_assignment: cluster_name: cluster_0 endpoints: @@ -66,3 +67,14 @@ static_resources: socket_address: address: 127.0.0.1 port_value: 10002 + transport_socket: + name: envoy.transport_sockets.quic + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicUpstreamTransport + upstream_tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: + filename: certs/servercert.pem + private_key: + filename: certs/serverkey.pem diff --git a/configs/raw_udp_tunneling_http2.yaml b/configs/raw_udp_tunneling_http2.yaml new file mode 100644 index 000000000000..30adcd619b7d --- /dev/null +++ b/configs/raw_udp_tunneling_http2.yaml @@ -0,0 +1,72 @@ +# This configuration takes incoming data on port 10000 and encapsulates it in a CONNECT +# request which is sent upstream port 10001. +# It can be used to test UDP tunneling as described in +# https://envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/upgrades + +admin: + address: + socket_address: + protocol: TCP + address: 127.0.0.1 + port_value: 9903 +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: UDP + address: 127.0.0.1 + port_value: 10000 + listener_filters: + - name: envoy.filters.udp_listener.udp_proxy + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig + stat_prefix: foo + matcher: + on_no_match: + action: + name: route + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: cluster_0 + session_filters: + - name: envoy.filters.udp.session.http_capsule + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3.FilterConfig + tunneling_config: + # note: proxy_host supports string substitution, for example setting "%FILTER_STATE(proxy.host.key:PLAIN)%" + # will take the target host value from the session's filter state. + proxy_host: proxy.host.com + # note: target_host supports string substitution, for example setting "%FILTER_STATE(target.host.key:PLAIN)%" + # will take the target host value from the session's filter state. + target_host: target.host.com + # note: The target port value can be overridden per-session by setting the required port value for + # the filter state key ``udp.connect.target_port``. + default_target_port: 443 + retry_options: + max_connect_attempts: 2 + buffer_options: + max_buffered_datagrams: 1024 + max_buffered_bytes: 16384 + headers_to_add: + - header: + key: original_dst_port + value: "%DOWNSTREAM_LOCAL_PORT%" + + clusters: + - name: cluster_0 + connect_timeout: 5s + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: cluster_0 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 10001 diff --git a/configs/terminate_http3_connect_udp.yaml b/configs/terminate_http3_connect_udp.yaml index 675df041e843..1f119d2b2074 100644 --- a/configs/terminate_http3_connect_udp.yaml +++ b/configs/terminate_http3_connect_udp.yaml @@ -49,8 +49,8 @@ static_resources: - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - http2_protocol_options: - allow_connect: true + http3_protocol_options: + allow_extended_connect: true upgrade_configs: - upgrade_type: CONNECT-UDP clusters: diff --git a/configs/terminate_http3_connect_udp_dns_resolution.yaml b/configs/terminate_http3_connect_udp_dns_resolution.yaml new file mode 100644 index 000000000000..b29b32233529 --- /dev/null +++ b/configs/terminate_http3_connect_udp_dns_resolution.yaml @@ -0,0 +1,72 @@ +# This configuration terminates a CONNECT-UDP request and sends UDP payloads directly over UDP to the target. +# The configured dynamic forward proxy enables DNS resolution at the proxy to suppot arbitrary target domain names. +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: UDP + address: 127.0.0.1 + port_value: 10001 + udp_listener_config: + quic_options: {} + downstream_socket_config: + prefer_gro: true + filter_chains: + - transport_socket: + name: envoy.transport_sockets.quic + typed_config: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicDownstreamTransport + downstream_tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: + filename: certs/servercert.pem + private_key: + filename: certs/serverkey.pem + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: HTTP3 + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + connect_matcher: + {} + route: + cluster: dynamic_forward_proxy_cluster + upgrade_configs: + - upgrade_type: CONNECT-UDP + connect_config: + {} + http_filters: + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: + name: dynamic_forward_proxy_cache_config + dns_lookup_family: V4_ONLY + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http3_protocol_options: + allow_extended_connect: true + upgrade_configs: + - upgrade_type: CONNECT-UDP + clusters: + - name: dynamic_forward_proxy_cluster + lb_policy: CLUSTER_PROVIDED + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + '@type': type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: dynamic_forward_proxy_cache_config + dns_lookup_family: V4_ONLY diff --git a/contrib/BUILD b/contrib/BUILD index f6813770abcc..34a896d22e40 100644 --- a/contrib/BUILD +++ b/contrib/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:utils.bzl", "json_data") +load("@envoy_toolshed//:macros.bzl", "json_data") load(":contrib_build_config.bzl", "CONTRIB_EXTENSIONS") licenses(["notice"]) # Apache 2 diff --git a/contrib/all_contrib_extensions.bzl b/contrib/all_contrib_extensions.bzl index 005cffb3c097..9d490cc79099 100644 --- a/contrib/all_contrib_extensions.bzl +++ b/contrib/all_contrib_extensions.bzl @@ -7,12 +7,17 @@ def envoy_contrib_linux_x86_64_constraints(): "@platforms//cpu:x86_64", ] +def envoy_contrib_linux_aarch64_constraints(): + return [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ] + ARM64_SKIP_CONTRIB_TARGETS = [ "envoy.tls.key_providers.cryptomb", "envoy.tls.key_providers.qat", - "envoy.matching.input_matchers.hyperscan", "envoy.network.connection_balance.dlb", - "envoy.regex_engines.hyperscan", + "envoy.compression.qatzip.compressor", ] PPC_SKIP_CONTRIB_TARGETS = [ "envoy.tls.key_providers.cryptomb", @@ -20,6 +25,7 @@ PPC_SKIP_CONTRIB_TARGETS = [ "envoy.matching.input_matchers.hyperscan", "envoy.network.connection_balance.dlb", "envoy.regex_engines.hyperscan", + "envoy.compression.qatzip.compressor", ] def envoy_all_contrib_extensions(denylist = []): diff --git a/contrib/checksum/filters/http/test/checksum_filter_test.cc b/contrib/checksum/filters/http/test/checksum_filter_test.cc index 823d5ee36d95..3f2f7e72d9f5 100644 --- a/contrib/checksum/filters/http/test/checksum_filter_test.cc +++ b/contrib/checksum/filters/http/test/checksum_filter_test.cc @@ -14,7 +14,6 @@ using testing::_; using testing::NiceMock; -using testing::Return; namespace Envoy { namespace Extensions { diff --git a/contrib/checksum/filters/http/test/config_test.cc b/contrib/checksum/filters/http/test/config_test.cc index 8329d623d990..718c734bb2d3 100644 --- a/contrib/checksum/filters/http/test/config_test.cc +++ b/contrib/checksum/filters/http/test/config_test.cc @@ -17,7 +17,8 @@ TEST(ChecksumFilterConfigTest, ChecksumFilter) { NiceMock context; ChecksumFilterFactory factory; envoy::extensions::filters::http::checksum::v3alpha::ChecksumConfig proto_config; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.cc b/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.cc index 6728eb12a63a..f0c28dbd0d4d 100644 --- a/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.cc +++ b/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.cc @@ -33,8 +33,10 @@ ClientSslAuthConfig::ClientSslAuthConfig( cm, config.auth_api_cluster(), dispatcher, random, std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, refresh_delay, 60000)), std::chrono::milliseconds(1000)), - tls_(tls.allocateSlot()), ip_allowlist_(config.ip_white_list()), - stats_(generateStats(scope, config.stat_prefix())) { + tls_(tls.allocateSlot()), stats_(generateStats(scope, config.stat_prefix())) { + auto list_or_error = Network::Address::IpList::create(config.ip_white_list()); + THROW_IF_STATUS_NOT_OK(list_or_error, throw); + ip_allowlist_ = std::move(list_or_error.value()); if (!cm.clusters().hasCluster(remote_cluster_name_)) { throw EnvoyException( diff --git a/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h b/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h index 23452b534237..e452ed158746 100644 --- a/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h +++ b/contrib/client_ssl_auth/filters/network/source/client_ssl_auth.h @@ -81,7 +81,7 @@ class ClientSslAuthConfig : public Http::RestApiFetcher { Event::Dispatcher& dispatcher, Stats::Scope& scope, Random::RandomGenerator& random); const AllowedPrincipals& allowedPrincipals(); - const Network::Address::IpList& ipAllowlist() { return ip_allowlist_; } + const Network::Address::IpList& ipAllowlist() { return *ip_allowlist_; } GlobalStats& stats() { return stats_; } private: @@ -99,7 +99,7 @@ class ClientSslAuthConfig : public Http::RestApiFetcher { void onFetchFailure(Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; ThreadLocal::SlotPtr tls_; - Network::Address::IpList ip_allowlist_; + std::unique_ptr ip_allowlist_; GlobalStats stats_; }; diff --git a/contrib/client_ssl_auth/filters/network/source/config.cc b/contrib/client_ssl_auth/filters/network/source/config.cc index bde7724057f0..1c0af35955f4 100644 --- a/contrib/client_ssl_auth/filters/network/source/config.cc +++ b/contrib/client_ssl_auth/filters/network/source/config.cc @@ -18,9 +18,12 @@ Network::FilterFactoryCb ClientSslAuthConfigFactory::createFilterFactoryFromProt ASSERT(!proto_config.auth_api_cluster().empty()); ASSERT(!proto_config.stat_prefix().empty()); + auto& server_context = context.serverFactoryContext(); + ClientSslAuthConfigSharedPtr filter_config(ClientSslAuthConfig::create( - proto_config, context.threadLocal(), context.clusterManager(), context.mainThreadDispatcher(), - context.scope(), context.api().randomGenerator())); + proto_config, server_context.threadLocal(), server_context.clusterManager(), + server_context.mainThreadDispatcher(), context.scope(), + server_context.api().randomGenerator())); return [filter_config](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(filter_config)); }; diff --git a/contrib/client_ssl_auth/filters/network/test/client_ssl_auth_test.cc b/contrib/client_ssl_auth/filters/network/test/client_ssl_auth_test.cc index 63e97ec041ea..b6b1aa25a786 100644 --- a/contrib/client_ssl_auth/filters/network/test/client_ssl_auth_test.cc +++ b/contrib/client_ssl_auth/filters/network/test/client_ssl_auth_test.cc @@ -176,8 +176,11 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { EXPECT_CALL(*interval_timer_, enableTimer(_, _)); Http::ResponseMessagePtr message(new Http::ResponseMessageImpl( Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "200"}}})); - message->body().add(api_->fileSystem().fileReadToEnd(TestEnvironment::runfilesPath( - "contrib/client_ssl_auth/filters/network/test/test_data/vpn_response_1.json"))); + message->body().add( + api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath( + "contrib/client_ssl_auth/filters/network/test/test_data/vpn_response_1.json")) + .value()); callbacks_->onSuccess(request_, std::move(message)); EXPECT_EQ(1U, stats_store_ diff --git a/contrib/client_ssl_auth/filters/network/test/config_test.cc b/contrib/client_ssl_auth/filters/network/test/config_test.cc index 649d3dba84a4..0c49cd5a740b 100644 --- a/contrib/client_ssl_auth/filters/network/test/config_test.cc +++ b/contrib/client_ssl_auth/filters/network/test/config_test.cc @@ -43,8 +43,8 @@ auth_api_cluster: fake_cluster envoy::extensions::filters::network::client_ssl_auth::v3::ClientSSLAuth proto_config; TestUtility::loadFromYamlAndValidate(yaml, proto_config); NiceMock context; - context.cluster_manager_.initializeClusters({"fake_cluster"}, {}); - context.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + context.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); ClientSslAuthConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); Network::MockConnection connection; @@ -62,8 +62,8 @@ auth_api_cluster: fake_cluster envoy::extensions::filters::network::client_ssl_auth::v3::ClientSSLAuth proto_config; TestUtility::loadFromYamlAndValidate(yaml, proto_config); NiceMock context; - context.cluster_manager_.initializeClusters({"fake_cluster"}, {}); - context.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + context.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); ClientSslAuthConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); Network::MockConnection connection; @@ -79,8 +79,8 @@ auth_api_cluster: fake_cluster )EOF" + GetParam(); NiceMock context; - context.cluster_manager_.initializeClusters({"fake_cluster"}, {}); - context.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + context.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); ClientSslAuthConfigFactory factory; envoy::extensions::filters::network::client_ssl_auth::v3::ClientSSLAuth proto_config = *dynamic_cast( diff --git a/contrib/contrib_build_config.bzl b/contrib/contrib_build_config.bzl index f2cdaea8bfdc..b8e7811a168c 100644 --- a/contrib/contrib_build_config.bzl +++ b/contrib/contrib_build_config.bzl @@ -1,5 +1,11 @@ # See bazel/README.md for details on how this system works. CONTRIB_EXTENSIONS = { + # + # Compression + # + + "envoy.compression.qatzip.compressor": "//contrib/qat/compression/qatzip/compressor/source:config", + # # HTTP filters # @@ -15,7 +21,7 @@ CONTRIB_EXTENSIONS = { # "envoy.filters.network.client_ssl_auth": "//contrib/client_ssl_auth/filters/network/source:config", - "envoy.filters.network.kafka_broker": "//contrib/kafka/filters/network/source:kafka_broker_config_lib", + "envoy.filters.network.kafka_broker": "//contrib/kafka/filters/network/source/broker:config_lib", "envoy.filters.network.kafka_mesh": "//contrib/kafka/filters/network/source/mesh:config_lib", "envoy.filters.network.mysql_proxy": "//contrib/mysql_proxy/filters/network/source:config", "envoy.filters.network.postgres_proxy": "//contrib/postgres_proxy/filters/network/source:config", @@ -66,6 +72,7 @@ CONTRIB_EXTENSIONS = { # "envoy.filters.generic.router": "//contrib/generic_proxy/filters/network/source/router:config", "envoy.generic_proxy.codecs.dubbo": "//contrib/generic_proxy/filters/network/source/codecs/dubbo:config", + "envoy.generic_proxy.codecs.kafka": "//contrib/generic_proxy/filters/network/source/codecs/kafka:config", # # xDS delegates diff --git a/contrib/cryptomb/private_key_providers/source/BUILD b/contrib/cryptomb/private_key_providers/source/BUILD index 199acb8fe411..610b66513373 100644 --- a/contrib/cryptomb/private_key_providers/source/BUILD +++ b/contrib/cryptomb/private_key_providers/source/BUILD @@ -22,7 +22,10 @@ envoy_cmake( defines = [ "OPENSSL_USE_STATIC_LIBS=TRUE", ], - lib_source = "@com_github_intel_ipp_crypto_crypto_mb//:all", + lib_source = select({ + "//bazel:boringssl_fips": "@com_github_intel_ipp_crypto_crypto_mb_fips//:all", + "//conditions:default": "@com_github_intel_ipp_crypto_crypto_mb//:all", + }), out_static_libs = ["libcrypto_mb.a"], tags = ["skip_on_windows"], target_compatible_with = envoy_contrib_linux_x86_64_constraints(), diff --git a/contrib/dlb/source/BUILD b/contrib/dlb/source/BUILD index b0952bb65138..cf8405c98f54 100644 --- a/contrib/dlb/source/BUILD +++ b/contrib/dlb/source/BUILD @@ -26,9 +26,10 @@ make( envoy_cc_contrib_extension( name = "connection_balancer", - srcs = [ - "connection_balancer_impl.cc", - ], + srcs = select({ + "//bazel:linux_x86_64": ["connection_balancer_impl.cc"], + "//conditions:default": [], + }), hdrs = ["connection_balancer_impl.h"], defines = select({ "//bazel:linux_x86_64": [], @@ -42,9 +43,9 @@ envoy_cc_contrib_extension( "//envoy/server:factory_context_interface", "//envoy/server:filter_config_interface", "//source/common/common:logger_lib", + "//source/common/listener_manager:active_tcp_listener", "//source/common/network:connection_balancer_lib", "//source/common/protobuf:utility_lib", - "//source/extensions/listener_managers/listener_manager:active_tcp_listener", "@envoy_api//contrib/envoy/extensions/network/connection_balance/dlb/v3alpha:pkg_cc_proto", ] + select({ "//bazel:linux_x86_64": [ diff --git a/contrib/dlb/source/connection_balancer_impl.cc b/contrib/dlb/source/connection_balancer_impl.cc index d91c093caca8..7ece065929d1 100644 --- a/contrib/dlb/source/connection_balancer_impl.cc +++ b/contrib/dlb/source/connection_balancer_impl.cc @@ -47,7 +47,7 @@ DlbConnectionBalanceFactory::createConnectionBalancerFromProto( fallback_policy = dlb_config.fallback_policy(); - const uint32_t worker_num = context.options().concurrency(); + const uint32_t worker_num = context.serverFactoryContext().options().concurrency(); if (worker_num > 32) { return fallback("Dlb connection balanncer only supports up to 32 worker threads, " diff --git a/contrib/dlb/source/connection_balancer_impl.h b/contrib/dlb/source/connection_balancer_impl.h index d2aebc865b26..77db11e14f2d 100644 --- a/contrib/dlb/source/connection_balancer_impl.h +++ b/contrib/dlb/source/connection_balancer_impl.h @@ -8,9 +8,9 @@ #include "envoy/server/filter_config.h" #include "source/common/api/os_sys_calls_impl.h" +#include "source/common/listener_manager/active_tcp_listener.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/protobuf/protobuf.h" -#include "source/extensions/listener_managers/listener_manager/active_tcp_listener.h" #include "contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/dlb.pb.h" #include "contrib/envoy/extensions/network/connection_balance/dlb/v3alpha/dlb.pb.validate.h" @@ -88,7 +88,7 @@ class DlbConnectionBalanceFactory : public Envoy::Network::ConnectionBalanceFact Envoy::Network::ConnectionBalancerSharedPtr createConnectionBalancerFromProto(const Protobuf::Message& config, - Server::Configuration::FactoryContext& context) override; + Server::Configuration::FactoryContext&) override; std::string name() const override { return "envoy.network.connection_balance.dlb"; } diff --git a/contrib/dlb/test/BUILD b/contrib/dlb/test/BUILD index bc1cca54cfba..de39044fe7e5 100644 --- a/contrib/dlb/test/BUILD +++ b/contrib/dlb/test/BUILD @@ -10,7 +10,10 @@ envoy_contrib_package() envoy_cc_test( name = "config_test", - srcs = ["config_test.cc"], + srcs = select({ + "//bazel:linux_x86_64": ["config_test.cc"], + "//conditions:default": [], + }), deps = [ "//contrib/dlb/source:connection_balancer", "//source/common/protobuf:utility_lib", diff --git a/contrib/dlb/test/config_test.cc b/contrib/dlb/test/config_test.cc index 7108f24738e8..be896ea6e8b6 100644 --- a/contrib/dlb/test/config_test.cc +++ b/contrib/dlb/test/config_test.cc @@ -138,7 +138,7 @@ TEST_F(DlbConnectionBalanceFactoryTest, TooManyThreads) { envoy::config::core::v3::TypedExtensionConfig typed_config; DlbConnectionBalanceFactory factory; NiceMock context; - context.options_.concurrency_ = 33; + context.server_factory_context_.options_.concurrency_ = 33; envoy::extensions::network::connection_balance::dlb::v3alpha::Dlb dlb; makeDlbConnectionBalanceConfig(typed_config, dlb); diff --git a/contrib/dynamo/filters/http/source/config.cc b/contrib/dynamo/filters/http/source/config.cc index 67d58734c362..5789def74927 100644 --- a/contrib/dynamo/filters/http/source/config.cc +++ b/contrib/dynamo/filters/http/source/config.cc @@ -18,8 +18,9 @@ Http::FilterFactoryCb DynamoFilterConfig::createFilterFactoryFromProtoTyped( Server::Configuration::FactoryContext& context) { auto stats = std::make_shared(context.scope(), stats_prefix); return [&context, stats](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter(std::make_shared( - context.runtime(), stats, context.mainThreadDispatcher().timeSource())); + callbacks.addStreamFilter( + std::make_shared(context.serverFactoryContext().runtime(), stats, + context.serverFactoryContext().timeSource())); }; } diff --git a/contrib/dynamo/filters/http/test/config_test.cc b/contrib/dynamo/filters/http/test/config_test.cc index b9295b013f4d..e29d32b3dbb3 100644 --- a/contrib/dynamo/filters/http/test/config_test.cc +++ b/contrib/dynamo/filters/http/test/config_test.cc @@ -18,7 +18,8 @@ TEST(DynamoFilterConfigTest, DynamoFilter) { NiceMock context; DynamoFilterConfig factory; envoy::extensions::filters::http::dynamo::v3::Dynamo proto_config; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/contrib/extensions_metadata.yaml b/contrib/extensions_metadata.yaml index 6086b1791d0d..8168b063da58 100644 --- a/contrib/extensions_metadata.yaml +++ b/contrib/extensions_metadata.yaml @@ -13,6 +13,11 @@ envoy.filters.http.golang: - envoy.filters.http security_posture: requires_trusted_downstream_and_upstream status: alpha +envoy.compression.qatzip.compressor: + categories: + - envoy.compression.compressor + security_posture: robust_to_untrusted_downstream_and_upstream + status: alpha envoy.filters.http.squash: categories: - envoy.filters.http @@ -127,6 +132,13 @@ envoy.generic_proxy.codecs.dubbo: status: wip type_urls: - envoy.extensions.filters.network.generic_proxy.codecs.dubbo.v3.DubboCodecConfig +envoy.generic_proxy.codecs.kafka: + categories: + - envoy.generic_proxy.codecs + security_posture: requires_trusted_downstream_and_upstream + status: wip + type_urls: + - envoy.extensions.filters.network.generic_proxy.codecs.kafka.v3.KafkaCodecConfig envoy.router.cluster_specifier_plugin.golang: categories: - envoy.router.cluster_specifier_plugin diff --git a/contrib/generic_proxy/filters/network/source/BUILD b/contrib/generic_proxy/filters/network/source/BUILD index e46ae6c4ba10..70f434f22603 100644 --- a/contrib/generic_proxy/filters/network/source/BUILD +++ b/contrib/generic_proxy/filters/network/source/BUILD @@ -21,7 +21,7 @@ envoy_cc_library( ":rds_lib", ":route_lib", ":stats_lib", - ":upstream_lib", + ":tracing_lib", "//contrib/generic_proxy/filters/network/source/interface:codec_interface", "//contrib/generic_proxy/filters/network/source/interface:proxy_config_interface", "//contrib/generic_proxy/filters/network/source/router:router_lib", @@ -171,4 +171,26 @@ envoy_cc_library( ":file_access_log_lib", "//contrib/generic_proxy/filters/network/source/interface:stream_interface", ], + # Ensure this factory in the source is always linked in. + alwayslink = 1, +) + +envoy_cc_library( + name = "tracing_lib", + srcs = [ + "tracing.cc", + ], + hdrs = [ + "tracing.h", + ], + external_deps = [ + "abseil_strings", + "abseil_optional", + "abseil_status", + ], + deps = [ + "//contrib/generic_proxy/filters/network/source/interface:stream_interface", + "//envoy/common:pure_lib", + "//envoy/tracing:trace_context_interface", + ], ) diff --git a/contrib/generic_proxy/filters/network/source/access_log.cc b/contrib/generic_proxy/filters/network/source/access_log.cc index b89a73780c8c..1e1f70feb868 100644 --- a/contrib/generic_proxy/filters/network/source/access_log.cc +++ b/contrib/generic_proxy/filters/network/source/access_log.cc @@ -116,7 +116,7 @@ class SimpleCommandParser : public CommandParser { if (!context.request_) { return absl::nullopt; } - auto optional_view = context.request_->getByKey(key); + auto optional_view = context.request_->get(key); if (!optional_view.has_value()) { return absl::nullopt; } @@ -132,7 +132,7 @@ class SimpleCommandParser : public CommandParser { if (!context.response_) { return absl::nullopt; } - auto optional_view = context.response_->getByKey(key); + auto optional_view = context.response_->get(key); if (!optional_view.has_value()) { return absl::nullopt; } diff --git a/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.cc b/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.cc index 441d28ad2996..4bbfa68e1b06 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.cc +++ b/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.cc @@ -16,28 +16,6 @@ namespace Dubbo { namespace { constexpr absl::string_view VERSION_KEY = "version"; -constexpr absl::string_view UNKNOWN_RESPONSE_STATUS = "UnknownResponseStatus"; - -#define ENUM_TO_STRING_VIEW(X) \ - case Common::Dubbo::ResponseStatus::X: \ - static constexpr absl::string_view X##_VIEW = #X; \ - return X##_VIEW; - -absl::string_view responseStatusToStringView(Common::Dubbo::ResponseStatus status) { - switch (status) { - ENUM_TO_STRING_VIEW(Ok); - ENUM_TO_STRING_VIEW(ClientTimeout); - ENUM_TO_STRING_VIEW(ServerTimeout); - ENUM_TO_STRING_VIEW(BadRequest); - ENUM_TO_STRING_VIEW(BadResponse); - ENUM_TO_STRING_VIEW(ServiceNotFound); - ENUM_TO_STRING_VIEW(ServiceError); - ENUM_TO_STRING_VIEW(ServerError); - ENUM_TO_STRING_VIEW(ClientError); - ENUM_TO_STRING_VIEW(ServerThreadpoolExhaustedError); - } - return UNKNOWN_RESPONSE_STATUS; -} Common::Dubbo::ResponseStatus genericStatusToStatus(StatusCode code) { switch (code) { @@ -50,29 +28,6 @@ Common::Dubbo::ResponseStatus genericStatusToStatus(StatusCode code) { } } -StatusCode statusToGenericStatus(Common::Dubbo::ResponseStatus status) { - switch (status) { - case Common::Dubbo::ResponseStatus::Ok: - return StatusCode::kOk; - case Common::Dubbo::ResponseStatus::ClientTimeout: - case Common::Dubbo::ResponseStatus::ServerTimeout: - return StatusCode::kUnknown; - case Common::Dubbo::ResponseStatus::BadRequest: - return StatusCode::kInvalidArgument; - case Common::Dubbo::ResponseStatus::BadResponse: - return StatusCode::kUnknown; - case Common::Dubbo::ResponseStatus::ServiceNotFound: - return StatusCode::kNotFound; - case Common::Dubbo::ResponseStatus::ServiceError: - case Common::Dubbo::ResponseStatus::ServerError: - case Common::Dubbo::ResponseStatus::ClientError: - return StatusCode::kUnavailable; - case Common::Dubbo::ResponseStatus::ServerThreadpoolExhaustedError: - return StatusCode::kResourceExhausted; - } - return StatusCode::kUnavailable; -} - } // namespace void DubboRequest::forEach(IterateCallback callback) const { @@ -94,7 +49,7 @@ void DubboRequest::forEach(IterateCallback callback) const { } } -absl::optional DubboRequest::getByKey(absl::string_view key) const { +absl::optional DubboRequest::get(absl::string_view key) const { if (key == VERSION_KEY) { return inner_metadata_->request().serviceVersion(); } @@ -105,7 +60,7 @@ absl::optional DubboRequest::getByKey(absl::string_view key) return typed_request->attachment().lookup(key); } -void DubboRequest::setByKey(absl::string_view key, absl::string_view val) { +void DubboRequest::set(absl::string_view key, absl::string_view val) { auto* typed_request = dynamic_cast(&inner_metadata_->mutableRequest()); ASSERT(typed_request != nullptr); @@ -113,7 +68,15 @@ void DubboRequest::setByKey(absl::string_view key, absl::string_view val) { typed_request->mutableAttachment()->insert(key, val); } -void DubboResponse::refreshGenericStatus() { +void DubboRequest::erase(absl::string_view key) { + auto* typed_request = + dynamic_cast(&inner_metadata_->mutableRequest()); + ASSERT(typed_request != nullptr); + + typed_request->mutableAttachment()->remove(key); +} + +void DubboResponse::refreshStatus() { ASSERT(inner_metadata_ != nullptr); ASSERT(inner_metadata_->hasResponse() && inner_metadata_->hasResponseStatus()); @@ -122,36 +85,40 @@ void DubboResponse::refreshGenericStatus() { const auto status = inner_metadata_->context().responseStatus(); const auto optional_type = inner_metadata_->response().responseType(); + // The final status is not ok if the response status is not ResponseStatus::Ok + // anyway. + bool response_ok = (status == Common::Dubbo::ResponseStatus::Ok); + + // The final status is not ok if the response type is ResponseWithException or + // ResponseWithExceptionWithAttachments even if the response status is Ok. if (status == Common::Dubbo::ResponseStatus::Ok) { ASSERT(optional_type.has_value()); auto type = optional_type.value_or(RpcResponseType::ResponseWithException); if (type == RpcResponseType::ResponseWithException || type == RpcResponseType::ResponseWithExceptionWithAttachments) { - status_ = Status(StatusCode::kUnavailable, "exception_via_upstream"); - return; + response_ok = false; } - status_ = absl::OkStatus(); - return; } - status_ = Status(statusToGenericStatus(status), responseStatusToStringView(status)); + status_ = StreamStatus(static_cast(status), response_ok); } DubboCodecBase::DubboCodecBase(Common::Dubbo::DubboCodecPtr codec) : codec_(std::move(codec)) {} -ResponsePtr DubboMessageCreator::response(Status status, const Request& origin_request) { +ResponsePtr DubboServerCodec::respond(Status status, absl::string_view, + const Request& origin_request) { const auto* typed_request = dynamic_cast(&origin_request); ASSERT(typed_request != nullptr); - Common::Dubbo::ResponseStatus response_status; + Common::Dubbo::ResponseStatus response_status = genericStatusToStatus(status.code()); + absl::optional optional_type; absl::string_view content; - if (status.ok()) { - response_status = Common::Dubbo::ResponseStatus::Ok; + + if (response_status == Common::Dubbo::ResponseStatus::Ok) { optional_type.emplace(Common::Dubbo::RpcResponseType::ResponseWithException); content = "exception_via_proxy"; } else { - response_status = genericStatusToStatus(status.code()); content = status.message(); } diff --git a/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h b/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h index 5983bc8b10d6..6ef3932be244 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h +++ b/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h @@ -33,17 +33,17 @@ class DubboRequest : public Request { // Request absl::string_view protocol() const override { return DubboProtocolName; } void forEach(IterateCallback callback) const override; - absl::optional getByKey(absl::string_view key) const override; - void setByKey(absl::string_view key, absl::string_view val) override; - void setByReferenceKey(absl::string_view key, absl::string_view val) override { - setByKey(key, val); - } - void setByReference(absl::string_view key, absl::string_view val) override { setByKey(key, val); } - + absl::optional get(absl::string_view key) const override; + void set(absl::string_view key, absl::string_view val) override; absl::string_view host() const override { return inner_metadata_->request().serviceName(); } absl::string_view path() const override { return inner_metadata_->request().serviceName(); } - absl::string_view method() const override { return inner_metadata_->request().methodName(); } + void erase(absl::string_view key) override; + + // StreamFrame + FrameFlags frameFlags() const override { return stream_frame_flags_; } + + FrameFlags stream_frame_flags_; Common::Dubbo::MessageMetadataSharedPtr inner_metadata_; }; @@ -55,24 +55,21 @@ class DubboResponse : public Response { ASSERT(inner_metadata_ != nullptr); ASSERT(inner_metadata_->hasContext()); ASSERT(inner_metadata_->hasResponse()); - refreshGenericStatus(); + refreshStatus(); } - void refreshGenericStatus(); + void refreshStatus(); // Response. absl::string_view protocol() const override { return DubboProtocolName; } - void forEach(IterateCallback) const override {} - absl::optional getByKey(absl::string_view) const override { - return absl::nullopt; - } - void setByKey(absl::string_view, absl::string_view) override{}; - void setByReferenceKey(absl::string_view, absl::string_view) override {} - void setByReference(absl::string_view, absl::string_view) override {} + StreamStatus status() const override { return status_; } + + // StreamFrame + FrameFlags frameFlags() const override { return stream_frame_flags_; } - Status status() const override { return status_; } + FrameFlags stream_frame_flags_; - Status status_; + StreamStatus status_; Common::Dubbo::MessageMetadataSharedPtr inner_metadata_; }; @@ -83,14 +80,14 @@ class DubboCodecBase : public Logger::Loggable { Common::Dubbo::DubboCodecPtr codec_; }; -template -class DubboDecoderBase : public DubboCodecBase, public DecoderType { +template +class DubboDecoderBase : public DubboCodecBase, public CodecType { public: using DubboCodecBase::DubboCodecBase; - void setDecoderCallback(CallBackType& callback) override { callback_ = &callback; } + void setCodecCallbacks(CallBackType& callback) override { callback_ = &callback; } - void decode(Buffer::Instance& buffer) override { + void decode(Buffer::Instance& buffer, bool) override { if (metadata_ == nullptr) { metadata_ = std::make_shared(); } @@ -121,10 +118,13 @@ class DubboDecoderBase : public DubboCodecBase, public DecoderType { } ASSERT(decode_status == Common::Dubbo::DecodeStatus::Success); - ExtendedOptions extended_options{metadata_->requestId(), metadata_->context().isTwoWay(), - false, metadata_->context().heartbeat()}; - callback_->onDecodingSuccess(std::make_unique(std::move(metadata_)), - extended_options); + + auto message = std::make_unique(metadata_); + message->stream_frame_flags_ = {{static_cast(metadata_->requestId()), + !metadata_->context().isTwoWay(), false, + metadata_->context().heartbeat()}, + true}; + callback_->onDecodingSuccess(std::move(message)); metadata_.reset(); } catch (const EnvoyException& error) { ENVOY_LOG(warn, "Dubbo codec: decoding error: {}", error.what()); @@ -133,69 +133,44 @@ class DubboDecoderBase : public DubboCodecBase, public DecoderType { } } - Common::Dubbo::MessageMetadataSharedPtr metadata_; - CallBackType* callback_{}; -}; - -using DubboRequestDecoder = DubboDecoderBase; -using DubboResponseDecoder = - DubboDecoderBase; - -class DubboRequestEncoder : public RequestEncoder, public DubboCodecBase { -public: - using DubboCodecBase::DubboCodecBase; - - void encode(const Request& request, RequestEncoderCallback& callback) override { - ASSERT(dynamic_cast(&request) != nullptr); - const auto* typed_request = static_cast(&request); + void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) override { + ASSERT(dynamic_cast(&frame) != nullptr); + const auto* typed_message = static_cast(&frame); Buffer::OwnedImpl buffer; - codec_->encode(buffer, *typed_request->inner_metadata_); - callback.onEncodingSuccess(buffer); + codec_->encode(buffer, *typed_message->inner_metadata_); + callbacks.onEncodingSuccess(buffer, true); } + + Common::Dubbo::MessageMetadataSharedPtr metadata_; + CallBackType* callback_{}; }; -class DubboResponseEncoder : public ResponseEncoder, public DubboCodecBase { +class DubboServerCodec + : public DubboDecoderBase { public: - using DubboCodecBase::DubboCodecBase; - - void encode(const Response& response, ResponseEncoderCallback& callback) override { - ASSERT(dynamic_cast(&response) != nullptr); - const auto* typed_response = static_cast(&response); + using DubboDecoderBase::DubboDecoderBase; - Buffer::OwnedImpl buffer; - codec_->encode(buffer, *typed_response->inner_metadata_); - callback.onEncodingSuccess(buffer); - } + ResponsePtr respond(absl::Status status, absl::string_view short_response_flags, + const Request& request) override; }; -class DubboMessageCreator : public MessageCreator { +class DubboClientCodec + : public DubboDecoderBase { public: - ResponsePtr response(Status status, const Request& origin_request) override; + using DubboDecoderBase::DubboDecoderBase; }; class DubboCodecFactory : public CodecFactory { public: - RequestDecoderPtr requestDecoder() const override { - return std::make_unique( - Common::Dubbo::DubboCodec::codecFromSerializeType(Common::Dubbo::SerializeType::Hessian2)); - } - ResponseDecoderPtr responseDecoder() const override { - return std::make_unique( + ServerCodecPtr createServerCodec() const override { + return std::make_unique( Common::Dubbo::DubboCodec::codecFromSerializeType(Common::Dubbo::SerializeType::Hessian2)); } - RequestEncoderPtr requestEncoder() const override { - return std::make_unique( + ClientCodecPtr createClientCodec() const override { + return std::make_unique( Common::Dubbo::DubboCodec::codecFromSerializeType(Common::Dubbo::SerializeType::Hessian2)); } - ResponseEncoderPtr responseEncoder() const override { - return std::make_unique( - Common::Dubbo::DubboCodec::codecFromSerializeType(Common::Dubbo::SerializeType::Hessian2)); - } - MessageCreatorPtr messageCreator() const override { - return std::make_unique(); - } - ProtocolOptions protocolOptions() const override { return {}; } }; class DubboCodecFactoryConfig : public CodecFactoryConfig { diff --git a/contrib/generic_proxy/filters/network/source/codecs/kafka/BUILD b/contrib/generic_proxy/filters/network/source/codecs/kafka/BUILD new file mode 100644 index 000000000000..807d8a1c160b --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/codecs/kafka/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_contrib_extension", + "envoy_contrib_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +envoy_cc_contrib_extension( + name = "config", + srcs = [ + "config.cc", + ], + hdrs = [ + "config.h", + ], + deps = [ + "//contrib/generic_proxy/filters/network/source/interface:codec_interface", + "//contrib/kafka/filters/network/source:kafka_request_codec_lib", + "//contrib/kafka/filters/network/source:kafka_response_codec_lib", + "@envoy_api//contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3:pkg_cc_proto", + ], +) diff --git a/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc new file mode 100644 index 000000000000..d5d600e2cf5d --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc @@ -0,0 +1,92 @@ +#include "contrib/generic_proxy/filters/network/source/codecs/kafka/config.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { +namespace Codec { +namespace Kafka { + +KafkaServerCodec::KafkaServerCodec() : response_encoder_(response_buffer_) {} + +void KafkaServerCodec::setCodecCallbacks(GenericProxy::ServerCodecCallbacks& callbacks) { + request_callbacks_ = std::make_shared(callbacks); + request_decoder_ = std::make_shared( + std::vector{request_callbacks_}); +} + +void KafkaServerCodec::decode(Envoy::Buffer::Instance& buffer, bool) { + request_buffer_.move(buffer); + request_decoder_->onData(request_buffer_); + // All data has been consumed, so we can drain the buffer. + request_buffer_.drain(request_buffer_.length()); +} + +void KafkaServerCodec::encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingCallbacks& callbacks) { + auto* typed_response = dynamic_cast(&frame); + if (typed_response == nullptr) { + ENVOY_LOG(error, "Kafka codec: invalid response frame type and cannot encode"); + return; + } + if (typed_response->response_ != nullptr) { + response_encoder_.encode(*typed_response->response_); + } else { + ENVOY_LOG(error, "Kafka codec: invalid empty response frame type and close connection"); + request_callbacks_->callbacks_.connection()->close(Network::ConnectionCloseType::FlushWrite); + return; + } + callbacks.onEncodingSuccess(response_buffer_, true); + // All data should be consumed by the generic proxy and send to the network. + ASSERT(response_buffer_.length() == 0); +} +GenericProxy::ResponsePtr KafkaServerCodec::respond(absl::Status, absl::string_view, + const GenericProxy::Request&) { + return std::make_unique(nullptr); +}; + +KafkaClientCodec::KafkaClientCodec() : request_encoder_(request_buffer_) {} + +void KafkaClientCodec::setCodecCallbacks(GenericProxy::ClientCodecCallbacks& callbacks) { + response_callbacks_ = std::make_shared(callbacks); + response_decoder_ = std::make_shared( + std::vector{response_callbacks_}); +} + +void KafkaClientCodec::decode(Envoy::Buffer::Instance& buffer, bool) { + response_buffer_.move(buffer); + response_decoder_->onData(response_buffer_); + // All data has been consumed, so we can drain the buffer. + response_buffer_.drain(response_buffer_.length()); +} + +void KafkaClientCodec::encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingCallbacks& callbacks) { + auto* typed_request = dynamic_cast(&frame); + if (typed_request == nullptr) { + ENVOY_LOG(error, "Kafka codec: invalid request frame type and cannot encode"); + return; + } + response_decoder_->expectResponse(typed_request->request_->request_header_.correlation_id_, + typed_request->request_->request_header_.api_key_, + typed_request->request_->request_header_.api_version_); + request_encoder_.encode(*typed_request->request_); + callbacks.onEncodingSuccess(request_buffer_, true); + // All data should be consumed by the generic proxy and send to the network. + ASSERT(request_buffer_.length() == 0); +} + +CodecFactoryPtr +KafkaCodecFactoryConfig::createCodecFactory(const Protobuf::Message&, + Envoy::Server::Configuration::FactoryContext&) { + return std::make_unique(); +} + +REGISTER_FACTORY(KafkaCodecFactoryConfig, CodecFactoryConfig); + +} // namespace Kafka +} // namespace Codec +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h new file mode 100644 index 000000000000..c9b7064459ac --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h @@ -0,0 +1,166 @@ +#pragma once + +#include "source/common/buffer/buffer_impl.h" + +#include "contrib/envoy/extensions/filters/network/generic_proxy/codecs/kafka/v3/kafka.pb.h" +#include "contrib/generic_proxy/filters/network/source/interface/codec.h" +#include "contrib/kafka/filters/network/source/request_codec.h" +#include "contrib/kafka/filters/network/source/response_codec.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { +namespace Codec { +namespace Kafka { + +using ProtoConfig = + envoy::extensions::filters::network::generic_proxy::codecs::kafka::v3::KafkaCodecConfig; + +class KafkaRequestFrame : public GenericProxy::StreamRequest { +public: + KafkaRequestFrame(NetworkFilters::Kafka::AbstractRequestSharedPtr request) + : request_(std::move(request)) { + ASSERT(request_ != nullptr); + } + + FrameFlags frameFlags() const override { + if (request_ == nullptr) { + return FrameFlags{}; + } + return FrameFlags{ + StreamFlags{static_cast(request_->request_header_.correlation_id_)}}; + } + + absl::string_view protocol() const override { return "kafka"; } + + NetworkFilters::Kafka::AbstractRequestSharedPtr request_; +}; + +class KafkaResponseFrame : public GenericProxy::StreamResponse { +public: + KafkaResponseFrame(NetworkFilters::Kafka::AbstractResponseSharedPtr response) + : response_(std::move(response)) {} + + FrameFlags frameFlags() const override { + if (response_ == nullptr) { + return FrameFlags{}; + } + return FrameFlags{StreamFlags{static_cast(response_->metadata_.correlation_id_)}}; + } + + absl::string_view protocol() const override { return "kafka"; } + + NetworkFilters::Kafka::AbstractResponseSharedPtr response_; +}; + +class KafkaRequestCallbacks : public NetworkFilters::Kafka::RequestCallback, + public Envoy::Logger::Loggable { +public: + KafkaRequestCallbacks(GenericProxy::ServerCodecCallbacks& callbacks) : callbacks_(callbacks) {} + + void onMessage(NetworkFilters::Kafka::AbstractRequestSharedPtr request) override { + ENVOY_CONN_LOG(debug, "Kafka codec: new request from downstream client", + callbacks_.connection().ref()); + callbacks_.onDecodingSuccess(std::make_unique(std::move(request))); + } + + void onFailedParse(NetworkFilters::Kafka::RequestParseFailureSharedPtr) override { + ENVOY_CONN_LOG(debug, "Kafka codec: failed to parse request from downstream client", + callbacks_.connection().ref()); + callbacks_.onDecodingFailure(); + } + + GenericProxy::ServerCodecCallbacks& callbacks_; +}; + +class KafkaResponseCallbacks : public NetworkFilters::Kafka::ResponseCallback, + public Envoy::Logger::Loggable { +public: + KafkaResponseCallbacks(GenericProxy::ClientCodecCallbacks& callbacks) : callbacks_(callbacks) {} + + void onMessage(NetworkFilters::Kafka::AbstractResponseSharedPtr response) override { + ENVOY_CONN_LOG(debug, "Kafka codec: new response from upstream server", + callbacks_.connection().ref()); + callbacks_.onDecodingSuccess(std::make_unique(std::move(response))); + } + + void onFailedParse(NetworkFilters::Kafka::ResponseMetadataSharedPtr) override { + ENVOY_CONN_LOG(debug, "Kafka codec: failed to parse response from upstream server", + callbacks_.connection().ref()); + callbacks_.onDecodingFailure(); + } + +private: + GenericProxy::ClientCodecCallbacks& callbacks_; +}; + +class KafkaServerCodec : public GenericProxy::ServerCodec, + public Envoy::Logger::Loggable { +public: + KafkaServerCodec(); + + void setCodecCallbacks(GenericProxy::ServerCodecCallbacks& callbacks) override; + void decode(Envoy::Buffer::Instance& buffer, bool end_stream) override; + void encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingCallbacks& callbacks) override; + GenericProxy::ResponsePtr respond(absl::Status, absl::string_view, + const GenericProxy::Request&) override; + + Envoy::Buffer::OwnedImpl request_buffer_; + Envoy::Buffer::OwnedImpl response_buffer_; + + NetworkFilters::Kafka::RequestDecoderSharedPtr request_decoder_; + NetworkFilters::Kafka::ResponseEncoder response_encoder_; + + std::shared_ptr request_callbacks_; +}; + +class KafkaClientCodec : public GenericProxy::ClientCodec, + public Envoy::Logger::Loggable { +public: + KafkaClientCodec(); + + void setCodecCallbacks(GenericProxy::ClientCodecCallbacks& callbacks) override; + void decode(Envoy::Buffer::Instance& buffer, bool end_stream) override; + void encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingCallbacks& callbacks) override; + + Envoy::Buffer::OwnedImpl request_buffer_; + Envoy::Buffer::OwnedImpl response_buffer_; + + NetworkFilters::Kafka::ResponseDecoderSharedPtr response_decoder_; + NetworkFilters::Kafka::RequestEncoder request_encoder_; + + std::shared_ptr response_callbacks_; +}; + +class KafkaCodecFactory : public GenericProxy::CodecFactory { +public: + GenericProxy::ClientCodecPtr createClientCodec() const override { + return std::make_unique(); + } + + GenericProxy::ServerCodecPtr createServerCodec() const override { + return std::make_unique(); + } +}; + +class KafkaCodecFactoryConfig : public GenericProxy::CodecFactoryConfig { +public: + // CodecFactoryConfig + GenericProxy::CodecFactoryPtr + createCodecFactory(const Envoy::Protobuf::Message& config, + Envoy::Server::Configuration::FactoryContext& context) override; + std::string name() const override { return "envoy.generic_proxy.codecs.kafka"; } + Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } +}; + +} // namespace Kafka +} // namespace Codec +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/source/config.cc b/contrib/generic_proxy/filters/network/source/config.cc index 1725dd262bdb..d6036138abaf 100644 --- a/contrib/generic_proxy/filters/network/source/config.cc +++ b/contrib/generic_proxy/filters/network/source/config.cc @@ -41,18 +41,19 @@ Factory::routeConfigProviderFromProto(const ProxyConfig& config, } return route_config_provider_manager.createRdsRouteConfigProvider( - config.generic_rds(), context.getServerFactoryContext(), config.stat_prefix(), + config.generic_rds(), context.serverFactoryContext(), config.stat_prefix(), context.initManager()); } else { return route_config_provider_manager.createStaticRouteConfigProvider( - config.route_config(), context.getServerFactoryContext()); + config.route_config(), context.serverFactoryContext()); } } -std::vector Factory::filtersFactoryFromProto( - const ProtobufWkt::RepeatedPtrField& filters, - const std::string stats_prefix, Envoy::Server::Configuration::FactoryContext& context) { - +std::vector +Factory::filtersFactoryFromProto(const ProtobufWkt::RepeatedPtrField& filters, + const TypedExtensionConfig& codec_config, + const std::string stats_prefix, + Envoy::Server::Configuration::FactoryContext& context) { std::vector factories; bool has_terminal_filter = false; std::string terminal_filter_name; @@ -64,6 +65,10 @@ std::vector Factory::filtersFactoryFromProto( auto& factory = Config::Utility::getAndCheckFactory(filter); + // Validate codec to see if this filter is compatible with the codec. + const auto validate_codec_status = factory.validateCodec(codec_config); + THROW_IF_NOT_OK_REF(validate_codec_status); + ProtobufTypes::MessagePtr message = factory.createEmptyConfigProto(); ASSERT(message != nullptr); Envoy::Config::Utility::translateOpaqueConfig(filter.typed_config(), @@ -87,11 +92,14 @@ std::vector Factory::filtersFactoryFromProto( Envoy::Network::FilterFactoryCb Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config, Envoy::Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); std::shared_ptr route_config_provider_manager = - context.singletonManager().getTyped( + server_context.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(generic_route_config_provider_manager), - [&context] { return std::make_shared(context.admin()); }); + [&server_context] { + return std::make_shared(server_context.admin()); + }); auto tracer_manager = Tracing::TracerManagerImpl::singleton(context); @@ -105,7 +113,7 @@ Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config, tracer = tracer_manager->getOrCreateTracer(&proto_config.tracing().provider()); } tracing_config = std::make_unique( - context.direction(), proto_config.tracing()); + context.listenerInfo().direction(), proto_config.tracing()); } // Access log configuration. @@ -119,10 +127,11 @@ Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config, const FilterConfigSharedPtr config = std::make_shared( proto_config.stat_prefix(), std::move(factories.first), routeConfigProviderFromProto(proto_config, context, *route_config_provider_manager), - filtersFactoryFromProto(proto_config.filters(), proto_config.stat_prefix(), context), + filtersFactoryFromProto(proto_config.filters(), proto_config.codec_config(), + proto_config.stat_prefix(), context), std::move(tracer), std::move(tracing_config), std::move(access_logs), context); - return [route_config_provider_manager, tracer_manager, config, &context, + return [route_config_provider_manager, tracer_manager, config, &server_context, custom_proxy_factory](Envoy::Network::FilterManager& filter_manager) -> void { // Create filter by the custom filter factory if the custom filter factory is not null. if (custom_proxy_factory != nullptr) { @@ -131,7 +140,7 @@ Factory::createFilterFactoryFromProtoTyped(const ProxyConfig& proto_config, } filter_manager.addReadFilter(std::make_shared( - config, context.mainThreadDispatcher().timeSource(), context.runtime())); + config, server_context.mainThreadDispatcher().timeSource(), server_context.runtime())); }; } diff --git a/contrib/generic_proxy/filters/network/source/config.h b/contrib/generic_proxy/filters/network/source/config.h index fdf04713b1c0..b6d59b69a454 100644 --- a/contrib/generic_proxy/filters/network/source/config.h +++ b/contrib/generic_proxy/filters/network/source/config.h @@ -23,7 +23,7 @@ class Factory : public Envoy::Extensions::NetworkFilters::Common::FactoryBase - factoriesFromProto(const envoy::config::core::v3::TypedExtensionConfig& codec_config, + factoriesFromProto(const TypedExtensionConfig& codec_config, Server::Configuration::FactoryContext& context); static Rds::RouteConfigProviderSharedPtr @@ -31,9 +31,10 @@ class Factory : public Envoy::Extensions::NetworkFilters::Common::FactoryBase filtersFactoryFromProto( - const ProtobufWkt::RepeatedPtrField& filters, - const std::string stats_prefix, Server::Configuration::FactoryContext& context); + static std::vector + filtersFactoryFromProto(const ProtobufWkt::RepeatedPtrField& filters, + const TypedExtensionConfig& codec_config, const std::string stats_prefix, + Server::Configuration::FactoryContext& context); }; } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/source/file_access_log.h b/contrib/generic_proxy/filters/network/source/file_access_log.h index 084d460fb775..796d57e86132 100644 --- a/contrib/generic_proxy/filters/network/source/file_access_log.h +++ b/contrib/generic_proxy/filters/network/source/file_access_log.h @@ -45,7 +45,7 @@ class FileAccessLogFactoryBase : public AccessLog::AccessLogInstanceFactoryBase< AccessLog::InstanceBaseSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterBasePtr&& filter, - Server::Configuration::CommonFactoryContext& context) override { + Server::Configuration::FactoryContext& context) override { const auto& typed_config = MessageUtil::downcastAndValidate< const envoy::extensions::access_loggers::file::v3::FileAccessLog&>( config, context.messageValidationVisitor()); @@ -93,15 +93,8 @@ class FileAccessLogFactoryBase : public AccessLog::AccessLogInstanceFactoryBase< Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, typed_config.path()}; return std::make_shared>( - file_info, std::move(filter), std::move(formatter), context.accessLogManager()); - } - - AccessLog::InstanceBaseSharedPtr createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterBasePtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); + file_info, std::move(filter), std::move(formatter), + context.serverFactoryContext().accessLogManager()); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/contrib/generic_proxy/filters/network/source/interface/codec.h b/contrib/generic_proxy/filters/network/source/interface/codec.h index 246ae6249a57..c5073d77a660 100644 --- a/contrib/generic_proxy/filters/network/source/interface/codec.h +++ b/contrib/generic_proxy/filters/network/source/interface/codec.h @@ -14,106 +14,79 @@ namespace NetworkFilters { namespace GenericProxy { /** - * Decoder of request. + * Server codec that used to decode downstream request and encode upstream response. + * This codec is used by downstream connection. */ -class RequestDecoder { +class ServerCodec { public: - virtual ~RequestDecoder() = default; + virtual ~ServerCodec() = default; - // The decode() method may be called multiple times for single request or response. - // So an independent setDecoderCallback() is used to set decoding callback. - virtual void setDecoderCallback(RequestDecoderCallback& callback) PURE; - virtual void decode(Buffer::Instance& buffer) PURE; -}; - -/** - * Decoder of response. - */ -class ResponseDecoder { -public: - virtual ~ResponseDecoder() = default; - - // The decode() method may be called multiple times for single request or response. - // So an independent setDecoderCallback() is used to set decoding callback. - virtual void setDecoderCallback(ResponseDecoderCallback& callback) PURE; - virtual void decode(Buffer::Instance& buffer) PURE; -}; - -/* - * Encoder of request. - */ -class RequestEncoder { -public: - virtual ~RequestEncoder() = default; - - virtual void encode(const Request&, RequestEncoderCallback& callback) PURE; -}; - -/* - * Encoder of response. - */ -class ResponseEncoder { -public: - virtual ~ResponseEncoder() = default; + /** + * Set callbacks of server codec. + * @param callbacks callbacks of server codec. This callback will have same lifetime + * as the server codec. + */ + virtual void setCodecCallbacks(ServerCodecCallbacks& callbacks) PURE; - virtual void encode(const Response&, ResponseEncoderCallback& callback) PURE; -}; + /** + * Decode request frame from downstream connection. + * @param buffer data to decode. + * @param end_stream whether this is the last data of the downstream connection. + */ + virtual void decode(Buffer::Instance& buffer, bool end_stream) PURE; -class MessageCreator { -public: - virtual ~MessageCreator() = default; + /** + * Encode response frame. + * @param frame response frame to encode. + * @param callbacks callbacks of encoding. This callback should be used to notify the + * generic proxy filter that the response is encoded and should be called only once. + */ + virtual void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) PURE; /** - * Create local response message for local reply. + * Create a response frame with specified status and flags. + * @param status status of the response. + * @param short_response_flags short flags of the response. + * @param request origin request that the response is created for. */ - virtual ResponsePtr response(Status status, const Request& origin_request) PURE; + virtual ResponsePtr respond(Status status, absl::string_view short_response_flags, + const Request& request) PURE; }; -using RequestDecoderPtr = std::unique_ptr; -using ResponseDecoderPtr = std::unique_ptr; -using RequestEncoderPtr = std::unique_ptr; -using ResponseEncoderPtr = std::unique_ptr; -using MessageCreatorPtr = std::unique_ptr; - /** - * Protocol specific options to control the behavior of the connection manager (generic proxy). + * Client codec that used to decode upstream response and encode downstream request. + * This codec is used by upstream connection. */ -class ProtocolOptions { +class ClientCodec { public: - ProtocolOptions(bool bind_upstream_connection) - : bind_upstream_connection_(bind_upstream_connection) {} - ProtocolOptions() = default; + virtual ~ClientCodec() = default; /** - * @return true if the upstream connection should be bound to the downstream connection, false - * otherwise. - * - * By default, one random upstream connection will be selected from the upstream connection pool - * and used for every request. And after the request is finished, the upstream connection will be - * released back to the upstream connection pool. - * - * If this option is true, the upstream connection will be bound to the downstream connection and - * have same lifetime as the downstream connection. The same upstream connection will be used for - * all requests from the same downstream connection. - * - * And if this options is true, one of the following requirements must be met: - * 1. The request must be handled one by one. That is, the next request can not be sent to the - * upstream until the previous request is finished. - * 2. Unique request id must be provided for each request and response. The request id must be - * unique for each request and response pair in same connection pair. And the request id must - * be the same for the corresponding request and response. - * TODO(wbpcode): add pipeline support in the future. - * - * This could be useful for some protocols that require the same upstream connection to be used - * for all requests from the same downstream connection. For example, the protocol using stateful - * connection. + * Set callbacks of client codec. + * @param callbacks callbacks of client codec. This callback will have same lifetime + * as the client codec. */ - bool bindUpstreamConnection() const { return bind_upstream_connection_; } + virtual void setCodecCallbacks(ClientCodecCallbacks& callbacks) PURE; -private: - bool bind_upstream_connection_{false}; + /** + * Decode response frame from upstream connection. + * @param buffer data to decode. + * @param end_stream whether this is the last data of the upstream connection. + */ + virtual void decode(Buffer::Instance& buffer, bool end_stream) PURE; + + /** + * Encode request frame. + * @param frame request frame to encode. + * @param callbacks callbacks of encoding. This callbacks should be used to notify the + * generic proxy filter that the request is encoded and should be called only once. + */ + virtual void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) PURE; }; +using ServerCodecPtr = std::unique_ptr; +using ClientCodecPtr = std::unique_ptr; + /** * Factory used to create generic stream encoder and decoder. If the developer wants to add * new protocol support to this proxy, they need to implement the corresponding codec factory for @@ -123,35 +96,15 @@ class CodecFactory { public: virtual ~CodecFactory() = default; - /* - * Create request decoder. - */ - virtual RequestDecoderPtr requestDecoder() const PURE; - - /* - * Create response decoder. - */ - virtual ResponseDecoderPtr responseDecoder() const PURE; - - /* - * Create request encoder. - */ - virtual RequestEncoderPtr requestEncoder() const PURE; - - /* - * Create response encoder. - */ - virtual ResponseEncoderPtr responseEncoder() const PURE; - /** - * Create message creator. + * Create a server codec instance. */ - virtual MessageCreatorPtr messageCreator() const PURE; + virtual ServerCodecPtr createServerCodec() const PURE; /** - * @return the options to control the behavior of generic proxy filter. + * Create a client codec instance. */ - virtual ProtocolOptions protocolOptions() const PURE; + virtual ClientCodecPtr createClientCodec() const PURE; }; using CodecFactoryPtr = std::unique_ptr; diff --git a/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h b/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h index 246bcb4f0021..899ce37623f8 100644 --- a/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h +++ b/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h @@ -13,75 +13,20 @@ namespace NetworkFilters { namespace GenericProxy { /** - * Extended options from request or response to control the behavior of the - * generic proxy filter. - * All these options are optional for the simple ping-pong use case. + * Callbacks of ServerCodec. */ -class ExtendedOptions { +class ServerCodecCallbacks { public: - ExtendedOptions(absl::optional stream_id, bool wait_response, bool drain_close, - bool is_heartbeat) - : stream_id_(stream_id.value_or(0)), has_stream_id_(stream_id.has_value()), - wait_response_(wait_response), drain_close_(drain_close), is_heartbeat_(is_heartbeat) {} - ExtendedOptions() = default; - - /** - * @return the stream id of the request or response. This is used to match the - * downstream request with the upstream response. - - * NOTE: In most cases, the stream id is not needed and will be ignored completely. - * The stream id is only used when we can't match the downstream request - * with the upstream response by the active stream instance self directly. - * For example, when the multiple downstream requests are multiplexed into one - * upstream connection. - */ - absl::optional streamId() const { - return has_stream_id_ ? absl::optional(stream_id_) : absl::nullopt; - } - - /** - * @return whether the current request requires an upstream response. - * NOTE: This is only used for the request. - */ - bool waitResponse() const { return wait_response_; } - - /** - * @return whether the downstream/upstream connection should be drained after - * current active requests are finished. - * NOTE: This is only used for the response. - */ - bool drainClose() const { return drain_close_; } - - /** - * @return whether the current request/response is a heartbeat request/response. - * NOTE: It would be better to handle heartbeat request/response by another L4 - * filter. Then the generic proxy filter can be used for the simple ping-pong - * use case. - */ - bool isHeartbeat() const { return is_heartbeat_; } - -private: - uint64_t stream_id_{0}; - bool has_stream_id_{false}; - - bool wait_response_{true}; - bool drain_close_{false}; - bool is_heartbeat_{false}; -}; - -/** - * Decoder callback of request. - */ -class RequestDecoderCallback { -public: - virtual ~RequestDecoderCallback() = default; - + virtual ~ServerCodecCallbacks() = default; /** * If request decoding success then this method will be called. - * @param request request from decoding. - * @param options extended options from request. + * @param frame request frame from decoding. Frist frame should be StreamRequest + * frame. + * NOTE: This method will be called multiple times for the multiple frames request. + * FrameFlags and embedded StreamFlags could be used to correlate frames of same + * request. */ - virtual void onDecodingSuccess(RequestPtr request, ExtendedOptions options) PURE; + virtual void onDecodingSuccess(StreamFramePtr frame) PURE; /** * If request decoding failure then this method will be called. @@ -106,18 +51,22 @@ class RequestDecoderCallback { }; /** - * Decoder callback of Response. + * Callbacks of ClientCodec. */ -class ResponseDecoderCallback { +class ClientCodecCallbacks { public: - virtual ~ResponseDecoderCallback() = default; + virtual ~ClientCodecCallbacks() = default; /** * If response decoding success then this method will be called. - * @param response response from decoding. - * @param options extended options from response. + * @param frame response frame from decoding. Frist frame should be StreamResponse + * frame. + * NOTE: This method will be called multiple times for the multiple frames response. + * FrameFlags and embedded StreamFlags could be used to correlate frames of same + * request. And the StreamFlags could also be used to correlate the response with + * the request. */ - virtual void onDecodingSuccess(ResponsePtr response, ExtendedOptions options) PURE; + virtual void onDecodingSuccess(StreamFramePtr frame) PURE; /** * If response decoding failure then this method will be called. @@ -142,31 +91,18 @@ class ResponseDecoderCallback { }; /** - * Encoder callback of request. - */ -class RequestEncoderCallback { -public: - virtual ~RequestEncoderCallback() = default; - - /** - * If request encoding success then this method will be called. - * @param buffer encoding result buffer. - */ - virtual void onEncodingSuccess(Buffer::Instance& buffer) PURE; -}; - -/** - * Encoder callback of Response. + * Callback of request/response frame. */ -class ResponseEncoderCallback { +class EncodingCallbacks { public: - virtual ~ResponseEncoderCallback() = default; + virtual ~EncodingCallbacks() = default; /** - * If response encoding success then this method will be called. + * If encoding success then this method will be called to notify the generic proxy. * @param buffer encoding result buffer. + * @param end_stream if last frame is encoded. */ - virtual void onEncodingSuccess(Buffer::Instance& buffer) PURE; + virtual void onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) PURE; }; } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/source/interface/config.h b/contrib/generic_proxy/filters/network/source/interface/config.h index 456f581eed83..c724f6286c01 100644 --- a/contrib/generic_proxy/filters/network/source/interface/config.h +++ b/contrib/generic_proxy/filters/network/source/interface/config.h @@ -10,6 +10,8 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { +using TypedExtensionConfig = envoy::config::core::v3::TypedExtensionConfig; + /** * Implemented by each generic filter and registered via Registry::registerFactory or the * convenience class RegisterFactory. @@ -41,6 +43,14 @@ class NamedFilterConfigFactory : public Config::TypedFactory { * @return bool true if this filter must be the last filter in a filter chain, false otherwise. */ virtual bool isTerminalFilter() PURE; + + /** + * @return absl::Status validate the codec config to see if it is compatible with the filter. + * If the codec config is not compatible with this filter, return an error status. + */ + virtual absl::Status validateCodec(const TypedExtensionConfig& /*config*/) { + return absl::OkStatus(); + } }; } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/source/interface/filter.h b/contrib/generic_proxy/filters/network/source/interface/filter.h index e8bcdb06c861..181183403cfb 100644 --- a/contrib/generic_proxy/filters/network/source/interface/filter.h +++ b/contrib/generic_proxy/filters/network/source/interface/filter.h @@ -15,6 +15,20 @@ namespace GenericProxy { using ResponseUpdateFunction = std::function; +/** + * StreamFrameHandler to handle the frames from the stream (if exists). + */ +class StreamFrameHandler { +public: + virtual ~StreamFrameHandler() = default; + + /** + * Handle the frame from the stream. + * @param frame frame from the stream. + */ + virtual void onStreamFrame(StreamFramePtr frame) PURE; +}; + /** * The stream filter callbacks are passed to all filters to use for writing response data and * interacting with the underlying stream in general. @@ -66,89 +80,43 @@ class StreamFilterCallbacks { */ virtual OptRef tracingConfig() const PURE; - /** - * @return absl::optional the extended options from downstream request. - */ - virtual absl::optional requestOptions() const PURE; - - /** - * @return absl::optional the extended options from upstream response. - */ - virtual absl::optional responseOptions() const PURE; - /** * @return const Network::Connection* downstream connection. */ virtual const Network::Connection* connection() const PURE; }; -class UpstreamBindingCallback { -public: - virtual ~UpstreamBindingCallback() = default; - - virtual void onBindFailure(ConnectionPool::PoolFailureReason reason, - absl::string_view transport_failure_reason, - Upstream::HostDescriptionConstSharedPtr host) PURE; - - virtual void onBindSuccess(Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr host) PURE; -}; - -class PendingResponseCallback : public ResponseDecoderCallback { -public: - virtual void onConnectionClose(Network::ConnectionEvent event) PURE; -}; - -/** - * The upstream manager is used to manage the upstream connection and the registered upstream - * or response callbacks. - */ -class UpstreamManager { +class DecoderFilterCallback : public virtual StreamFilterCallbacks { public: - virtual ~UpstreamManager() = default; + virtual void sendLocalReply(Status status, ResponseUpdateFunction&& cb = nullptr) PURE; - /** - * @param callbacks supplies the callback to be called when the upstream connection is ready. - */ - virtual void registerUpstreamCallback(uint64_t stream_id, UpstreamBindingCallback& cb) PURE; + virtual void continueDecoding() PURE; /** - * @param stream_id supplies the stream id of request. + * Called when the upstream response frame is received. This should only be called once. + * @param response supplies the upstream response frame. */ - virtual void unregisterUpstreamCallback(uint64_t stream_id) PURE; + virtual void onResponseStart(StreamResponsePtr response) PURE; /** - * @param cb supplies the callback to be called when the upstream response is ready. + * Called when the upstream response frame is received. This should only be called once. + * @param frame supplies the upstream frame. */ - virtual void registerResponseCallback(uint64_t stream_id, PendingResponseCallback& cb) PURE; + virtual void onResponseFrame(StreamFramePtr frame) PURE; /** - * @param stream_id supplies the stream id of request. + * Register a request frames handler to used to handle the request frames (except the special + * StreamRequest frame). + * This handler will be Called when the filter chain is completed. + * @param handler supplies the request frames handler. + * + * TODO(wbpcode): this is used by the terminal filter the handle the request frames because + * the filter chain doesn't support to handle extra frames. We should remove this when the + * filter chain supports to handle extra frames. */ - virtual void unregisterResponseCallback(uint64_t stream_id) PURE; -}; - -class DecoderFilterCallback : public virtual StreamFilterCallbacks { -public: - virtual void sendLocalReply(Status status, ResponseUpdateFunction&& cb = nullptr) PURE; - - virtual void continueDecoding() PURE; - - virtual void upstreamResponse(ResponsePtr response, ExtendedOptions options) PURE; + virtual void setRequestFramesHandler(StreamFrameHandler& handler) PURE; virtual void completeDirectly() PURE; - - /** - * Try to create a new upstream connection and bind it to the current downstream connection. - * This should be called only once for each downstream connection. - * @param tcp_pool_data supplies the upstream connection pool. - */ - virtual void bindUpstreamConn(Upstream::TcpPoolData&& tcp_pool_data) PURE; - - /** - * @return OptRef the upstream manager for the current downstream connection. - */ - virtual OptRef boundUpstreamConn() PURE; }; class EncoderFilterCallback : public virtual StreamFilterCallbacks { @@ -165,7 +133,7 @@ class DecoderFilter { virtual void onDestroy() PURE; virtual void setDecoderFilterCallbacks(DecoderFilterCallback& callbacks) PURE; - virtual FilterStatus onStreamDecoded(Request& request) PURE; + virtual FilterStatus onStreamDecoded(StreamRequest& request) PURE; }; class EncoderFilter { @@ -175,7 +143,7 @@ class EncoderFilter { virtual void onDestroy() PURE; virtual void setEncoderFilterCallbacks(EncoderFilterCallback& callbacks) PURE; - virtual FilterStatus onStreamEncoded(Response& response) PURE; + virtual FilterStatus onStreamEncoded(StreamResponse& response) PURE; }; class StreamFilter : public DecoderFilter, public EncoderFilter {}; diff --git a/contrib/generic_proxy/filters/network/source/interface/stream.h b/contrib/generic_proxy/filters/network/source/interface/stream.h index c8a628fcd918..fcd7debb7660 100644 --- a/contrib/generic_proxy/filters/network/source/interface/stream.h +++ b/contrib/generic_proxy/filters/network/source/interface/stream.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -16,10 +17,126 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { -class StreamBase { +/** + * Stream flags from request or response to control the behavior of the + * generic proxy filter. This is mainly used as part of FrameFlags. + * All these flags could be ignored for the simple ping-pong use case. + */ +class StreamFlags { public: - virtual ~StreamBase() = default; + StreamFlags(uint64_t stream_id = 0, bool one_way_stream = false, bool drain_close = false, + bool is_heartbeat = false) + : stream_id_(stream_id), one_way_stream_(one_way_stream), drain_close_(drain_close), + is_heartbeat_(is_heartbeat) {} + + /** + * @return the stream id of the request or response. This is used to match the + * downstream request with the upstream response. + + * NOTE: In most cases, the stream id is not needed and will be ignored completely. + * The stream id is only used when we can't match the downstream request + * with the upstream response by the active stream instance self directly. + * For example, when the multiple downstream requests are multiplexed into one + * upstream connection. + */ + uint64_t streamId() const { return stream_id_; } + + /** + * @return whether the stream is one way stream. If request is one way stream, the + * generic proxy filter will not wait for the response from the upstream. + */ + bool oneWayStream() const { return one_way_stream_; } + + /** + * @return whether the downstream/upstream connection should be drained after + * current active stream are finished. + */ + bool drainClose() const { return drain_close_; } + + /** + * @return whether the current request/response is a heartbeat request/response. + * NOTE: It would be better to handle heartbeat request/response by another L4 + * filter. Then the generic proxy filter can be used for the simple ping-pong + * use case. + */ + bool isHeartbeat() const { return is_heartbeat_; } + +private: + uint64_t stream_id_{0}; + bool one_way_stream_{false}; + bool drain_close_{false}; + bool is_heartbeat_{false}; +}; + +/** + * Flags of stream frame. This is used to control the behavior of the generic proxy filter. + * All these flags could be ignored for the simple ping-pong use case. + */ +class FrameFlags { +public: + /** + * Construct FrameFlags with stream flags and end stream flag. The stream flags MUST be + * same for all frames of the same stream. + * @param stream_flags StreamFlags of the stream. + * @param end_stream whether the current frame is the last frame of the request or response. + * @param frame_tags frame tags of the current frame. The meaning of the frame tags is + * application protocol specific. + */ + FrameFlags(StreamFlags stream_flags = StreamFlags(), bool end_stream = true, + uint32_t frame_tags = 0) + : stream_flags_(stream_flags), end_stream_(end_stream), frame_tags_(frame_tags) {} + + /** + * Get flags of stream that the frame belongs to. The flags MUST be same for all frames of the + * same stream. Copy semantics is used because the flags are lightweight (only 16 bytes for now). + * @return StreamFlags of the stream. + */ + StreamFlags streamFlags() const { return stream_flags_; } + + /** + * @return whether the current frame is the last frame of the request or response. + */ + bool endStream() const { return end_stream_; } + + /** + * @return frame tags of the current frame. The meaning of the frame tags is application + * protocol specific. This allows the creator of the frame to attach additional information to the + * frame and get it by the receiver of the frame without parsing the frame payload or dynamic + * cast. + * For example, the frame tags could be used to indicate the type of the frame by the server + * codec. Then the client codec could get the frame type without dynamic cast. + */ + uint32_t frameTags() const { return frame_tags_; } + +private: + StreamFlags stream_flags_{}; + + // Default to true for backward compatibility. + bool end_stream_{true}; + uint32_t frame_tags_{}; +}; + +/** + * Stream frame interface. This is used to represent the stream frame of request or response. + */ +class StreamFrame { +public: + virtual ~StreamFrame() = default; + + /** + * Get stream frame flags of current frame. The default implementation returns empty flags + * that could be used for the simple ping-pong use case. + * @return FrameFlags of the current frame. + */ + virtual FrameFlags frameFlags() const { return {}; } +}; + +using StreamFramePtr = std::unique_ptr; +using StreamFrameSharedPtr = std::shared_ptr; + +class StreamBase : public StreamFrame { +public: using IterateCallback = std::function; /** @@ -35,7 +152,7 @@ class StreamBase { * * @param callback supplies the iteration callback. */ - virtual void forEach(IterateCallback callback) const PURE; + virtual void forEach(IterateCallback /*callback*/) const {}; /** * Get generic stream metadata value by key. @@ -43,7 +160,7 @@ class StreamBase { * @param key The metadata key of string view type. * @return The optional metadata value of string_view type. */ - virtual absl::optional getByKey(absl::string_view key) const PURE; + virtual absl::optional get(absl::string_view /*key*/) const { return {}; } /** * Set new generic stream metadata key/value pair. @@ -51,39 +168,59 @@ class StreamBase { * @param key The metadata key of string view type. * @param val The metadata value of string view type. */ - virtual void setByKey(absl::string_view key, absl::string_view val) PURE; + virtual void set(absl::string_view /*key*/, absl::string_view /*val*/) {} /** - * Set new generic stream metadata key/value pair. The key MUST point to data that will live - * beyond the lifetime of any generic stream that using the string. - * + * Erase generic stream metadata by key. * @param key The metadata key of string view type. - * @param val The metadata value of string view type. - */ - virtual void setByReferenceKey(absl::string_view key, absl::string_view val) PURE; - - /** - * Set new generic stream metadata key/value pair. Both key and val MUST point to data that - * will live beyond the lifetime of any generic stream that using the string. - * - * @param key The metadata key of string view type. - * @param val The metadata value of string view type. */ - virtual void setByReference(absl::string_view key, absl::string_view val) PURE; + virtual void erase(absl::string_view /*key*/) {} // Used for matcher. static constexpr absl::string_view name() { return "generic_proxy"; } }; /** - * Using interface that provided by the TraceContext as the interface of generic request. + * Interface of generic request. This is derived from StreamFrame that contains the request + * specific information. First frame of the request MUST be a StreamRequest. + * + * NOTE: using interface that provided by the TraceContext as the interface of generic request here + * to simplify the tracing integration. This is not a good design. This should be changed in the + * future. */ -class Request : public Tracing::TraceContext { +class StreamRequest : public StreamBase { public: - // Used for matcher. - static constexpr absl::string_view name() { return "generic_proxy"; } + /** + * Get request host. + * + * @return The host of generic request. The meaning of the return value may be different For + * different application protocols. It typically should be domain, VIP, or service name that + * used to represents target service instances. + */ + virtual absl::string_view host() const { return {}; } + + /** + * Get request path. + * + * @return The path of generic request. The meaning of the return value may be different For + * different application protocols. It typically should be RPC service name that used to + * represents set of method or functionality provided by target service. + */ + virtual absl::string_view path() const { return {}; } + + /** + * Get request method. + * + * @return The method of generic request. The meaning of the return value may be different For + * different application protocols. + */ + virtual absl::string_view method() const { return {}; } }; +using StreamRequestPtr = std::unique_ptr; +using StreamRequestSharedPtr = std::shared_ptr; +// Alias for backward compatibility. +using Request = StreamRequest; using RequestPtr = std::unique_ptr; using RequestSharedPtr = std::shared_ptr; @@ -95,22 +232,79 @@ enum class Event { ConnectionFailure, }; +/** + * The Status type is used by the generic proxy to indicate statuses or error types + * to the application protocol codec. This is application protocol independent. + */ using Status = absl::Status; using StatusCode = absl::StatusCode; -class Response : public StreamBase { +/** + * Generic stream status. The Status is used by the application protocol codec to + * indicate the status of the response. The meaning of status code is application + * protocol specific. + */ +struct StreamStatus { +public: + StreamStatus() = default; + StreamStatus(uint32_t code, bool ok) : code_(code), ok_(ok) {} + + // Returns true if the status indicates success. This will be used for tracing, logging + // or stats purposes. + ABSL_MUST_USE_RESULT bool ok() const { return ok_; } + + // Returns the status code value. The code will be used for tracing, logging or stats + // purposes. The specific code value is application protocol specific. + ABSL_MUST_USE_RESULT uint32_t code() const { return code_; } + +private: + uint32_t code_{}; + bool ok_{true}; +}; + +/** + * Interface of generic response. This is derived from StreamFrame that contains the response + * specific information. First frame of the response MUST be a StreamResponse. + */ +class StreamResponse : public StreamBase { public: /** * Get response status. * * @return generic response status. */ - virtual Status status() const PURE; + virtual StreamStatus status() const { return {}; } }; +using StreamResponsePtr = std::unique_ptr; +using StreamResponseSharedPtr = std::shared_ptr; +// Alias for backward compatibility. +using Response = StreamResponse; using ResponsePtr = std::unique_ptr; using ResponseSharedPtr = std::shared_ptr; +template class StreamFramePtrHelper { +public: + StreamFramePtrHelper(StreamFramePtr frame) { + auto frame_ptr = frame.release(); + + auto typed_frame_ptr = dynamic_cast(frame_ptr); + + if (typed_frame_ptr == nullptr) { + // If the frame is not the expected type, wrap it + // in the original StreamFramePtr. + frame_ = StreamFramePtr{frame_ptr}; + } else { + // If the frame is the expected type, wrap it + // in the typed frame unique pointer. + typed_frame_ = std::unique_ptr{typed_frame_ptr}; + } + } + + StreamFramePtr frame_; + std::unique_ptr typed_frame_; +}; + } // namespace GenericProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/generic_proxy/filters/network/source/match.cc b/contrib/generic_proxy/filters/network/source/match.cc index 83719c2b6c4d..d4849322c19d 100644 --- a/contrib/generic_proxy/filters/network/source/match.cc +++ b/contrib/generic_proxy/filters/network/source/match.cc @@ -76,7 +76,7 @@ bool RequestMatchInputMatcher::match(const Request& request) { } for (const auto& property : properties_) { - if (auto val = request.getByKey(property.first); val.has_value()) { + if (auto val = request.get(property.first); val.has_value()) { if (!property.second->match(val.value())) { // Property does not match. return false; diff --git a/contrib/generic_proxy/filters/network/source/match.h b/contrib/generic_proxy/filters/network/source/match.h index 447e44a4a7c6..88fe8a2644e7 100644 --- a/contrib/generic_proxy/filters/network/source/match.h +++ b/contrib/generic_proxy/filters/network/source/match.h @@ -136,7 +136,7 @@ class PropertyMatchDataInput : public Matcher::DataInput { PropertyMatchDataInput(const std::string& property_name) : name_(property_name) {} Matcher::DataInputGetResult get(const Request& data) const override { - const auto value = data.getByKey(name_); + const auto value = data.get(name_); Matcher::MatchingDataType matching_data = value.has_value() ? Matcher::MatchingDataType(std::string(value.value())) : absl::monostate(); diff --git a/contrib/generic_proxy/filters/network/source/proxy.cc b/contrib/generic_proxy/filters/network/source/proxy.cc index 3dd7fb484023..b3d29529d335 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.cc +++ b/contrib/generic_proxy/filters/network/source/proxy.cc @@ -33,13 +33,19 @@ Tracing::Decision tracingDecision(const Tracing::ConnectionManagerTracingConfig& } // namespace -ActiveStream::ActiveStream(Filter& parent, RequestPtr request, ExtendedOptions options) - : parent_(parent), downstream_request_stream_(std::move(request)), - downstream_request_options_(options), +ActiveStream::ActiveStream(Filter& parent, StreamRequestPtr request) + : parent_(parent), request_stream_(std::move(request)), + request_stream_end_(request_stream_->frameFlags().endStream()), stream_info_(parent_.time_source_, parent_.callbacks_->connection().connectionInfoProviderSharedPtr()), request_timer_(new Stats::HistogramCompletableTimespanImpl(parent_.stats_.request_time_ms_, parent_.time_source_)) { + if (!request_stream_end_) { + // If the request is not fully received, register the stream to the frame handler map. + parent_.registerFrameHandler(requestStreamId(), this); + registered_in_frame_handlers_ = true; + } + parent_.stats_.request_.inc(); parent_.stats_.request_active_.inc(); @@ -55,7 +61,9 @@ ActiveStream::ActiveStream(Filter& parent, RequestPtr request, ExtendedOptions o if (decision.traced) { stream_info_.setTraceReason(decision.reason); } - active_span_ = tracer->startSpan(*this, *downstream_request_stream_, stream_info_, decision); + + TraceContextBridge trace_context{*request_stream_}; + active_span_ = tracer->startSpan(*this, trace_context, stream_info_, decision); } Tracing::OperationName ActiveStream::operationName() const { @@ -78,6 +86,11 @@ uint32_t ActiveStream::maxPathTagLength() const { return connection_manager_tracing_config_->maxPathTagLength(); } +bool ActiveStream::spawnUpstreamSpan() const { + ASSERT(connection_manager_tracing_config_.has_value()); + return connection_manager_tracing_config_->spawnUpstreamSpan(); +} + Envoy::Event::Dispatcher& ActiveStream::dispatcher() { return parent_.downstreamConnection().dispatcher(); } @@ -90,33 +103,82 @@ void ActiveStream::resetStream() { parent_.deferredStream(*this); } +void ActiveStream::sendResponseStartToDownstream() { + ASSERT(response_stream_ != nullptr); + response_filter_chain_complete_ = true; + + parent_.sendFrameToDownstream(*response_stream_, *this); +} + +void ActiveStream::sendResponseFrameToDownstream() { + if (!response_filter_chain_complete_) { + // Wait for the response header frame to be sent first. It may be blocked by + // the filter chain. + return; + } + + while (!response_stream_frames_.empty()) { + // Pop the first frame from the queue. + auto frame = std::move(response_stream_frames_.front()); + response_stream_frames_.pop_front(); + + // Send the frame to downstream. + parent_.sendFrameToDownstream(*frame, *this); + } +} + +void ActiveStream::sendRequestFrameToUpstream() { + if (!request_filter_chain_complete_) { + // Wait for the request header frame to be sent first. It may be blocked by + // the filter chain. + return; + } + + if (request_stream_frame_handler_ == nullptr) { + // The request stream frame handler is not ready yet. + return; + } + + while (!request_stream_frames_.empty()) { + // Pop the first frame from the queue. + auto frame = std::move(request_stream_frames_.front()); + request_stream_frames_.pop_front(); + + // Send the frame to upstream. + request_stream_frame_handler_->onStreamFrame(std::move(frame)); + } +} + +// TODO(wbpcode): add the short_response_flags support to the sendLocalReply +// method. void ActiveStream::sendLocalReply(Status status, ResponseUpdateFunction&& func) { - ASSERT(parent_.message_creator_ != nullptr); - local_or_upstream_response_stream_ = - parent_.message_creator_->response(status, *downstream_request_stream_); + response_stream_ = parent_.server_codec_->respond(status, "", *request_stream_); + response_stream_frames_.clear(); + // Only one frame is allowed in the local reply. + response_stream_end_ = true; - ASSERT(local_or_upstream_response_stream_ != nullptr); + ASSERT(response_stream_ != nullptr); if (func != nullptr) { - func(*local_or_upstream_response_stream_); + func(*response_stream_); } - parent_.sendReplyDownstream(*local_or_upstream_response_stream_, *this); + sendResponseStartToDownstream(); } void ActiveStream::continueDecoding() { - if (active_stream_reset_ || downstream_request_stream_ == nullptr) { + if (active_stream_reset_ || request_stream_ == nullptr) { return; } if (cached_route_entry_ == nullptr) { - cached_route_entry_ = parent_.config_->routeEntry(*downstream_request_stream_); + cached_route_entry_ = parent_.config_->routeEntry(*request_stream_); } - ASSERT(downstream_request_stream_ != nullptr); + ASSERT(request_stream_ != nullptr); for (; next_decoder_filter_index_ < decoder_filters_.size();) { - auto status = decoder_filters_[next_decoder_filter_index_]->filter_->onStreamDecoded( - *downstream_request_stream_); + auto status = + decoder_filters_[next_decoder_filter_index_]->filter_->onStreamDecoded(*request_stream_); next_decoder_filter_index_++; if (status == FilterStatus::StopIteration) { break; @@ -124,38 +186,59 @@ void ActiveStream::continueDecoding() { } if (next_decoder_filter_index_ == decoder_filters_.size()) { ENVOY_LOG(debug, "Complete decoder filters"); + request_filter_chain_complete_ = true; + sendRequestFrameToUpstream(); } } -void ActiveStream::upstreamResponse(ResponsePtr response, ExtendedOptions options) { - local_or_upstream_response_stream_ = std::move(response); - local_or_upstream_response_options_ = options; - parent_.stream_drain_decision_ = options.drainClose(); - continueEncoding(); -} +void ActiveStream::onRequestFrame(StreamFramePtr frame) { + request_stream_end_ = frame->frameFlags().endStream(); + request_stream_frames_.emplace_back(std::move(frame)); -void ActiveStream::completeDirectly() { parent_.deferredStream(*this); }; + ASSERT(registered_in_frame_handlers_); + if (request_stream_end_) { + // If the request is fully received, remove the stream from the + // frame handler map. + parent_.unregisterFrameHandler(requestStreamId()); + registered_in_frame_handlers_ = false; + } -void ActiveStream::ActiveDecoderFilter::bindUpstreamConn(Upstream::TcpPoolData&& pool_data) { - parent_.parent_.bindUpstreamConn(std::move(pool_data)); + // Try to send the frame to upstream immediately. + sendRequestFrameToUpstream(); } -OptRef ActiveStream::ActiveDecoderFilter::boundUpstreamConn() { - return parent_.parent_.boundUpstreamConn(); + +void ActiveStream::onResponseStart(ResponsePtr response) { + response_stream_ = std::move(response); + response_stream_end_ = response_stream_->frameFlags().endStream(); + parent_.stream_drain_decision_ = response_stream_->frameFlags().streamFlags().drainClose(); + continueEncoding(); +} + +void ActiveStream::onResponseFrame(StreamFramePtr frame) { + response_stream_end_ = frame->frameFlags().endStream(); + response_stream_frames_.emplace_back(std::move(frame)); + // Try to send the frame to downstream immediately. + sendResponseFrameToDownstream(); } +void ActiveStream::completeDirectly() { + response_stream_end_ = true; + parent_.deferredStream(*this); +}; + const Network::Connection* ActiveStream::ActiveFilterBase::connection() const { return &parent_.parent_.downstreamConnection(); } void ActiveStream::continueEncoding() { - if (active_stream_reset_ || local_or_upstream_response_stream_ == nullptr) { + if (active_stream_reset_ || response_stream_ == nullptr) { return; } - ASSERT(local_or_upstream_response_stream_ != nullptr); + ASSERT(response_stream_ != nullptr); for (; next_encoder_filter_index_ < encoder_filters_.size();) { - auto status = encoder_filters_[next_encoder_filter_index_]->filter_->onStreamEncoded( - *local_or_upstream_response_stream_); + auto status = + encoder_filters_[next_encoder_filter_index_]->filter_->onStreamEncoded(*response_stream_); next_encoder_filter_index_++; if (status == FilterStatus::StopIteration) { break; @@ -164,13 +247,24 @@ void ActiveStream::continueEncoding() { if (next_encoder_filter_index_ == encoder_filters_.size()) { ENVOY_LOG(debug, "Complete encoder filters"); - parent_.sendReplyDownstream(*local_or_upstream_response_stream_, *this); + sendResponseStartToDownstream(); + sendResponseFrameToDownstream(); } } -void ActiveStream::onEncodingSuccess(Buffer::Instance& buffer) { +void ActiveStream::onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) { ASSERT(parent_.downstreamConnection().state() == Network::Connection::State::Open); parent_.downstreamConnection().write(buffer, false); + + if (!end_stream) { + return; + } + + ENVOY_LOG(debug, "Generic proxy: downstream response complete"); + + ASSERT(response_stream_end_); + ASSERT(response_stream_frames_.empty()); + parent_.stats_.response_.inc(); parent_.deferredStream(*this); } @@ -183,19 +277,23 @@ void ActiveStream::initializeFilterChain(FilterChainFactory& factory) { } void ActiveStream::completeRequest() { + if (registered_in_frame_handlers_) { + parent_.unregisterFrameHandler(requestStreamId()); + registered_in_frame_handlers_ = false; + } + stream_info_.onRequestComplete(); request_timer_->complete(); parent_.stats_.request_active_.dec(); if (active_span_) { - Tracing::TracerUtility::finalizeSpan(*active_span_, *downstream_request_stream_, stream_info_, - *this, false); + TraceContextBridge trace_context{*request_stream_}; + Tracing::TracerUtility::finalizeSpan(*active_span_, trace_context, stream_info_, *this, false); } for (const auto& access_log : parent_.config_->accessLogs()) { - access_log->log({downstream_request_stream_.get(), local_or_upstream_response_stream_.get()}, - stream_info_); + access_log->log({request_stream_.get(), response_stream_.get()}, stream_info_); } for (auto& filter : decoder_filters_) { @@ -209,176 +307,38 @@ void ActiveStream::completeRequest() { } } -UpstreamManagerImpl::UpstreamManagerImpl(Filter& parent, Upstream::TcpPoolData&& tcp_pool_data) - : UpstreamConnection(std::move(tcp_pool_data), - parent.config_->codecFactory().responseDecoder()), - parent_(parent) { - response_decoder_->setDecoderCallback(*this); -} - -void UpstreamManagerImpl::registerResponseCallback(uint64_t stream_id, - PendingResponseCallback& cb) { - // The stream id is already registered and it should not happen. We treat it as - // request decoding failure. - // All pending streams will be reset and downstream connection will be closed and - // bound upstream connection will be cleaned up. - if (registered_response_callbacks_.find(stream_id) != registered_response_callbacks_.end()) { - ENVOY_LOG(error, "generic proxy: stream_id {} already registered", stream_id); - parent_.onDecodingFailure(); - return; - } - - registered_response_callbacks_[stream_id] = &cb; -} - -void UpstreamManagerImpl::registerUpstreamCallback(uint64_t stream_id, - UpstreamBindingCallback& cb) { - // Connection is already bound to a downstream connection and use it directly. - if (owned_conn_data_ != nullptr) { - cb.onBindSuccess(owned_conn_data_->connection(), upstream_host_); - return; - } - - // The stream id is already registered and it should not happen. We treat it as - // request decoding failure. - // All pending streams will be reset and downstream connection will be closed and - // bound upstream connection will be cleaned up. - if (registered_upstream_callbacks_.find(stream_id) != registered_upstream_callbacks_.end()) { - ENVOY_LOG(error, "generic proxy: stream_id {} already registered", stream_id); - parent_.onDecodingFailure(); - return; - } - - registered_upstream_callbacks_[stream_id] = &cb; -} - -void UpstreamManagerImpl::unregisterResponseCallback(uint64_t stream_id) { - registered_response_callbacks_.erase(stream_id); -} - -void UpstreamManagerImpl::unregisterUpstreamCallback(uint64_t stream_id) { - registered_upstream_callbacks_.erase(stream_id); -} - -void UpstreamManagerImpl::onEventImpl(Network::ConnectionEvent event) { - if (event == Network::ConnectionEvent::Connected || - event == Network::ConnectionEvent::ConnectedZeroRtt) { - return; - } - - // If the connection event is consumed by this upstream manager, it means that - // the upstream connection is ready and onPoolReady()/onPoolSuccessImpl() have - // been called. So the registered upstream callbacks should be empty. - ASSERT(registered_upstream_callbacks_.empty()); - - while (!registered_response_callbacks_.empty()) { - auto it = registered_response_callbacks_.begin(); - auto cb = it->second; - registered_response_callbacks_.erase(it); - - cb->onConnectionClose(event); - } - - parent_.onBoundUpstreamConnectionEvent(event); -} - -void UpstreamManagerImpl::onPoolSuccessImpl() { - ASSERT(registered_response_callbacks_.empty()); - - while (!registered_upstream_callbacks_.empty()) { - auto iter = registered_upstream_callbacks_.begin(); - auto* cb = iter->second; - registered_upstream_callbacks_.erase(iter); - - cb->onBindSuccess(owned_conn_data_->connection(), upstream_host_); - } -} - -void UpstreamManagerImpl::onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, - absl::string_view transport_failure_reason) { - ASSERT(registered_response_callbacks_.empty()); - - while (!registered_upstream_callbacks_.empty()) { - auto iter = registered_upstream_callbacks_.begin(); - auto* cb = iter->second; - registered_upstream_callbacks_.erase(iter); - - cb->onBindFailure(reason, transport_failure_reason, upstream_host_); +Envoy::Network::FilterStatus Filter::onData(Envoy::Buffer::Instance& data, bool end_stream) { + if (downstream_connection_closed_) { + return Envoy::Network::FilterStatus::StopIteration; } - parent_.onBoundUpstreamConnectionEvent(Network::ConnectionEvent::RemoteClose); + // Basically this end_stream should always be false because we don't support half-close. + server_codec_->decode(data, end_stream); + return Envoy::Network::FilterStatus::StopIteration; } -void UpstreamManagerImpl::onDecodingSuccess(ResponsePtr response, ExtendedOptions options) { - // registered_upstream_callbacks_ should be empty because after upstream connection is ready. - ASSERT(registered_upstream_callbacks_.empty()); - - uint64_t stream_id = options.streamId().value_or(0); - - auto it = registered_response_callbacks_.find(stream_id); - if (it == registered_response_callbacks_.end()) { - ENVOY_LOG(error, "generic proxy: stream_id {} not found", stream_id); +void Filter::onDecodingSuccess(StreamFramePtr request) { + const uint64_t stream_id = request->frameFlags().streamFlags().streamId(); + // One existing stream expects this frame. + if (auto iter = frame_handlers_.find(stream_id); iter != frame_handlers_.end()) { + iter->second->onRequestFrame(std::move(request)); return; } - auto cb = it->second; - registered_response_callbacks_.erase(it); - - cb->onDecodingSuccess(std::move(response), options); -} - -void UpstreamManagerImpl::onDecodingFailure() { - // registered_upstream_callbacks_ should be empty because after upstream connection is ready. - ASSERT(registered_upstream_callbacks_.empty()); - - ENVOY_LOG(error, "generic proxy bound upstream manager: decoding failure"); - - parent_.stats_.response_decoding_error_.inc(); - - while (!registered_response_callbacks_.empty()) { - auto it = registered_response_callbacks_.begin(); - auto cb = it->second; - registered_response_callbacks_.erase(it); + StreamFramePtrHelper helper(std::move(request)); - cb->onDecodingFailure(); - } - - // Close both the upstream and downstream connections by this call. - parent_.onBoundUpstreamConnectionEvent(Network::ConnectionEvent::RemoteClose); -} - -void UpstreamManagerImpl::writeToConnection(Buffer::Instance& buffer) { - if (is_cleaned_up_) { + // Create a new active stream for the leading StreamRequest frame. + if (helper.typed_frame_ != nullptr) { + newDownstreamRequest(std::move(helper.typed_frame_)); return; } - if (owned_conn_data_ != nullptr) { - ASSERT(owned_conn_data_->connection().state() == Network::Connection::State::Open); - owned_conn_data_->connection().write(buffer, false); - } -} - -OptRef UpstreamManagerImpl::connection() { - if (is_cleaned_up_) { - return {}; - } - if (owned_conn_data_ != nullptr) { - return {owned_conn_data_->connection()}; - } - return {}; -} - -Envoy::Network::FilterStatus Filter::onData(Envoy::Buffer::Instance& data, bool) { - if (downstream_connection_closed_) { - return Envoy::Network::FilterStatus::StopIteration; - } - - request_decoder_->decode(data); - return Envoy::Network::FilterStatus::StopIteration; -} - -void Filter::onDecodingSuccess(RequestPtr request, ExtendedOptions options) { - newDownstreamRequest(std::move(request), options); + ASSERT(helper.frame_ != nullptr); + // No existing stream expects this non-leading frame. It should not happen. + // We treat it as request decoding failure. + ENVOY_LOG(error, "generic proxy: id {} not found for stream frame", + helper.frame_->frameFlags().streamFlags().streamId()); + onDecodingFailure(); } void Filter::onDecodingFailure() { @@ -402,12 +362,27 @@ OptRef Filter::connection() { return {downstreamConnection()}; } -void Filter::sendReplyDownstream(Response& response, ResponseEncoderCallback& callback) { - response_encoder_->encode(response, callback); +void Filter::sendFrameToDownstream(StreamFrame& frame, EncodingCallbacks& callbacks) { + server_codec_->encode(frame, callbacks); +} + +void Filter::registerFrameHandler(uint64_t stream_id, ActiveStream* raw_stream) { + // If the stream expects variable length frames, then add it to the frame + // handler map. + // This map entry will be removed when the request or response end frame is + // received. + if (frame_handlers_.contains(stream_id)) { + ENVOY_LOG(error, "generic proxy: repetitive stream id: {} at same time", stream_id); + onDecodingFailure(); + return; + } + frame_handlers_[stream_id] = raw_stream; } -void Filter::newDownstreamRequest(RequestPtr request, ExtendedOptions options) { - auto stream = std::make_unique(*this, std::move(request), options); +void Filter::unregisterFrameHandler(uint64_t stream_id) { frame_handlers_.erase(stream_id); } + +void Filter::newDownstreamRequest(StreamRequestPtr request) { + auto stream = std::make_unique(*this, std::move(request)); auto raw_stream = stream.get(); LinkedList::moveIntoList(std::move(stream), active_streams_); @@ -450,35 +425,6 @@ void Filter::mayBeDrainClose() { // Default implementation for connection draining. void Filter::onDrainCloseAndNoActiveStreams() { closeDownstreamConnection(); } -void Filter::bindUpstreamConn(Upstream::TcpPoolData&& tcp_pool_data) { - ASSERT(config_->codecFactory().protocolOptions().bindUpstreamConnection()); - ASSERT(upstream_manager_ == nullptr); - upstream_manager_ = std::make_unique(*this, std::move(tcp_pool_data)); - upstream_manager_->newConnection(); -} - -void Filter::onBoundUpstreamConnectionEvent(Network::ConnectionEvent event) { - if (event == Network::ConnectionEvent::RemoteClose || - event == Network::ConnectionEvent::LocalClose) { - ENVOY_LOG(debug, "generic proxy: bound upstream connection closed."); - // All pending streams should be reset by the upstream connection manager. - // In case there are still pending streams, we reset them here again. - resetStreamsForUnexpectedError(); - - if (upstream_manager_ != nullptr) { - // Clean up upstream connection manager. Always set the close_connection - // flag to true to ensure the upstream connection is closed in case of - // the onBoundUpstreamConnectionEvent() is called for other reasons. - upstream_manager_->cleanUp(true); - downstreamConnection().dispatcher().deferredDelete(std::move(upstream_manager_)); - upstream_manager_ = nullptr; - } - - // Close downstream connection. - closeDownstreamConnection(); - } -} - } // namespace GenericProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/generic_proxy/filters/network/source/proxy.h b/contrib/generic_proxy/filters/network/source/proxy.h index cbfdaf66b069..3dc1eeb77db4 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.h +++ b/contrib/generic_proxy/filters/network/source/proxy.h @@ -30,8 +30,7 @@ #include "contrib/generic_proxy/filters/network/source/rds_impl.h" #include "contrib/generic_proxy/filters/network/source/route.h" #include "contrib/generic_proxy/filters/network/source/stats.h" -#include "contrib/generic_proxy/filters/network/source/upstream.h" -#include "stats.h" +#include "contrib/generic_proxy/filters/network/source/tracing.h" namespace Envoy { namespace Extensions { @@ -58,12 +57,13 @@ class FilterConfigImpl : public FilterConfig { Tracing::ConnectionManagerTracingConfigPtr tracing_config, std::vector&& access_logs, Envoy::Server::Configuration::FactoryContext& context) - : stat_prefix_(stat_prefix), + : stat_prefix_(fmt::format("generic_proxy.{}.", stat_prefix)), stats_(GenericFilterStats::generateStats(stat_prefix_, context.scope())), codec_factory_(std::move(codec)), route_config_provider_(std::move(route_config_provider)), factories_(std::move(factories)), drain_decision_(context.drainDecision()), tracer_(std::move(tracer)), tracing_config_(std::move(tracing_config)), - access_logs_(std::move(access_logs)), time_source_(context.timeSource()) {} + access_logs_(std::move(access_logs)), + time_source_(context.serverFactoryContext().timeSource()) {} // FilterConfig RouteEntryConstSharedPtr routeEntry(const Request& request) const override { @@ -115,7 +115,7 @@ class FilterConfigImpl : public FilterConfig { class ActiveStream : public FilterChainManager, public LinkedObject, public Envoy::Event::DeferredDeletable, - public ResponseEncoderCallback, + public EncodingCallbacks, public Tracing::Config, Logger::Loggable { public: @@ -139,12 +139,6 @@ class ActiveStream : public FilterChainManager, StreamInfo::StreamInfo& streamInfo() override { return parent_.stream_info_; } Tracing::Span& activeSpan() override { return parent_.activeSpan(); } OptRef tracingConfig() const override { return parent_.tracingConfig(); } - absl::optional requestOptions() const override { - return parent_.downstream_request_options_; - } - absl::optional responseOptions() const override { - return parent_.local_or_upstream_response_options_; - } const Network::Connection* connection() const override; bool isDualFilter() const { return is_dual_; } @@ -167,12 +161,18 @@ class ActiveStream : public FilterChainManager, parent_.sendLocalReply(status, std::move(func)); } void continueDecoding() override { parent_.continueDecoding(); } - void upstreamResponse(ResponsePtr response, ExtendedOptions options) override { - parent_.upstreamResponse(std::move(response), options); + void onResponseStart(ResponsePtr response) override { + parent_.onResponseStart(std::move(response)); + } + void onResponseFrame(StreamFramePtr frame) override { + parent_.onResponseFrame(std::move(frame)); + } + void setRequestFramesHandler(StreamFrameHandler& handler) override { + ASSERT(parent_.request_stream_frame_handler_ == nullptr, + "request frames handler is already set"); + parent_.request_stream_frame_handler_ = &handler; } void completeDirectly() override { parent_.completeDirectly(); } - void bindUpstreamConn(Upstream::TcpPoolData&& pool_data) override; - OptRef boundUpstreamConn() override; DecoderFilterSharedPtr filter_; }; @@ -219,7 +219,7 @@ class ActiveStream : public FilterChainManager, FilterContext context_; }; - ActiveStream(Filter& parent, RequestPtr request, ExtendedOptions request_options); + ActiveStream(Filter& parent, StreamRequestPtr request); void addDecoderFilter(ActiveDecoderFilterPtr filter) { decoder_filters_.emplace_back(std::move(filter)); @@ -237,7 +237,8 @@ class ActiveStream : public FilterChainManager, void sendLocalReply(Status status, ResponseUpdateFunction&&); void continueDecoding(); - void upstreamResponse(ResponsePtr response, ExtendedOptions options); + void onResponseStart(StreamResponsePtr response); + void onResponseFrame(StreamFramePtr frame); void completeDirectly(); void continueEncoding(); @@ -249,7 +250,9 @@ class ActiveStream : public FilterChainManager, } // ResponseEncoderCallback - void onEncodingSuccess(Buffer::Instance& buffer) override; + void onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) override; + + void onRequestFrame(StreamFramePtr frame); std::vector& decoderFiltersForTest() { return decoder_filters_; } std::vector& encoderFiltersForTest() { return encoder_filters_; } @@ -273,6 +276,10 @@ class ActiveStream : public FilterChainManager, void completeRequest(); + uint64_t requestStreamId() const { + return request_stream_->frameFlags().streamFlags().streamId(); + } + private: // Keep these methods private to ensure that these methods are only called by the reference // returned by the public tracingConfig() method. @@ -281,16 +288,30 @@ class ActiveStream : public FilterChainManager, const Tracing::CustomTagMap* customTags() const override; bool verbose() const override; uint32_t maxPathTagLength() const override; + bool spawnUpstreamSpan() const override; + + void sendRequestFrameToUpstream(); + + void sendResponseStartToDownstream(); + void sendResponseFrameToDownstream(); bool active_stream_reset_{false}; + bool registered_in_frame_handlers_{false}; + Filter& parent_; - RequestPtr downstream_request_stream_; - absl::optional downstream_request_options_; + StreamRequestPtr request_stream_; + std::list request_stream_frames_; + bool request_stream_end_{false}; + bool request_filter_chain_complete_{false}; - ResponsePtr local_or_upstream_response_stream_; - absl::optional local_or_upstream_response_options_; + StreamFrameHandler* request_stream_frame_handler_{nullptr}; + + StreamResponsePtr response_stream_; + std::list response_stream_frames_; + bool response_stream_end_{false}; + bool response_filter_chain_complete_{false}; RouteEntryConstSharedPtr cached_route_entry_; @@ -309,49 +330,16 @@ class ActiveStream : public FilterChainManager, }; using ActiveStreamPtr = std::unique_ptr; -class UpstreamManagerImpl : public UpstreamConnection, - public UpstreamManager, - public ResponseDecoderCallback { -public: - UpstreamManagerImpl(Filter& parent, Upstream::TcpPoolData&& tcp_pool_data); - - // UpstreamConnection - void onPoolSuccessImpl() override; - void onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, - absl::string_view transport_failure_reason) override; - void onEventImpl(Network::ConnectionEvent event) override; - - // ResponseDecoderCallback - void onDecodingSuccess(ResponsePtr response, ExtendedOptions options) override; - void onDecodingFailure() override; - void writeToConnection(Buffer::Instance& buffer) override; - OptRef connection() override; - - // UpstreamManager - void registerResponseCallback(uint64_t stream_id, PendingResponseCallback& cb) override; - void registerUpstreamCallback(uint64_t stream_id, UpstreamBindingCallback& cb) override; - void unregisterResponseCallback(uint64_t stream_id) override; - void unregisterUpstreamCallback(uint64_t stream_id) override; - - Filter& parent_; - - absl::flat_hash_map registered_response_callbacks_; - absl::flat_hash_map registered_upstream_callbacks_; -}; - class Filter : public Envoy::Network::ReadFilter, public Network::ConnectionCallbacks, public Envoy::Logger::Loggable, - public RequestDecoderCallback { + public ServerCodecCallbacks { public: Filter(FilterConfigSharedPtr config, TimeSource& time_source, Runtime::Loader& runtime) : config_(std::move(config)), stats_(config_->stats()), drain_decision_(config_->drainDecision()), time_source_(time_source), runtime_(runtime) { - request_decoder_ = config_->codecFactory().requestDecoder(); - request_decoder_->setDecoderCallback(*this); - response_encoder_ = config_->codecFactory().responseEncoder(); - message_creator_ = config_->codecFactory().messageCreator(); - protocol_options_ = config_->codecFactory().protocolOptions(); + server_codec_ = config_->codecFactory().createServerCodec(); + server_codec_->setCodecCallbacks(*this); } // Envoy::Network::ReadFilter @@ -365,7 +353,7 @@ class Filter : public Envoy::Network::ReadFilter, } // RequestDecoderCallback - void onDecodingSuccess(RequestPtr request, ExtendedOptions options) override; + void onDecodingSuccess(StreamFramePtr request) override; void onDecodingFailure() override; void writeToConnection(Buffer::Instance& buffer) override; OptRef connection() override; @@ -379,18 +367,11 @@ class Filter : public Envoy::Network::ReadFilter, } downstream_connection_closed_ = true; resetStreamsForUnexpectedError(); - - // If these is bound upstream connection, clean it up. - if (upstream_manager_ != nullptr) { - upstream_manager_->cleanUp(true); - downstreamConnection().dispatcher().deferredDelete(std::move(upstream_manager_)); - upstream_manager_ = nullptr; - } } void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} - void sendReplyDownstream(Response& response, ResponseEncoderCallback& callback); + void sendFrameToDownstream(StreamFrame& frame, EncodingCallbacks& callbacks); Network::Connection& downstreamConnection() { ASSERT(callbacks_ != nullptr); @@ -400,9 +381,8 @@ class Filter : public Envoy::Network::ReadFilter, /** * Create a new active stream and add it to the active stream list. * @param request the request to be processed. - * @param options the extended options for the request. */ - void newDownstreamRequest(RequestPtr request, ExtendedOptions options); + void newDownstreamRequest(StreamRequestPtr request); /** * Move the stream to the deferred delete stream list. This is called when the stream is reset @@ -416,6 +396,7 @@ class Filter : public Envoy::Network::ReadFilter, } std::list& activeStreamsForTest() { return active_streams_; } + const auto& frameHandlersForTest() { return frame_handlers_; } // This may be called multiple times in some scenarios. But it is safe. void resetStreamsForUnexpectedError(); @@ -424,13 +405,6 @@ class Filter : public Envoy::Network::ReadFilter, void mayBeDrainClose(); - void bindUpstreamConn(Upstream::TcpPoolData&& tcp_pool_data); - OptRef boundUpstreamConn() { - return makeOptRefFromPtr(upstream_manager_.get()); - } - - void onBoundUpstreamConnectionEvent(Network::ConnectionEvent event); - protected: // This will be called when drain decision is made and all active streams are handled. // This is a virtual method so that it can be overridden by derived classes. @@ -442,6 +416,9 @@ class Filter : public Envoy::Network::ReadFilter, friend class ActiveStream; friend class UpstreamManagerImpl; + void registerFrameHandler(uint64_t stream_id, ActiveStream* stream); + void unregisterFrameHandler(uint64_t stream_id); + bool downstream_connection_closed_{}; FilterConfigSharedPtr config_{}; @@ -451,17 +428,12 @@ class Filter : public Envoy::Network::ReadFilter, TimeSource& time_source_; Runtime::Loader& runtime_; - RequestDecoderPtr request_decoder_; - ResponseEncoderPtr response_encoder_; - MessageCreatorPtr message_creator_; - ProtocolOptions protocol_options_; + ServerCodecPtr server_codec_; Buffer::OwnedImpl response_buffer_; - // The upstream connection manager. - std::unique_ptr upstream_manager_; - std::list active_streams_; + absl::flat_hash_map frame_handlers_; }; } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/source/router/BUILD b/contrib/generic_proxy/filters/network/source/router/BUILD index 0b06176425b0..f6995ba144bd 100644 --- a/contrib/generic_proxy/filters/network/source/router/BUILD +++ b/contrib/generic_proxy/filters/network/source/router/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( "router.h", ], deps = [ + "//contrib/generic_proxy/filters/network/source:tracing_lib", "//contrib/generic_proxy/filters/network/source:upstream_lib", "//contrib/generic_proxy/filters/network/source/interface:codec_interface", "//contrib/generic_proxy/filters/network/source/interface:config_interface", @@ -30,6 +31,7 @@ envoy_cc_library( "//source/common/stream_info:stream_info_lib", "//source/common/tracing:tracer_lib", "//source/common/upstream:load_balancer_lib", + "@envoy_api//contrib/envoy/extensions/filters/network/generic_proxy/router/v3:pkg_cc_proto", ], ) diff --git a/contrib/generic_proxy/filters/network/source/router/config.cc b/contrib/generic_proxy/filters/network/source/router/config.cc index fff6713512c2..625a4ea582b8 100644 --- a/contrib/generic_proxy/filters/network/source/router/config.cc +++ b/contrib/generic_proxy/filters/network/source/router/config.cc @@ -9,10 +9,16 @@ namespace GenericProxy { namespace Router { FilterFactoryCb -RouterFactory::createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, +RouterFactory::createFilterFactoryFromProto(const Protobuf::Message& config, const std::string&, Server::Configuration::FactoryContext& context) { - return [&context](FilterChainFactoryCallbacks& callbacks) { - callbacks.addDecoderFilter(std::make_shared(context)); + const auto& typed_config = MessageUtil::downcastAndValidate< + const envoy::extensions::filters::network::generic_proxy::router::v3::Router&>( + config, context.serverFactoryContext().messageValidationVisitor()); + + auto router_config = std::make_shared(typed_config); + + return [&context, router_config](FilterChainFactoryCallbacks& callbacks) { + callbacks.addDecoderFilter(std::make_shared(router_config, context)); }; } diff --git a/contrib/generic_proxy/filters/network/source/router/router.cc b/contrib/generic_proxy/filters/network/source/router/router.cc index f6fdd0c550e9..e125fd3b58ba 100644 --- a/contrib/generic_proxy/filters/network/source/router/router.cc +++ b/contrib/generic_proxy/filters/network/source/router/router.cc @@ -1,5 +1,7 @@ #include "contrib/generic_proxy/filters/network/source/router/router.h" +#include + #include "envoy/common/conn_pool.h" #include "envoy/network/connection.h" @@ -9,6 +11,7 @@ #include "source/common/tracing/tracer_impl.h" #include "contrib/generic_proxy/filters/network/source/interface/filter.h" +#include "contrib/generic_proxy/filters/network/source/tracing.h" namespace Envoy { namespace Extensions { @@ -22,76 +25,275 @@ absl::string_view resetReasonToStringView(StreamResetReason reason) { "overflow", "protocol_error"}; return Reasons[static_cast(reason)]; } + +constexpr absl::string_view RouterFilterName = "envoy.filters.generic.router"; + } // namespace -UpstreamManagerImpl::UpstreamManagerImpl(UpstreamRequest& parent, Upstream::TcpPoolData&& pool) - : UpstreamConnection(std::move(pool), - parent.decoder_callbacks_.downstreamCodec().responseDecoder()), - parent_(parent) {} +void GenericUpstream::writeToConnection(Buffer::Instance& buffer) { + if (is_cleaned_up_) { + return; + } + + if (owned_conn_data_ != nullptr) { + ASSERT(owned_conn_data_->connection().state() == Network::Connection::State::Open); + owned_conn_data_->connection().write(buffer, false); + } +} + +OptRef GenericUpstream::connection() { + if (is_cleaned_up_) { + return {}; + } + if (owned_conn_data_ != nullptr) { + return {owned_conn_data_->connection()}; + } + return {}; +} + +BoundGenericUpstream::BoundGenericUpstream(const CodecFactory& codec_factory, + Envoy::Upstream::TcpPoolData&& tcp_pool_data, + Network::Connection& downstream_connection) + : GenericUpstream(std::move(tcp_pool_data), codec_factory.createClientCodec()), + downstream_connection_(downstream_connection) { + + connection_event_watcher_ = std::make_unique(*this); + downstream_connection_.addConnectionCallbacks(*connection_event_watcher_); +} + +void BoundGenericUpstream::onDownstreamConnectionEvent(Network::ConnectionEvent event) { + if (event == Network::ConnectionEvent::LocalClose || + event == Network::ConnectionEvent::RemoteClose) { + // The event should be handled first by the generic proxy first. So all pending + // requests will be cleaned up by the downstream connection close event. + ASSERT(waiting_upstream_requests_.empty()); + ASSERT(waiting_response_requests_.empty()); + + // Close upstream connection and this will trigger the upstream connection close event. + cleanUp(true); + } +} + +void BoundGenericUpstream::insertUpstreamRequest(uint64_t stream_id, + UpstreamRequest* pending_request) { + if (upstream_connection_ready_.has_value()) { + // Upstream connection is already ready. If the upstream connection is failed then + // all pending requests will be reset and no new upstream request will be created. + ASSERT(upstream_connection_ready_.value()); + if (!upstream_connection_ready_.value()) { + return; + } + + ASSERT(waiting_upstream_requests_.empty()); + + if (waiting_response_requests_.contains(stream_id)) { + ENVOY_LOG(error, "generic proxy: stream_id {} already registered for response", stream_id); + // Close downstream connection because we treat this as request decoding failure. + // The downstream closing will trigger the upstream connection closing. + downstream_connection_.close(Network::ConnectionCloseType::FlushWrite); + return; + } + + waiting_response_requests_[stream_id] = pending_request; + pending_request->onUpstreamSuccess(upstream_host_); + } else { + // Waiting for the upstream connection to be ready. + if (waiting_upstream_requests_.contains(stream_id)) { + ENVOY_LOG(error, "generic proxy: stream_id {} already registered for upstream", stream_id); + // Close downstream connection because we treat this as request decoding failure. + // The downstream closing will trigger the upstream connection closing. + downstream_connection_.close(Network::ConnectionCloseType::FlushWrite); + return; + } + + waiting_upstream_requests_[stream_id] = pending_request; + + // Try to initialize the upstream connection after there is at least one pending request. + // If the upstream connection is already initialized, this is a no-op. + initialize(); + } +} + +void BoundGenericUpstream::removeUpstreamRequest(uint64_t stream_id) { + waiting_upstream_requests_.erase(stream_id); + waiting_response_requests_.erase(stream_id); +} + +void BoundGenericUpstream::onEventImpl(Network::ConnectionEvent event) { + if (event == Network::ConnectionEvent::Connected || + event == Network::ConnectionEvent::ConnectedZeroRtt) { + return; + } + + ASSERT(waiting_upstream_requests_.empty()); + + while (!waiting_response_requests_.empty()) { + auto it = waiting_response_requests_.begin(); + auto cb = it->second; + waiting_response_requests_.erase(it); + + cb->onConnectionClose(event); + } + + // If the downstream connection is not closed, close it. + if (downstream_connection_.state() == Network::Connection::State::Open) { + downstream_connection_.close(Network::ConnectionCloseType::FlushWrite); + } +} + +void BoundGenericUpstream::cleanUp(bool close_connection) { + // Shared upstream manager never release the connection back to the pool + // because the connection is bound to the downstream connection. + if (!close_connection) { + return; + } + // Only actually do the cleanup when we want to close the connection. + UpstreamConnection::cleanUp(true); +} + +void BoundGenericUpstream::onPoolSuccessImpl() { + // This should be called only once and all pending requests should be notified. + // After this is called, the upstream connection is ready and new upstream requests + // should be notified directly. + + ASSERT(!upstream_connection_ready_.has_value()); + upstream_connection_ready_ = true; + + ASSERT(waiting_response_requests_.empty()); + + while (!waiting_upstream_requests_.empty()) { + auto it = waiting_upstream_requests_.begin(); + auto cb = it->second; + + // Insert it to the waiting response list and remove it from the waiting upstream list. + waiting_response_requests_[it->first] = cb; + waiting_upstream_requests_.erase(it); + + // Now, notify the upstream request that the upstream connection is ready. + cb->onUpstreamSuccess(upstream_host_); + } +} + +void BoundGenericUpstream::onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, + absl::string_view transport_failure_reason) { + // This should be called only once and all pending requests should be notified. + // Then the downstream connection will be closed. + + ASSERT(!upstream_connection_ready_.has_value()); + upstream_connection_ready_ = false; + + ASSERT(waiting_response_requests_.empty()); + + while (!waiting_upstream_requests_.empty()) { + auto it = waiting_upstream_requests_.begin(); + auto cb = it->second; + + // Remove it from the waiting upstream list. + waiting_upstream_requests_.erase(it); + + // Now, notify the upstream request that the upstream connection is failed. + cb->onUpstreamFailure(reason, transport_failure_reason, upstream_host_); + } + + // If the downstream connection is not closed, close it. + downstream_connection_.close(Network::ConnectionCloseType::FlushWrite); +} + +void BoundGenericUpstream::onDecodingSuccess(StreamFramePtr response) { + const uint64_t stream_id = response->frameFlags().streamFlags().streamId(); + const bool end_stream = response->frameFlags().endStream(); + + auto it = waiting_response_requests_.find(stream_id); + if (it == waiting_response_requests_.end()) { + ENVOY_LOG(error, "generic proxy: id {} not found for frame", stream_id); + return; + } + + auto cb = it->second; + + // If the response is end, remove the callback from the map. + if (end_stream) { + waiting_response_requests_.erase(it); + } + + return cb->onDecodingSuccess(std::move(response)); +} + +void BoundGenericUpstream::onDecodingFailure() { + ENVOY_LOG(error, "generic proxy bound upstream manager: decoding failure"); + + // This will trigger the upstream connection close event and all pending requests will be reset + // by the upstream connection close event. + cleanUp(true); -void UpstreamManagerImpl::onEventImpl(Network::ConnectionEvent event) { + // All pending streams will be reset by the upstream connection close event. + ASSERT(waiting_response_requests_.empty()); +} + +OwnedGenericUpstream::OwnedGenericUpstream(const CodecFactory& codec_factory, + Envoy::Upstream::TcpPoolData&& tcp_pool_data) + : GenericUpstream(std::move(tcp_pool_data), codec_factory.createClientCodec()) {} + +void OwnedGenericUpstream::insertUpstreamRequest(uint64_t, UpstreamRequest* pending_request) { + upstream_request_ = pending_request; + initialize(); +} + +void OwnedGenericUpstream::onEventImpl(Network::ConnectionEvent event) { if (event == Network::ConnectionEvent::Connected || event == Network::ConnectionEvent::ConnectedZeroRtt) { return; } - parent_.onConnectionClose(event); + ASSERT(upstream_request_ != nullptr); + upstream_request_->onConnectionClose(event); } -void UpstreamManagerImpl::onPoolSuccessImpl() { - parent_.onBindSuccess(owned_conn_data_->connection(), upstream_host_); +void OwnedGenericUpstream::onPoolSuccessImpl() { + ASSERT(upstream_request_ != nullptr); + upstream_request_->onUpstreamSuccess(upstream_host_); } -void UpstreamManagerImpl::onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, - absl::string_view transport_failure_reason) { - parent_.onBindFailure(reason, transport_failure_reason, upstream_host_); +void OwnedGenericUpstream::onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, + absl::string_view transport_failure_reason) { + ASSERT(upstream_request_ != nullptr); + upstream_request_->onUpstreamFailure(reason, transport_failure_reason, upstream_host_); } -void UpstreamManagerImpl::setResponseCallback() { response_decoder_->setDecoderCallback(parent_); } +// ResponseDecoderCallback +void OwnedGenericUpstream::onDecodingSuccess(StreamFramePtr response) { + ASSERT(upstream_request_ != nullptr); + upstream_request_->onDecodingSuccess(std::move(response)); +} +void OwnedGenericUpstream::onDecodingFailure() { + ASSERT(upstream_request_ != nullptr); + upstream_request_->onDecodingFailure(); +} -UpstreamRequest::UpstreamRequest(RouterFilter& parent, - absl::optional tcp_pool_data) - : parent_(parent), decoder_callbacks_(*parent_.callbacks_), - tcp_pool_data_(std::move(tcp_pool_data)), - stream_info_(parent.context_.mainThreadDispatcher().timeSource(), nullptr) { +UpstreamRequest::UpstreamRequest(RouterFilter& parent, GenericUpstreamSharedPtr generic_upstream) + : parent_(parent), generic_upstream_(std::move(generic_upstream)), + stream_info_(parent.context_.serverFactoryContext().timeSource(), nullptr) { // Set the upstream info for the stream info. stream_info_.setUpstreamInfo(std::make_shared()); - decoder_callbacks_.streamInfo().setUpstreamInfo(stream_info_.upstreamInfo()); - stream_info_.healthCheck(decoder_callbacks_.streamInfo().healthCheck()); + parent_.callbacks_->streamInfo().setUpstreamInfo(stream_info_.upstreamInfo()); + stream_info_.healthCheck(parent_.callbacks_->streamInfo().healthCheck()); stream_info_.setUpstreamClusterInfo(parent_.cluster_); // Set request options. - auto options = decoder_callbacks_.requestOptions(); - ASSERT(options.has_value()); - stream_id_ = options->streamId().value_or(0); - wait_response_ = options->waitResponse(); + auto options = parent_.request_stream_->frameFlags().streamFlags(); + stream_id_ = options.streamId(); + expects_response_ = !options.oneWayStream(); // Set tracing config. - tracing_config_ = decoder_callbacks_.tracingConfig(); - if (tracing_config_.has_value()) { - span_ = decoder_callbacks_.activeSpan().spawnChild( + if (tracing_config_ = parent_.callbacks_->tracingConfig(); tracing_config_.has_value()) { + span_ = parent_.callbacks_->activeSpan().spawnChild( tracing_config_.value().get(), absl::StrCat("router ", parent_.cluster_->observabilityName(), " egress"), - parent.context_.mainThreadDispatcher().timeSource().systemTime()); + parent.context_.serverFactoryContext().timeSource().systemTime()); } } -void UpstreamRequest::startStream() { - if (!tcp_pool_data_.has_value()) { - // Iff the upstream connection binding is enabled, the upstream connection should be - // managed by the generic proxy directly. Then register the upstream callbacks to the - // generic proxy and wait for the bound upstream connection. - ASSERT(decoder_callbacks_.boundUpstreamConn().has_value()); - decoder_callbacks_.boundUpstreamConn()->registerUpstreamCallback(stream_id_, *this); - return; - } - - // If the tcp_pool_data_ has value, it means we should get or create an upstream connection - // for the request. - upstream_manager_ = - std::make_unique(*this, std::move(tcp_pool_data_.value())); - upstream_manager_->newConnection(); -} +void UpstreamRequest::startStream() { generic_upstream_->insertUpstreamRequest(stream_id_, this); } void UpstreamRequest::resetStream(StreamResetReason reason) { if (stream_reset_) { @@ -101,24 +303,14 @@ void UpstreamRequest::resetStream(StreamResetReason reason) { ENVOY_LOG(debug, "generic proxy upstream request: reset upstream request"); - if (upstream_manager_ != nullptr) { - // If the upstream connection is managed by the upstream request self, we should clean - // up the upstream connection. - upstream_manager_->cleanUp(true); - decoder_callbacks_.dispatcher().deferredDelete(std::move(upstream_manager_)); - upstream_manager_ = nullptr; - } else { - // If the upstream connection is not managed by the generic proxy, we should unregister - // the related callbacks from the generic proxy. - ASSERT(decoder_callbacks_.boundUpstreamConn().has_value()); - decoder_callbacks_.boundUpstreamConn()->unregisterUpstreamCallback(stream_id_); - decoder_callbacks_.boundUpstreamConn()->unregisterResponseCallback(stream_id_); - } + generic_upstream_->removeUpstreamRequest(stream_id_); + generic_upstream_->cleanUp(true); if (span_ != nullptr) { span_->setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); span_->setTag(Tracing::Tags::get().ErrorReason, resetReasonToStringView(reason)); - Tracing::TracerUtility::finalizeSpan(*span_, *parent_.request_, stream_info_, + TraceContextBridge trace_context{*parent_.request_stream_}; + Tracing::TracerUtility::finalizeSpan(*span_, trace_context, stream_info_, tracing_config_.value().get(), true); } @@ -137,15 +329,13 @@ void UpstreamRequest::clearStream(bool close_connection) { ENVOY_LOG(debug, "generic proxy upstream request: complete upstream request"); if (span_ != nullptr) { - Tracing::TracerUtility::finalizeSpan(*span_, *parent_.request_, stream_info_, + TraceContextBridge trace_context{*parent_.request_stream_}; + Tracing::TracerUtility::finalizeSpan(*span_, trace_context, stream_info_, tracing_config_.value().get(), true); } - if (upstream_manager_ != nullptr) { - upstream_manager_->cleanUp(close_connection); - decoder_callbacks_.dispatcher().deferredDelete(std::move(upstream_manager_)); - upstream_manager_ = nullptr; - } + generic_upstream_->removeUpstreamRequest(stream_id_); + generic_upstream_->cleanUp(close_connection); // Remove this stream form the parent's list because this upstream request is complete. deferredDelete(); @@ -155,35 +345,52 @@ void UpstreamRequest::deferredDelete() { if (inserted()) { // Remove this stream from the parent's list of upstream requests and delete it at // next event loop iteration. - decoder_callbacks_.dispatcher().deferredDelete(removeFromList(parent_.upstream_requests_)); + parent_.callbacks_->dispatcher().deferredDelete(removeFromList(parent_.upstream_requests_)); } } -void UpstreamRequest::onEncodingSuccess(Buffer::Instance& buffer) { - ENVOY_LOG(debug, "upstream request encoding success"); +void UpstreamRequest::sendRequestStartToUpstream() { + request_stream_header_sent_ = true; + ASSERT(generic_upstream_ != nullptr); + generic_upstream_->clientCodec()->encode(*parent_.request_stream_, *this); +} + +void UpstreamRequest::sendRequestFrameToUpstream() { + if (!request_stream_header_sent_) { + // Do not send request frame to upstream until the request header is sent. It may be blocked + // by the upstream connecting. + return; + } + + while (!parent_.request_stream_frames_.empty()) { + auto frame = std::move(parent_.request_stream_frames_.front()); + parent_.request_stream_frames_.pop_front(); + + ASSERT(generic_upstream_ != nullptr); + generic_upstream_->clientCodec()->encode(*frame, *this); + } +} + +void UpstreamRequest::onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) { encodeBufferToUpstream(buffer); + if (!end_stream) { + return; + } + + // Request is complete. + ENVOY_LOG(debug, "upstream request encoding success"); + // Need not to wait for the upstream response and complete directly. - if (!wait_response_) { + if (!expects_response_) { clearStream(false); parent_.completeDirectly(); return; } - - // If the upstream connection manager is null, it means the upstream - // connection is managed by the generic proxy directly. Register the - // response callback to the generic proxy and wait for the upstream - // response. - if (upstream_manager_ == nullptr) { - ASSERT(decoder_callbacks_.boundUpstreamConn().has_value()); - decoder_callbacks_.boundUpstreamConn()->registerResponseCallback(stream_id_, *this); - } else { - upstream_manager_->setResponseCallback(); - } } -void UpstreamRequest::onBindFailure(ConnectionPool::PoolFailureReason reason, absl::string_view, - Upstream::HostDescriptionConstSharedPtr host) { +void UpstreamRequest::onUpstreamFailure(ConnectionPool::PoolFailureReason reason, absl::string_view, + Upstream::HostDescriptionConstSharedPtr host) { ENVOY_LOG(debug, "upstream request: tcp connection (bound or owned) failure"); // Mimic an upstream reset. @@ -197,49 +404,43 @@ void UpstreamRequest::onBindFailure(ConnectionPool::PoolFailureReason reason, ab resetStream(StreamResetReason::ConnectionFailure); } -void UpstreamRequest::onBindSuccess(Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr host) { +void UpstreamRequest::onUpstreamSuccess(Upstream::HostDescriptionConstSharedPtr host) { ENVOY_LOG(debug, "upstream request: {} tcp connection has ready", - upstream_manager_ != nullptr ? "owned" : "bound"); + parent_.config_->bindUpstreamConnection() ? "bound" : "owned"); onUpstreamHostSelected(std::move(host)); - upstream_conn_ = &conn; if (span_ != nullptr) { - span_->injectContext(*parent_.request_, upstream_host_); + TraceContextBridge trace_context{*parent_.request_stream_}; + span_->injectContext(trace_context, upstream_host_); } - parent_.request_encoder_->encode(*parent_.request_, *this); + sendRequestStartToUpstream(); + sendRequestFrameToUpstream(); } -void UpstreamRequest::onDecodingSuccess(ResponsePtr response, ExtendedOptions options) { - clearStream(options.drainClose()); - parent_.onUpstreamResponse(std::move(response), options); -} - -void UpstreamRequest::onDecodingFailure() { resetStream(StreamResetReason::ProtocolError); } +void UpstreamRequest::onDecodingSuccess(StreamFramePtr response) { + const bool end_stream = response->frameFlags().endStream(); + if (end_stream) { + clearStream(response->frameFlags().streamFlags().drainClose()); + } -void UpstreamRequest::writeToConnection(Buffer::Instance& buffer) { - // If the upstream response is complete or the upstream request is reset then - // ignore the write. - if (stream_reset_ || response_complete_) { + if (response_stream_header_received_) { + parent_.onResponseFrame(std::move(response)); return; } - if (upstream_conn_ != nullptr) { - ASSERT(upstream_conn_->state() == Network::Connection::State::Open); - upstream_conn_->write(buffer, false); + StreamFramePtrHelper helper(std::move(response)); + if (helper.typed_frame_ == nullptr) { + ENVOY_LOG(error, "upstream request: first frame is not StreamResponse"); + resetStream(StreamResetReason::ProtocolError); + return; } + response_stream_header_received_ = true; + parent_.onResponseStart(std::move(helper.typed_frame_)); } -OptRef UpstreamRequest::connection() { - if (stream_reset_ || response_complete_) { - return {}; - } - - return upstream_conn_ != nullptr ? OptRef(*upstream_conn_) - : OptRef(); -} +void UpstreamRequest::onDecodingFailure() { resetStream(StreamResetReason::ProtocolError); } void UpstreamRequest::onConnectionClose(Network::ConnectionEvent event) { // If the upstream response is complete or the upstream request is reset then @@ -266,16 +467,21 @@ void UpstreamRequest::onUpstreamHostSelected(Upstream::HostDescriptionConstShare } void UpstreamRequest::encodeBufferToUpstream(Buffer::Instance& buffer) { - ASSERT(upstream_conn_ != nullptr); - ENVOY_LOG(trace, "proxying {} bytes", buffer.length()); - upstream_conn_->write(buffer, false); + ASSERT(generic_upstream_ != nullptr); + generic_upstream_->writeToConnection(buffer); } -void RouterFilter::onUpstreamResponse(ResponsePtr response, ExtendedOptions options) { - filter_complete_ = true; - callbacks_->upstreamResponse(std::move(response), std::move(options)); +void RouterFilter::onResponseStart(ResponsePtr response) { + filter_complete_ = response->frameFlags().endStream(); + callbacks_->onResponseStart(std::move(response)); +} + +void RouterFilter::onResponseFrame(StreamFramePtr frame) { + ASSERT(!filter_complete_, "response frame received after response complete"); + filter_complete_ = frame->frameFlags().endStream(); + callbacks_->onResponseFrame(std::move(frame)); } void RouterFilter::completeDirectly() { @@ -338,7 +544,8 @@ void RouterFilter::resetStream(StreamResetReason reason) { void RouterFilter::kickOffNewUpstreamRequest() { const auto& cluster_name = route_entry_->clusterName(); - auto thread_local_cluster = context_.clusterManager().getThreadLocalCluster(cluster_name); + auto thread_local_cluster = + context_.serverFactoryContext().clusterManager().getThreadLocalCluster(cluster_name); if (thread_local_cluster == nullptr) { filter_complete_ = true; callbacks_->sendLocalReply(Status(StatusCode::kNotFound, "cluster_not_found")); @@ -354,55 +561,80 @@ void RouterFilter::kickOffNewUpstreamRequest() { return; } - if (callbacks_->boundUpstreamConn().has_value()) { - // Upstream connection binding is enabled and the upstream connection is already bound. - // Create a new upstream request without a connection pool and start the request. - auto upstream_request = std::make_unique(*this, absl::nullopt); - auto raw_upstream_request = upstream_request.get(); - LinkedList::moveIntoList(std::move(upstream_request), upstream_requests_); - raw_upstream_request->startStream(); - return; + GenericUpstreamSharedPtr generic_upstream; + + if (config_->bindUpstreamConnection()) { + // If the upstream connection binding is enabled. + + const auto* const_downstream_connection = callbacks_->connection(); + ASSERT(const_downstream_connection != nullptr); + auto downstream_connection = const_cast(const_downstream_connection); + + auto* bound_upstream = + downstream_connection->streamInfo().filterState()->getDataMutable( + RouterFilterName); + if (bound_upstream == nullptr) { + // The upstream connection is not bound yet and create a new bound upstream connection. + auto pool_data = thread_local_cluster->tcpConnPool(Upstream::ResourcePriority::Default, this); + if (!pool_data.has_value()) { + filter_complete_ = true; + callbacks_->sendLocalReply(Status(StatusCode::kUnavailable, "no_healthy_upstream")); + return; + } + auto new_bound_upstream = std::make_shared( + callbacks_->downstreamCodec(), std::move(pool_data.value()), *downstream_connection); + bound_upstream = new_bound_upstream.get(); + downstream_connection->streamInfo().filterState()->setData( + RouterFilterName, std::move(new_bound_upstream), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + } + + ASSERT(bound_upstream != nullptr); + generic_upstream = bound_upstream->shared_from_this(); + } else { + // Upstream connection binding is disabled and create a new upstream connection. + auto pool_data = thread_local_cluster->tcpConnPool(Upstream::ResourcePriority::Default, this); + if (!pool_data.has_value()) { + filter_complete_ = true; + callbacks_->sendLocalReply(Status(StatusCode::kUnavailable, "no_healthy_upstream")); + return; + } + generic_upstream = std::make_shared(callbacks_->downstreamCodec(), + std::move(pool_data.value())); } - auto pool_data = thread_local_cluster->tcpConnPool(Upstream::ResourcePriority::Default, this); - if (!pool_data.has_value()) { - filter_complete_ = true; - callbacks_->sendLocalReply(Status(StatusCode::kUnavailable, "no_healthy_upstream")); - return; - } + auto upstream_request = std::make_unique(*this, std::move(generic_upstream)); + auto raw_upstream_request = upstream_request.get(); + LinkedList::moveIntoList(std::move(upstream_request), upstream_requests_); + raw_upstream_request->startStream(); +} - if (protocol_options_.bindUpstreamConnection()) { - // Upstream connection binding is enabled and the upstream connection is not bound yet. - // Bind the upstream connection and start the request. - callbacks_->bindUpstreamConn(std::move(pool_data.value())); - auto upstream_request = std::make_unique(*this, absl::nullopt); - auto raw_upstream_request = upstream_request.get(); - LinkedList::moveIntoList(std::move(upstream_request), upstream_requests_); - raw_upstream_request->startStream(); +void RouterFilter::onStreamFrame(StreamFramePtr frame) { + request_stream_end_ = frame->frameFlags().endStream(); + request_stream_frames_.emplace_back(std::move(frame)); + + if (upstream_requests_.empty()) { return; } - // Normal upstream request. - auto upstream_request = std::make_unique(*this, std::move(pool_data.value())); - auto raw_upstream_request = upstream_request.get(); - LinkedList::moveIntoList(std::move(upstream_request), upstream_requests_); - raw_upstream_request->startStream(); + upstream_requests_.front()->sendRequestFrameToUpstream(); } -FilterStatus RouterFilter::onStreamDecoded(Request& request) { +FilterStatus RouterFilter::onStreamDecoded(StreamRequest& request) { ENVOY_LOG(debug, "Try route request to the upstream based on the route entry"); setRouteEntry(callbacks_->routeEntry()); - request_ = &request; + request_stream_end_ = request.frameFlags().endStream(); + request_stream_ = &request; - if (route_entry_ == nullptr) { - ENVOY_LOG(debug, "No route for current request and send local reply"); - callbacks_->sendLocalReply(Status(StatusCode::kNotFound, "route_not_found")); + if (route_entry_ != nullptr) { + kickOffNewUpstreamRequest(); return FilterStatus::StopIteration; } - request_encoder_ = callbacks_->downstreamCodec().requestEncoder(); - kickOffNewUpstreamRequest(); + ENVOY_LOG(debug, "No route for current request and send local reply"); + callbacks_->sendLocalReply(Status(StatusCode::kNotFound, "route_not_found")); return FilterStatus::StopIteration; } diff --git a/contrib/generic_proxy/filters/network/source/router/router.h b/contrib/generic_proxy/filters/network/source/router/router.h index d64d9bdb8e9d..a96d12d7024f 100644 --- a/contrib/generic_proxy/filters/network/source/router/router.h +++ b/contrib/generic_proxy/filters/network/source/router/router.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/network/connection.h" #include "envoy/server/factory_context.h" @@ -8,6 +10,8 @@ #include "source/common/stream_info/stream_info_impl.h" #include "source/common/upstream/load_balancer_impl.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/router/v3/router.pb.h" +#include "contrib/envoy/extensions/filters/network/generic_proxy/router/v3/router.pb.validate.h" #include "contrib/generic_proxy/filters/network/source/interface/codec.h" #include "contrib/generic_proxy/filters/network/source/interface/filter.h" #include "contrib/generic_proxy/filters/network/source/interface/stream.h" @@ -37,34 +41,106 @@ enum class StreamResetReason : uint32_t { class RouterFilter; class UpstreamRequest; -/** - * We needn't to inherit from UpstreamManager interfaces because - * the Router::UpstreamManagerImpl is only used by Router::UpstreamRequest and - * only one request is allowed to be processed at the same time. - * We still inherit from UpstreamConnection to avoid code duplication. - */ -class UpstreamManagerImpl : public UpstreamConnection { +class GenericUpstream : public UpstreamConnection, public ClientCodecCallbacks { +public: + GenericUpstream(Upstream::TcpPoolData&& tcp_pool_data, ClientCodecPtr&& client_codec) + : UpstreamConnection(std::move(tcp_pool_data), std::move(client_codec)) { + client_codec_->setCodecCallbacks(*this); + } + + // ResponseDecoderCallback + void writeToConnection(Buffer::Instance& buffer) override; + OptRef connection() override; + + virtual void insertUpstreamRequest(uint64_t stream_id, UpstreamRequest* pending_request) PURE; + virtual void removeUpstreamRequest(uint64_t stream_id) PURE; +}; +using GenericUpstreamSharedPtr = std::shared_ptr; + +class BoundGenericUpstream : public GenericUpstream, + public StreamInfo::FilterState::Object, + public std::enable_shared_from_this { +public: + BoundGenericUpstream(const CodecFactory& codec_factory, + Envoy::Upstream::TcpPoolData&& tcp_pool_data, + Network::Connection& downstream_connection); + + void onDownstreamConnectionEvent(Network::ConnectionEvent event); + + // UpstreamConnection + void onPoolSuccessImpl() override; + void onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, + absl::string_view transport_failure_reason) override; + void onEventImpl(Network::ConnectionEvent event) override; + void cleanUp(bool close_connection) override; + + // ResponseDecoderCallback + void onDecodingSuccess(StreamFramePtr response) override; + void onDecodingFailure() override; + + // GenericUpstream + void insertUpstreamRequest(uint64_t stream_id, UpstreamRequest* pending_request) override; + void removeUpstreamRequest(uint64_t stream_id) override; + + const auto& waitingResponseRequestsForTest() const { return waiting_response_requests_; } + const auto& waitingUpstreamRequestsForTest() const { return waiting_upstream_requests_; } + +private: + struct EventWatcher : public Network::ConnectionCallbacks { + EventWatcher(BoundGenericUpstream& parent) : parent_(parent) {} + BoundGenericUpstream& parent_; + + // Network::ConnectionCallbacks + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} + void onEvent(Network::ConnectionEvent downstream_event) override { + parent_.onDownstreamConnectionEvent(downstream_event); + } + }; + + Network::Connection& downstream_connection_; + + std::unique_ptr connection_event_watcher_; + + absl::optional upstream_connection_ready_; + + // Pending upstream requests that are waiting for the upstream response to be received. + absl::flat_hash_map waiting_response_requests_; + // Pending upstream requests that are waiting for the upstream connection to be ready. + absl::flat_hash_map waiting_upstream_requests_; +}; + +class OwnedGenericUpstream : public GenericUpstream { public: - UpstreamManagerImpl(UpstreamRequest& parent, Upstream::TcpPoolData&& pool); + OwnedGenericUpstream(const CodecFactory& codec_factory, + Envoy::Upstream::TcpPoolData&& tcp_pool_data); void setResponseCallback(); + // UpstreamConnection void onEventImpl(Network::ConnectionEvent event) override; void onPoolSuccessImpl() override; void onPoolFailureImpl(ConnectionPool::PoolFailureReason reason, absl::string_view transport_failure_reason) override; - UpstreamRequest& parent_; + // ResponseDecoderCallback + void onDecodingSuccess(StreamFramePtr response) override; + void onDecodingFailure() override; + + // GenericUpstream + void insertUpstreamRequest(uint64_t stream_id, UpstreamRequest* pending_request) override; + void removeUpstreamRequest(uint64_t) override {} + +private: + UpstreamRequest* upstream_request_{}; }; -class UpstreamRequest : public UpstreamBindingCallback, - public LinkedObject, +class UpstreamRequest : public LinkedObject, public Envoy::Event::DeferredDeletable, - public RequestEncoderCallback, - public PendingResponseCallback, + public EncodingCallbacks, Logger::Loggable { public: - UpstreamRequest(RouterFilter& parent, absl::optional tcp_data); + UpstreamRequest(RouterFilter& parent, GenericUpstreamSharedPtr generic_upstream); void startStream(); void resetStream(StreamResetReason reason); @@ -73,67 +149,80 @@ class UpstreamRequest : public UpstreamBindingCallback, // Called when the stream has been reset or completed. void deferredDelete(); - // UpstreamBindingCallback - void onBindFailure(ConnectionPool::PoolFailureReason reason, - absl::string_view transport_failure_reason, - Upstream::HostDescriptionConstSharedPtr host) override; - void onBindSuccess(Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr host) override; + void onUpstreamFailure(ConnectionPool::PoolFailureReason reason, + absl::string_view transport_failure_reason, + Upstream::HostDescriptionConstSharedPtr host); + void onUpstreamSuccess(Upstream::HostDescriptionConstSharedPtr host); - // PendingResponseCallback - void onDecodingSuccess(ResponsePtr response, ExtendedOptions options) override; - void onDecodingFailure() override; - void writeToConnection(Buffer::Instance& buffer) override; - OptRef connection() override; - void onConnectionClose(Network::ConnectionEvent event) override; + void onConnectionClose(Network::ConnectionEvent event); + + void onDecodingSuccess(StreamFramePtr response); + void onDecodingFailure(); // RequestEncoderCallback - void onEncodingSuccess(Buffer::Instance& buffer) override; + void onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) override; void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host); void encodeBufferToUpstream(Buffer::Instance& buffer); - bool stream_reset_{}; + void sendRequestStartToUpstream(); + void sendRequestFrameToUpstream(); RouterFilter& parent_; - DecoderFilterCallback& decoder_callbacks_; - uint64_t stream_id_{}; - bool wait_response_{}; - absl::optional tcp_pool_data_; - std::unique_ptr upstream_manager_; - - Network::ClientConnection* upstream_conn_{}; + GenericUpstreamSharedPtr generic_upstream_; Upstream::HostDescriptionConstSharedPtr upstream_host_; - bool response_complete_{}; - Buffer::OwnedImpl upstream_request_buffer_; StreamInfo::StreamInfoImpl stream_info_; - OptRef tracing_config_; Tracing::SpanPtr span_; + + // One of these flags should be set to true when the request is complete. + bool stream_reset_{}; + bool response_complete_{}; + + bool expects_response_{}; + + bool request_stream_header_sent_{}; + bool response_stream_header_received_{}; }; using UpstreamRequestPtr = std::unique_ptr; +class RouterConfig { +public: + RouterConfig(const envoy::extensions::filters::network::generic_proxy::router::v3::Router& config) + : bind_upstream_connection_(config.bind_upstream_connection()) {} + + bool bindUpstreamConnection() const { return bind_upstream_connection_; } + +private: + const bool bind_upstream_connection_{}; +}; +using RouterConfigSharedPtr = std::shared_ptr; + class RouterFilter : public DecoderFilter, public Upstream::LoadBalancerContextBase, + public StreamFrameHandler, Logger::Loggable { public: - RouterFilter(Server::Configuration::FactoryContext& context) : context_(context) {} + RouterFilter(RouterConfigSharedPtr config, Server::Configuration::FactoryContext& context) + : config_(std::move(config)), context_(context) {} // DecoderFilter void onDestroy() override; void setDecoderFilterCallbacks(DecoderFilterCallback& callbacks) override { callbacks_ = &callbacks; - protocol_options_ = callbacks_->downstreamCodec().protocolOptions(); + // Set handler for following request frames. + callbacks_->setRequestFramesHandler(*this); } - FilterStatus onStreamDecoded(Request& request) override; + FilterStatus onStreamDecoded(StreamRequest& request) override; - void onUpstreamResponse(ResponsePtr response, ExtendedOptions options); + void onResponseStart(StreamResponsePtr response); + void onResponseFrame(StreamFramePtr frame); void completeDirectly(); void onUpstreamRequestReset(UpstreamRequest& upstream_request, StreamResetReason reason); @@ -147,6 +236,9 @@ class RouterFilter : public DecoderFilter, const Envoy::Router::MetadataMatchCriteria* metadataMatchCriteria() override; const Network::Connection* downstreamConnection() const override; + // StreamFrameHandler + void onStreamFrame(StreamFramePtr frame) override; + private: friend class UpstreamRequest; friend class UpstreamManagerImpl; @@ -162,17 +254,17 @@ class RouterFilter : public DecoderFilter, const RouteEntry* route_entry_{}; Upstream::ClusterInfoConstSharedPtr cluster_; - Request* request_{}; + Request* request_stream_{}; + std::list request_stream_frames_; + bool request_stream_end_{}; Envoy::Router::MetadataMatchCriteriaConstPtr metadata_match_; - RequestEncoderPtr request_encoder_; - std::list upstream_requests_; DecoderFilterCallback* callbacks_{}; - ProtocolOptions protocol_options_; + RouterConfigSharedPtr config_; Server::Configuration::FactoryContext& context_; }; diff --git a/contrib/generic_proxy/filters/network/source/tracing.cc b/contrib/generic_proxy/filters/network/source/tracing.cc new file mode 100644 index 000000000000..10d0d7bd4af4 --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/tracing.cc @@ -0,0 +1,24 @@ +#include "contrib/generic_proxy/filters/network/source/tracing.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { + +absl::string_view TraceContextBridge::protocol() const { return request_.protocol(); } +absl::string_view TraceContextBridge::host() const { return request_.host(); } +absl::string_view TraceContextBridge::path() const { return request_.path(); } +absl::string_view TraceContextBridge::method() const { return request_.method(); } +void TraceContextBridge::forEach(IterateCallback callback) const { request_.forEach(callback); } +absl::optional TraceContextBridge::get(absl::string_view key) const { + return request_.get(key); +} +void TraceContextBridge::set(absl::string_view key, absl::string_view val) { + request_.set(key, val); +} +void TraceContextBridge::remove(absl::string_view key) { request_.erase(key); } + +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/source/tracing.h b/contrib/generic_proxy/filters/network/source/tracing.h new file mode 100644 index 000000000000..986bd8fe3dc6 --- /dev/null +++ b/contrib/generic_proxy/filters/network/source/tracing.h @@ -0,0 +1,36 @@ +#pragma once + +#include "envoy/tracing/trace_context.h" + +#include "contrib/generic_proxy/filters/network/source/interface/stream.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { + +/* + * Simple wrapper around a StreamRequest that provides the TraceContext interface. + */ +class TraceContextBridge : public Tracing::TraceContext { +public: + TraceContextBridge(StreamRequest& request) : request_(request) {} + + // Tracing::TraceContext + absl::string_view protocol() const override; + absl::string_view host() const override; + absl::string_view path() const override; + absl::string_view method() const override; + void forEach(IterateCallback callback) const override; + absl::optional get(absl::string_view key) const override; + void set(absl::string_view key, absl::string_view val) override; + void remove(absl::string_view key) override; + +private: + StreamRequest& request_; +}; + +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/source/upstream.cc b/contrib/generic_proxy/filters/network/source/upstream.cc index cd8aa1f33cf4..d92ef81d1456 100644 --- a/contrib/generic_proxy/filters/network/source/upstream.cc +++ b/contrib/generic_proxy/filters/network/source/upstream.cc @@ -8,7 +8,14 @@ namespace GenericProxy { UpstreamConnection::~UpstreamConnection() { // Do clean up here again to ensure the cleanUp is called. This is safe to call // multiple times because of the is_cleand_up_ flag. - cleanUp(true); + this->cleanUp(true); +} + +void UpstreamConnection::initialize() { + if (!initialized_) { + initialized_ = true; + newConnection(); + } } void UpstreamConnection::cleanUp(bool close_connection) { @@ -37,15 +44,14 @@ void UpstreamConnection::cleanUp(bool close_connection) { } } -void UpstreamConnection::onUpstreamData(Buffer::Instance& data, bool) { +void UpstreamConnection::onUpstreamData(Buffer::Instance& data, bool end_stream) { ASSERT(!is_cleaned_up_); - ASSERT(response_decoder_ != nullptr); if (data.length() == 0) { return; } - response_decoder_->decode(data); + client_codec_->decode(data, end_stream); } void UpstreamConnection::onPoolFailure(ConnectionPool::PoolFailureReason reason, @@ -75,13 +81,7 @@ void UpstreamConnection::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& co onPoolSuccessImpl(); } -void UpstreamConnection::onEvent(Network::ConnectionEvent event) { - // Ignore events if the connection is already cleaned up. - if (is_cleaned_up_) { - return; - } - onEventImpl(event); -} +void UpstreamConnection::onEvent(Network::ConnectionEvent event) { onEventImpl(event); } } // namespace GenericProxy } // namespace NetworkFilters diff --git a/contrib/generic_proxy/filters/network/source/upstream.h b/contrib/generic_proxy/filters/network/source/upstream.h index b0a7778840af..dc88b29d72cf 100644 --- a/contrib/generic_proxy/filters/network/source/upstream.h +++ b/contrib/generic_proxy/filters/network/source/upstream.h @@ -21,7 +21,8 @@ class UpstreamConnection : public Envoy::Event::DeferredDeletable, void newConnection() { tcp_pool_handle_ = tcp_pool_data_.newConnection(*this); } - void cleanUp(bool close_connection); + void initialize(); + virtual void cleanUp(bool close_connection); // Tcp::ConnectionPool::Callbacks void onPoolFailure(ConnectionPool::PoolFailureReason, absl::string_view, @@ -35,9 +36,11 @@ class UpstreamConnection : public Envoy::Event::DeferredDeletable, void onUpstreamData(Buffer::Instance& data, bool end_stream) override; void onEvent(Network::ConnectionEvent) override; + ClientCodecPtr& clientCodec() { return client_codec_; } + protected: - UpstreamConnection(Upstream::TcpPoolData&& tcp_pool_data, ResponseDecoderPtr&& response_decoder) - : tcp_pool_data_(std::move(tcp_pool_data)), response_decoder_(std::move(response_decoder)) {} + UpstreamConnection(Upstream::TcpPoolData&& tcp_pool_data, ClientCodecPtr&& client_codec) + : tcp_pool_data_(std::move(tcp_pool_data)), client_codec_(std::move(client_codec)) {} virtual void onEventImpl(Network::ConnectionEvent event) PURE; virtual void onPoolSuccessImpl() PURE; @@ -45,9 +48,12 @@ class UpstreamConnection : public Envoy::Event::DeferredDeletable, absl::string_view transport_failure_reason) PURE; Upstream::TcpPoolData tcp_pool_data_; - ResponseDecoderPtr response_decoder_; + ClientCodecPtr client_codec_; bool is_cleaned_up_{}; + // Whether the upstream connection is created. This will be set to true when the initialize() + // is called. + bool initialized_{}; Tcp::ConnectionPool::Cancellable* tcp_pool_handle_{}; Tcp::ConnectionPool::ConnectionDataPtr owned_conn_data_; diff --git a/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc b/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc index 9e5a6c8b8401..6a4e3abb8614 100644 --- a/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc @@ -68,21 +68,17 @@ TEST(DubboRequestTest, DubboRequestTest) { EXPECT_EQ("fake_service", request.host()); EXPECT_EQ("fake_service", request.path()); EXPECT_EQ("fake_method", request.method()); - EXPECT_EQ("fake_version", request.getByKey("version").value()); + EXPECT_EQ("fake_version", request.get("version").value()); } // Get and set headers. { - EXPECT_EQ("fake_group", request.getByKey("group").value()); + EXPECT_EQ("fake_group", request.get("group").value()); - EXPECT_EQ(false, request.getByKey("custom_key").has_value()); + EXPECT_EQ(false, request.get("custom_key").has_value()); - request.setByKey("custom_key", "custom_value"); - EXPECT_EQ("custom_value", request.getByKey("custom_key").value()); - request.setByReference("custom_key1", "custom_value1"); - EXPECT_EQ("custom_value1", request.getByKey("custom_key1").value()); - request.setByReferenceKey("custom_key2", "custom_value2"); - EXPECT_EQ("custom_value2", request.getByKey("custom_key2").value()); + request.set("custom_key", "custom_value"); + EXPECT_EQ("custom_value", request.get("custom_key").value()); } // Iterate headers. @@ -92,8 +88,8 @@ TEST(DubboRequestTest, DubboRequestTest) { attachment_size++; return true; }); - // Version is not part of attachments. So there are only 4 attachments. - EXPECT_EQ(4, attachment_size); + // Version is not part of attachments. So there are only 2 attachments. + EXPECT_EQ(2, attachment_size); } } @@ -111,73 +107,61 @@ TEST(DubboResponseTest, DubboResponseTest) { { DubboResponse response( createDubboResponse(request, ResponseStatus::Ok, RpcResponseType::ResponseWithValue)); - EXPECT_EQ(StatusCode::kOk, response.status().code()); + EXPECT_EQ(20, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::Ok, RpcResponseType::ResponseWithException)); - EXPECT_EQ(StatusCode::kUnavailable, response.status().code()); - EXPECT_EQ("exception_via_upstream", response.status().message()); + EXPECT_EQ(20, response.status().code()); } { DubboResponse response(createDubboResponse( request, ResponseStatus::Ok, RpcResponseType::ResponseWithExceptionWithAttachments)); - EXPECT_EQ(StatusCode::kUnavailable, response.status().code()); - EXPECT_EQ("exception_via_upstream", response.status().message()); + EXPECT_EQ(20, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::ClientTimeout, absl::nullopt)); - EXPECT_EQ(StatusCode::kUnknown, response.status().code()); - EXPECT_EQ("ClientTimeout", response.status().message()); + EXPECT_EQ(30, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::ServerTimeout, absl::nullopt)); - EXPECT_EQ(StatusCode::kUnknown, response.status().code()); - EXPECT_EQ(StatusCode::kUnknown, response.status().code()); - EXPECT_EQ("ServerTimeout", response.status().message()); + EXPECT_EQ(31, response.status().code()); } { DubboResponse response(createDubboResponse(request, ResponseStatus::BadRequest, absl::nullopt)); - EXPECT_EQ(StatusCode::kInvalidArgument, response.status().code()); - EXPECT_EQ("BadRequest", response.status().message()); + EXPECT_EQ(40, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::BadResponse, absl::nullopt)); - EXPECT_EQ(StatusCode::kUnknown, response.status().code()); - EXPECT_EQ("BadResponse", response.status().message()); + EXPECT_EQ(50, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::ServiceNotFound, absl::nullopt)); - EXPECT_EQ(StatusCode::kNotFound, response.status().code()); - EXPECT_EQ("ServiceNotFound", response.status().message()); + EXPECT_EQ(60, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::ServiceError, absl::nullopt)); - EXPECT_EQ(StatusCode::kUnavailable, response.status().code()); - EXPECT_EQ("ServiceError", response.status().message()); + EXPECT_EQ(70, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::ServerError, absl::nullopt)); - EXPECT_EQ(StatusCode::kUnavailable, response.status().code()); - EXPECT_EQ("ServerError", response.status().message()); + EXPECT_EQ(80, response.status().code()); } { DubboResponse response( createDubboResponse(request, ResponseStatus::ClientError, absl::nullopt)); - EXPECT_EQ(StatusCode::kUnavailable, response.status().code()); - EXPECT_EQ("ClientError", response.status().message()); + EXPECT_EQ(90, response.status().code()); } { DubboResponse response(createDubboResponse( request, ResponseStatus::ServerThreadpoolExhaustedError, absl::nullopt)); - EXPECT_EQ(StatusCode::kResourceExhausted, response.status().code()); - EXPECT_EQ("ServerThreadpoolExhaustedError", response.status().message()); + EXPECT_EQ(100, response.status().code()); } // Getter and setter do nothing for response. @@ -185,13 +169,9 @@ TEST(DubboResponseTest, DubboResponseTest) { DubboResponse response( createDubboResponse(request, ResponseStatus::Ok, RpcResponseType::ResponseWithValue)); - EXPECT_EQ(false, response.getByKey("custom_key").has_value()); - response.setByKey("custom_key", "custom_value"); - EXPECT_EQ(false, response.getByKey("custom_key").has_value()); - response.setByReference("custom_key", "custom_value"); - EXPECT_EQ(false, response.getByKey("custom_key").has_value()); - response.setByReferenceKey("custom_key", "custom_value"); - EXPECT_EQ(false, response.getByKey("custom_key").has_value()); + EXPECT_EQ(false, response.get("custom_key").has_value()); + response.set("custom_key", "custom_value"); + EXPECT_EQ(false, response.get("custom_key").has_value()); } // Iterate headers. @@ -208,49 +188,56 @@ TEST(DubboResponseTest, DubboResponseTest) { } } -TEST(RequestDecoderTest, RequestDecoderTest) { +TEST(DubboServerCodecTest, DubboServerCodecTest) { auto codec = std::make_unique(); codec->initilize(std::make_unique()); - MockRequestDecoderCallback callback; - DubboRequestDecoder decoder(std::move(codec)); - decoder.setDecoderCallback(callback); + MockServerCodecCallbacks callbacks; + DubboServerCodec server_codec(std::move(codec)); + server_codec.setCodecCallbacks(callbacks); auto raw_serializer = const_cast( - dynamic_cast(decoder.codec_->serializer().get())); + dynamic_cast(server_codec.codec_->serializer().get())); // Decode failure. { + server_codec.metadata_.reset(); Buffer::OwnedImpl buffer; buffer.writeBEInt(0); buffer.writeBEInt(0); - EXPECT_CALL(callback, onDecodingFailure()); - decoder.decode(buffer); + EXPECT_CALL(callbacks, onDecodingFailure()); + server_codec.decode(buffer, false); } // Waiting for header. { + server_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\xc2', 0x00})); // No enough header bytes and do nothing. - decoder.decode(buffer); + server_codec.decode(buffer, false); } // Waiting for data. { + server_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\xc2', 0x00})); buffer.writeBEInt(1); buffer.writeBEInt(8); // No enough body bytes and do nothing. - decoder.decode(buffer); + server_codec.decode(buffer, false); } // Decode request. { + server_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\xc2', 0x00})); buffer.writeBEInt(1); @@ -260,54 +247,118 @@ TEST(RequestDecoderTest, RequestDecoderTest) { EXPECT_CALL(*raw_serializer, deserializeRpcRequest(_, _)) .WillOnce(Return(ByMove(std::make_unique()))); - EXPECT_CALL(callback, onDecodingSuccess(_, _)); - decoder.decode(buffer); + EXPECT_CALL(callbacks, onDecodingSuccess(_)); + server_codec.decode(buffer, false); + } + + // Encode response. + { + + MockEncodingCallbacks encoding_callbacks; + DubboRequest request(createDubboRequst(false)); + DubboResponse response( + createDubboResponse(request, ResponseStatus::Ok, RpcResponseType::ResponseWithValue)); + + EXPECT_CALL(*raw_serializer, serializeRpcResponse(_, _)); + EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); + + server_codec.encode(response, encoding_callbacks); + } + + { + Status status = absl::OkStatus(); + DubboRequest request(createDubboRequst(false)); + + auto response = server_codec.respond(status, "", request); + auto* typed_response = static_cast(response.get()); + auto* typed_inner_response = + static_cast(&typed_response->inner_metadata_->mutableResponse()); + + EXPECT_EQ(ResponseStatus::Ok, typed_response->inner_metadata_->responseStatus()); + EXPECT_EQ(RpcResponseType::ResponseWithException, typed_inner_response->responseType().value()); + EXPECT_EQ("exception_via_proxy", typed_inner_response->localRawMessage().value()); + } + + { + Status status(StatusCode::kInvalidArgument, "test_message"); + DubboRequest request(createDubboRequst(false)); + + auto response = server_codec.respond(status, "", request); + auto* typed_response = static_cast(response.get()); + auto* typed_inner_response = + static_cast(&typed_response->inner_metadata_->mutableResponse()); + + EXPECT_EQ(ResponseStatus::BadRequest, typed_response->inner_metadata_->responseStatus()); + EXPECT_EQ(false, typed_inner_response->responseType().has_value()); + EXPECT_EQ("test_message", typed_inner_response->localRawMessage().value()); + } + + { + Status status(StatusCode::kAborted, "test_message2"); + DubboRequest request(createDubboRequst(false)); + + auto response = server_codec.respond(status, "", request); + auto* typed_response = static_cast(response.get()); + auto* typed_inner_response = + static_cast(&typed_response->inner_metadata_->mutableResponse()); + + EXPECT_EQ(ResponseStatus::ServerError, typed_response->inner_metadata_->responseStatus()); + EXPECT_EQ(false, typed_inner_response->responseType().has_value()); + EXPECT_EQ("test_message2", typed_inner_response->localRawMessage().value()); } } -TEST(ResponseDecoderTest, ResponseDecoderTest) { +TEST(DubboClientCodecTest, DubboClientCodecTest) { auto codec = std::make_unique(); codec->initilize(std::make_unique()); - MockResponseDecoderCallback callback; - DubboResponseDecoder decoder(std::move(codec)); - decoder.setDecoderCallback(callback); + MockClientCodecCallbacks callbacks; + DubboClientCodec client_codec(std::move(codec)); + client_codec.setCodecCallbacks(callbacks); auto raw_serializer = const_cast( - dynamic_cast(decoder.codec_->serializer().get())); + dynamic_cast(client_codec.codec_->serializer().get())); // Decode failure. { + client_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.writeBEInt(0); buffer.writeBEInt(0); - EXPECT_CALL(callback, onDecodingFailure()); - decoder.decode(buffer); + EXPECT_CALL(callbacks, onDecodingFailure()); + client_codec.decode(buffer, false); } // Waiting for header. { + client_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\x02', 20})); // No enough header bytes and do nothing. - decoder.decode(buffer); + client_codec.decode(buffer, false); } // Waiting for data. { + client_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\x02', 20})); buffer.writeBEInt(1); buffer.writeBEInt(8); // No enough body bytes and do nothing. - decoder.decode(buffer); + client_codec.decode(buffer, false); } // Decode response. { + client_codec.metadata_.reset(); + Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\x02', 20})); buffer.writeBEInt(1); @@ -320,119 +371,40 @@ TEST(ResponseDecoderTest, ResponseDecoderTest) { EXPECT_CALL(*raw_serializer, deserializeRpcResponse(_, _)) .WillOnce(Return(ByMove(std::move(response)))); - EXPECT_CALL(callback, onDecodingSuccess(_, _)); - decoder.decode(buffer); + EXPECT_CALL(callbacks, onDecodingSuccess(_)); + client_codec.decode(buffer, false); } -} - -TEST(RequestEncoderTest, RequestEncoderTest) { - auto codec = std::make_unique(); - codec->initilize(std::make_unique()); - - MockRequestEncoderCallback callback; - DubboRequestEncoder encoder(std::move(codec)); - auto raw_serializer = const_cast( - dynamic_cast(encoder.codec_->serializer().get())); - - // Normal request. + // Encode normal request. { + MockEncodingCallbacks encoding_callbacks; + DubboRequest request(createDubboRequst(false)); EXPECT_CALL(*raw_serializer, serializeRpcRequest(_, _)); - EXPECT_CALL(callback, onEncodingSuccess(_)); + EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); - encoder.encode(request, callback); + client_codec.encode(request, encoding_callbacks); } - // One-way request. + // Encode one-way request. { + MockEncodingCallbacks encoding_callbacks; + DubboRequest request(createDubboRequst(true)); EXPECT_CALL(*raw_serializer, serializeRpcRequest(_, _)); - EXPECT_CALL(callback, onEncodingSuccess(_)); - - encoder.encode(request, callback); - } -} - -TEST(ResponseEncoderTest, ResponseEncoderTest) { - auto codec = std::make_unique(); - codec->initilize(std::make_unique()); - - MockResponseEncoderCallback callback; - DubboResponseEncoder encoder(std::move(codec)); - - auto raw_serializer = const_cast( - dynamic_cast(encoder.codec_->serializer().get())); + EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); - // Normal response. - { - DubboRequest request(createDubboRequst(false)); - DubboResponse response( - createDubboResponse(request, ResponseStatus::Ok, RpcResponseType::ResponseWithValue)); - - EXPECT_CALL(*raw_serializer, serializeRpcResponse(_, _)); - EXPECT_CALL(callback, onEncodingSuccess(_)); - - encoder.encode(response, callback); - } -} - -TEST(DubboMessageCreatorTest, DubboMessageCreatorTest) { - DubboMessageCreator creator; - - { - Status status = absl::OkStatus(); - DubboRequest request(createDubboRequst(false)); - - auto response = creator.response(status, request); - auto* typed_response = static_cast(response.get()); - auto* typed_inner_response = - static_cast(&typed_response->inner_metadata_->mutableResponse()); - - EXPECT_EQ(ResponseStatus::Ok, typed_response->inner_metadata_->responseStatus()); - EXPECT_EQ(RpcResponseType::ResponseWithException, typed_inner_response->responseType().value()); - EXPECT_EQ("exception_via_proxy", typed_inner_response->localRawMessage().value()); - } - - { - Status status(StatusCode::kInvalidArgument, "test_message"); - DubboRequest request(createDubboRequst(false)); - - auto response = creator.response(status, request); - auto* typed_response = static_cast(response.get()); - auto* typed_inner_response = - static_cast(&typed_response->inner_metadata_->mutableResponse()); - - EXPECT_EQ(ResponseStatus::BadRequest, typed_response->inner_metadata_->responseStatus()); - EXPECT_EQ(false, typed_inner_response->responseType().has_value()); - EXPECT_EQ("test_message", typed_inner_response->localRawMessage().value()); - } - - { - Status status(StatusCode::kAborted, "test_message2"); - DubboRequest request(createDubboRequst(false)); - - auto response = creator.response(status, request); - auto* typed_response = static_cast(response.get()); - auto* typed_inner_response = - static_cast(&typed_response->inner_metadata_->mutableResponse()); - - EXPECT_EQ(ResponseStatus::ServerError, typed_response->inner_metadata_->responseStatus()); - EXPECT_EQ(false, typed_inner_response->responseType().has_value()); - EXPECT_EQ("test_message2", typed_inner_response->localRawMessage().value()); + client_codec.encode(request, encoding_callbacks); } } TEST(DubboCodecFactoryTest, DubboCodecFactoryTest) { DubboCodecFactory factory; - EXPECT_NE(nullptr, factory.messageCreator().get()); - EXPECT_NE(nullptr, factory.requestDecoder().get()); - EXPECT_NE(nullptr, factory.requestEncoder().get()); - EXPECT_NE(nullptr, factory.responseDecoder().get()); - EXPECT_NE(nullptr, factory.responseEncoder().get()); + EXPECT_NE(nullptr, factory.createClientCodec().get()); + EXPECT_NE(nullptr, factory.createServerCodec().get()); } TEST(DubboCodecFactoryConfigTest, DubboCodecFactoryConfigTest) { diff --git a/contrib/generic_proxy/filters/network/test/codecs/kafka/BUILD b/contrib/generic_proxy/filters/network/test/codecs/kafka/BUILD new file mode 100644 index 000000000000..d3c54211c8ea --- /dev/null +++ b/contrib/generic_proxy/filters/network/test/codecs/kafka/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_contrib_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +envoy_cc_test( + name = "config_test", + srcs = [ + "config_test.cc", + ], + deps = [ + "//contrib/generic_proxy/filters/network/source/codecs/kafka:config", + "//contrib/generic_proxy/filters/network/test/mocks:codec_mocks", + "//test/mocks/server:factory_context_mocks", + ], +) diff --git a/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc b/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc new file mode 100644 index 000000000000..7bee7cecd916 --- /dev/null +++ b/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc @@ -0,0 +1,282 @@ +#include +#include + +#include "test/mocks/server/factory_context.h" + +#include "contrib/generic_proxy/filters/network/source/codecs/kafka/config.h" +#include "contrib/generic_proxy/filters/network/test/mocks/codec.h" +#include "contrib/kafka/filters/network/source/external/requests.h" +#include "contrib/kafka/filters/network/source/external/responses.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace GenericProxy { +namespace Codec { +namespace Kafka { +namespace { + +using testing::NiceMock; + +TEST(KafkaCodecTest, SimpleFrameTest) { + + { + auto request = + std::make_shared>( + NetworkFilters::Kafka::RequestHeader(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, 3, + absl::nullopt), + NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); + + KafkaRequestFrame frame(request); + EXPECT_EQ(frame.frameFlags().streamFlags().streamId(), 3); + } + + { + KafkaResponseFrame frame(nullptr); + EXPECT_EQ(frame.protocol(), "kafka"); + EXPECT_EQ(frame.frameFlags().streamFlags().streamId(), 0); + } + + { + auto response = + std::make_shared>( + NetworkFilters::Kafka::ResponseMetadata(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, + 3), + NetworkFilters::Kafka::FetchResponse({}, {})); + + KafkaResponseFrame frame(response); + EXPECT_EQ(frame.frameFlags().streamFlags().streamId(), 3); + } +} + +TEST(KafkaCodecTest, KafkaRequestCallbacksTest) { + NiceMock callbacks; + + KafkaRequestCallbacks request_callbacks(callbacks); + + { + EXPECT_CALL(callbacks, onDecodingSuccess(_)); + + auto request = + std::make_shared>( + NetworkFilters::Kafka::RequestHeader(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, 3, + absl::nullopt), + NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); + + request_callbacks.onMessage(request); + } + + { + EXPECT_CALL(callbacks, onDecodingFailure()); + request_callbacks.onFailedParse(nullptr); + } +} + +TEST(KafkaCodecTest, KafkaResponseCallbacksTest) { + NiceMock callbacks; + + KafkaResponseCallbacks response_callbacks(callbacks); + + { + EXPECT_CALL(callbacks, onDecodingSuccess(_)); + + auto response = + std::make_shared>( + NetworkFilters::Kafka::ResponseMetadata(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, + 3), + NetworkFilters::Kafka::FetchResponse({}, {})); + + response_callbacks.onMessage(response); + } + + { + EXPECT_CALL(callbacks, onDecodingFailure()); + response_callbacks.onFailedParse(nullptr); + } +} + +TEST(KafkaCodecTest, KafkaServerCodecTest) { + NiceMock callbacks; + + KafkaServerCodec server_codec; + server_codec.setCodecCallbacks(callbacks); + + { + // Test respond() method. + + auto request = + std::make_shared>( + NetworkFilters::Kafka::RequestHeader(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, 3, + absl::nullopt), + NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); + + KafkaRequestFrame frame(request); + auto local_response = server_codec.respond(absl::OkStatus(), "", frame); + + EXPECT_NE(local_response, nullptr); + EXPECT_EQ(dynamic_cast(local_response.get())->response_, nullptr); + } + + { + // Test decode() method. + EXPECT_CALL(callbacks, onDecodingSuccess(_)) + .WillOnce(testing::Invoke([](StreamFramePtr request) { + EXPECT_EQ(dynamic_cast(request.get()) + ->request_->request_header_.correlation_id_, + 3); + })); + + auto request = + std::make_shared>( + NetworkFilters::Kafka::RequestHeader(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, 3, + absl::nullopt), + NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); + + Buffer::OwnedImpl buffer; + const uint32_t size = htobe32(request->computeSize()); + buffer.add(&size, sizeof(size)); // Encode data length. + + request->encode(buffer); + server_codec.decode(buffer, false); + } + + { + // Test encode() method with non-response frame. + + NiceMock encoding_callbacks; + + auto request = + std::make_shared>( + NetworkFilters::Kafka::RequestHeader(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, 3, + absl::nullopt), + NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); + KafkaRequestFrame request_frame(request); + + // Do nothiing. + server_codec.encode(request_frame, encoding_callbacks); + } + + { + // Test encode() method without actual response. + + NiceMock encoding_callbacks; + NiceMock mock_connection; + + KafkaResponseFrame response_frame(nullptr); + + // Expect close connection. + EXPECT_CALL(callbacks, connection()) + .WillOnce(testing::Return(makeOptRef(mock_connection))); + EXPECT_CALL(mock_connection, close(Network::ConnectionCloseType::FlushWrite)); + + server_codec.encode(response_frame, encoding_callbacks); + } + + { + // Test encode() method with response. + + NiceMock encoding_callbacks; + + auto response = + std::make_shared>( + NetworkFilters::Kafka::ResponseMetadata(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, + 3), + NetworkFilters::Kafka::FetchResponse({}, {})); + + KafkaResponseFrame response_frame(response); + + Envoy::Buffer::OwnedImpl dst_buffer; + const uint32_t size = htobe32(response->computeSize()); + dst_buffer.add(&size, sizeof(size)); // Encode data length. + response->encode(dst_buffer); + + EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) + .WillOnce(testing::Invoke([&](Buffer::Instance& buffer, bool) { + EXPECT_EQ(buffer.toString(), dst_buffer.toString()); + })); + server_codec.encode(response_frame, encoding_callbacks); + } +} + +TEST(KafkaCodecTest, KafkaClientCodecTest) { + NiceMock callbacks; + + KafkaClientCodec client_codec; + client_codec.setCodecCallbacks(callbacks); + + { + // Test decode() method. + EXPECT_CALL(callbacks, onDecodingSuccess(_)) + .WillOnce(testing::Invoke([](StreamFramePtr response) { + EXPECT_EQ(dynamic_cast(response.get()) + ->response_->metadata_.correlation_id_, + 3); + })); + + auto response = + std::make_shared>( + NetworkFilters::Kafka::ResponseMetadata(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, + 3), + NetworkFilters::Kafka::FetchResponse({}, {})); + + Buffer::OwnedImpl buffer; + const uint32_t size = htobe32(response->computeSize()); + buffer.add(&size, sizeof(size)); // Encode data length. + + response->encode(buffer); + + client_codec.response_decoder_->expectResponse(3, 0, 0); + client_codec.decode(buffer, false); + } + + { + // Test encode() method with non-request frame. + + NiceMock encoding_callbacks; + + auto response = + std::make_shared>( + NetworkFilters::Kafka::ResponseMetadata(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, + 3), + NetworkFilters::Kafka::FetchResponse({}, {})); + KafkaResponseFrame response_frame(response); + + // Do nothiing. + client_codec.encode(response_frame, encoding_callbacks); + } + + { + // Test encode() method with request. + + NiceMock encoding_callbacks; + + auto request = + std::make_shared>( + NetworkFilters::Kafka::RequestHeader(NetworkFilters::Kafka::FETCH_REQUEST_API_KEY, 0, 3, + absl::nullopt), + NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); + + KafkaRequestFrame request_frame(request); + + Envoy::Buffer::OwnedImpl dst_buffer; + const uint32_t size = htobe32(request->computeSize()); + dst_buffer.add(&size, sizeof(size)); // Encode data length. + request->encode(dst_buffer); + + EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) + .WillOnce(testing::Invoke([&](Buffer::Instance& buffer, bool) { + EXPECT_EQ(buffer.toString(), dst_buffer.toString()); + })); + + client_codec.encode(request_frame, encoding_callbacks); + } +} + +} // namespace +} // namespace Kafka +} // namespace Codec +} // namespace GenericProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/generic_proxy/filters/network/test/config_test.cc b/contrib/generic_proxy/filters/network/test/config_test.cc index 276666182325..873d5c2fc82d 100644 --- a/contrib/generic_proxy/filters/network/test/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/config_test.cc @@ -128,9 +128,8 @@ version_info: "1" factory_context.server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ ->onConfigUpdate(decoded_resources.refvec_, response.version_info()) .ok()); - auto message_ptr = - factory_context.admin_.config_tracker_.config_tracker_callbacks_["genericrds_routes"]( - universal_name_matcher); + auto message_ptr = factory_context.server_factory_context_.admin_.config_tracker_ + .config_tracker_callbacks_["genericrds_routes"](universal_name_matcher); const auto& dump = TestUtility::downcastAndValidate(*message_ptr); EXPECT_EQ(1, dump.dynamic_route_configs().size()); @@ -274,6 +273,7 @@ TEST(BasicFilterConfigTest, CreatingFilterFactories) { NiceMock factory_context; ProtobufWkt::RepeatedPtrField filters_proto_config; + envoy::config::core::v3::TypedExtensionConfig codec_config; const std::string yaml_config_0 = R"EOF( name: mock_generic_proxy_filter_name_0 @@ -309,9 +309,10 @@ TEST(BasicFilterConfigTest, CreatingFilterFactories) { // No terminal filter. { - EXPECT_THROW_WITH_MESSAGE( - Factory::filtersFactoryFromProto(filters_proto_config, "test", factory_context), - EnvoyException, "A terminal L7 filter is necessary for generic proxy"); + EXPECT_THROW_WITH_MESSAGE(Factory::filtersFactoryFromProto(filters_proto_config, codec_config, + "test", factory_context), + EnvoyException, + "A terminal L7 filter is necessary for generic proxy"); } // Error terminal filter position. @@ -319,17 +320,32 @@ TEST(BasicFilterConfigTest, CreatingFilterFactories) { ON_CALL(mock_filter_config_0, isTerminalFilter()).WillByDefault(Return(true)); EXPECT_THROW_WITH_MESSAGE( - Factory::filtersFactoryFromProto(filters_proto_config, "test", factory_context), + Factory::filtersFactoryFromProto(filters_proto_config, codec_config, "test", + factory_context), EnvoyException, "Terminal filter: mock_generic_proxy_filter_name_0 must be the last generic L7 " "filter"); } + // Codec validation error. + { + ON_CALL(mock_filter_config_0, isTerminalFilter()).WillByDefault(Return(false)); + ON_CALL(mock_filter_config_0, validateCodec(_)) + .WillByDefault(Return(absl::InvalidArgumentError("codec validation error"))); + + EXPECT_THROW_WITH_MESSAGE(Factory::filtersFactoryFromProto(filters_proto_config, codec_config, + "test", factory_context), + EnvoyException, "codec validation error"); + } + { ON_CALL(mock_filter_config_0, isTerminalFilter()).WillByDefault(Return(false)); ON_CALL(mock_filter_config_1, isTerminalFilter()).WillByDefault(Return(true)); - auto factories = - Factory::filtersFactoryFromProto(filters_proto_config, "test", factory_context); + ON_CALL(mock_filter_config_0, validateCodec(_)).WillByDefault(Return(absl::OkStatus())); + ON_CALL(mock_filter_config_1, validateCodec(_)).WillByDefault(Return(absl::OkStatus())); + + auto factories = Factory::filtersFactoryFromProto(filters_proto_config, codec_config, "test", + factory_context); EXPECT_EQ(2, factories.size()); } } @@ -364,7 +380,9 @@ TEST(BasicFilterConfigTest, TestConfigurationWithTracing) { NiceMock codec_factory_config; Registry::InjectFactory registration(codec_factory_config); + NiceMock listener_info; NiceMock factory_context; + ON_CALL(factory_context, listenerInfo()).WillByDefault(testing::ReturnRef(listener_info)); factory_context.server_factory_context_.cluster_manager_.initializeClusters({"zipkin"}, {}); factory_context.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( {"zipkin"}); diff --git a/contrib/generic_proxy/filters/network/test/fake_codec.cc b/contrib/generic_proxy/filters/network/test/fake_codec.cc index 85edb7e88c7d..ab838ed1b610 100644 --- a/contrib/generic_proxy/filters/network/test/fake_codec.cc +++ b/contrib/generic_proxy/filters/network/test/fake_codec.cc @@ -7,30 +7,18 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { -RequestDecoderPtr FakeStreamCodecFactory::requestDecoder() const { - return std::make_unique(); +ServerCodecPtr FakeStreamCodecFactory::createServerCodec() const { + return std::make_unique(); } -ResponseDecoderPtr FakeStreamCodecFactory::responseDecoder() const { - return std::make_unique(); +ClientCodecPtr FakeStreamCodecFactory::createClientCodec() const { + return std::make_unique(); } -RequestEncoderPtr FakeStreamCodecFactory::requestEncoder() const { - return std::make_unique(); -} -ResponseEncoderPtr FakeStreamCodecFactory::responseEncoder() const { - return std::make_unique(); -} -MessageCreatorPtr FakeStreamCodecFactory::messageCreator() const { - return std::make_unique(); -} -ProtocolOptions FakeStreamCodecFactory::protocolOptions() const { return protocol_options_; } CodecFactoryPtr FakeStreamCodecFactoryConfig::createCodecFactory(const Protobuf::Message&, Envoy::Server::Configuration::FactoryContext&) { - auto factory = std::make_unique(); - factory->protocol_options_ = protocol_options_; - return factory; + return std::make_unique(); } } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/test/fake_codec.h b/contrib/generic_proxy/filters/network/test/fake_codec.h index de3f9ae35678..702abb47d25e 100644 --- a/contrib/generic_proxy/filters/network/test/fake_codec.h +++ b/contrib/generic_proxy/filters/network/test/fake_codec.h @@ -18,20 +18,20 @@ template class FakeStreamBase : public InterfaceType { callback(pair.first, pair.second); } } - absl::optional getByKey(absl::string_view key) const override { + absl::optional get(absl::string_view key) const override { auto iter = data_.find(key); if (iter == data_.end()) { return absl::nullopt; } return absl::make_optional(iter->second); } - void setByKey(absl::string_view key, absl::string_view val) override { - data_[key] = std::string(val); - } - void setByReferenceKey(absl::string_view key, absl::string_view val) override { - setByKey(key, val); - } - void setByReference(absl::string_view key, absl::string_view val) override { setByKey(key, val); } + void set(absl::string_view key, absl::string_view val) override { data_[key] = std::string(val); } + void erase(absl::string_view key) override { data_.erase(key); } + + // StreamFrame + FrameFlags frameFlags() const override { return stream_frame_flags_; } + + FrameFlags stream_frame_flags_; absl::flat_hash_map data_; }; @@ -63,13 +63,14 @@ class FakeStreamCodecFactory : public CodecFactory { class FakeResponse : public FakeStreamBase { public: absl::string_view protocol() const override { return protocol_; } - Status status() const override { return status_; } + StreamStatus status() const override { return status_; } std::string protocol_; - Status status_; + StreamStatus status_; + std::string message_; }; - class FakeRequestDecoder : public RequestDecoder { + class FakeServerCodec : public ServerCodec { public: bool parseRequestBody() { std::string body(message_size_.value(), 0); @@ -93,21 +94,29 @@ class FakeStreamCodecFactory : public CodecFactory { request->data_.emplace(pair); } absl::optional stream_id; - bool wait_response = true; + bool one_way_stream = false; if (auto it = request->data_.find("stream_id"); it != request->data_.end()) { stream_id = std::stoull(it->second); } - if (auto it = request->data_.find("wait_response"); it != request->data_.end()) { - wait_response = it->second == "true"; + if (auto it = request->data_.find("one_way"); it != request->data_.end()) { + one_way_stream = it->second == "true"; } - ExtendedOptions request_options{stream_id, wait_response, false, false}; - callback_->onDecodingSuccess(std::move(request), request_options); + // Mock multiple frames in one request. + bool end_stream = true; + if (auto it = request->data_.find("end_stream"); it != request->data_.end()) { + end_stream = it->second == "true"; + } + + request->stream_frame_flags_ = + FrameFlags(StreamFlags(stream_id.value_or(0), one_way_stream, false, false), end_stream); + + callback_->onDecodingSuccess(std::move(request)); return true; } - void setDecoderCallback(RequestDecoderCallback& callback) override { callback_ = &callback; } - void decode(Buffer::Instance& buffer) override { + void setCodecCallbacks(ServerCodecCallbacks& callback) override { callback_ = &callback; } + void decode(Buffer::Instance& buffer, bool) override { buffer_.move(buffer); while (true) { if (!message_size_.has_value()) { @@ -131,12 +140,40 @@ class FakeStreamCodecFactory : public CodecFactory { } } + void encode(const StreamFrame& response, EncodingCallbacks& callback) override { + const FakeResponse* typed_response = dynamic_cast(&response); + ASSERT(typed_response != nullptr); + + std::string body; + body.reserve(512); + body = typed_response->protocol_ + "|" + typed_response->message_ + "|"; + for (const auto& pair : typed_response->data_) { + body += pair.first + ":" + pair.second + ";"; + } + // Additional 4 bytes for status. + encoding_buffer_.writeBEInt(body.size() + 4); + encoding_buffer_.writeBEInt(static_cast(typed_response->status_.code())); + encoding_buffer_.add(body); + + callback.onEncodingSuccess(encoding_buffer_, response.frameFlags().endStream()); + } + + ResponsePtr respond(Status status, absl::string_view, const Request&) override { + auto response = std::make_unique(); + response->status_ = {static_cast(status.code()), + status.code() == absl::StatusCode::kOk}; + response->message_ = status.message(); + response->protocol_ = "fake_protocol_for_test"; + return response; + } + absl::optional message_size_; Buffer::OwnedImpl buffer_; - RequestDecoderCallback* callback_{}; + Buffer::OwnedImpl encoding_buffer_; + ServerCodecCallbacks* callback_{}; }; - class FakeResponseDecoder : public ResponseDecoder { + class FakeClientCodec : public ClientCodec { public: bool parseResponseBody() { int32_t status_code = buffer_.peekBEInt(); @@ -155,7 +192,8 @@ class FakeStreamCodecFactory : public CodecFactory { } auto response = std::make_unique(); - response->status_ = Status(StatusCode(status_code), result[1]); + response->status_ = {static_cast(status_code), + static_cast(status_code) == absl::StatusCode::kOk}; response->protocol_ = std::string(result[0]); for (absl::string_view pair_str : absl::StrSplit(result[2], ';', absl::SkipEmpty())) { auto pair = absl::StrSplit(pair_str, absl::MaxSplits(':', 1)); @@ -170,14 +208,22 @@ class FakeStreamCodecFactory : public CodecFactory { if (auto it = response->data_.find("close_connection"); it != response->data_.end()) { close_connection = it->second == "true"; } - ExtendedOptions response_options{stream_id, false, close_connection, false}; - callback_->onDecodingSuccess(std::move(response), response_options); + // Mock multiple frames in one response. + bool end_stream = true; + if (auto it = response->data_.find("end_stream"); it != response->data_.end()) { + end_stream = it->second == "true"; + } + + response->stream_frame_flags_ = FrameFlags( + StreamFlags(stream_id.value_or(0), false, close_connection, false), end_stream); + + callback_->onDecodingSuccess(std::move(response)); return true; } - void setDecoderCallback(ResponseDecoderCallback& callback) override { callback_ = &callback; } - void decode(Buffer::Instance& buffer) override { + void setCodecCallbacks(ClientCodecCallbacks& callback) override { callback_ = &callback; } + void decode(Buffer::Instance& buffer, bool) override { buffer_.move(buffer); while (true) { if (!message_size_.has_value()) { @@ -207,14 +253,7 @@ class FakeStreamCodecFactory : public CodecFactory { } } - absl::optional message_size_; - Buffer::OwnedImpl buffer_; - ResponseDecoderCallback* callback_{}; - }; - - class FakeRequestEncoder : public RequestEncoder { - public: - void encode(const Request& request, RequestEncoderCallback& callback) override { + void encode(const StreamFrame& request, EncodingCallbacks& callback) override { const FakeRequest* typed_request = dynamic_cast(&request); ASSERT(typed_request != nullptr); @@ -225,56 +264,20 @@ class FakeStreamCodecFactory : public CodecFactory { for (const auto& pair : typed_request->data_) { body += pair.first + ":" + pair.second + ";"; } - buffer_.writeBEInt(body.size()); - buffer_.add(body); + encoding_buffer_.writeBEInt(body.size()); + encoding_buffer_.add(body); - callback.onEncodingSuccess(buffer_); + callback.onEncodingSuccess(encoding_buffer_, request.frameFlags().endStream()); } + absl::optional message_size_; Buffer::OwnedImpl buffer_; + Buffer::OwnedImpl encoding_buffer_; + ClientCodecCallbacks* callback_{}; }; - class FakeResponseEncoder : public ResponseEncoder { - public: - void encode(const Response& response, ResponseEncoderCallback& callback) override { - const FakeResponse* typed_response = dynamic_cast(&response); - ASSERT(typed_response != nullptr); - - std::string body; - body.reserve(512); - body = typed_response->protocol_ + "|" + std::string(typed_response->status_.message()) + "|"; - for (const auto& pair : typed_response->data_) { - body += pair.first + ":" + pair.second + ";"; - } - // Additional 4 bytes for status. - buffer_.writeBEInt(body.size() + 4); - buffer_.writeBEInt(static_cast(typed_response->status_.raw_code())); - buffer_.add(body); - - callback.onEncodingSuccess(buffer_); - } - - Buffer::OwnedImpl buffer_; - }; - - class FakeMessageCreator : public MessageCreator { - public: - ResponsePtr response(Status status, const Request&) override { - auto response = std::make_unique(); - response->status_ = status; - response->protocol_ = "fake_protocol_for_test"; - return response; - } - }; - - RequestDecoderPtr requestDecoder() const override; - ResponseDecoderPtr responseDecoder() const override; - RequestEncoderPtr requestEncoder() const override; - ResponseEncoderPtr responseEncoder() const override; - MessageCreatorPtr messageCreator() const override; - ProtocolOptions protocolOptions() const override; - - ProtocolOptions protocol_options_; + ServerCodecPtr createServerCodec() const override; + ClientCodecPtr createClientCodec() const override; }; class FakeStreamCodecFactoryConfig : public CodecFactoryConfig { @@ -288,8 +291,6 @@ class FakeStreamCodecFactoryConfig : public CodecFactoryConfig { } std::set configTypes() override { return {"envoy.generic_proxy.codecs.fake.type"}; } std::string name() const override { return "envoy.generic_proxy.codecs.fake"; } - - ProtocolOptions protocol_options_; }; } // namespace GenericProxy diff --git a/contrib/generic_proxy/filters/network/test/integration_test.cc b/contrib/generic_proxy/filters/network/test/integration_test.cc index c919f738c21e..65a15c246caf 100644 --- a/contrib/generic_proxy/filters/network/test/integration_test.cc +++ b/contrib/generic_proxy/filters/network/test/integration_test.cc @@ -54,8 +54,8 @@ class IntegrationTest : public testing::TestWithParamdecode(data); + Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override { + parent_.client_codec_->decode(data, end_stream); return Network::FilterStatus::Continue; } Network::FilterStatus onNewConnection() override { return Network::FilterStatus::Continue; } @@ -65,37 +65,46 @@ class IntegrationTest : public testing::TestWithParam; - struct TestRequestEncoderCallback : public RequestEncoderCallback { - void onEncodingSuccess(Buffer::Instance& buffer) override { - buffer_.move(buffer); - complete_ = true; - request_bytes_ = buffer_.length(); - } - bool complete_{}; - size_t request_bytes_{}; + struct TestRequestEncoderCallback : public EncodingCallbacks { + void onEncodingSuccess(Buffer::Instance& buffer, bool) override { buffer_.move(buffer); } Buffer::OwnedImpl buffer_; }; using TestRequestEncoderCallbackSharedPtr = std::shared_ptr; - struct TestResponseEncoderCallback : public ResponseEncoderCallback { - void onEncodingSuccess(Buffer::Instance& buffer) override { - buffer_.move(buffer); - complete_ = true; - response_bytes_ = buffer_.length(); - } - bool complete_{}; - size_t response_bytes_{}; + struct TestResponseEncoderCallback : public EncodingCallbacks { + void onEncodingSuccess(Buffer::Instance& buffer, bool) override { buffer_.move(buffer); } Buffer::OwnedImpl buffer_; }; using TestResponseEncoderCallbackSharedPtr = std::shared_ptr; - struct TestResponseDecoderCallback : public ResponseDecoderCallback { + struct TestResponseDecoderCallback : public ClientCodecCallbacks { TestResponseDecoderCallback(IntegrationTest& parent) : parent_(parent) {} - void onDecodingSuccess(ResponsePtr response, ExtendedOptions) override { - response_ = std::move(response); - complete_ = true; - parent_.integration_->dispatcher_->exit(); + struct SingleResponse { + bool end_stream_{}; + ResponsePtr response_; + std::list response_frames_; + }; + + void onDecodingSuccess(StreamFramePtr response_frame) override { + auto& response = responses_[response_frame->frameFlags().streamFlags().streamId()]; + + ASSERT(!response.end_stream_); + response.end_stream_ = response_frame->frameFlags().endStream(); + + if (response.response_ != nullptr) { + response.response_frames_.push_back(std::move(response_frame)); + } else { + ASSERT(response.response_frames_.empty()); + StreamFramePtrHelper helper(std::move(response_frame)); + ASSERT(helper.typed_frame_ != nullptr); + response.response_ = std::move(helper.typed_frame_); + } + + // Exit dispatcher if we have received all the expected response frames. + if (responses_[waiting_for_stream_id_].end_stream_) { + parent_.integration_->dispatcher_->exit(); + } } void onDecodingFailure() override {} void writeToConnection(Buffer::Instance&) override {} @@ -106,8 +115,8 @@ class IntegrationTest : public testing::TestWithParam responses_; IntegrationTest& parent_; }; using TestResponseDecoderCallbackSharedPtr = std::shared_ptr; @@ -116,19 +125,21 @@ class IntegrationTest : public testing::TestWithParam(config_yaml); integration_->initialize(); - // Create codec for downstream client. + // Create codec for downstream client to encode request and decode response. codec_factory_ = std::move(codec_factory); - request_encoder_ = codec_factory_->requestEncoder(); - response_decoder_ = codec_factory_->responseDecoder(); - response_encoder_ = codec_factory_->responseEncoder(); + client_codec_ = codec_factory_->createClientCodec(); + request_encoder_callback_ = std::make_shared(); response_decoder_callback_ = std::make_shared(*this); + client_codec_->setCodecCallbacks(*response_decoder_callback_); + + // Helper codec for upstream server to encode response. + server_codec_ = codec_factory_->createServerCodec(); response_encoder_callback_ = std::make_shared(); - response_decoder_->setDecoderCallback(*response_decoder_callback_); } - std::string defaultConfig() { - return absl::StrCat(ConfigHelper::baseConfig(false), R"EOF( + std::string defaultConfig(bool bind_upstream_connection = false) { + return absl::StrCat(ConfigHelper::baseConfig(false), fmt::format(R"EOF( filter_chains: filters: name: meta @@ -139,12 +150,13 @@ class IntegrationTest : public testing::TestWithParamencode(request, *request_encoder_callback_); - RELEASE_ASSERT(request_encoder_callback_->complete_, "Encoding should complete Immediately"); + void sendRequestForTest(StreamFrame& request) { + client_codec_->encode(request, *request_encoder_callback_); client_connection_->write(request_encoder_callback_->buffer_, false); client_connection_->dispatcher().run(Envoy::Event::Dispatcher::RunType::NonBlock); + // Clear buffer for next encoding. + request_encoder_callback_->buffer_.drain(request_encoder_callback_->buffer_.length()); } // Waiting upstream connection to be created. @@ -208,39 +222,46 @@ class IntegrationTest : public testing::TestWithParamwaitForData(num_bytes, data); + void + waitForUpstreamRequestForTest(const std::function& data_validator) { + auto result = upstream_connection_->waitForData(data_validator, nullptr); RELEASE_ASSERT(result, result.failure_message()); // Clear data for next test. upstream_connection_->clearData(); } // Send upstream response. - void sendResponseForTest(const Response& response) { - response_encoder_->encode(response, *response_encoder_callback_); - RELEASE_ASSERT(response_encoder_callback_->complete_, "Encoding should complete Immediately"); + void sendResponseForTest(const StreamFrame& response) { + server_codec_->encode(response, *response_encoder_callback_); auto result = upstream_connection_->write(response_encoder_callback_->buffer_.toString(), false); + // Clear buffer for next encoding. + response_encoder_callback_->buffer_.drain(response_encoder_callback_->buffer_.length()); RELEASE_ASSERT(result, result.failure_message()); } // Waiting for downstream response. - AssertionResult waitDownstreamResponseForTest(std::chrono::milliseconds timeout) { + AssertionResult waitDownstreamResponseForTest(std::chrono::milliseconds timeout, + uint64_t stream_id) { bool timer_fired = false; - if (!response_decoder_callback_->complete_) { + if (!response_decoder_callback_->responses_[stream_id].end_stream_) { Envoy::Event::TimerPtr timer( integration_->dispatcher_->createTimer([this, &timer_fired]() -> void { timer_fired = true; integration_->dispatcher_->exit(); })); timer->enableTimer(timeout); + response_decoder_callback_->waiting_for_stream_id_ = stream_id; integration_->dispatcher_->run(Envoy::Event::Dispatcher::RunType::Block); if (timer_fired) { return AssertionFailure() << "Timed out waiting for response"; } + if (timer->enabled()) { + timer->disableTimer(); + } } - if (!response_decoder_callback_->complete_) { + if (!response_decoder_callback_->responses_[stream_id].end_stream_) { return AssertionFailure() << "No response or response not complete"; } return AssertionSuccess(); @@ -261,9 +282,9 @@ class IntegrationTest : public testing::TestWithParamresponse_, nullptr); - EXPECT_EQ(response_decoder_callback_->response_->status().message(), "route_not_found"); + EXPECT_NE(response_decoder_callback_->responses_[0].response_, nullptr); + EXPECT_EQ(response_decoder_callback_->responses_[0].response_->status().code(), + static_cast(absl::StatusCode::kNotFound)); cleanup(); } @@ -334,34 +356,34 @@ TEST_P(IntegrationTest, RequestAndResponse) { sendRequestForTest(request); waitForUpstreamConnectionForTest(); - waitForUpstreamRequestForTest(request_encoder_callback_->request_bytes_, nullptr); + const std::function data_validator = + [](const std::string& data) -> bool { return data.find("v1") != std::string::npos; }; + waitForUpstreamRequestForTest(data_validator); FakeStreamCodecFactory::FakeResponse response; response.protocol_ = "fake_fake_fake"; - response.status_ = Status(); + response.status_ = StreamStatus(); response.data_["zzzz"] = "xxxx"; sendResponseForTest(response); - RELEASE_ASSERT(waitDownstreamResponseForTest(std::chrono::milliseconds(200)), + RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 0), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->response_, nullptr); - EXPECT_EQ(response_decoder_callback_->response_->status().code(), StatusCode::kOk); - EXPECT_EQ(response_decoder_callback_->response_->getByKey("zzzz"), "xxxx"); + EXPECT_NE(response_decoder_callback_->responses_[0].response_, nullptr); + EXPECT_EQ(response_decoder_callback_->responses_[0].response_->status().code(), 0); + EXPECT_EQ(response_decoder_callback_->responses_[0].response_->get("zzzz"), "xxxx"); cleanup(); } TEST_P(IntegrationTest, MultipleRequestsWithSameStreamId) { FakeStreamCodecFactoryConfig codec_factory_config; - codec_factory_config.protocol_options_ = ProtocolOptions{true}; Registry::InjectFactory registration(codec_factory_config); auto codec_factory = std::make_unique(); - codec_factory->protocol_options_ = ProtocolOptions{true}; - initialize(defaultConfig(), std::move(codec_factory)); + initialize(defaultConfig(true), std::move(codec_factory)); EXPECT_TRUE(makeClientConnectionForTest()); @@ -375,7 +397,9 @@ TEST_P(IntegrationTest, MultipleRequestsWithSameStreamId) { sendRequestForTest(request_1); waitForUpstreamConnectionForTest(); - waitForUpstreamRequestForTest(request_encoder_callback_->request_bytes_, nullptr); + const std::function data_validator = + [](const std::string& data) -> bool { return data.find("v1") != std::string::npos; }; + waitForUpstreamRequestForTest(data_validator); FakeStreamCodecFactory::FakeRequest request_2; request_2.host_ = "service_name_0"; @@ -396,13 +420,11 @@ TEST_P(IntegrationTest, MultipleRequestsWithSameStreamId) { TEST_P(IntegrationTest, MultipleRequests) { FakeStreamCodecFactoryConfig codec_factory_config; - codec_factory_config.protocol_options_ = ProtocolOptions{true}; Registry::InjectFactory registration(codec_factory_config); auto codec_factory = std::make_unique(); - codec_factory->protocol_options_ = ProtocolOptions{true}; - initialize(defaultConfig(), std::move(codec_factory)); + initialize(defaultConfig(true), std::move(codec_factory)); EXPECT_TRUE(makeClientConnectionForTest()); @@ -411,60 +433,203 @@ TEST_P(IntegrationTest, MultipleRequests) { request_1.method_ = "hello"; request_1.path_ = "/path_or_anything"; request_1.protocol_ = "fake_fake_fake"; - request_1.data_ = {{"version", "v1"}, {"stream_id", "1"}}; + request_1.data_ = {{"version", "v1"}, {"stream_id", "1"}, {"frame", "1_header"}}; sendRequestForTest(request_1); waitForUpstreamConnectionForTest(); - waitForUpstreamRequestForTest(request_encoder_callback_->request_bytes_, nullptr); + const std::function data_validator_1 = + [](const std::string& data) -> bool { + return data.find("frame:1_header") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_1); FakeStreamCodecFactory::FakeRequest request_2; request_2.host_ = "service_name_0"; request_2.method_ = "hello"; request_2.path_ = "/path_or_anything"; request_2.protocol_ = "fake_fake_fake"; - request_2.data_ = {{"version", "v1"}, {"stream_id", "2"}}; + request_2.data_ = {{"version", "v1"}, {"stream_id", "2"}, {"frame", "2_header"}}; // Reset request encoder callback. request_encoder_callback_ = std::make_shared(); // Send the second request with the different stream id and expect the connection to be alive. sendRequestForTest(request_2); - waitForUpstreamRequestForTest(request_encoder_callback_->request_bytes_, nullptr); + const std::function data_validator_2 = + [](const std::string& data) -> bool { + return data.find("frame:2_header") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_2); FakeStreamCodecFactory::FakeResponse response_2; response_2.protocol_ = "fake_fake_fake"; - response_2.status_ = Status(); + response_2.status_ = StreamStatus(); response_2.data_["zzzz"] = "xxxx"; response_2.data_["stream_id"] = "2"; sendResponseForTest(response_2); - RELEASE_ASSERT(waitDownstreamResponseForTest(std::chrono::milliseconds(200)), + RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 2), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->response_, nullptr); - EXPECT_EQ(response_decoder_callback_->response_->status().code(), StatusCode::kOk); - EXPECT_EQ(response_decoder_callback_->response_->getByKey("zzzz"), "xxxx"); - EXPECT_EQ(response_decoder_callback_->response_->getByKey("stream_id"), "2"); + EXPECT_NE(response_decoder_callback_->responses_[2].response_, nullptr); + EXPECT_EQ(response_decoder_callback_->responses_[2].response_->status().code(), 0); + EXPECT_EQ(response_decoder_callback_->responses_[2].response_->get("zzzz"), "xxxx"); + EXPECT_EQ(response_decoder_callback_->responses_[2].response_->get("stream_id"), "2"); + + FakeStreamCodecFactory::FakeResponse response_1; + response_1.protocol_ = "fake_fake_fake"; + response_1.status_ = StreamStatus(); + response_1.data_["zzzz"] = "yyyy"; + response_1.data_["stream_id"] = "1"; + + sendResponseForTest(response_1); + + RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 1), + "unexpected timeout"); + + EXPECT_NE(response_decoder_callback_->responses_[1].response_, nullptr); + EXPECT_EQ(response_decoder_callback_->responses_[1].response_->status().code(), 0); + EXPECT_EQ(response_decoder_callback_->responses_[1].response_->get("zzzz"), "yyyy"); + EXPECT_EQ(response_decoder_callback_->responses_[1].response_->get("stream_id"), "1"); + + cleanup(); +} + +TEST_P(IntegrationTest, MultipleRequestsWithMultipleFrames) { + FakeStreamCodecFactoryConfig codec_factory_config; + Registry::InjectFactory registration(codec_factory_config); + + auto codec_factory = std::make_unique(); + + initialize(defaultConfig(true), std::move(codec_factory)); + + EXPECT_TRUE(makeClientConnectionForTest()); + + FakeStreamCodecFactory::FakeRequest request_1; + request_1.host_ = "service_name_0"; + request_1.method_ = "hello"; + request_1.path_ = "/path_or_anything"; + request_1.protocol_ = "fake_fake_fake"; + request_1.data_ = { + {"version", "v1"}, {"stream_id", "1"}, {"end_stream", "false"}, {"frame", "1_header"}}; + + FakeStreamCodecFactory::FakeRequest request_1_frame_1; + request_1_frame_1.data_ = {{"stream_id", "1"}, {"end_stream", "false"}, {"frame", "1_frame_1"}}; + + FakeStreamCodecFactory::FakeRequest request_1_frame_2; + request_1_frame_2.data_ = {{"stream_id", "1"}, {"end_stream", "true"}, {"frame", "1_frame_2"}}; + + FakeStreamCodecFactory::FakeRequest request_2; + request_2.host_ = "service_name_0"; + request_2.method_ = "hello"; + request_2.path_ = "/path_or_anything"; + request_2.protocol_ = "fake_fake_fake"; + request_2.data_ = { + {"version", "v1"}, {"stream_id", "2"}, {"end_stream", "false"}, {"frame", "2_header"}}; + + FakeStreamCodecFactory::FakeRequest request_2_frame_1; + request_2_frame_1.data_ = {{"stream_id", "2"}, {"end_stream", "false"}, {"frame", "2_frame_1"}}; - response_decoder_callback_->complete_ = false; + FakeStreamCodecFactory::FakeRequest request_2_frame_2; + request_2_frame_2.data_ = {{"stream_id", "2"}, {"end_stream", "true"}, {"frame", "2_frame_2"}}; + + // We handle frame one by one to make sure the order is correct. + + sendRequestForTest(request_1); + waitForUpstreamConnectionForTest(); + + // First frame of request 1. + const std::function data_validator_1 = + [](const std::string& data) -> bool { + return data.find("frame:1_header") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_1); + + // Second frame of request 1. + sendRequestForTest(request_1_frame_1); + const std::function data_validator_1_frame_1 = + [](const std::string& data) -> bool { + return data.find("frame:1_frame_1") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_1_frame_1); + + // First frame of request 2. + sendRequestForTest(request_2); + const std::function data_validator_2 = + [](const std::string& data) -> bool { + return data.find("frame:2_header") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_2); + + // Second frame of request 2. + sendRequestForTest(request_2_frame_1); + const std::function data_validator_2_frame_1 = + [](const std::string& data) -> bool { + return data.find("frame:2_frame_1") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_2_frame_1); + + // Third frame of request 1. + sendRequestForTest(request_1_frame_2); + const std::function data_validator_1_frame_2 = + [](const std::string& data) -> bool { + return data.find("frame:1_frame_2") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_1_frame_2); + + // Third frame of request 2. + sendRequestForTest(request_2_frame_2); + const std::function data_validator_2_frame_2 = + [](const std::string& data) -> bool { + return data.find("frame:2_frame_2") != std::string::npos; + }; + waitForUpstreamRequestForTest(data_validator_2_frame_2); + + FakeStreamCodecFactory::FakeResponse response_2; + response_2.protocol_ = "fake_fake_fake"; + response_2.status_ = StreamStatus(); + response_2.data_["zzzz"] = "xxxx"; + response_2.data_["stream_id"] = "2"; + response_2.data_["end_stream"] = "false"; + + FakeStreamCodecFactory::FakeResponse response_2_frame_1; + response_2_frame_1.data_["stream_id"] = "2"; + response_2_frame_1.data_["end_stream"] = "true"; + + sendResponseForTest(response_2); + sendResponseForTest(response_2_frame_1); + + RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 2), + "unexpected timeout"); + + EXPECT_NE(response_decoder_callback_->responses_[2].response_, nullptr); + EXPECT_EQ(response_decoder_callback_->responses_[2].response_->status().code(), 0); + EXPECT_EQ(response_decoder_callback_->responses_[2].response_->get("zzzz"), "xxxx"); + EXPECT_EQ(response_decoder_callback_->responses_[2].response_->get("stream_id"), "2"); FakeStreamCodecFactory::FakeResponse response_1; response_1.protocol_ = "fake_fake_fake"; - response_1.status_ = Status(); + response_1.status_ = StreamStatus(); response_1.data_["zzzz"] = "yyyy"; response_1.data_["stream_id"] = "1"; + response_1.data_["end_stream"] = "false"; + + FakeStreamCodecFactory::FakeResponse response_1_frame_1; + response_1_frame_1.data_["stream_id"] = "1"; + response_1_frame_1.data_["end_stream"] = "true"; sendResponseForTest(response_1); + sendResponseForTest(response_1_frame_1); - RELEASE_ASSERT(waitDownstreamResponseForTest(std::chrono::milliseconds(200)), + RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 1), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->response_, nullptr); - EXPECT_EQ(response_decoder_callback_->response_->status().code(), StatusCode::kOk); - EXPECT_EQ(response_decoder_callback_->response_->getByKey("zzzz"), "yyyy"); - EXPECT_EQ(response_decoder_callback_->response_->getByKey("stream_id"), "1"); + EXPECT_NE(response_decoder_callback_->responses_[1].response_, nullptr); + EXPECT_EQ(response_decoder_callback_->responses_[1].response_->status().code(), 0); + EXPECT_EQ(response_decoder_callback_->responses_[1].response_->get("zzzz"), "yyyy"); + EXPECT_EQ(response_decoder_callback_->responses_[1].response_->get("stream_id"), "1"); cleanup(); } diff --git a/contrib/generic_proxy/filters/network/test/match_test.cc b/contrib/generic_proxy/filters/network/test/match_test.cc index e2daf8a0d55d..e640ba1571e8 100644 --- a/contrib/generic_proxy/filters/network/test/match_test.cc +++ b/contrib/generic_proxy/filters/network/test/match_test.cc @@ -119,8 +119,8 @@ TEST(RequestMatchInputMatcherTest, RequestMatchInputMatcherTest) { NiceMock factory_context; RequestMatchDataInputMatcherFactory factory; auto proto_config = factory.createEmptyConfigProto(); - auto matcher = factory.createInputMatcherFactoryCb(*proto_config, - factory_context.getServerFactoryContext())(); + auto matcher = + factory.createInputMatcherFactoryCb(*proto_config, factory_context.serverFactoryContext())(); { Matcher::MatchingDataType input; diff --git a/contrib/generic_proxy/filters/network/test/mocks/codec.cc b/contrib/generic_proxy/filters/network/test/mocks/codec.cc index 552eb926d1fa..049076ef808c 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/codec.cc +++ b/contrib/generic_proxy/filters/network/test/mocks/codec.cc @@ -12,17 +12,10 @@ namespace NetworkFilters { namespace GenericProxy { MockCodecFactory::MockCodecFactory() { - ON_CALL(*this, requestDecoder()) - .WillByDefault(Return(ByMove(std::make_unique>()))); - ON_CALL(*this, responseDecoder()) - .WillByDefault(Return(ByMove(std::make_unique>()))); - ON_CALL(*this, requestEncoder()) - .WillByDefault(Return(ByMove(std::make_unique>()))); - ON_CALL(*this, responseEncoder()) - .WillByDefault(Return(ByMove(std::make_unique>()))); - ON_CALL(*this, messageCreator()) - .WillByDefault(Return(ByMove(std::make_unique>()))); - ON_CALL(*this, protocolOptions()).WillByDefault(Return(ProtocolOptions{false})); + ON_CALL(*this, createServerCodec()) + .WillByDefault(Return(ByMove(std::make_unique>()))); + ON_CALL(*this, createClientCodec()) + .WillByDefault(Return(ByMove(std::make_unique>()))); } MockProxyFactory::MockProxyFactory() = default; diff --git a/contrib/generic_proxy/filters/network/test/mocks/codec.h b/contrib/generic_proxy/filters/network/test/mocks/codec.h index 650e9d054b0b..6277ecdf1f5c 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/codec.h +++ b/contrib/generic_proxy/filters/network/test/mocks/codec.h @@ -8,72 +8,48 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { -class MockRequestDecoderCallback : public RequestDecoderCallback { +class MockServerCodecCallbacks : public ServerCodecCallbacks { public: - MOCK_METHOD(void, onDecodingSuccess, (RequestPtr request, ExtendedOptions)); + MOCK_METHOD(void, onDecodingSuccess, (StreamFramePtr request)); MOCK_METHOD(void, onDecodingFailure, ()); MOCK_METHOD(void, writeToConnection, (Buffer::Instance & buffer)); MOCK_METHOD(OptRef, connection, ()); }; -class MockResponseDecoderCallback : public ResponseDecoderCallback { +class MockClientCodecCallbacks : public ClientCodecCallbacks { public: - MOCK_METHOD(void, onDecodingSuccess, (ResponsePtr response, ExtendedOptions)); + MOCK_METHOD(void, onDecodingSuccess, (StreamFramePtr response)); MOCK_METHOD(void, onDecodingFailure, ()); MOCK_METHOD(void, writeToConnection, (Buffer::Instance & buffer)); MOCK_METHOD(OptRef, connection, ()); }; -class MockRequestEncoderCallback : public RequestEncoderCallback { +class MockEncodingCallbacks : public EncodingCallbacks { public: - MOCK_METHOD(void, onEncodingSuccess, (Buffer::Instance & buffer)); + MOCK_METHOD(void, onEncodingSuccess, (Buffer::Instance & buffer, bool end_stream)); }; -/** - * Encoder callback of Response. - */ -class MockResponseEncoderCallback : public ResponseEncoderCallback { +class MockServerCodec : public ServerCodec { public: - MOCK_METHOD(void, onEncodingSuccess, (Buffer::Instance & buffer)); + MOCK_METHOD(void, setCodecCallbacks, (ServerCodecCallbacks & callbacks)); + MOCK_METHOD(void, decode, (Buffer::Instance & buffer, bool end_stream)); + MOCK_METHOD(void, encode, (const StreamFrame&, EncodingCallbacks& callbacks)); + MOCK_METHOD(ResponsePtr, respond, (Status status, absl::string_view, const Request&)); }; -class MockRequestDecoder : public RequestDecoder { +class MockClientCodec : public ClientCodec { public: - MOCK_METHOD(void, setDecoderCallback, (RequestDecoderCallback & callback)); - MOCK_METHOD(void, decode, (Buffer::Instance & buffer)); -}; - -class MockResponseDecoder : public ResponseDecoder { -public: - MOCK_METHOD(void, setDecoderCallback, (ResponseDecoderCallback & callback)); - MOCK_METHOD(void, decode, (Buffer::Instance & buffer)); -}; - -class MockRequestEncoder : public RequestEncoder { -public: - MOCK_METHOD(void, encode, (const Request&, RequestEncoderCallback& callback)); -}; - -class MockResponseEncoder : public ResponseEncoder { -public: - MOCK_METHOD(void, encode, (const Response&, ResponseEncoderCallback& callback)); -}; - -class MockMessageCreator : public MessageCreator { -public: - MOCK_METHOD(ResponsePtr, response, (Status status, const Request&)); + MOCK_METHOD(void, setCodecCallbacks, (ClientCodecCallbacks & callbacks)); + MOCK_METHOD(void, decode, (Buffer::Instance & buffer, bool end_stream)); + MOCK_METHOD(void, encode, (const StreamFrame&, EncodingCallbacks& callbacks)); }; class MockCodecFactory : public CodecFactory { public: MockCodecFactory(); - MOCK_METHOD(RequestDecoderPtr, requestDecoder, (), (const)); - MOCK_METHOD(ResponseDecoderPtr, responseDecoder, (), (const)); - MOCK_METHOD(RequestEncoderPtr, requestEncoder, (), (const)); - MOCK_METHOD(ResponseEncoderPtr, responseEncoder, (), (const)); - MOCK_METHOD(MessageCreatorPtr, messageCreator, (), (const)); - MOCK_METHOD(ProtocolOptions, protocolOptions, (), (const)); + MOCK_METHOD(ServerCodecPtr, createServerCodec, (), (const)); + MOCK_METHOD(ClientCodecPtr, createClientCodec, (), (const)); }; class MockProxyFactory : public ProxyFactory { diff --git a/contrib/generic_proxy/filters/network/test/mocks/filter.cc b/contrib/generic_proxy/filters/network/test/mocks/filter.cc index 7a2bba0efc3d..aa94963b5b2c 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/filter.cc +++ b/contrib/generic_proxy/filters/network/test/mocks/filter.cc @@ -13,6 +13,8 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { +MockStreamFrameHandler::MockStreamFrameHandler() = default; + MockStreamFilterConfig::MockStreamFilterConfig() { ON_CALL(*this, createEmptyConfigProto()).WillByDefault(Invoke([]() { return std::make_unique(); @@ -44,47 +46,7 @@ MockStreamFilter::MockStreamFilter() { ON_CALL(*this, onStreamDecoded(_)).WillByDefault(Return(FilterStatus::Continue)); } -MockPendingResponseCallback::MockPendingResponseCallback() = default; - -MockUpstreamBindingCallback::MockUpstreamBindingCallback() = default; - -MockUpstreamManager::MockUpstreamManager() { - ON_CALL(*this, registerUpstreamCallback(_, _)) - .WillByDefault(Invoke([this](uint64_t stream_id, UpstreamBindingCallback& cb) { - if (call_on_bind_success_immediately_) { - cb.onBindSuccess(upstream_conn_, upstream_host_); - return; - } - - if (call_on_bind_failure_immediately_) { - cb.onBindFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure, "", - upstream_host_); - return; - } - upstream_callbacks_[stream_id] = &cb; - })); - ON_CALL(*this, unregisterUpstreamCallback(_)).WillByDefault(Invoke([this](uint64_t stream_id) { - upstream_callbacks_.erase(stream_id); - })); - - ON_CALL(*this, registerResponseCallback(_, _)) - .WillByDefault(Invoke([this](uint64_t stream_id, PendingResponseCallback& cb) { - response_callbacks_[stream_id] = &cb; - })); - - ON_CALL(*this, unregisterResponseCallback(_)).WillByDefault(Invoke([this](uint64_t stream_id) { - response_callbacks_.erase(stream_id); - })); -} - -MockDecoderFilterCallback::MockDecoderFilterCallback() { - ON_CALL(*this, boundUpstreamConn()).WillByDefault(Invoke([this]() -> OptRef { - if (has_upstream_manager_) { - return upstream_manager_; - } - return {}; - })); -} +MockDecoderFilterCallback::MockDecoderFilterCallback() = default; } // namespace GenericProxy } // namespace NetworkFilters diff --git a/contrib/generic_proxy/filters/network/test/mocks/filter.h b/contrib/generic_proxy/filters/network/test/mocks/filter.h index 4bd32066c82f..de1af24efd60 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/filter.h +++ b/contrib/generic_proxy/filters/network/test/mocks/filter.h @@ -12,6 +12,13 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { +class MockStreamFrameHandler : public StreamFrameHandler { +public: + MockStreamFrameHandler(); + + MOCK_METHOD(void, onStreamFrame, (StreamFramePtr frame)); +}; + class MockDecoderFilter : public DecoderFilter { public: MockDecoderFilter(); @@ -58,6 +65,7 @@ class MockStreamFilterConfig : public NamedFilterConfigFactory { (const Protobuf::Message&, Server::Configuration::ServerFactoryContext&, ProtobufMessage::ValidationVisitor&)); MOCK_METHOD(std::string, name, (), (const)); + MOCK_METHOD(absl::Status, validateCodec, (const TypedExtensionConfig&)); MOCK_METHOD(std::set, configTypes, ()); MOCK_METHOD(bool, isTerminalFilter, ()); }; @@ -71,28 +79,6 @@ class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { MOCK_METHOD(void, addFilter, (StreamFilterSharedPtr filter)); }; -class MockUpstreamBindingCallback : public UpstreamBindingCallback { -public: - MockUpstreamBindingCallback(); - - MOCK_METHOD(void, onBindFailure, - (ConnectionPool::PoolFailureReason reason, absl::string_view transport_failure_reason, - Upstream::HostDescriptionConstSharedPtr host)); - MOCK_METHOD(void, onBindSuccess, - (Network::ClientConnection & conn, Upstream::HostDescriptionConstSharedPtr host)); -}; - -class MockPendingResponseCallback : public PendingResponseCallback { -public: - MockPendingResponseCallback(); - - MOCK_METHOD(void, onDecodingSuccess, (ResponsePtr response, ExtendedOptions options)); - MOCK_METHOD(void, onDecodingFailure, ()); - MOCK_METHOD(void, writeToConnection, (Buffer::Instance & buffer)); - MOCK_METHOD(OptRef, connection, ()); - MOCK_METHOD(void, onConnectionClose, (Network::ConnectionEvent event)); -}; - class MockFilterChainManager : public FilterChainManager { public: MockFilterChainManager(); @@ -114,94 +100,19 @@ template class MockStreamFilterCallbacks : public Base { MOCK_METHOD(StreamInfo::StreamInfo&, streamInfo, ()); MOCK_METHOD(Tracing::Span&, activeSpan, ()); MOCK_METHOD(OptRef, tracingConfig, (), (const)); - MOCK_METHOD(absl::optional, requestOptions, (), (const)); - MOCK_METHOD(absl::optional, responseOptions, (), (const)); MOCK_METHOD(const Network::Connection*, connection, (), (const)); }; -class MockUpstreamManager : public UpstreamManager { -public: - MockUpstreamManager(); - - MOCK_METHOD(void, registerUpstreamCallback, (uint64_t stream_id, UpstreamBindingCallback& cb)); - MOCK_METHOD(void, unregisterUpstreamCallback, (uint64_t stream_id)); - - MOCK_METHOD(void, registerResponseCallback, (uint64_t stream_id, PendingResponseCallback& cb)); - MOCK_METHOD(void, unregisterResponseCallback, (uint64_t stream_id)); - - void setupConnectionPool(Upstream::TcpPoolData&& data) { - // Mock the connection creation and callbacks. - upstream_cancellable_ = data.newConnection(pool_callbacks_); - } - - void callOnBindSuccess(uint64_t stream_id) { - auto it = upstream_callbacks_.find(stream_id); - auto cb = it->second; - - upstream_callbacks_.erase(it); - cb->onBindSuccess(upstream_conn_, upstream_host_); - } - - void callOnBindFailure(uint64_t stream_id, ConnectionPool::PoolFailureReason reason) { - auto it = upstream_callbacks_.find(stream_id); - auto cb = it->second; - - upstream_callbacks_.erase(it); - cb->onBindFailure(reason, "", upstream_host_); - } - - void callOnDecodingSuccess(uint64_t stream_id, ResponsePtr response, ExtendedOptions options) { - auto it = response_callbacks_.find(stream_id); - auto cb = it->second; - - response_callbacks_.erase(it); - - cb->onDecodingSuccess(std::move(response), options); - } - - void callOnConnectionClose(uint64_t stream_id, Network::ConnectionEvent event) { - auto it = response_callbacks_.find(stream_id); - auto cb = it->second; - - response_callbacks_.erase(it); - - cb->onConnectionClose(event); - } - - void callOnDecodingFailure(uint64_t stream_id) { - auto it = response_callbacks_.find(stream_id); - auto cb = it->second; - - response_callbacks_.erase(it); - cb->onDecodingFailure(); - } - - bool call_on_bind_success_immediately_{}; - bool call_on_bind_failure_immediately_{}; - - std::shared_ptr> upstream_host_; - testing::NiceMock upstream_conn_; - - Tcp::ConnectionPool::Cancellable* upstream_cancellable_{}; - testing::NiceMock pool_callbacks_; - - absl::flat_hash_map upstream_callbacks_; - absl::flat_hash_map response_callbacks_; -}; - class MockDecoderFilterCallback : public MockStreamFilterCallbacks { public: MockDecoderFilterCallback(); MOCK_METHOD(void, sendLocalReply, (Status, ResponseUpdateFunction&&)); MOCK_METHOD(void, continueDecoding, ()); - MOCK_METHOD(void, upstreamResponse, (ResponsePtr response, ExtendedOptions options)); + MOCK_METHOD(void, onResponseStart, (StreamResponsePtr response)); + MOCK_METHOD(void, onResponseFrame, (StreamFramePtr frame)); + MOCK_METHOD(void, setRequestFramesHandler, (StreamFrameHandler & handler)); MOCK_METHOD(void, completeDirectly, ()); - MOCK_METHOD(void, bindUpstreamConn, (Upstream::TcpPoolData &&)); - MOCK_METHOD(OptRef, boundUpstreamConn, ()); - - bool has_upstream_manager_{}; - testing::NiceMock upstream_manager_; }; class MockEncoderFilterCallback : public MockStreamFilterCallbacks { diff --git a/contrib/generic_proxy/filters/network/test/proxy_test.cc b/contrib/generic_proxy/filters/network/test/proxy_test.cc index 10253829422a..ecd06a1b344d 100644 --- a/contrib/generic_proxy/filters/network/test/proxy_test.cc +++ b/contrib/generic_proxy/filters/network/test/proxy_test.cc @@ -102,9 +102,9 @@ class FilterConfigTest : public testing::Test { Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig( sff_config, factory_context_); - return std::make_shared(Filesystem::FilePathAndType{}, nullptr, - std::move(formatter), - factory_context_.accessLogManager()); + return std::make_shared( + Filesystem::FilePathAndType{}, nullptr, std::move(formatter), + factory_context_.server_factory_context_.accessLogManager()); } std::shared_ptr> tracer_; @@ -176,47 +176,32 @@ TEST_F(FilterConfigTest, CodecFactory) { class FilterTest : public FilterConfigTest { public: - void initializeFilter(bool with_tracing = false, bool bind_upstream = false, - AccessLogInstanceSharedPtr logger = {}) { + void initializeFilter(bool with_tracing = false, AccessLogInstanceSharedPtr logger = {}) { FilterConfigTest::initializeFilterConfig(with_tracing, logger); - auto encoder = std::make_unique>(); - encoder_ = encoder.get(); - EXPECT_CALL(*codec_factory_, responseEncoder()).WillOnce(Return(ByMove(std::move(encoder)))); + auto server_codec = std::make_unique>(); + server_codec_ = server_codec.get(); + EXPECT_CALL(*codec_factory_, createServerCodec()) + .WillOnce(Return(ByMove(std::move(server_codec)))); - auto decoder = std::make_unique>(); - decoder_ = decoder.get(); - EXPECT_CALL(*codec_factory_, requestDecoder()).WillOnce(Return(ByMove(std::move(decoder)))); - - auto creator = std::make_unique>(); - creator_ = creator.get(); - EXPECT_CALL(*codec_factory_, messageCreator()).WillOnce(Return(ByMove(std::move(creator)))); - - ProtocolOptions protocol_options{bind_upstream}; - ON_CALL(*codec_factory_, protocolOptions()).WillByDefault(Return(protocol_options)); - - EXPECT_CALL(*decoder_, setDecoderCallback(_)) + EXPECT_CALL(*server_codec_, setCodecCallbacks(_)) .WillOnce( - Invoke([this](RequestDecoderCallback& callback) { decoder_callback_ = &callback; })); + Invoke([this](ServerCodecCallbacks& callback) { decoder_callback_ = &callback; })); - filter_ = std::make_shared(filter_config_, factory_context_.time_system_, - factory_context_.runtime_loader_); + filter_ = std::make_shared(filter_config_, + factory_context_.server_factory_context_.time_system_, + factory_context_.server_factory_context_.runtime_loader_); EXPECT_EQ(filter_.get(), decoder_callback_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } - NiceMock tcp_conn_pool_; - NiceMock upstream_connection_; - std::shared_ptr filter_; - RequestDecoderCallback* decoder_callback_{}; + ServerCodecCallbacks* decoder_callback_{}; - NiceMock* decoder_; - NiceMock* encoder_; - NiceMock* creator_; + NiceMock* server_codec_{}; NiceMock filter_callbacks_; }; @@ -231,7 +216,7 @@ TEST_F(FilterTest, SimpleOnData) { Buffer::OwnedImpl fake_empty_buffer; - EXPECT_CALL(*decoder_, decode(_)); + EXPECT_CALL(*server_codec_, decode(_, _)); filter_->onData(fake_empty_buffer, false); } @@ -240,7 +225,7 @@ TEST_F(FilterTest, OnDecodingFailureWithoutActiveStreams) { Buffer::OwnedImpl fake_empty_buffer; - EXPECT_CALL(*decoder_, decode(_)); + EXPECT_CALL(*server_codec_, decode(_, _)); filter_->onData(fake_empty_buffer, false); EXPECT_CALL(filter_callbacks_.connection_, close(_)); @@ -257,7 +242,7 @@ TEST_F(FilterTest, OnDecodingSuccessWithNormalRequest) { Buffer::OwnedImpl fake_empty_buffer; - EXPECT_CALL(*decoder_, decode(_)); + EXPECT_CALL(*server_codec_, decode(_, _)); filter_->onData(fake_empty_buffer, false); auto request = std::make_unique(); @@ -265,7 +250,7 @@ TEST_F(FilterTest, OnDecodingSuccessWithNormalRequest) { // Three mock factories was added. EXPECT_CALL(*mock_stream_filter, onStreamDecoded(_)).Times(3); - decoder_callback_->onDecodingSuccess(std::move(request), ExtendedOptions()); + decoder_callback_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); } @@ -290,7 +275,7 @@ TEST_F(FilterTest, OnConnectionClosedEvent) { TEST_F(FilterTest, SendReplyDownstream) { initializeFilter(); - NiceMock encoder_callback; + NiceMock encoder_callback; auto response = std::make_unique(); @@ -298,18 +283,19 @@ TEST_F(FilterTest, SendReplyDownstream) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(encoder_callback, onEncodingSuccess(_)) - .WillOnce(Invoke( - [&](Buffer::Instance& buffer) { filter_callbacks_.connection_.write(buffer, false); })); + EXPECT_CALL(encoder_callback, onEncodingSuccess(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + filter_callbacks_.connection_.write(buffer, false); + })); - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response&, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); - filter_->sendReplyDownstream(*response, encoder_callback); + filter_->sendFrameToDownstream(*response, encoder_callback); } TEST_F(FilterTest, GetConnection) { @@ -323,7 +309,7 @@ TEST_F(FilterTest, NewStreamAndResetStream) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -346,7 +332,7 @@ TEST_F(FilterTest, NewStreamAndResetStreamFromFilter) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -363,7 +349,7 @@ TEST_F(FilterTest, NewStreamAndDispatcher) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -375,15 +361,15 @@ TEST_F(FilterTest, OnDecodingFailureWithActiveStreams) { initializeFilter(); auto request_0 = std::make_unique(); - filter_->newDownstreamRequest(std::move(request_0), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request_0)); auto request_1 = std::make_unique(); - filter_->newDownstreamRequest(std::move(request_1), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request_1)); EXPECT_EQ(2, filter_->activeStreamsForTest().size()); Buffer::OwnedImpl fake_empty_buffer; - EXPECT_CALL(*decoder_, decode(_)); + EXPECT_CALL(*server_codec_, decode(_, _)); filter_->onData(fake_empty_buffer, false); EXPECT_CALL(filter_callbacks_.connection_, close(_)); @@ -397,7 +383,7 @@ TEST_F(FilterTest, ActiveStreamRouteEntry) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -413,7 +399,7 @@ TEST_F(FilterTest, ActiveStreamPerFilterConfig) { EXPECT_CALL(*route_matcher_, routeEntry(_)).WillOnce(Return(mock_route_entry_)); auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -437,7 +423,7 @@ TEST_F(FilterTest, ActiveStreamPerFilterConfigNoRouteEntry) { EXPECT_CALL(*route_matcher_, routeEntry(_)).WillOnce(Return(nullptr)); auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -457,7 +443,7 @@ TEST_F(FilterTest, ActiveStreamConnection) { initializeFilter(); auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -476,7 +462,7 @@ TEST_F(FilterTest, ActiveStreamAddFilters) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -524,7 +510,7 @@ TEST_F(FilterTest, ActiveStreamAddFiltersOrder) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -562,7 +548,7 @@ TEST_F(FilterTest, ActiveStreamFiltersContinueDecoding) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -601,7 +587,7 @@ TEST_F(FilterTest, ActiveStreamFiltersContinueEncoding) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -614,19 +600,19 @@ TEST_F(FilterTest, ActiveStreamFiltersContinueEncoding) { EXPECT_EQ(0, active_stream->nextEncoderFilterIndexForTest()); auto response = std::make_unique(); - // `continueEncoding` will be called in the `upstreamResponse`. - active_stream->upstreamResponse(std::move(response), ExtendedOptions()); + // `continueEncoding` will be called in the `onResponseStart`. + active_stream->onResponseStart(std::move(response)); // Encoding will be stopped when `onStreamEncoded` of `mock_stream_filter_1` is called. EXPECT_EQ(2, active_stream->nextEncoderFilterIndexForTest()); EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response&, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); active_stream->encoderFiltersForTest()[1]->continueEncoding(); @@ -637,26 +623,29 @@ TEST_F(FilterTest, ActiveStreamSendLocalReply) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); - EXPECT_CALL(*creator_, response(_, _)) - .WillOnce(Invoke([&](Status status, const Request&) -> ResponsePtr { + EXPECT_CALL(*server_codec_, respond(_, _, _)) + .WillOnce(Invoke([&](Status status, absl::string_view, const Request&) -> ResponsePtr { auto response = std::make_unique(); - response->status_ = std::move(status); + response->status_ = {static_cast(status.code()), + status.code() == StatusCode::kOk}; + response->message_ = status.message(); return response; })); EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response& response, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame& response, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; - EXPECT_EQ(response.status().message(), "test_detail"); + EXPECT_EQ(dynamic_cast(&response)->status().code(), + static_cast(StatusCode::kUnknown)); buffer.add("test"); - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); active_stream->sendLocalReply(Status(StatusCode::kUnknown, "test_detail"), [](Response&) {}); @@ -667,7 +656,7 @@ TEST_F(FilterTest, ActiveStreamCompleteDirectly) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -684,7 +673,7 @@ TEST_F(FilterTest, ActiveStreamCompleteDirectlyFromFilter) { auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); @@ -699,7 +688,7 @@ TEST_F(FilterTest, NewStreamAndReplyNormally) { mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; // The logger is used to test the log format. - initializeFilter(false, false, loggerFormFormat()); + initializeFilter(false, loggerFormFormat()); auto request = std::make_unique(); request->host_ = "host-value"; @@ -708,22 +697,22 @@ TEST_F(FilterTest, NewStreamAndReplyNormally) { request->protocol_ = "protocol-value"; request->data_["request-key"] = "request-value"; - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response&, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); EXPECT_CALL( - *factory_context_.access_log_manager_.file_, + *factory_context_.server_factory_context_.access_log_manager_.file_, write("host-value /path-value method-value protocol-value request-value response-value -")); EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); @@ -732,603 +721,193 @@ TEST_F(FilterTest, NewStreamAndReplyNormally) { auto response = std::make_unique(); response->data_["response-key"] = "response-value"; - active_stream->upstreamResponse(std::move(response), ExtendedOptions()); + active_stream->onResponseStart(std::move(response)); } -TEST_F(FilterTest, NewStreamAndReplyNormallyWithDrainClose) { +TEST_F(FilterTest, NewStreamAndReplyNormallyWithMultipleFrames) { auto mock_decoder_filter_0 = std::make_shared>(); mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - initializeFilter(); - - auto request = std::make_unique(); + NiceMock mock_stream_frame_handler; - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); - EXPECT_EQ(1, filter_->activeStreamsForTest().size()); - - auto active_stream = filter_->activeStreamsForTest().begin()->get(); - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response&, ResponseEncoderCallback& callback) { - Buffer::OwnedImpl buffer; - buffer.add("test"); - callback.onEncodingSuccess(buffer); + EXPECT_CALL(*mock_decoder_filter_0, setDecoderFilterCallbacks(_)) + .WillOnce(Invoke([&mock_stream_frame_handler](DecoderFilterCallback& callbacks) { + callbacks.setRequestFramesHandler(mock_stream_frame_handler); })); - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(true)); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - - auto response = std::make_unique(); - active_stream->upstreamResponse(std::move(response), ExtendedOptions()); -} - -TEST_F(FilterTest, NewStreamAndReplyNormallyWithStreamDrainClose) { - auto mock_decoder_filter_0 = std::make_shared>(); - mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - - initializeFilter(); + // The logger is used to test the log format. + initializeFilter(false, loggerFormFormat()); auto request = std::make_unique(); + request->host_ = "host-value"; + request->path_ = "/path-value"; + request->method_ = "method-value"; + request->protocol_ = "protocol-value"; + request->data_["request-key"] = "request-value"; + request->stream_frame_flags_ = FrameFlags(StreamFlags(), false); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + // The first frame is not the end stream and we will create a frame handler for it. + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); + EXPECT_EQ(1, filter_->frameHandlersForTest().size()); - auto active_stream = filter_->activeStreamsForTest().begin()->get(); - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response&, ResponseEncoderCallback& callback) { - Buffer::OwnedImpl buffer; - buffer.add("test"); - callback.onEncodingSuccess(buffer); - })); + // stream_frame_handler will be called twice to handle the two frames (except the first + // StreamRequest frame). + EXPECT_CALL(mock_stream_frame_handler, onStreamFrame(_)).Times(2); - // The drain close of factory_context_.drain_manager_ is false, but the drain close of - // active_stream is true. - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - - auto response = std::make_unique(); - active_stream->upstreamResponse(std::move(response), - ExtendedOptions({}, false, /*drain_close*/ true, false)); -} - -TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) { - auto mock_decoder_filter_0 = std::make_shared>(); - mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - - initializeFilter(true); - - auto request = std::make_unique(); - - auto* span = new NiceMock(); - EXPECT_CALL(*tracer_, startSpan_(_, _, _, _)) - .WillOnce( - Invoke([&](const Tracing::Config& config, Tracing::TraceContext&, - const StreamInfo::StreamInfo&, const Tracing::Decision) -> Tracing::Span* { - EXPECT_EQ(Tracing::OperationName::Egress, config.operationName()); - return span; - })); - - filter_->newDownstreamRequest(std::move(request), ExtendedOptions()); + auto request_frame_1 = std::make_unique(); + request_frame_1->stream_frame_flags_ = FrameFlags(StreamFlags(), false); + filter_->onDecodingSuccess(std::move(request_frame_1)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); + EXPECT_EQ(1, filter_->frameHandlersForTest().size()); - auto active_stream = filter_->activeStreamsForTest().begin()->get(); - - EXPECT_CALL(*span, setTag(_, _)).Times(testing::AnyNumber()); - EXPECT_CALL(*span, finishSpan()); - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response&, ResponseEncoderCallback& callback) { - Buffer::OwnedImpl buffer; - buffer.add("test"); - callback.onEncodingSuccess(buffer); - })); - - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); - - auto response = std::make_unique(); - active_stream->upstreamResponse(std::move(response), ExtendedOptions()); -} - -TEST_F(FilterTest, BindUpstreamConnectionFailure) { - auto mock_decoder_filter_0 = std::make_shared>(); - mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - - initializeFilter(false, true); - - auto request = std::make_unique(); - - filter_->newDownstreamRequest(std::move(request), ExtendedOptions(123, true, false, false)); + // When the last frame is the end stream, we will delete the frame handler. + auto request_frame_2 = std::make_unique(); + request_frame_2->stream_frame_flags_ = FrameFlags(StreamFlags(), true); + filter_->onDecodingSuccess(std::move(request_frame_2)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); + EXPECT_EQ(0, filter_->frameHandlersForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback; - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback); - - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); + EXPECT_CALL( + *factory_context_.server_factory_context_.access_log_manager_.file_, + write("host-value /path-value method-value protocol-value request-value response-value -")); EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - // One for the upstream_manager_ and one for the active stream. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response& response, ResponseEncoderCallback& callback) { + EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)).Times(2); + EXPECT_CALL(*server_codec_, encode(_, _)) + .Times(2) + .WillRepeatedly(Invoke([&](const StreamFrame& frame, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; - EXPECT_EQ("test_detail", response.status().message()); buffer.add("test"); - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, frame.frameFlags().endStream()); })); - EXPECT_CALL(*creator_, response(_, _)) - .WillOnce(Invoke([&](Status status, const Request&) -> ResponsePtr { - auto response = std::make_unique(); - response->status_ = std::move(status); - return response; - })); + auto response = std::make_unique(); + response->data_["response-key"] = "response-value"; + response->stream_frame_flags_ = FrameFlags(StreamFlags(), false); - EXPECT_CALL(upstream_callback, onBindFailure(_, _, _)) - .WillOnce(Invoke([&](ConnectionPool::PoolFailureReason reason, absl::string_view, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(ConnectionPool::PoolFailureReason::RemoteConnectionFailure, reason); + active_stream->onResponseStart(std::move(response)); - active_stream->sendLocalReply(Status(StatusCode::kUnknown, "test_detail"), - [](Response&) {}); - })); + auto response_frame_1 = std::make_unique(); + response_frame_1->stream_frame_flags_ = FrameFlags(StreamFlags(), true); - tcp_conn_pool_.poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + active_stream->onResponseFrame(std::move(response_frame_1)); } -TEST_F(FilterTest, BindUpstreamConnectionSuccessButCloseBeforeUpstreamResponse) { +TEST_F(FilterTest, NewStreamAndReplyNormallyWithDrainClose) { auto mock_decoder_filter_0 = std::make_shared>(); mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - initializeFilter(false, true); + initializeFilter(); auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions(123, true, false, false)); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback; - NiceMock response_callback; - - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback); - - auto typed_upstream_manager = - dynamic_cast(filter_->boundUpstreamConn().ptr()); - - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - // One for the upstream_manager_ and one for the active stream. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response& response, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; - EXPECT_EQ("test_detail", response.status().message()); buffer.add("test"); - callback.onEncodingSuccess(buffer); - })); - - EXPECT_CALL(*creator_, response(_, _)) - .WillOnce(Invoke([&](Status status, const Request&) -> ResponsePtr { - auto response = std::make_unique(); - response->status_ = std::move(status); - return response; - })); - - EXPECT_CALL(response_callback, onConnectionClose(_)) - .WillOnce(Invoke([&](const Network::ConnectionEvent& event) { - EXPECT_EQ(Network::ConnectionEvent::RemoteClose, event); - active_stream->sendLocalReply(Status(StatusCode::kUnknown, "test_detail"), - [](Response&) {}); + callback.onEncodingSuccess(buffer, true); })); - EXPECT_CALL(upstream_callback, onBindSuccess(_, _)) - .WillOnce(Invoke([&](Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(&upstream_connection_, &conn); - filter_->boundUpstreamConn()->registerResponseCallback(123, response_callback); // NOLINT - })); + EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(true)); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); + EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - tcp_conn_pool_.poolReady(upstream_connection_); - typed_upstream_manager->onEvent(Network::ConnectionEvent::RemoteClose); + auto response = std::make_unique(); + active_stream->onResponseStart(std::move(response)); } -TEST_F(FilterTest, BindUpstreamConnectionSuccessButDecodingFailure) { +TEST_F(FilterTest, NewStreamAndReplyNormallyWithStreamDrainClose) { auto mock_decoder_filter_0 = std::make_shared>(); mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - initializeFilter(false, true); + initializeFilter(); auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions(123, true, false, false)); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback; - NiceMock response_callback; - - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback); - - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - // One for the upstream_manager_ and one for the active stream. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response& response, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; - EXPECT_EQ("test_detail", response.status().message()); buffer.add("test"); - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); - EXPECT_CALL(*creator_, response(_, _)) - .WillOnce(Invoke([&](Status status, const Request&) -> ResponsePtr { - auto response = std::make_unique(); - response->status_ = std::move(status); - return response; - })); - - EXPECT_CALL(response_callback, onDecodingFailure()).WillOnce(Invoke([&]() { - active_stream->sendLocalReply(Status(StatusCode::kUnknown, "test_detail"), [](Response&) {}); - })); - - EXPECT_CALL(upstream_callback, onBindSuccess(_, _)) - .WillOnce(Invoke([&](Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(&upstream_connection_, &conn); - filter_->boundUpstreamConn()->registerResponseCallback(123, response_callback); // NOLINT - })); - - tcp_conn_pool_.poolReady(upstream_connection_); - response_decoder_callback->onDecodingFailure(); -} - -TEST_F(FilterTest, BindUpstreamConnectionSuccessAndDecodingSuccess) { - auto mock_decoder_filter_0 = std::make_shared>(); - mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - - initializeFilter(false, true); - - auto request = std::make_unique(); - - filter_->newDownstreamRequest(std::move(request), ExtendedOptions(123, true, false, false)); - EXPECT_EQ(1, filter_->activeStreamsForTest().size()); - - auto active_stream = filter_->activeStreamsForTest().begin()->get(); - - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback; - NiceMock response_callback; - - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback); - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - + // The drain close of factory_context_.drain_manager_ is false, but the drain close of + // active_stream is true. EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - // For the active stream. EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response& response, ResponseEncoderCallback& callback) { - Buffer::OwnedImpl buffer; - EXPECT_EQ("response_2", response.status().message()); - buffer.add("test"); - callback.onEncodingSuccess(buffer); - })); - - EXPECT_CALL(response_callback, onDecodingSuccess(_, _)) - .WillOnce(Invoke([&](ResponsePtr response, ExtendedOptions options) { - active_stream->upstreamResponse(std::move(response), options); - })); - - EXPECT_CALL(upstream_callback, onBindSuccess(_, _)) - .WillOnce(Invoke([&](Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(&upstream_connection_, &conn); - filter_->boundUpstreamConn()->registerResponseCallback(123, response_callback); // NOLINT - })); - - tcp_conn_pool_.poolReady(upstream_connection_); - - auto response_1 = std::make_unique(); - response_1->status_ = Status(StatusCode::kUnknown, "response_1"); - - // This response will be ignored because the there is no related callback registered for it. - response_decoder_callback->onDecodingSuccess(std::move(response_1), - ExtendedOptions(321, false, false, false)); - - auto response_2 = std::make_unique(); - response_2->status_ = Status(StatusCode::kUnknown, "response_2"); - - response_decoder_callback->onDecodingSuccess(std::move(response_2), - ExtendedOptions(123, false, false, false)); -} - -TEST_F(FilterTest, BindUpstreamConnectionSuccessAndMultipleDecodingSuccess) { - auto mock_decoder_filter_0 = std::make_shared>(); - mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - - initializeFilter(false, true); - - auto request_1 = std::make_unique(); - auto request_2 = std::make_unique(); - - filter_->newDownstreamRequest(std::move(request_1), ExtendedOptions(123, true, false, false)); - filter_->newDownstreamRequest(std::move(request_2), ExtendedOptions(321, true, false, false)); - - EXPECT_EQ(2, filter_->activeStreamsForTest().size()); - - auto active_stream_1 = (++filter_->activeStreamsForTest().begin())->get(); - auto active_stream_2 = (filter_->activeStreamsForTest().begin())->get(); - - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback_1; - NiceMock upstream_callback_2; - - NiceMock response_callback_1; - NiceMock response_callback_2; - - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback_1); - filter_->boundUpstreamConn()->registerUpstreamCallback(321, upstream_callback_2); - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)).Times(2); - - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).Times(2).WillRepeatedly(Return(false)); - // Both for the active streams. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); - - EXPECT_CALL(*encoder_, encode(_, _)) - .Times(2) - .WillRepeatedly(Invoke([&](const Response&, ResponseEncoderCallback& callback) { - Buffer::OwnedImpl buffer; - buffer.add("test"); - callback.onEncodingSuccess(buffer); - })); - - EXPECT_CALL(response_callback_1, onDecodingSuccess(_, _)) - .WillOnce(Invoke([&](ResponsePtr response, ExtendedOptions options) { - EXPECT_EQ(123, options.streamId().value()); - active_stream_1->upstreamResponse(std::move(response), options); - })); - - EXPECT_CALL(response_callback_2, onDecodingSuccess(_, _)) - .WillOnce(Invoke([&](ResponsePtr response, ExtendedOptions options) { - EXPECT_EQ(321, options.streamId().value()); - active_stream_2->upstreamResponse(std::move(response), options); - })); - - EXPECT_CALL(upstream_callback_1, onBindSuccess(_, _)) - .WillOnce(Invoke([&](Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(&upstream_connection_, &conn); - filter_->boundUpstreamConn()->registerResponseCallback(123, response_callback_1); // NOLINT - })); - - EXPECT_CALL(upstream_callback_2, onBindSuccess(_, _)) - .WillOnce(Invoke([&](Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(&upstream_connection_, &conn); - filter_->boundUpstreamConn()->registerResponseCallback(321, response_callback_2); // NOLINT - })); - - tcp_conn_pool_.poolReady(upstream_connection_); - - auto response_1 = std::make_unique(); - response_1->status_ = Status(StatusCode::kUnknown, "response_1"); - - response_decoder_callback->onDecodingSuccess(std::move(response_1), - ExtendedOptions(123, false, false, false)); - - auto response_2 = std::make_unique(); - response_2->status_ = Status(StatusCode::kUnknown, "response_2"); - - response_decoder_callback->onDecodingSuccess(std::move(response_2), - ExtendedOptions(321, false, false, false)); -} - -TEST_F(FilterTest, BindUpstreamConnectionSuccessButMultipleRequestHasSameStreamId) { - auto mock_decoder_filter_0 = std::make_shared>(); - mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - - initializeFilter(false, true); - - auto request_1 = std::make_unique(); - auto request_2 = std::make_unique(); - - filter_->newDownstreamRequest(std::move(request_1), ExtendedOptions(123, true, false, false)); - filter_->newDownstreamRequest(std::move(request_2), ExtendedOptions(123, true, false, false)); - - EXPECT_EQ(2, filter_->activeStreamsForTest().size()); - - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback_1; - NiceMock upstream_callback_2; - - NiceMock response_callback_1; - NiceMock response_callback_2; - - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback_1); - - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).Times(2).WillRepeatedly(Return(false)); - // One for upstream_manager_ and two for the active streams. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(3); - // The second request has the same stream id as the first one and this will cause the connection - // to be closed. EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback_2); + + auto response = std::make_unique(); + response->stream_frame_flags_ = FrameFlags(StreamFlags(0, false, true, false), true); + active_stream->onResponseStart(std::move(response)); } -TEST_F(FilterTest, BindUpstreamConnectionSuccessAndWriteSomethinToConnection) { +TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) { auto mock_decoder_filter_0 = std::make_shared>(); mock_decoder_filters_ = {{"mock_0", mock_decoder_filter_0}}; - initializeFilter(false, true); + initializeFilter(true); auto request = std::make_unique(); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions(123, true, false, false)); + auto* span = new NiceMock(); + EXPECT_CALL(*tracer_, startSpan_(_, _, _, _)) + .WillOnce( + Invoke([&](const Tracing::Config& config, Tracing::TraceContext&, + const StreamInfo::StreamInfo&, const Tracing::Decision) -> Tracing::Span* { + EXPECT_EQ(Tracing::OperationName::Egress, config.operationName()); + return span; + })); + + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); auto active_stream = filter_->activeStreamsForTest().begin()->get(); - auto response_decoder = std::make_unique>(); - auto raw_response_decoder = response_decoder.get(); - ResponseDecoderCallback* response_decoder_callback{}; - EXPECT_CALL(*codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - EXPECT_CALL(*raw_response_decoder, setDecoderCallback(_)) - .WillOnce(Invoke( - [&](ResponseDecoderCallback& callback) { response_decoder_callback = &callback; })); - - NiceMock upstream_callback; - NiceMock response_callback; - - { - EXPECT_CALL(filter_callbacks_.connection_, - write(BufferStringEqual("anything_to_downstream"), false)); - Buffer::OwnedImpl buffer; - buffer.add("anything_to_downstream"); - filter_->writeToConnection(buffer); - } - - filter_->bindUpstreamConn(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_)); - filter_->boundUpstreamConn()->registerUpstreamCallback(123, upstream_callback); - - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); + EXPECT_CALL(*span, setTag(_, _)).Times(testing::AnyNumber()); + EXPECT_CALL(*span, finishSpan()); EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); - // One for the upstream_manager_ and one for the active stream. - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); - - EXPECT_CALL(*encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Response& response, ResponseEncoderCallback& callback) { + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { Buffer::OwnedImpl buffer; - EXPECT_EQ("test_detail", response.status().message()); buffer.add("test"); - callback.onEncodingSuccess(buffer); - })); - - EXPECT_CALL(*creator_, response(_, _)) - .WillOnce(Invoke([&](Status status, const Request&) -> ResponsePtr { - auto response = std::make_unique(); - response->status_ = std::move(status); - return response; + callback.onEncodingSuccess(buffer, true); })); - EXPECT_CALL(response_callback, onDecodingFailure()).WillOnce(Invoke([&]() { - active_stream->sendLocalReply(Status(StatusCode::kUnknown, "test_detail"), [](Response&) {}); - })); - - EXPECT_CALL(upstream_callback, onBindSuccess(_, _)) - .WillOnce(Invoke([&](Network::ClientConnection& conn, - Upstream::HostDescriptionConstSharedPtr) { - EXPECT_EQ(&upstream_connection_, &conn); - filter_->boundUpstreamConn()->registerResponseCallback(123, response_callback); // NOLINT - })); - - tcp_conn_pool_.poolReady(upstream_connection_); - - { - EXPECT_CALL(upstream_connection_, write(BufferStringEqual("anything_to_upstream"), false)); - Buffer::OwnedImpl buffer; - buffer.add("anything_to_upstream"); - response_decoder_callback->writeToConnection(buffer); - } + EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); - response_decoder_callback->onDecodingFailure(); + auto response = std::make_unique(); + active_stream->onResponseStart(std::move(response)); } TEST_F(FilterTest, TestStats) { - initializeFilter(false, true); + initializeFilter(false); auto request = std::make_unique(); + request->stream_frame_flags_ = FrameFlags(StreamFlags(123, false, false, false), true); - filter_->newDownstreamRequest(std::move(request), ExtendedOptions(123, true, false, false)); + filter_->onDecodingSuccess(std::move(request)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); EXPECT_EQ(1, filter_config_->stats().request_.value()); EXPECT_EQ(1, filter_config_->stats().request_active_.value()); @@ -1336,7 +915,9 @@ TEST_F(FilterTest, TestStats) { auto active_stream = filter_->activeStreamsForTest().begin()->get(); Buffer::OwnedImpl buffer; buffer.add("123"); - active_stream->onEncodingSuccess(buffer); + // Mock response. + active_stream->onResponseStart(std::make_unique()); + active_stream->onEncodingSuccess(buffer, true); EXPECT_EQ(1, filter_config_->stats().response_.value()); EXPECT_EQ(0, filter_config_->stats().request_active_.value()); } diff --git a/contrib/generic_proxy/filters/network/test/router/config_test.cc b/contrib/generic_proxy/filters/network/test/router/config_test.cc index a873531709ad..d9384a5d8a8d 100644 --- a/contrib/generic_proxy/filters/network/test/router/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/router/config_test.cc @@ -15,18 +15,19 @@ TEST(RouterFactoryTest, RouterFactoryTest) { NiceMock factory_context; RouterFactory factory; - ProtobufWkt::Struct proto_config; + envoy::extensions::filters::network::generic_proxy::router::v3::Router proto_config; EXPECT_NO_THROW(factory.createFilterFactoryFromProto(proto_config, "test", factory_context)); EXPECT_NE(nullptr, factory.createEmptyConfigProto()); EXPECT_EQ(nullptr, factory.createEmptyRouteConfigProto()); EXPECT_EQ(nullptr, factory.createRouteSpecificFilterConfig( - proto_config, factory_context.getServerFactoryContext(), + proto_config, factory_context.serverFactoryContext(), factory_context.messageValidationVisitor())); EXPECT_EQ("envoy.filters.generic.router", factory.name()); EXPECT_EQ(true, factory.isTerminalFilter()); + proto_config.set_bind_upstream_connection(true); auto fn = factory.createFilterFactoryFromProto(proto_config, "test", factory_context); NiceMock mock_cb; diff --git a/contrib/generic_proxy/filters/network/test/router/router_test.cc b/contrib/generic_proxy/filters/network/test/router/router_test.cc index 62384272689e..43ae75503def 100644 --- a/contrib/generic_proxy/filters/network/test/router/router_test.cc +++ b/contrib/generic_proxy/filters/network/test/router/router_test.cc @@ -1,3 +1,5 @@ +#include + #include "source/common/tracing/common_values.h" #include "test/mocks/server/factory_context.h" @@ -30,13 +32,11 @@ namespace { struct TestParameters { bool operator!=(const TestParameters& other) const { - return with_tracing != other.with_tracing || bind_upstream != other.bind_upstream || - bound_already != other.bound_already; + return with_tracing != other.with_tracing || bind_upstream != other.bind_upstream; } bool with_tracing{}; bool bind_upstream{}; - bool bound_already{}; }; class RouterFilterTest : public testing::TestWithParam { @@ -47,23 +47,27 @@ class RouterFilterTest : public testing::TestWithParam { ON_CALL(mock_filter_callback_, activeSpan()).WillByDefault(ReturnRef(active_span_)); ON_CALL(mock_filter_callback_, downstreamCodec()).WillByDefault(ReturnRef(mock_codec_factory_)); ON_CALL(mock_filter_callback_, streamInfo()).WillByDefault(ReturnRef(mock_stream_info_)); - } + ON_CALL(mock_filter_callback_, connection()) + .WillByDefault(Return(&mock_downstream_connection_)); - void setup(ExtendedOptions request_options = ExtendedOptions{}) { auto parameter = GetParam(); - protocol_options_ = ProtocolOptions{parameter.bind_upstream}; - bound_already_ = parameter.bound_already; - with_tracing_ = parameter.with_tracing; - request_options_ = request_options; + mock_downstream_connection_.stream_info_.filter_state_ = + std::make_shared( + StreamInfo::FilterState::LifeSpan::Connection); - ON_CALL(mock_codec_factory_, protocolOptions()).WillByDefault(Return(protocol_options_)); - ON_CALL(mock_filter_callback_, requestOptions()).WillByDefault(Return(request_options_)); + envoy::extensions::filters::network::generic_proxy::router::v3::Router router_config; + router_config.set_bind_upstream_connection(parameter.bind_upstream); + config_ = std::make_shared(router_config); + with_tracing_ = parameter.with_tracing; + } - filter_ = std::make_shared(factory_context_); + void setup(FrameFlags frame_flags = FrameFlags{}) { + filter_ = std::make_shared(config_, factory_context_); filter_->setDecoderFilterCallbacks(mock_filter_callback_); request_ = std::make_unique(); + request_->stream_frame_flags_ = frame_flags; } void cleanUp() { @@ -72,177 +76,163 @@ class RouterFilterTest : public testing::TestWithParam { request_.reset(); } - void expectSetUpstreamCallback() { - - if (!protocol_options_.bindUpstreamConnection()) { - // New connection and response decoder will be created for this upstream request. - auto response_decoder = std::make_unique>(); - mock_response_decoder_ = response_decoder.get(); - EXPECT_CALL(mock_codec_factory_, responseDecoder()) - .WillOnce(Return(ByMove(std::move(response_decoder)))); - - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, - newConnection(_)); - return; - } - - // Set if the bound upstream manager is set. - if (!bound_already_) { - EXPECT_CALL(mock_filter_callback_, bindUpstreamConn(_)) - .WillOnce(Invoke([this](Upstream::TcpPoolData&&) { - mock_filter_callback_.has_upstream_manager_ = true; - })); - } else { - mock_filter_callback_.has_upstream_manager_ = true; - } - auto& upstream_manager = mock_filter_callback_.upstream_manager_; - EXPECT_CALL(upstream_manager, registerUpstreamCallback(_, _)) - .WillOnce(Invoke([&upstream_manager](uint64_t stream_id, UpstreamBindingCallback& cb) { - if (upstream_manager.call_on_bind_success_immediately_) { - cb.onBindSuccess(upstream_manager.upstream_conn_, upstream_manager.upstream_host_); - return; - } - - if (upstream_manager.call_on_bind_failure_immediately_) { - cb.onBindFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure, "", - upstream_manager.upstream_host_); - return; - } - upstream_manager.upstream_callbacks_[stream_id] = &cb; - })); + BoundGenericUpstream* boundUpstreamConnection() { + return mock_downstream_connection_.stream_info_.filter_state_ + ->getDataMutable("envoy.filters.generic.router"); } - void expectSetResponseCallback(PendingResponseCallback* expected_cb) { - if (!protocol_options_.bindUpstreamConnection()) { - EXPECT_CALL(*mock_response_decoder_, setDecoderCallback(_)) - .WillOnce( - Invoke([expected_cb](ResponseDecoderCallback& cb) { EXPECT_EQ(expected_cb, &cb); })); - } else { - ON_CALL(mock_filter_callback_.upstream_manager_, registerResponseCallback(_, _)) - .WillByDefault( - Invoke([this, expected_cb](uint64_t stream_id, PendingResponseCallback& cb) { - EXPECT_EQ(expected_cb, &cb); - mock_filter_callback_.upstream_manager_.response_callbacks_[stream_id] = &cb; - })); - } + void expectCreateConnection() { + creating_connection_ = true; + // New connection and response decoder will be created for this upstream request. + auto client_codec = std::make_unique>(); + mock_client_codec_ = client_codec.get(); + EXPECT_CALL(mock_codec_factory_, createClientCodec()) + .WillOnce(Return(ByMove(std::move(client_codec)))); + EXPECT_CALL(*mock_client_codec_, setCodecCallbacks(_)) + .WillOnce(Invoke([this](ClientCodecCallbacks& cb) { client_cb_ = &cb; })); + + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_, + newConnection(_)); } void expectCancelConnect() { - if (!protocol_options_.bindUpstreamConnection()) { - EXPECT_CALL( - factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.handles_.back(), - cancel(_)); - } else { - auto& upstream_manager = mock_filter_callback_.upstream_manager_; - EXPECT_CALL(upstream_manager, unregisterUpstreamCallback(_)) - .WillOnce(Invoke([&upstream_manager](uint64_t stream_id) { - upstream_manager.upstream_callbacks_.erase(stream_id); - })); - EXPECT_CALL(upstream_manager, unregisterResponseCallback(_)) - .WillOnce(Invoke([&upstream_manager](uint64_t stream_id) { - upstream_manager.response_callbacks_.erase(stream_id); - })); + if (creating_connection_) { + creating_connection_ = false; + + // Only cancel the connection if it is owned by the upstream request. If the connection is + // bound to the downstream connection, then this won't be called. + if (!config_->bindUpstreamConnection()) { + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.handles_.back(), + cancel(_)); + } } } - void expectConnectionClose() { - if (!protocol_options_.bindUpstreamConnection()) { - EXPECT_CALL(mock_upstream_connection_, close(Network::ConnectionCloseType::FlushWrite)); - } else { - auto& upstream_manager = mock_filter_callback_.upstream_manager_; - EXPECT_CALL(upstream_manager, unregisterUpstreamCallback(_)) - .WillOnce(Invoke([&upstream_manager](uint64_t stream_id) { - upstream_manager.upstream_callbacks_.erase(stream_id); - })); - EXPECT_CALL(upstream_manager, unregisterResponseCallback(_)) - .WillOnce(Invoke([&upstream_manager](uint64_t stream_id) { - upstream_manager.response_callbacks_.erase(stream_id); - })); - } + void expectUpstreamConnectionClose() { + EXPECT_CALL(mock_upstream_connection_, close(Network::ConnectionCloseType::FlushWrite)); } void notifyPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason) { - if (!protocol_options_.bindUpstreamConnection()) { - factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure(reason); - } else { - ASSERT(!mock_filter_callback_.upstream_manager_.upstream_callbacks_.empty()); - mock_filter_callback_.upstream_manager_.callOnBindFailure(0, reason); + if (creating_connection_) { + creating_connection_ = false; + + if (config_->bindUpstreamConnection()) { + EXPECT_TRUE(!boundUpstreamConnection()->waitingUpstreamRequestsForTest().empty()); + EXPECT_TRUE(boundUpstreamConnection()->waitingResponseRequestsForTest().empty()); + EXPECT_CALL(mock_downstream_connection_, close(Network::ConnectionCloseType::FlushWrite)); + } + + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(reason); + + if (config_->bindUpstreamConnection()) { + EXPECT_TRUE(boundUpstreamConnection()->waitingUpstreamRequestsForTest().empty()); + EXPECT_TRUE(boundUpstreamConnection()->waitingResponseRequestsForTest().empty()); + } } } void notifyPoolReady() { - if (!protocol_options_.bindUpstreamConnection()) { - EXPECT_CALL(mock_upstream_connection_, write(_, _)); - factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - mock_upstream_connection_); - } else { - ASSERT(!mock_filter_callback_.upstream_manager_.upstream_callbacks_.empty()); - EXPECT_CALL(mock_filter_callback_.upstream_manager_.upstream_conn_, write(_, _)); - mock_filter_callback_.upstream_manager_.callOnBindSuccess(0); + if (creating_connection_) { + creating_connection_ = false; + + if (config_->bindUpstreamConnection()) { + EXPECT_TRUE(!boundUpstreamConnection()->waitingUpstreamRequestsForTest().empty()); + EXPECT_TRUE(boundUpstreamConnection()->waitingResponseRequestsForTest().empty()); + } + + EXPECT_CALL(mock_upstream_connection_, write(_, _)).Times(testing::AtLeast(1)); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(mock_upstream_connection_); + + if (config_->bindUpstreamConnection()) { + EXPECT_TRUE(boundUpstreamConnection()->waitingUpstreamRequestsForTest().empty()); + } } } void notifyConnectionClose(Network::ConnectionEvent event) { - if (!protocol_options_.bindUpstreamConnection()) { - ASSERT(!filter_->upstreamRequestsForTest().empty()); - auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - upstream_request->upstream_manager_->onEvent(event); - } else { - ASSERT(!mock_filter_callback_.upstream_manager_.response_callbacks_.empty()); - mock_filter_callback_.upstream_manager_.callOnConnectionClose(0, event); + ASSERT(!filter_->upstreamRequestsForTest().empty()); + auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); + + if (config_->bindUpstreamConnection()) { + EXPECT_TRUE(boundUpstreamConnection()->waitingUpstreamRequestsForTest().empty()); + EXPECT_TRUE(!boundUpstreamConnection()->waitingResponseRequestsForTest().empty()); + EXPECT_CALL(mock_downstream_connection_, close(Network::ConnectionCloseType::FlushWrite)); + } + + upstream_request->generic_upstream_->onEvent(event); + + if (config_->bindUpstreamConnection()) { + EXPECT_TRUE(boundUpstreamConnection()->waitingUpstreamRequestsForTest().empty()); + EXPECT_TRUE(boundUpstreamConnection()->waitingResponseRequestsForTest().empty()); } } - void notifyDecodingSuccess(ResponsePtr&& response, ExtendedOptions response_options) { - if (!protocol_options_.bindUpstreamConnection()) { - ASSERT(!filter_->upstreamRequestsForTest().empty()); + void notifyDecodingSuccess(StreamFramePtr&& response) { + ASSERT(!filter_->upstreamRequestsForTest().empty()); - auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); + auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - EXPECT_CALL(*mock_response_decoder_, decode(BufferStringEqual("test_1"))) - .WillOnce(Invoke([&](Buffer::Instance& buffer) { - buffer.drain(buffer.length()); - upstream_request->onDecodingSuccess(std::move(response), response_options); - })); + EXPECT_CALL(*mock_client_codec_, decode(BufferStringEqual("test_1"), _)) + .WillOnce(Invoke([this, resp = std::make_shared(std::move(response))]( + Buffer::Instance& buffer, bool) { + buffer.drain(buffer.length()); - Buffer::OwnedImpl test_buffer; - test_buffer.add("test_1"); + const bool end_stream = (*resp)->frameFlags().endStream(); + int pending_request_size = 0; + if (config_->bindUpstreamConnection()) { + pending_request_size = + boundUpstreamConnection()->waitingResponseRequestsForTest().size(); + } - upstream_request->upstream_manager_->onUpstreamData(test_buffer, false); - } else { - ASSERT(!mock_filter_callback_.upstream_manager_.response_callbacks_.empty()); - mock_filter_callback_.upstream_manager_.callOnDecodingSuccess(0, std::move(response), - response_options); - } + client_cb_->onDecodingSuccess(std::move(*resp)); + + if (config_->bindUpstreamConnection()) { + EXPECT_EQ(pending_request_size - (end_stream ? 1 : 0), + boundUpstreamConnection()->waitingResponseRequestsForTest().size()); + } + })); + + Buffer::OwnedImpl test_buffer; + test_buffer.add("test_1"); + + upstream_request->generic_upstream_->onUpstreamData(test_buffer, false); } void notifyDecodingFailure() { - if (!protocol_options_.bindUpstreamConnection()) { - ASSERT(!filter_->upstreamRequestsForTest().empty()); + ASSERT(!filter_->upstreamRequestsForTest().empty()); - auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); + auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - EXPECT_CALL(*mock_response_decoder_, decode(BufferStringEqual("test_1"))) - .WillOnce(Invoke([&](Buffer::Instance& buffer) { - buffer.drain(buffer.length()); - upstream_request->onDecodingFailure(); - })); + if (config_->bindUpstreamConnection()) { + // If upstream connection binding is enabled, the downstream connection will be closed + // when the upstream connection is closed. + EXPECT_CALL(mock_downstream_connection_, close(Network::ConnectionCloseType::FlushWrite)); + } - Buffer::OwnedImpl test_buffer; - test_buffer.add("test_1"); + EXPECT_CALL(mock_upstream_connection_, close(Network::ConnectionCloseType::FlushWrite)) + .WillOnce(Invoke([upstream_request](Network::ConnectionCloseType) { + // Mock clean up closing. + upstream_request->generic_upstream_->onEvent(Network::ConnectionEvent::LocalClose); + })); - upstream_request->upstream_manager_->onUpstreamData(test_buffer, false); - } else { - ASSERT(!mock_filter_callback_.upstream_manager_.response_callbacks_.empty()); - mock_filter_callback_.upstream_manager_.callOnDecodingFailure(0); - } + EXPECT_CALL(*mock_client_codec_, decode(BufferStringEqual("test_1"), _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + buffer.drain(buffer.length()); + client_cb_->onDecodingFailure(); + })); + + Buffer::OwnedImpl test_buffer; + test_buffer.add("test_1"); + + upstream_request->generic_upstream_->onUpstreamData(test_buffer, false); } /** * Kick off a new upstream request. - * @param with_tracing whether to set up tracing. - * @param with_bound_upstream whether to set up bound upstream. This is only make sense when - * protocol_options_.bindUpstreamConnection() is true. */ void kickOffNewUpstreamRequest() { EXPECT_CALL(mock_filter_callback_, routeEntry()).WillOnce(Return(&mock_route_entry_)); @@ -250,9 +240,13 @@ class RouterFilterTest : public testing::TestWithParam { const std::string cluster_name = "cluster_0"; EXPECT_CALL(mock_route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name)); - factory_context_.cluster_manager_.initializeThreadLocalClusters({cluster_name}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {cluster_name}); - expectSetUpstreamCallback(); + if (boundUpstreamConnection() == nullptr) { + // Upstream binding is disabled or not set up yet, try to create a new connection. + expectCreateConnection(); + } if (with_tracing_) { EXPECT_CALL(mock_filter_callback_, tracingConfig()) @@ -267,11 +261,6 @@ class RouterFilterTest : public testing::TestWithParam { .WillOnce(Return(OptRef{})); } - auto request_encoder = std::make_unique>(); - mock_request_encoder_ = request_encoder.get(); - EXPECT_CALL(mock_codec_factory_, requestEncoder()) - .WillOnce(Return(testing::ByMove(std::move(request_encoder)))); - EXPECT_EQ(filter_->onStreamDecoded(*request_), FilterStatus::StopIteration); EXPECT_EQ(1, filter_->upstreamRequestsForTest().size()); } @@ -317,42 +306,43 @@ class RouterFilterTest : public testing::TestWithParam { NiceMock mock_filter_callback_; NiceMock mock_stream_info_; + + NiceMock mock_downstream_connection_; NiceMock mock_upstream_connection_; + NiceMock mock_codec_factory_; - NiceMock* mock_request_encoder_{}; - NiceMock* mock_response_decoder_{}; + NiceMock* mock_client_codec_{}; + + ClientCodecCallbacks* client_cb_{}; NiceMock mock_route_entry_; - std::shared_ptr filter_; - ProtocolOptions protocol_options_; - bool bound_already_{}; + std::shared_ptr config_; - ExtendedOptions request_options_; + std::shared_ptr filter_; std::unique_ptr request_; NiceMock tracing_config_; NiceMock active_span_; NiceMock* child_span_{}; bool with_tracing_{}; + bool creating_connection_{}; }; std::vector getTestParameters() { std::vector ret; - ret.push_back({false, false, false}); - ret.push_back({true, true, false}); - ret.push_back({true, true, true}); + ret.push_back({false, false}); + ret.push_back({true, true}); return ret; } std::string testParameterToString(const testing::TestParamInfo& params) { - return fmt::format("with_tracing_{}_bind_upstream_{}_bound_already_{}", + return fmt::format("with_tracing_{}_bind_upstream_{}", params.param.with_tracing ? "true" : "false", - params.param.bind_upstream ? "true" : "false", - params.param.bound_already ? "true" : "false"); + params.param.bind_upstream ? "true" : "false"); } INSTANTIATE_TEST_SUITE_P(GenericRoute, RouterFilterTest, testing::ValuesIn(getTestParameters()), @@ -369,6 +359,9 @@ TEST_P(RouterFilterTest, OnStreamDecodedAndNoRouteEntry) { EXPECT_EQ(filter_->onStreamDecoded(*request_), FilterStatus::StopIteration); cleanUp(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, NoUpstreamCluster) { @@ -388,6 +381,9 @@ TEST_P(RouterFilterTest, NoUpstreamCluster) { filter_->onStreamDecoded(*request_); cleanUp(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamClusterMaintainMode) { @@ -399,10 +395,12 @@ TEST_P(RouterFilterTest, UpstreamClusterMaintainMode) { EXPECT_CALL(mock_route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name)); - factory_context_.cluster_manager_.initializeThreadLocalClusters({cluster_name}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {cluster_name}); // Maintain mode. - EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + EXPECT_CALL(*factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_, maintenanceMode()) .WillOnce(Return(true)); EXPECT_CALL(mock_filter_callback_, sendLocalReply(_, _)) @@ -412,6 +410,9 @@ TEST_P(RouterFilterTest, UpstreamClusterMaintainMode) { filter_->onStreamDecoded(*request_); cleanUp(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamClusterNoHealthyUpstream) { @@ -423,10 +424,12 @@ TEST_P(RouterFilterTest, UpstreamClusterNoHealthyUpstream) { EXPECT_CALL(mock_route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name)); - factory_context_.cluster_manager_.initializeThreadLocalClusters({cluster_name}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {cluster_name}); // No conn pool. - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); EXPECT_CALL(mock_filter_callback_, sendLocalReply(_, _)) @@ -437,6 +440,9 @@ TEST_P(RouterFilterTest, UpstreamClusterNoHealthyUpstream) { filter_->onStreamDecoded(*request_); cleanUp(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, KickOffNormalUpstreamRequest) { @@ -444,6 +450,9 @@ TEST_P(RouterFilterTest, KickOffNormalUpstreamRequest) { kickOffNewUpstreamRequest(); cleanUp(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestResetBeforePoolCallback) { @@ -472,6 +481,9 @@ TEST_P(RouterFilterTest, UpstreamRequestResetBeforePoolCallback) { filter_->upstreamRequestsForTest().begin()->get()->resetStream(StreamResetReason::LocalReset); EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolFailureConnctionOverflow) { @@ -495,6 +507,9 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolFailureConnctionOverflow) { })); notifyPoolFailure(ConnectionPool::PoolFailureReason::Overflow); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolFailureConnctionTimeout) { @@ -518,22 +533,25 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolFailureConnctionTimeout) { })); notifyPoolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndExpectNoResponse) { - setup(ExtendedOptions{{}, false, false, false}); + setup(FrameFlags(StreamFlags(0, true, false, false), true)); kickOffNewUpstreamRequest(); EXPECT_CALL(mock_filter_callback_, completeDirectly()).WillOnce(Invoke([this]() -> void { EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); })); - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect no response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); if (with_tracing_) { @@ -545,6 +563,9 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndExpectNoResponse) { notifyPoolReady(); EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButConnectionErrorBeforeResponse) { @@ -553,19 +574,17 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButConnectionErrorBeforeRespons auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - expectSetResponseCallback(upstream_request); - - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); notifyPoolReady(); - EXPECT_NE(upstream_request->upstream_conn_, nullptr); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); EXPECT_CALL(mock_filter_callback_, sendLocalReply(_, _)) .WillOnce(Invoke([this](Status status, ResponseUpdateFunction&&) { @@ -575,6 +594,9 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButConnectionErrorBeforeRespons // Mock connection close event. notifyConnectionClose(Network::ConnectionEvent::LocalClose); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButConnectionTerminationBeforeResponse) { @@ -583,19 +605,17 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButConnectionTerminationBeforeR auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - expectSetResponseCallback(upstream_request); - - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); notifyPoolReady(); - EXPECT_NE(upstream_request->upstream_conn_, nullptr); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); EXPECT_CALL(mock_filter_callback_, sendLocalReply(_, _)) .WillOnce(Invoke([this](Status status, ResponseUpdateFunction&&) { @@ -605,6 +625,9 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButConnectionTerminationBeforeR // Mock connection close event. notifyConnectionClose(Network::ConnectionEvent::RemoteClose); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButStreamDestroyBeforeResponse) { @@ -613,25 +636,26 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyButStreamDestroyBeforeResponse) auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - expectSetResponseCallback(upstream_request); - - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); notifyPoolReady(); - EXPECT_NE(upstream_request->upstream_conn_, nullptr); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); - expectConnectionClose(); + expectUpstreamConnectionClose(); filter_->onDestroy(); // Do nothing for the second call. filter_->onDestroy(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponse) { @@ -640,14 +664,12 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponse) { auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - expectSetResponseCallback(upstream_request); - - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); if (with_tracing_) { @@ -657,39 +679,164 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponse) { notifyPoolReady(); - EXPECT_NE(upstream_request->upstream_conn_, nullptr); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); if (with_tracing_) { EXPECT_CALL(*child_span_, setTag(_, _)).Times(testing::AnyNumber()); EXPECT_CALL(*child_span_, finishSpan()); } - EXPECT_CALL(mock_filter_callback_, upstreamResponse(_, _)) - .WillOnce(Invoke([this](ResponsePtr, ExtendedOptions) { - // When the response is sent to callback, the upstream request should be removed. - EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + // When the response is sent to callback, the upstream request should be removed. + EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + })); + + auto response = std::make_unique(); + notifyDecodingSuccess(std::move(response)); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); +} + +TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseAndMultipleRequest) { + for (size_t i = 0; i < 5; i++) { + setup(FrameFlags(StreamFlags(i))); + + // Expect immediate encoding. + if (GetParam().bind_upstream && i > 0) { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { + Buffer::OwnedImpl buffer; + buffer.add("hello"); + // Expect response. + callback.onEncodingSuccess(buffer, true); + })); + } + + kickOffNewUpstreamRequest(); + + // Expect encoding after pool ready. + if (!GetParam().bind_upstream || i == 0) { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { + Buffer::OwnedImpl buffer; + buffer.add("hello"); + // Expect response. + callback.onEncodingSuccess(buffer, true); + })); + } + + auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); + + notifyPoolReady(); + + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); + + EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + // When the response is sent to callback, the upstream request should be removed. + EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + })); + + auto response = std::make_unique(); + response->stream_frame_flags_ = FrameFlags(StreamFlags(i)); + notifyDecodingSuccess(std::move(response)); + + cleanUp(); + } + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); +} + +TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithMultipleFrames) { + // There are multiple frames in the request. + setup(FrameFlags(StreamFlags(0, false, false, true), /*end_stream*/ false)); + kickOffNewUpstreamRequest(); + + auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); + + if (with_tracing_) { + // Inject tracing context. + EXPECT_CALL(*child_span_, injectContext(_, _)); + } + + auto frame_1 = std::make_unique(); + frame_1->stream_frame_flags_ = FrameFlags(StreamFlags(0, false, false, true), false); + + // This only store the frame and does nothing else because the pool is not ready yet. + filter_->onStreamFrame(std::move(frame_1)); + + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .Times(2) + .WillRepeatedly(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { + Buffer::OwnedImpl buffer; + buffer.add("hello"); + // Expect response. + callback.onEncodingSuccess(buffer, false); + })); + + // This will trigger two frames to be sent. + notifyPoolReady(); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); + + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { + Buffer::OwnedImpl buffer; + buffer.add("hello"); + // Expect response. + callback.onEncodingSuccess(buffer, true); + })); + + // End stream is set to true by default. + auto frame_2 = std::make_unique(); + // This will trigger the last frame to be sent directly because connection is ready and other + // frames are already sent. + filter_->onStreamFrame(std::move(frame_2)); + + if (with_tracing_) { + EXPECT_CALL(*child_span_, setTag(_, _)).Times(testing::AnyNumber()); + EXPECT_CALL(*child_span_, finishSpan()); + } + + EXPECT_CALL(mock_filter_callback_, onResponseStart(_)); + EXPECT_CALL(mock_filter_callback_, onResponseFrame(_)) + .Times(2) + .WillRepeatedly(Invoke([this](StreamFramePtr frame) { + // When the entire response is sent to callback, the upstream request should be removed. + if (frame->frameFlags().endStream()) { + EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + } else { + EXPECT_EQ(1, filter_->upstreamRequestsForTest().size()); + } })); auto response = std::make_unique(); - notifyDecodingSuccess(std::move(response), ExtendedOptions()); + response->stream_frame_flags_ = FrameFlags(StreamFlags(0, false, false, false), false); + notifyDecodingSuccess(std::move(response)); + + auto response_frame_1 = std::make_unique(); + response_frame_1->stream_frame_flags_ = FrameFlags(StreamFlags(0, false, false, false), false); + notifyDecodingSuccess(std::move(response_frame_1)); + + // End stream is set to true by default. + auto response_frame_2 = std::make_unique(); + notifyDecodingSuccess(std::move(response_frame_2)); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithDrainCloseSetInResponse) { - ONLY_RUN_TEST_WITH_PARAM((TestParameters{false, false, false})); - setup(); kickOffNewUpstreamRequest(); auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - expectSetResponseCallback(upstream_request); - - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); if (with_tracing_) { @@ -699,18 +846,21 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithDrainCloseSetInR notifyPoolReady(); - EXPECT_NE(upstream_request->upstream_conn_, nullptr); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); - EXPECT_CALL(mock_filter_callback_, upstreamResponse(_, _)) - .WillOnce(Invoke([this](ResponsePtr, ExtendedOptions) { - // When the response is sent to callback, the upstream request should be removed. - EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); - })); + EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + // When the response is sent to callback, the upstream request should be removed. + EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); + })); EXPECT_CALL(mock_upstream_connection_, close(Network::ConnectionCloseType::FlushWrite)); auto response = std::make_unique(); - notifyDecodingSuccess(std::move(response), ExtendedOptions({}, false, true, false)); + response->stream_frame_flags_ = FrameFlags(StreamFlags(0, false, true, false), true); + notifyDecodingSuccess(std::move(response)); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseDecodingFailure) { @@ -719,28 +869,30 @@ TEST_P(RouterFilterTest, UpstreamRequestPoolReadyAndResponseDecodingFailure) { auto upstream_request = filter_->upstreamRequestsForTest().begin()->get(); - expectSetResponseCallback(upstream_request); - - EXPECT_CALL(*mock_request_encoder_, encode(_, _)) - .WillOnce(Invoke([&](const Request&, RequestEncoderCallback& callback) -> void { + EXPECT_CALL(*mock_client_codec_, encode(_, _)) + .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { Buffer::OwnedImpl buffer; buffer.add("hello"); // Expect response. - callback.onEncodingSuccess(buffer); + callback.onEncodingSuccess(buffer, true); })); notifyPoolReady(); - EXPECT_NE(upstream_request->upstream_conn_, nullptr); + EXPECT_NE(nullptr, upstream_request->generic_upstream_->connection().ptr()); EXPECT_CALL(mock_filter_callback_, sendLocalReply(_, _)) .WillOnce(Invoke([this](Status status, ResponseUpdateFunction&&) { EXPECT_EQ(0, filter_->upstreamRequestsForTest().size()); - EXPECT_EQ(status.message(), "protocol_error"); + // Decoding error of bound upstream connection will not be notified to every requests + // and will be treated as local reset. + EXPECT_TRUE(status.message() == "protocol_error" || status.message() == "local_reset"); })); - expectConnectionClose(); notifyDecodingFailure(); + + // Mock downstream closing. + mock_downstream_connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } TEST_P(RouterFilterTest, LoadBalancerContextDownstreamConnection) { diff --git a/contrib/golang/common/dso/dso.cc b/contrib/golang/common/dso/dso.cc index 2071b551c28b..ee8b0abda452 100644 --- a/contrib/golang/common/dso/dso.cc +++ b/contrib/golang/common/dso/dso.cc @@ -76,9 +76,9 @@ GoUint64 HttpFilterDsoImpl::envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUi return envoy_go_filter_merge_http_plugin_config_(p0, p1, p2, p3); } -void HttpFilterDsoImpl::envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0) { +void HttpFilterDsoImpl::envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0, GoInt p1) { ASSERT(envoy_go_filter_destroy_http_plugin_config_ != nullptr); - return envoy_go_filter_destroy_http_plugin_config_(p0); + return envoy_go_filter_destroy_http_plugin_config_(p0, p1); } GoUint64 HttpFilterDsoImpl::envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2, @@ -196,15 +196,15 @@ GoUint64 NetworkFilterDsoImpl::envoyGoFilterOnDownstreamWrite(void* w, GoUint64 return envoy_go_filter_on_downstream_write_(w, data_size, data_ptr, slice_num, end_of_stream); } -void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamConnectionReady(void* w, GoUint64 connID) { +void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamConnectionReady(void* w, GoUint64 conn_id) { ASSERT(envoy_go_filter_on_upstream_connection_ready_ != nullptr); - envoy_go_filter_on_upstream_connection_ready_(w, connID); + envoy_go_filter_on_upstream_connection_ready_(w, conn_id); } void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason, - GoUint64 connID) { + GoUint64 conn_id) { ASSERT(envoy_go_filter_on_upstream_connection_failure_ != nullptr); - envoy_go_filter_on_upstream_connection_failure_(w, reason, connID); + envoy_go_filter_on_upstream_connection_failure_(w, reason, conn_id); } void NetworkFilterDsoImpl::envoyGoFilterOnUpstreamData(void* w, GoUint64 data_size, diff --git a/contrib/golang/common/dso/dso.h b/contrib/golang/common/dso/dso.h index fc1e7a0e4071..2ad0be8ada66 100644 --- a/contrib/golang/common/dso/dso.h +++ b/contrib/golang/common/dso/dso.h @@ -34,7 +34,7 @@ class HttpFilterDso : public Dso { virtual GoUint64 envoyGoFilterNewHttpPluginConfig(httpConfig* p0) PURE; virtual GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) PURE; - virtual void envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0) PURE; + virtual void envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0, GoInt p1) PURE; virtual GoUint64 envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) PURE; virtual GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2, @@ -52,7 +52,7 @@ class HttpFilterDsoImpl : public HttpFilterDso { GoUint64 envoyGoFilterNewHttpPluginConfig(httpConfig* p0) override; GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override; - void envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0) override; + void envoyGoFilterDestroyHttpPluginConfig(GoUint64 p0, GoInt p1) override; GoUint64 envoyGoFilterOnHttpHeader(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override; GoUint64 envoyGoFilterOnHttpData(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) override; @@ -64,7 +64,7 @@ class HttpFilterDsoImpl : public HttpFilterDso { GoUint64 (*envoy_go_filter_new_http_plugin_config_)(httpConfig* p0) = {nullptr}; GoUint64 (*envoy_go_filter_merge_http_plugin_config_)(GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) = {nullptr}; - void (*envoy_go_filter_destroy_http_plugin_config_)(GoUint64 p0) = {nullptr}; + void (*envoy_go_filter_destroy_http_plugin_config_)(GoUint64 p0, GoInt p1) = {nullptr}; GoUint64 (*envoy_go_filter_on_http_header_)(httpRequest* p0, GoUint64 p1, GoUint64 p2, GoUint64 p3) = {nullptr}; GoUint64 (*envoy_go_filter_on_http_data_)(httpRequest* p0, GoUint64 p1, GoUint64 p2, @@ -123,9 +123,9 @@ class NetworkFilterDso : public Dso { virtual GoUint64 envoyGoFilterOnDownstreamWrite(void* w, GoUint64 data_size, GoUint64 data_ptr, GoInt slice_num, GoInt end_of_stream) PURE; - virtual void envoyGoFilterOnUpstreamConnectionReady(void* w, GoUint64 connID) PURE; + virtual void envoyGoFilterOnUpstreamConnectionReady(void* w, GoUint64 conn_id) PURE; virtual void envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason, - GoUint64 connID) PURE; + GoUint64 conn_id) PURE; virtual void envoyGoFilterOnUpstreamData(void* w, GoUint64 data_size, GoUint64 data_ptr, GoInt slice_num, GoInt end_of_stream) PURE; virtual void envoyGoFilterOnUpstreamEvent(void* w, GoInt event) PURE; @@ -149,8 +149,8 @@ class NetworkFilterDsoImpl : public NetworkFilterDso { GoUint64 envoyGoFilterOnDownstreamWrite(void* w, GoUint64 data_size, GoUint64 data_ptr, GoInt slice_num, GoInt end_of_stream) override; - void envoyGoFilterOnUpstreamConnectionReady(void* w, GoUint64 connID) override; - void envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason, GoUint64 connID) override; + void envoyGoFilterOnUpstreamConnectionReady(void* w, GoUint64 conn_id) override; + void envoyGoFilterOnUpstreamConnectionFailure(void* w, GoInt reason, GoUint64 conn_id) override; void envoyGoFilterOnUpstreamData(void* w, GoUint64 data_size, GoUint64 data_ptr, GoInt slice_num, GoInt end_of_stream) override; void envoyGoFilterOnUpstreamEvent(void* w, GoInt event) override; @@ -172,9 +172,9 @@ class NetworkFilterDsoImpl : public NetworkFilterDso { GoInt slice_num, GoInt end_of_stream) = {nullptr}; - void (*envoy_go_filter_on_upstream_connection_ready_)(void* w, GoUint64 connID) = {nullptr}; + void (*envoy_go_filter_on_upstream_connection_ready_)(void* w, GoUint64 conn_id) = {nullptr}; void (*envoy_go_filter_on_upstream_connection_failure_)(void* w, GoInt reason, - GoUint64 connID) = {nullptr}; + GoUint64 conn_id) = {nullptr}; void (*envoy_go_filter_on_upstream_data_)(void* w, GoUint64 data_size, GoUint64 data_ptr, GoInt slice_num, GoInt end_of_stream) = {nullptr}; void (*envoy_go_filter_on_upstream_event_)(void* w, GoInt event) = {nullptr}; diff --git a/contrib/golang/common/dso/libgolang.h b/contrib/golang/common/dso/libgolang.h index 96467474711b..7d400fbc1e30 100644 --- a/contrib/golang/common/dso/libgolang.h +++ b/contrib/golang/common/dso/libgolang.h @@ -97,35 +97,26 @@ extern "C" { // go:linkname envoyGoFilterNewHttpPluginConfig // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterNewHttpPluginConfig -extern GoUint64 -envoyGoFilterNewHttpPluginConfig(httpConfig* p0); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterNewHttpPluginConfig(httpConfig* p0); // go:linkname envoyGoFilterDestroyHttpPluginConfig // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterDestroyHttpPluginConfig -extern void envoyGoFilterDestroyHttpPluginConfig(GoUint64 id); +extern void envoyGoFilterDestroyHttpPluginConfig(GoUint64 id, GoInt need_delay); // go:linkname envoyGoFilterMergeHttpPluginConfig // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterMergeHttpPluginConfig -extern GoUint64 -envoyGoFilterMergeHttpPluginConfig(GoUint64 namePtr, // NOLINT(readability-identifier-naming) - GoUint64 nameLen, // NOLINT(readability-identifier-naming) - GoUint64 parentId, // NOLINT(readability-identifier-naming) - GoUint64 childId); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterMergeHttpPluginConfig(GoUint64 name_ptr, GoUint64 name_len, + GoUint64 parent_id, GoUint64 child_id); // go:linkname envoyGoFilterOnHttpHeader // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterOnHttpHeader -extern GoUint64 -envoyGoFilterOnHttpHeader(httpRequest* r, - GoUint64 endStream, // NOLINT(readability-identifier-naming) - GoUint64 headerNum, // NOLINT(readability-identifier-naming) - GoUint64 headerBytes); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnHttpHeader(httpRequest* r, GoUint64 end_stream, GoUint64 header_num, + GoUint64 header_bytes); // go:linkname envoyGoFilterOnHttpData // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterOnHttpData -extern GoUint64 envoyGoFilterOnHttpData(httpRequest* r, - GoUint64 endStream, // NOLINT(readability-identifier-naming) - GoUint64 buffer, - GoUint64 length); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnHttpData(httpRequest* r, GoUint64 end_stream, GoUint64 buffer, + GoUint64 length); // go:linkname envoyGoFilterOnHttpLog // github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http.envoyGoFilterOnHttpLog @@ -141,83 +132,53 @@ extern void envoyGoRequestSemaDec(httpRequest* r); // go:linkname envoyGoOnClusterSpecify // github.com/envoyproxy/envoy/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier.envoyGoOnClusterSpecify -extern GoInt64 envoyGoOnClusterSpecify(GoUint64 pluginPtr, // NOLINT(readability-identifier-naming) - GoUint64 headerPtr, // NOLINT(readability-identifier-naming) - GoUint64 pluginId, // NOLINT(readability-identifier-naming) - GoUint64 bufferPtr, // NOLINT(readability-identifier-naming) - GoUint64 bufferLen); // NOLINT(readability-identifier-naming) +extern GoInt64 envoyGoOnClusterSpecify(GoUint64 plugin_ptr, GoUint64 header_ptr, GoUint64 plugin_id, + GoUint64 buffer_ptr, GoUint64 buffer_len); // go:linkname envoyGoClusterSpecifierNewPlugin // github.com/envoyproxy/envoy/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier.envoyGoClusterSpecifierNewPlugin -extern GoUint64 -envoyGoClusterSpecifierNewPlugin(GoUint64 configPtr, // NOLINT(readability-identifier-naming) - GoUint64 configLen); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoClusterSpecifierNewPlugin(GoUint64 config_ptr, GoUint64 config_len); // go:linkname envoyGoFilterOnNetworkFilterConfig // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnNetworkFilterConfig -extern GoUint64 -envoyGoFilterOnNetworkFilterConfig(GoUint64 libraryIDPtr, // NOLINT(readability-identifier-naming) - GoUint64 libraryIDLen, // NOLINT(readability-identifier-naming) - GoUint64 configPtr, // NOLINT(readability-identifier-naming) - GoUint64 configLen); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnNetworkFilterConfig(GoUint64 library_id_ptr, GoUint64 library_id_len, + GoUint64 config_ptr, GoUint64 config_len); // go:linkname envoyGoFilterOnDownstreamConnection // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamConnection -extern GoUint64 -envoyGoFilterOnDownstreamConnection(void* f, - GoUint64 pluginNamePtr, // NOLINT(readability-identifier-naming) - GoUint64 pluginNameLen, // NOLINT(readability-identifier-naming) - GoUint64 configID); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnDownstreamConnection(void* f, GoUint64 plugin_name_ptr, + GoUint64 plugin_name_len, GoUint64 config_id); // go:linkname envoyGoFilterOnDownstreamData // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamData -extern GoUint64 -envoyGoFilterOnDownstreamData(void* f, - GoUint64 dataSize, // NOLINT(readability-identifier-naming) - GoUint64 dataPtr, // NOLINT(readability-identifier-naming) - GoInt sliceNum, // NOLINT(readability-identifier-naming) - GoInt endOfStream); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnDownstreamData(void* f, GoUint64 data_size, GoUint64 data_ptr, + GoInt slice_num, GoInt end_of_stream); // go:linkname envoyGoFilterOnDownstreamWrite // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamWrite -extern GoUint64 -envoyGoFilterOnDownstreamWrite(void* f, - GoUint64 dataSize, // NOLINT(readability-identifier-naming) - GoUint64 dataPtr, // NOLINT(readability-identifier-naming) - GoInt sliceNum, // NOLINT(readability-identifier-naming) - GoInt endOfStream); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnDownstreamWrite(void* f, GoUint64 data_size, GoUint64 data_ptr, + GoInt slice_num, GoInt end_of_stream); // go:linkname envoyGoFilterOnDownstreamEvent // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnDownstreamEvent -extern void envoyGoFilterOnDownstreamEvent(void* f, - GoInt event); // NOLINT(readability-identifier-naming) +extern void envoyGoFilterOnDownstreamEvent(void* f, GoInt event); // go:linkname envoyGoFilterOnUpstreamConnectionReady // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamConnectionReady -extern void -envoyGoFilterOnUpstreamConnectionReady(void* f, - GoUint64 connID); // NOLINT(readability-identifier-naming) +extern void envoyGoFilterOnUpstreamConnectionReady(void* f, GoUint64 conn_id); // go:linkname envoyGoFilterOnUpstreamConnectionFailure // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamConnectionFailure -extern void -envoyGoFilterOnUpstreamConnectionFailure(void* f, - GoInt reason, // NOLINT(readability-identifier-naming) - GoUint64 connID); // NOLINT(readability-identifier-naming) +extern void envoyGoFilterOnUpstreamConnectionFailure(void* f, GoInt reason, GoUint64 conn_id); // go:linkname envoyGoFilterOnUpstreamData // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamData -extern GoUint64 -envoyGoFilterOnUpstreamData(void* f, - GoUint64 dataSize, // NOLINT(readability-identifier-naming) - GoUint64 dataPtr, // NOLINT(readability-identifier-naming) - GoInt sliceNum, // NOLINT(readability-identifier-naming) - GoInt endOfStream); // NOLINT(readability-identifier-naming) +extern GoUint64 envoyGoFilterOnUpstreamData(void* f, GoUint64 data_size, GoUint64 data_ptr, + GoInt slice_num, GoInt end_of_stream); // go:linkname envoyGoFilterOnUpstreamEvent // github.com/envoyproxy/envoy/contrib/golang/filters/network/source/go/pkg/network.envoyGoFilterOnUpstreamEvent -extern void envoyGoFilterOnUpstreamEvent(void* f, - GoInt event); // NOLINT(readability-identifier-naming) +extern void envoyGoFilterOnUpstreamEvent(void* f, GoInt event); #ifdef __cplusplus } diff --git a/contrib/golang/common/dso/test/dso_test.cc b/contrib/golang/common/dso/test/dso_test.cc index ebc44e8d9c77..2954e20da6d2 100644 --- a/contrib/golang/common/dso/test/dso_test.cc +++ b/contrib/golang/common/dso/test/dso_test.cc @@ -98,13 +98,13 @@ TEST(DsoInstanceTest, RemovePluginConfig) { EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(config), 0); // remove it - dso->envoyGoFilterDestroyHttpPluginConfig(300); + dso->envoyGoFilterDestroyHttpPluginConfig(300, 0); // new again, after removed. EXPECT_EQ(dso->envoyGoFilterNewHttpPluginConfig(config), 300); // remove twice should be ok - dso->envoyGoFilterDestroyHttpPluginConfig(300); - dso->envoyGoFilterDestroyHttpPluginConfig(300); + dso->envoyGoFilterDestroyHttpPluginConfig(300, 0); + dso->envoyGoFilterDestroyHttpPluginConfig(300, 0); } } // namespace diff --git a/contrib/golang/common/dso/test/mocks.h b/contrib/golang/common/dso/test/mocks.h index ab688967135d..777068961d26 100644 --- a/contrib/golang/common/dso/test/mocks.h +++ b/contrib/golang/common/dso/test/mocks.h @@ -16,7 +16,7 @@ class MockHttpFilterDsoImpl : public HttpFilterDso { MOCK_METHOD(GoUint64, envoyGoFilterNewHttpPluginConfig, (httpConfig * p0)); MOCK_METHOD(GoUint64, envoyGoFilterMergeHttpPluginConfig, (GoUint64 p0, GoUint64 p1, GoUint64 p2, GoUint64 p3)); - MOCK_METHOD(void, envoyGoFilterDestroyHttpPluginConfig, (GoUint64 p0)); + MOCK_METHOD(void, envoyGoFilterDestroyHttpPluginConfig, (GoUint64 p0, GoInt p1)); MOCK_METHOD(GoUint64, envoyGoFilterOnHttpHeader, (httpRequest * p0, GoUint64 p1, GoUint64 p2, GoUint64 p3)); MOCK_METHOD(GoUint64, envoyGoFilterOnHttpData, @@ -42,9 +42,9 @@ class MockNetworkFilterDsoImpl : public NetworkFilterDso { MOCK_METHOD(GoUint64, envoyGoFilterOnDownstreamWrite, (void* w, GoUint64 dataSize, GoUint64 dataPtr, GoInt sliceNum, GoInt endOfStream)); - MOCK_METHOD(void, envoyGoFilterOnUpstreamConnectionReady, (void* w, GoUint64 connID)); + MOCK_METHOD(void, envoyGoFilterOnUpstreamConnectionReady, (void* w, GoUint64 conn_id)); MOCK_METHOD(void, envoyGoFilterOnUpstreamConnectionFailure, - (void* w, GoInt reason, GoUint64 connID)); + (void* w, GoInt reason, GoUint64 conn_id)); MOCK_METHOD(void, envoyGoFilterOnUpstreamData, (void* w, GoUint64 dataSize, GoUint64 dataPtr, GoInt sliceNum, GoInt endOfStream)); MOCK_METHOD(void, envoyGoFilterOnUpstreamEvent, (void* w, GoInt event)); diff --git a/contrib/golang/common/dso/test/test_data/simple.go b/contrib/golang/common/dso/test/test_data/simple.go index 01fb808f71b3..1af57bfb6b8e 100644 --- a/contrib/golang/common/dso/test/test_data/simple.go +++ b/contrib/golang/common/dso/test/test_data/simple.go @@ -33,7 +33,7 @@ func envoyGoFilterNewHttpPluginConfig(c *C.httpConfig) uint64 { } //export envoyGoFilterDestroyHttpPluginConfig -func envoyGoFilterDestroyHttpPluginConfig(id uint64) { +func envoyGoFilterDestroyHttpPluginConfig(id uint64, needDelay int) { configCache.Delete(id) } diff --git a/contrib/golang/common/go/api/BUILD b/contrib/golang/common/go/api/BUILD index 2a9245ed571d..9d3d70db763e 100644 --- a/contrib/golang/common/go/api/BUILD +++ b/contrib/golang/common/go/api/BUILD @@ -10,7 +10,6 @@ go_library( srcs = [ "api.h", "capi.go", - "cgocheck.go", "filter.go", "logger.go", "type.go", diff --git a/contrib/golang/common/go/api/api.h b/contrib/golang/common/go/api/api.h index 312916ad0528..504d412d3e14 100644 --- a/contrib/golang/common/go/api/api.h +++ b/contrib/golang/common/go/api/api.h @@ -6,24 +6,24 @@ extern "C" { #endif -#include +#include // NOLINT(modernize-deprecated-headers) typedef struct { // NOLINT(modernize-use-using) const char* data; - unsigned long long int len; + uint64_t len; } Cstring; typedef struct { // NOLINT(modernize-use-using) Cstring plugin_name; - unsigned long long int configId; + uint64_t configId; int phase; } httpRequest; typedef struct { // NOLINT(modernize-use-using) - unsigned long long int plugin_name_ptr; - unsigned long long int plugin_name_len; - unsigned long long int config_ptr; - unsigned long long int config_len; + uint64_t plugin_name_ptr; + uint64_t plugin_name_len; + uint64_t config_ptr; + uint64_t config_len; int is_route_config; } httpConfig; @@ -52,57 +52,67 @@ typedef enum { // NOLINT(modernize-use-using) } CAPIStatus; CAPIStatus envoyGoFilterHttpContinue(void* r, int status); -CAPIStatus envoyGoFilterHttpSendLocalReply(void* r, int response_code, void* body_text, - void* headers, long long int grpc_status, void* details); -CAPIStatus envoyGoFilterHttpSendPanicReply(void* r, void* details); - -CAPIStatus envoyGoFilterHttpGetHeader(void* r, void* key, void* value); +CAPIStatus envoyGoFilterHttpSendLocalReply(void* r, int response_code, void* body_text_data, + int body_text_len, void* headers, int headers_num, + long long int grpc_status, void* details_data, + int details_len); +CAPIStatus envoyGoFilterHttpSendPanicReply(void* r, void* details_data, int details_len); + +CAPIStatus envoyGoFilterHttpGetHeader(void* r, void* key_data, int key_len, uint64_t* value_data, + int* value_len); CAPIStatus envoyGoFilterHttpCopyHeaders(void* r, void* strs, void* buf); -CAPIStatus envoyGoFilterHttpSetHeaderHelper(void* r, void* key, void* value, headerAction action); -CAPIStatus envoyGoFilterHttpRemoveHeader(void* r, void* key); +CAPIStatus envoyGoFilterHttpSetHeaderHelper(void* r, void* key_data, int key_len, void* value_data, + int value_len, headerAction action); +CAPIStatus envoyGoFilterHttpRemoveHeader(void* r, void* key_data, int key_len); -CAPIStatus envoyGoFilterHttpGetBuffer(void* r, unsigned long long int buffer, void* value); -CAPIStatus envoyGoFilterHttpSetBufferHelper(void* r, unsigned long long int buffer, void* data, - int length, bufferAction action); +CAPIStatus envoyGoFilterHttpGetBuffer(void* r, uint64_t buffer, void* value); +CAPIStatus envoyGoFilterHttpDrainBuffer(void* r, uint64_t buffer, uint64_t length); +CAPIStatus envoyGoFilterHttpSetBufferHelper(void* r, uint64_t buffer, void* data, int length, + bufferAction action); CAPIStatus envoyGoFilterHttpCopyTrailers(void* r, void* strs, void* buf); -CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value, headerAction action); -CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key); +CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key_data, int key_len, void* value, + int value_len, headerAction action); +CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key_data, int key_len); -CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value); -CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, void* value); +CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, uint64_t* value_data, int* value_len); +CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, uint64_t* value); -CAPIStatus envoyGoFilterHttpGetDynamicMetadata(void* r, void* name, void* hand); -CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name, void* key, void* buf); +CAPIStatus envoyGoFilterHttpGetDynamicMetadata(void* r, void* name_data, int name_len, + uint64_t* value_data, int* value_len); +CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name_data, int name_len, + void* key_data, int key_len, void* buf_data, + int buf_len); -void envoyGoFilterLog(uint32_t level, void* message); +void envoyGoFilterLog(uint32_t level, void* message_data, int message_len); uint32_t envoyGoFilterLogLevel(); void envoyGoFilterHttpFinalize(void* r, int reason); void envoyGoConfigHttpFinalize(void* c); -CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value, int state_type, +CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key_data, int key_len, + void* value_data, int value_len, int state_type, int life_span, int stream_sharing); -CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key, void* value); -CAPIStatus envoyGoFilterHttpGetStringProperty(void* r, void* key, void* value, int* rc); +CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key_data, int key_len, + uint64_t* value_data, int* value_len); +CAPIStatus envoyGoFilterHttpGetStringProperty(void* r, void* key_data, int key_len, + uint64_t* value_data, int* value_len, int* rc); -CAPIStatus envoyGoFilterHttpDefineMetric(void* c, uint32_t metric_type, void* name, - void* metric_id); +CAPIStatus envoyGoFilterHttpDefineMetric(void* c, uint32_t metric_type, void* name_data, + int name_len, uint32_t* metric_id); CAPIStatus envoyGoFilterHttpIncrementMetric(void* c, uint32_t metric_id, int64_t offset); -CAPIStatus envoyGoFilterHttpGetMetric(void* c, uint32_t metric_id, void* value); +CAPIStatus envoyGoFilterHttpGetMetric(void* c, uint32_t metric_id, uint64_t* value); CAPIStatus envoyGoFilterHttpRecordMetric(void* c, uint32_t metric_id, uint64_t value); // downstream -CAPIStatus envoyGoFilterDownstreamClose(void* wrapper, int closeType); -CAPIStatus envoyGoFilterDownstreamWrite(void* wrapper, void* buffers, int buffersNum, - int endStream); +CAPIStatus envoyGoFilterDownstreamClose(void* wrapper, int close_type); +CAPIStatus envoyGoFilterDownstreamWrite(void* f, void* buffer_ptr, int buffer_len, int end_stream); void envoyGoFilterDownstreamFinalize(void* wrapper, int reason); CAPIStatus envoyGoFilterDownstreamInfo(void* wrapper, int t, void* ret); -// upstream -void* envoyGoFilterUpstreamConnect(void* libraryID, void* addr, unsigned long long int connID); -CAPIStatus envoyGoFilterUpstreamWrite(void* wrapper, void* buffers, int buffersNum, int endStream); -CAPIStatus envoyGoFilterUpstreamClose(void* wrapper, int closeType); +void* envoyGoFilterUpstreamConnect(void* library_id, void* addr, uint64_t conn_id); +CAPIStatus envoyGoFilterUpstreamWrite(void* u, void* buffer_ptr, int buffer_len, int end_stream); +CAPIStatus envoyGoFilterUpstreamClose(void* wrapper, int close_type); void envoyGoFilterUpstreamFinalize(void* wrapper, int reason); CAPIStatus envoyGoFilterUpstreamInfo(void* wrapper, int t, void* ret); diff --git a/contrib/golang/common/go/api/capi.go b/contrib/golang/common/go/api/capi.go index 6bd99ea0294e..2ff034c7f510 100644 --- a/contrib/golang/common/go/api/capi.go +++ b/contrib/golang/common/go/api/capi.go @@ -21,23 +21,25 @@ import "unsafe" type HttpCAPI interface { HttpContinue(r unsafe.Pointer, status uint64) - HttpSendLocalReply(r unsafe.Pointer, responseCode int, bodyText string, headers map[string]string, grpcStatus int64, details string) + HttpSendLocalReply(r unsafe.Pointer, responseCode int, bodyText string, headers map[string][]string, grpcStatus int64, details string) // Send a specialized reply that indicates that the filter has failed on the go side. Internally this is used for // when unhandled panics are detected. HttpSendPanicReply(r unsafe.Pointer, details string) // experience api, memory unsafe - HttpGetHeader(r unsafe.Pointer, key *string, value *string) + HttpGetHeader(r unsafe.Pointer, key string) string HttpCopyHeaders(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string - HttpSetHeader(r unsafe.Pointer, key *string, value *string, add bool) - HttpRemoveHeader(r unsafe.Pointer, key *string) + HttpSetHeader(r unsafe.Pointer, key string, value string, add bool) + HttpRemoveHeader(r unsafe.Pointer, key string) - HttpGetBuffer(r unsafe.Pointer, bufferPtr uint64, value *string, length uint64) + HttpGetBuffer(r unsafe.Pointer, bufferPtr uint64, length uint64) []byte + HttpDrainBuffer(r unsafe.Pointer, bufferPtr uint64, length uint64) HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action BufferAction) + HttpSetBytesBufferHelper(r unsafe.Pointer, bufferPtr uint64, value []byte, action BufferAction) HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string - HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool) - HttpRemoveTrailer(r unsafe.Pointer, key *string) + HttpSetTrailer(r unsafe.Pointer, key string, value string, add bool) + HttpRemoveTrailer(r unsafe.Pointer, key string) HttpGetStringValue(r unsafe.Pointer, id int) (string, bool) HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool) @@ -87,3 +89,17 @@ type NetworkCAPI interface { // UpstreamInfo gets the upstream connection info of infoType UpstreamInfo(f unsafe.Pointer, infoType int) string } + +type CommonCAPI interface { + Log(level LogType, message string) + LogLevel() LogType +} + +type commonCApiImpl struct{} + +var cAPI CommonCAPI = &commonCApiImpl{} + +// SetCommonCAPI for mock cAPI +func SetCommonCAPI(api CommonCAPI) { + cAPI = api +} diff --git a/contrib/golang/common/go/api/cgocheck.go b/contrib/golang/common/go/api/cgocheck.go deleted file mode 100644 index 01c6f84c8c40..000000000000 --- a/contrib/golang/common/go/api/cgocheck.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "os" - "strings" -) - -func CgoCheckDisabled() bool { - env := os.Getenv("GODEBUG") - // TODO: handle compile-time GODEBUG var after Go 1.21 is released - if strings.Index(env, "cgocheck=0") != -1 { - return true - } - return false -} diff --git a/contrib/golang/common/go/api/filter.go b/contrib/golang/common/go/api/filter.go index 6950cbb8f441..20e913cbb8ac 100644 --- a/contrib/golang/common/go/api/filter.go +++ b/contrib/golang/common/go/api/filter.go @@ -152,7 +152,7 @@ type FilterCallbacks interface { StreamFilterCallbacks // Continue or SendLocalReply should be last API invoked, no more code after them. Continue(StatusType) - SendLocalReply(responseCode int, bodyText string, headers map[string]string, grpcStatus int64, details string) + SendLocalReply(responseCode int, bodyText string, headers map[string][]string, grpcStatus int64, details string) // RecoverPanic recover panic in defer and terminate the request by SendLocalReply with 500 status code. RecoverPanic() Log(level LogType, msg string) @@ -162,7 +162,11 @@ type FilterCallbacks interface { // If the fetch succeeded, a string will be returned. // If the value is a timestamp, it is returned as a timestamp string like "2023-07-31T07:21:40.695646+00:00". // If the fetch failed (including the value is not found), an error will be returned. - // Currently, fetching requests/response attributes are mostly unsupported. + // + // The error can be one of: + // * ErrInternalFailure + // * ErrSerializationFailure (Currently, fetching attributes in List/Map type are unsupported) + // * ErrValueNotFound GetProperty(key string) (string, error) // TODO add more for filter callbacks } diff --git a/contrib/golang/common/go/api/logger.go b/contrib/golang/common/go/api/logger.go index 03dc81d9668a..b2732b922863 100644 --- a/contrib/golang/common/go/api/logger.go +++ b/contrib/golang/common/go/api/logger.go @@ -17,49 +17,38 @@ package api -/* -// ref https://github.com/golang/go/issues/25832 - -#cgo linux LDFLAGS: -Wl,-unresolved-symbols=ignore-all -#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup - -#include -#include +import "fmt" -#include "api.h" - -*/ -import "C" -import ( - "fmt" - "unsafe" -) +func (c *commonCApiImpl) Log(level LogType, message string) { + panic("To implement") +} -// The default log format is: -// [2023-08-09 03:04:15.985][1390][critical][golang] [contrib/golang/common/log/cgo.cc:27] msg +func (c *commonCApiImpl) LogLevel() LogType { + panic("To implement") +} func LogTrace(message string) { - C.envoyGoFilterLog(C.uint32_t(Trace), unsafe.Pointer(&message)) + cAPI.Log(Trace, message) } func LogDebug(message string) { - C.envoyGoFilterLog(C.uint32_t(Debug), unsafe.Pointer(&message)) + cAPI.Log(Debug, message) } func LogInfo(message string) { - C.envoyGoFilterLog(C.uint32_t(Info), unsafe.Pointer(&message)) + cAPI.Log(Info, message) } func LogWarn(message string) { - C.envoyGoFilterLog(C.uint32_t(Warn), unsafe.Pointer(&message)) + cAPI.Log(Warn, message) } func LogError(message string) { - C.envoyGoFilterLog(C.uint32_t(Error), unsafe.Pointer(&message)) + cAPI.Log(Error, message) } func LogCritical(message string) { - C.envoyGoFilterLog(C.uint32_t(Critical), unsafe.Pointer(&message)) + cAPI.Log(Critical, message) } func LogTracef(format string, v ...any) { @@ -87,5 +76,5 @@ func LogCriticalf(format string, v ...any) { } func GetLogLevel() LogType { - return LogType(C.envoyGoFilterLogLevel()) + return cAPI.LogLevel() } diff --git a/contrib/golang/common/go/api/type.go b/contrib/golang/common/go/api/type.go index 522f8166000a..bcecdff69256 100644 --- a/contrib/golang/common/go/api/type.go +++ b/contrib/golang/common/go/api/type.go @@ -17,6 +17,8 @@ package api +import "errors" + // ****************** filter status start ******************// type StatusType int @@ -126,14 +128,10 @@ type HeaderMap interface { // RangeWithCopy calls f sequentially for each key and value copied from the map. RangeWithCopy(f func(key, value string) bool) - - // ByteSize return size of HeaderMap - ByteSize() uint64 } type RequestHeaderMap interface { HeaderMap - Protocol() string Scheme() string Method() string Host() string @@ -200,12 +198,6 @@ type DataBufferBase interface { // buffer becomes too large, Write will panic with ErrTooLarge. WriteUint64(p uint64) error - // Peek returns n bytes from buffer, without draining any buffered data. - // If n > readable buffer, nil will be returned. - // It can be used in codec to check first-n-bytes magic bytes - // Note: do not change content in return bytes, use write instead - Peek(n int) []byte - // Bytes returns all bytes from buffer, without draining any buffered data. // It can be used to get fixed-length content, such as headers, body. // Note: do not change content in return bytes, use write instead @@ -432,3 +424,13 @@ func (t ConnectionInfoType) String() string { } return "unknown" } + +// *************** errors start **************// +var ( + ErrInternalFailure = errors.New("internal failure") + ErrValueNotFound = errors.New("value not found") + // Failed to serialize the value when we fetch the value as string + ErrSerializationFailure = errors.New("serialization failure") +) + +// *************** errors end **************// diff --git a/contrib/golang/common/go/api_impl/BUILD b/contrib/golang/common/go/api_impl/BUILD new file mode 100644 index 000000000000..3813bf257793 --- /dev/null +++ b/contrib/golang/common/go/api_impl/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +licenses(["notice"]) # Apache 2 + +go_library( + name = "api_impl", + srcs = [ + "api.h", + "capi_impl.go", + ], + cgo = True, + clinkopts = select({ + "@io_bazel_rules_go//go/platform:android": [ + "-Wl,-unresolved-symbols=ignore-all", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "-Wl,-undefined,dynamic_lookup", + ], + "@io_bazel_rules_go//go/platform:ios": [ + "-Wl,-undefined,dynamic_lookup", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "-Wl,-unresolved-symbols=ignore-all", + ], + "//conditions:default": [], + }), + importpath = "github.com/envoyproxy/envoy/contrib/golang/common/go/api_impl", + visibility = ["//visibility:public"], + deps = [ + "//contrib/golang/common/go/api", + ], +) diff --git a/contrib/golang/common/go/api_impl/api.h b/contrib/golang/common/go/api_impl/api.h new file mode 120000 index 000000000000..7ccbc18e959f --- /dev/null +++ b/contrib/golang/common/go/api_impl/api.h @@ -0,0 +1 @@ +../api/api.h \ No newline at end of file diff --git a/contrib/golang/common/go/api_impl/capi_impl.go b/contrib/golang/common/go/api_impl/capi_impl.go new file mode 100644 index 000000000000..6b1762ee8758 --- /dev/null +++ b/contrib/golang/common/go/api_impl/capi_impl.go @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api_impl + +/* +// ref https://github.com/golang/go/issues/25832 + +#cgo CFLAGS: -I../api +#cgo linux LDFLAGS: -Wl,-unresolved-symbols=ignore-all +#cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup + +#include +#include + +#include "api.h" + +*/ +import "C" +import ( + "unsafe" + + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" +) + +type commonCApiImpl struct{} + +// The default log format is: +// [2023-08-09 03:04:15.985][1390][critical][golang] [contrib/golang/common/log/cgo.cc:27] msg + +func (c *commonCApiImpl) Log(level api.LogType, message string) { + C.envoyGoFilterLog(C.uint32_t(level), unsafe.Pointer(unsafe.StringData(message)), C.int(len(message))) +} + +func (c *commonCApiImpl) LogLevel() api.LogType { + return api.LogType(C.envoyGoFilterLogLevel()) +} + +func init() { + api.SetCommonCAPI(&commonCApiImpl{}) +} diff --git a/contrib/golang/common/log/cgo.cc b/contrib/golang/common/log/cgo.cc index 97b26b9aeaf5..0aa799b3ea7f 100644 --- a/contrib/golang/common/log/cgo.cc +++ b/contrib/golang/common/log/cgo.cc @@ -42,22 +42,18 @@ uint32_t FilterLogger::level() const { return static_cast(ENVOY_LOGGER const FilterLogger& getFilterLogger() { CONSTRUCT_ON_FIRST_USE(FilterLogger); } -// The returned absl::string_view only refer to the GoString, won't copy the string content into -// C++, should not use it after the current cgo call returns. -absl::string_view referGoString(void* str) { - if (str == nullptr) { - return ""; - } - auto go_str = reinterpret_cast(str); - return {go_str->p, static_cast(go_str->n)}; +// The returned absl::string_view only refer to Go memory, +// should not use it after the current cgo call returns. +absl::string_view stringViewFromGoPointer(void* p, int len) { + return {static_cast(p), static_cast(len)}; } #ifdef __cplusplus extern "C" { #endif -void envoyGoFilterLog(uint32_t level, void* message) { - auto mesg = referGoString(message); +void envoyGoFilterLog(uint32_t level, void* message_data, int message_len) { + auto mesg = stringViewFromGoPointer(message_data, message_len); getFilterLogger().log(level, mesg); } diff --git a/contrib/golang/filters/http/source/cgo.cc b/contrib/golang/filters/http/source/cgo.cc index e4035beed84f..5cd8e047ba43 100644 --- a/contrib/golang/filters/http/source/cgo.cc +++ b/contrib/golang/filters/http/source/cgo.cc @@ -12,24 +12,16 @@ namespace Golang { // thread. // -// Deep copy GoString into std::string, including the string content, +// Deep copy Go memory into std::string, // it's safe to use it after the current cgo call returns. -std::string copyGoString(void* str) { - if (str == nullptr) { - return ""; - } - auto go_str = reinterpret_cast(str); - return {go_str->p, static_cast(go_str->n)}; +std::string copyStringFromGoPointer(void* p, int len) { + return {static_cast(p), static_cast(len)}; } -// The returned absl::string_view only refer to the GoString, won't copy the string content into -// C++, should not use it after the current cgo call returns. -absl::string_view referGoString(void* str) { - if (str == nullptr) { - return ""; - } - auto go_str = reinterpret_cast(str); - return {go_str->p, static_cast(go_str->n)}; +// The returned absl::string_view only refer to Go memory, +// should not use it after the current cgo call returns. +absl::string_view stringViewFromGoPointer(void* p, int len) { + return {static_cast(p), static_cast(len)}; } absl::string_view stringViewFromGoSlice(void* slice) { @@ -40,18 +32,15 @@ absl::string_view stringViewFromGoSlice(void* slice) { return {static_cast(go_slice->data), static_cast(go_slice->len)}; } -std::vector stringsFromGoSlice(void* slice) { +std::vector stringsFromGoSlice(void* slice_data, int slice_len) { std::vector list; - if (slice == nullptr) { + if (slice_len == 0) { return list; } - auto go_slice = reinterpret_cast(slice); - auto str = reinterpret_cast(go_slice->data); - for (auto i = 0; i < go_slice->len; i += 2) { - auto key = std::string(static_cast(str->p), str->n); - str++; - auto value = std::string(static_cast(str->p), str->n); - str++; + auto strs = reinterpret_cast(slice_data); + for (auto i = 0; i < slice_len; i += 2) { + auto key = std::string(strs[i + 0]); + auto value = std::string(strs[i + 1]); list.push_back(key); list.push_back(value); } @@ -88,21 +77,22 @@ CAPIStatus envoyGoFilterHttpContinue(void* r, int status) { }); } -CAPIStatus envoyGoFilterHttpSendLocalReply(void* r, int response_code, void* body_text, - void* headers, long long int grpc_status, - void* details) { +CAPIStatus envoyGoFilterHttpSendLocalReply(void* r, int response_code, void* body_text_data, + int body_text_len, void* headers, int headers_num, + long long int grpc_status, void* details_data, + int details_len) { return envoyGoFilterHandlerWrapper( r, - [response_code, body_text, headers, grpc_status, - details](std::shared_ptr& filter) -> CAPIStatus { - auto header_values = stringsFromGoSlice(headers); + [response_code, body_text_data, body_text_len, headers, headers_num, grpc_status, + details_data, details_len](std::shared_ptr& filter) -> CAPIStatus { + auto header_values = stringsFromGoSlice(headers, headers_num); std::function modify_headers = [header_values](Http::ResponseHeaderMap& headers) -> void { for (size_t i = 0; i < header_values.size(); i += 2) { const auto& key = header_values[i]; const auto& value = header_values[i + 1]; if (value.length() > 0) { - headers.setCopy(Http::LowerCaseString(key), value); + headers.addCopy(Http::LowerCaseString(key), value); } } }; @@ -111,19 +101,20 @@ CAPIStatus envoyGoFilterHttpSendLocalReply(void* r, int response_code, void* bod // Deep clone the GoString into C++, since the GoString may be freed after the function // returns, while they may still be used in the callback. return filter->sendLocalReply(static_cast(response_code), - copyGoString(body_text), modify_headers, status, - copyGoString(details)); + copyStringFromGoPointer(body_text_data, body_text_len), + modify_headers, status, + copyStringFromGoPointer(details_data, details_len)); }); } // unsafe API, without copy memory from c to go. -CAPIStatus envoyGoFilterHttpGetHeader(void* r, void* key, void* value) { - return envoyGoFilterHandlerWrapper(r, - [key, value](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto go_value = reinterpret_cast(value); - return filter->getHeader(key_str, go_value); - }); +CAPIStatus envoyGoFilterHttpGetHeader(void* r, void* key_data, int key_len, uint64_t* value_data, + int* value_len) { + return envoyGoFilterHandlerWrapper( + r, [key_data, key_len, value_data, value_len](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + return filter->getHeader(key_str, value_data, value_len); + }); } CAPIStatus envoyGoFilterHttpCopyHeaders(void* r, void* strs, void* buf) { @@ -134,23 +125,27 @@ CAPIStatus envoyGoFilterHttpCopyHeaders(void* r, void* strs, void* buf) { }); } -CAPIStatus envoyGoFilterHttpSetHeaderHelper(void* r, void* key, void* value, headerAction act) { - return envoyGoFilterHandlerWrapper( - r, [key, value, act](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto value_str = referGoString(value); - return filter->setHeader(key_str, value_str, act); - }); +CAPIStatus envoyGoFilterHttpSetHeaderHelper(void* r, void* key_data, int key_len, void* value_data, + int value_len, headerAction act) { + return envoyGoFilterHandlerWrapper(r, + [key_data, key_len, value_data, value_len, + act](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + auto value_str = + stringViewFromGoPointer(value_data, value_len); + return filter->setHeader(key_str, value_str, act); + }); } -CAPIStatus envoyGoFilterHttpRemoveHeader(void* r, void* key) { - return envoyGoFilterHandlerWrapper(r, [key](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - return filter->removeHeader(key_str); - }); +CAPIStatus envoyGoFilterHttpRemoveHeader(void* r, void* key_data, int key_len) { + return envoyGoFilterHandlerWrapper( + r, [key_data, key_len](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + return filter->removeHeader(key_str); + }); } -CAPIStatus envoyGoFilterHttpGetBuffer(void* r, unsigned long long int buffer_ptr, void* data) { +CAPIStatus envoyGoFilterHttpGetBuffer(void* r, uint64_t buffer_ptr, void* data) { return envoyGoFilterHandlerWrapper( r, [buffer_ptr, data](std::shared_ptr& filter) -> CAPIStatus { auto buffer = reinterpret_cast(buffer_ptr); @@ -158,12 +153,20 @@ CAPIStatus envoyGoFilterHttpGetBuffer(void* r, unsigned long long int buffer_ptr }); } -CAPIStatus envoyGoFilterHttpSetBufferHelper(void* r, unsigned long long int buffer_ptr, void* data, - int length, bufferAction action) { +CAPIStatus envoyGoFilterHttpDrainBuffer(void* r, uint64_t buffer_ptr, uint64_t length) { + return envoyGoFilterHandlerWrapper( + r, [buffer_ptr, length](std::shared_ptr& filter) -> CAPIStatus { + auto buffer = reinterpret_cast(buffer_ptr); + return filter->drainBuffer(buffer, length); + }); +} + +CAPIStatus envoyGoFilterHttpSetBufferHelper(void* r, uint64_t buffer_ptr, void* data, int length, + bufferAction action) { return envoyGoFilterHandlerWrapper( r, [buffer_ptr, data, length, action](std::shared_ptr& filter) -> CAPIStatus { auto buffer = reinterpret_cast(buffer_ptr); - auto value = absl::string_view(reinterpret_cast(data), length); + auto value = stringViewFromGoPointer(data, length); return filter->setBufferHelper(buffer, value, action); }); } @@ -176,54 +179,62 @@ CAPIStatus envoyGoFilterHttpCopyTrailers(void* r, void* strs, void* buf) { }); } -CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key, void* value, headerAction act) { - return envoyGoFilterHandlerWrapper( - r, [key, value, act](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto value_str = referGoString(value); - return filter->setTrailer(key_str, value_str, act); - }); +CAPIStatus envoyGoFilterHttpSetTrailer(void* r, void* key_data, int key_len, void* value_data, + int value_len, headerAction act) { + return envoyGoFilterHandlerWrapper(r, + [key_data, key_len, value_data, value_len, + act](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + auto value_str = + stringViewFromGoPointer(value_data, value_len); + return filter->setTrailer(key_str, value_str, act); + }); } -CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key) { - return envoyGoFilterHandlerWrapper(r, [key](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - return filter->removeTrailer(key_str); - }); +CAPIStatus envoyGoFilterHttpRemoveTrailer(void* r, void* key_data, int key_len) { + return envoyGoFilterHandlerWrapper( + r, [key_data, key_len](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + return filter->removeTrailer(key_str); + }); } -CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, void* value) { - return envoyGoFilterHandlerWrapper(r, [id, value](std::shared_ptr& filter) -> CAPIStatus { - auto value_str = reinterpret_cast(value); - return filter->getStringValue(id, value_str); - }); +CAPIStatus envoyGoFilterHttpGetStringValue(void* r, int id, uint64_t* value_data, int* value_len) { + return envoyGoFilterHandlerWrapper( + r, [id, value_data, value_len](std::shared_ptr& filter) -> CAPIStatus { + return filter->getStringValue(id, value_data, value_len); + }); } -CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, void* value) { +CAPIStatus envoyGoFilterHttpGetIntegerValue(void* r, int id, uint64_t* value) { return envoyGoFilterHandlerWrapper(r, [id, value](std::shared_ptr& filter) -> CAPIStatus { - auto value_int = reinterpret_cast(value); - return filter->getIntegerValue(id, value_int); - }); -} - -CAPIStatus envoyGoFilterHttpGetDynamicMetadata(void* r, void* name, void* buf) { - return envoyGoFilterHandlerWrapper(r, [name, buf](std::shared_ptr& filter) -> CAPIStatus { - auto name_str = copyGoString(name); - auto buf_slice = reinterpret_cast(buf); - return filter->getDynamicMetadata(name_str, buf_slice); + return filter->getIntegerValue(id, value); }); } -CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name, void* key, void* buf) { +CAPIStatus envoyGoFilterHttpGetDynamicMetadata(void* r, void* name_data, int name_len, + uint64_t* buf_data, int* buf_len) { return envoyGoFilterHandlerWrapper( - r, [name, key, buf](std::shared_ptr& filter) -> CAPIStatus { - auto name_str = copyGoString(name); - auto key_str = copyGoString(key); - auto buf_str = stringViewFromGoSlice(buf); - return filter->setDynamicMetadata(name_str, key_str, buf_str); + r, [name_data, name_len, buf_data, buf_len](std::shared_ptr& filter) -> CAPIStatus { + auto name_str = copyStringFromGoPointer(name_data, name_len); + return filter->getDynamicMetadata(name_str, buf_data, buf_len); }); } +CAPIStatus envoyGoFilterHttpSetDynamicMetadata(void* r, void* name_data, int name_len, + void* key_data, int key_len, void* buf_data, + int buf_len) { + return envoyGoFilterHandlerWrapper(r, + [name_data, name_len, key_data, key_len, buf_data, + buf_len](std::shared_ptr& filter) -> CAPIStatus { + auto name_str = copyStringFromGoPointer(name_data, name_len); + auto key_str = copyStringFromGoPointer(key_data, key_len); + auto buf_str = stringViewFromGoPointer(buf_data, buf_len); + return filter->setDynamicMetadata(name_str, key_str, + buf_str); + }); +} + void envoyGoFilterHttpFinalize(void* r, int reason) { UNREFERENCED_PARAMETER(reason); // req is used by go, so need to use raw memory and then it is safe to release at the gc finalize @@ -239,52 +250,57 @@ void envoyGoConfigHttpFinalize(void* c) { delete config; } -CAPIStatus envoyGoFilterHttpSendPanicReply(void* r, void* details) { - return envoyGoFilterHandlerWrapper(r, [details](std::shared_ptr& filter) -> CAPIStatus { - // Since this is only used for logs we don't need to deep copy. - return filter->sendPanicReply(referGoString(details)); - }); +CAPIStatus envoyGoFilterHttpSendPanicReply(void* r, void* details_data, int details_len) { + return envoyGoFilterHandlerWrapper( + r, [details_data, details_len](std::shared_ptr& filter) -> CAPIStatus { + // Since this is only used for logs we don't need to deep copy. + auto details = stringViewFromGoPointer(details_data, details_len); + return filter->sendPanicReply(details); + }); } -CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key, void* value, int state_type, +CAPIStatus envoyGoFilterHttpSetStringFilterState(void* r, void* key_data, int key_len, + void* value_data, int value_len, int state_type, int life_span, int stream_sharing) { - return envoyGoFilterHandlerWrapper(r, - [key, value, state_type, life_span, stream_sharing]( - std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto value_str = referGoString(value); - return filter->setStringFilterState(key_str, value_str, - state_type, life_span, - stream_sharing); - }); -} - -CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key, void* value) { - return envoyGoFilterHandlerWrapper(r, - [key, value](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto value_str = reinterpret_cast(value); - return filter->getStringFilterState(key_str, value_str); - }); + return envoyGoFilterHandlerWrapper( + r, + [key_data, key_len, value_data, value_len, state_type, life_span, + stream_sharing](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + auto value_str = stringViewFromGoPointer(value_data, value_len); + return filter->setStringFilterState(key_str, value_str, state_type, life_span, + stream_sharing); + }); } -CAPIStatus envoyGoFilterHttpGetStringProperty(void* r, void* key, void* value, int* rc) { +CAPIStatus envoyGoFilterHttpGetStringFilterState(void* r, void* key_data, int key_len, + uint64_t* value_data, int* value_len) { return envoyGoFilterHandlerWrapper( - r, [key, value, rc](std::shared_ptr& filter) -> CAPIStatus { - auto key_str = referGoString(key); - auto value_str = reinterpret_cast(value); - return filter->getStringProperty(key_str, value_str, rc); + r, [key_data, key_len, value_data, value_len](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + return filter->getStringFilterState(key_str, value_data, value_len); }); } -CAPIStatus envoyGoFilterHttpDefineMetric(void* c, uint32_t metric_type, void* name, - void* metric_id) { +CAPIStatus envoyGoFilterHttpGetStringProperty(void* r, void* key_data, int key_len, + uint64_t* value_data, int* value_len, int* rc) { + return envoyGoFilterHandlerWrapper(r, + [key_data, key_len, value_data, value_len, + rc](std::shared_ptr& filter) -> CAPIStatus { + auto key_str = stringViewFromGoPointer(key_data, key_len); + return filter->getStringProperty(key_str, value_data, + value_len, rc); + }); +} + +CAPIStatus envoyGoFilterHttpDefineMetric(void* c, uint32_t metric_type, void* name_data, + int name_len, uint32_t* metric_id) { return envoyGoConfigHandlerWrapper( c, - [metric_type, name, metric_id](std::shared_ptr& filter_config) -> CAPIStatus { - auto name_str = referGoString(name); - auto metric_id_int = reinterpret_cast(metric_id); - return filter_config->defineMetric(metric_type, name_str, metric_id_int); + [metric_type, name_data, name_len, + metric_id](std::shared_ptr& filter_config) -> CAPIStatus { + auto name_str = stringViewFromGoPointer(name_data, name_len); + return filter_config->defineMetric(metric_type, name_str, metric_id); }); } @@ -295,11 +311,10 @@ CAPIStatus envoyGoFilterHttpIncrementMetric(void* c, uint32_t metric_id, int64_t }); } -CAPIStatus envoyGoFilterHttpGetMetric(void* c, uint32_t metric_id, void* value) { +CAPIStatus envoyGoFilterHttpGetMetric(void* c, uint32_t metric_id, uint64_t* value) { return envoyGoConfigHandlerWrapper( c, [metric_id, value](std::shared_ptr& filter_config) -> CAPIStatus { - auto value_int = reinterpret_cast(value); - return filter_config->getMetric(metric_id, value_int); + return filter_config->getMetric(metric_id, value); }); } diff --git a/contrib/golang/filters/http/source/go/pkg/http/BUILD b/contrib/golang/filters/http/source/go/pkg/http/BUILD index 0c1b726ce318..ae49c9487324 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/BUILD +++ b/contrib/golang/filters/http/source/go/pkg/http/BUILD @@ -34,6 +34,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//contrib/golang/common/go/api", + "//contrib/golang/common/go/api_impl", "//contrib/golang/common/go/utils", "@org_golang_google_protobuf//proto", "@org_golang_google_protobuf//types/known/anypb", diff --git a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go index 8cecb683c5e8..a195c3b26162 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go +++ b/contrib/golang/filters/http/source/go/pkg/http/capi_impl.go @@ -33,7 +33,6 @@ package http import "C" import ( "errors" - "reflect" "runtime" "strings" "sync/atomic" @@ -43,6 +42,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + _ "github.com/envoyproxy/envoy/contrib/golang/common/go/api_impl" ) const ( @@ -93,15 +93,22 @@ func capiStatusToStr(status C.CAPIStatus) string { return errNotInGo case C.CAPIInvalidPhase: return errInvalidPhase + } + + return "unknown status" +} + +func capiStatusToErr(status C.CAPIStatus) error { + switch status { case C.CAPIValueNotFound: - return errValueNotFound + return api.ErrValueNotFound case C.CAPIInternalFailure: - return errInternalFailure + return api.ErrInternalFailure case C.CAPISerializationFailure: - return errSerializationFailure + return api.ErrSerializationFailure } - return "unknown status" + return errors.New("unknown status") } func (c *httpCApiImpl) HttpContinue(r unsafe.Pointer, status uint64) { @@ -111,24 +118,40 @@ func (c *httpCApiImpl) HttpContinue(r unsafe.Pointer, status uint64) { // Only may panic with errRequestFinished, errFilterDestroyed or errNotInGo, // won't panic with errInvalidPhase and others, otherwise will cause deadloop, see RecoverPanic for the details. -func (c *httpCApiImpl) HttpSendLocalReply(r unsafe.Pointer, response_code int, body_text string, headers map[string]string, grpc_status int64, details string) { +func (c *httpCApiImpl) HttpSendLocalReply(r unsafe.Pointer, responseCode int, bodyText string, headers map[string][]string, grpcStatus int64, details string) { hLen := len(headers) - strs := make([]string, 0, hLen) - for k, v := range headers { - strs = append(strs, k, v) + strs := make([]*C.char, 0, hLen*2) + defer func() { + for _, s := range strs { + C.free(unsafe.Pointer(s)) + } + }() + // TODO: use runtime.Pinner after go1.22 release for better performance. + for k, h := range headers { + for _, v := range h { + keyStr := C.CString(k) + valueStr := C.CString(v) + strs = append(strs, keyStr, valueStr) + } } - res := C.envoyGoFilterHttpSendLocalReply(r, C.int(response_code), unsafe.Pointer(&body_text), unsafe.Pointer(&strs), C.longlong(grpc_status), unsafe.Pointer(&details)) + res := C.envoyGoFilterHttpSendLocalReply(r, C.int(responseCode), + unsafe.Pointer(unsafe.StringData(bodyText)), C.int(len(bodyText)), + unsafe.Pointer(unsafe.SliceData(strs)), C.int(len(strs)), + C.longlong(grpcStatus), unsafe.Pointer(unsafe.StringData(details)), C.int(len(details))) handleCApiStatus(res) } func (c *httpCApiImpl) HttpSendPanicReply(r unsafe.Pointer, details string) { - res := C.envoyGoFilterHttpSendPanicReply(r, unsafe.Pointer(&details)) + res := C.envoyGoFilterHttpSendPanicReply(r, unsafe.Pointer(unsafe.StringData(details)), C.int(len(details))) handleCApiStatus(res) } -func (c *httpCApiImpl) HttpGetHeader(r unsafe.Pointer, key *string, value *string) { - res := C.envoyGoFilterHttpGetHeader(r, unsafe.Pointer(key), unsafe.Pointer(value)) +func (c *httpCApiImpl) HttpGetHeader(r unsafe.Pointer, key string) string { + var valueData C.uint64_t + var valueLen C.int + res := C.envoyGoFilterHttpGetHeader(r, unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), &valueData, &valueLen) handleCApiStatus(res) + return unsafe.String((*byte)(unsafe.Pointer(uintptr(valueData))), int(valueLen)) } func (c *httpCApiImpl) HttpCopyHeaders(r unsafe.Pointer, num uint64, bytes uint64) map[string][]string { @@ -146,10 +169,7 @@ func (c *httpCApiImpl) HttpCopyHeaders(r unsafe.Pointer, num uint64, bytes uint6 // we have to make sure the all strings is not using before reusing, // but strings may be alive beyond the request life. buf := make([]byte, bytes) - sHeader := (*reflect.SliceHeader)(unsafe.Pointer(&strs)) - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - - res := C.envoyGoFilterHttpCopyHeaders(r, unsafe.Pointer(sHeader.Data), unsafe.Pointer(bHeader.Data)) + res := C.envoyGoFilterHttpCopyHeaders(r, unsafe.Pointer(unsafe.SliceData(strs)), unsafe.Pointer(unsafe.SliceData(buf))) handleCApiStatus(res) m := make(map[string][]string, num) @@ -167,34 +187,44 @@ func (c *httpCApiImpl) HttpCopyHeaders(r unsafe.Pointer, num uint64, bytes uint6 return m } -func (c *httpCApiImpl) HttpSetHeader(r unsafe.Pointer, key *string, value *string, add bool) { +func (c *httpCApiImpl) HttpSetHeader(r unsafe.Pointer, key string, value string, add bool) { var act C.headerAction if add { act = C.HeaderAdd } else { act = C.HeaderSet } - res := C.envoyGoFilterHttpSetHeaderHelper(r, unsafe.Pointer(key), unsafe.Pointer(value), act) + res := C.envoyGoFilterHttpSetHeaderHelper(r, unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), + unsafe.Pointer(unsafe.StringData(value)), C.int(len(value)), act) handleCApiStatus(res) } -func (c *httpCApiImpl) HttpRemoveHeader(r unsafe.Pointer, key *string) { - res := C.envoyGoFilterHttpRemoveHeader(r, unsafe.Pointer(key)) +func (c *httpCApiImpl) HttpRemoveHeader(r unsafe.Pointer, key string) { + res := C.envoyGoFilterHttpRemoveHeader(r, unsafe.Pointer(unsafe.StringData(key)), C.int(len(key))) handleCApiStatus(res) } -func (c *httpCApiImpl) HttpGetBuffer(r unsafe.Pointer, bufferPtr uint64, value *string, length uint64) { +func (c *httpCApiImpl) HttpGetBuffer(r unsafe.Pointer, bufferPtr uint64, length uint64) []byte { buf := make([]byte, length) - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - sHeader := (*reflect.StringHeader)(unsafe.Pointer(value)) - sHeader.Data = bHeader.Data - sHeader.Len = int(length) - res := C.envoyGoFilterHttpGetBuffer(r, C.ulonglong(bufferPtr), unsafe.Pointer(bHeader.Data)) + res := C.envoyGoFilterHttpGetBuffer(r, C.uint64_t(bufferPtr), unsafe.Pointer(unsafe.SliceData(buf))) + handleCApiStatus(res) + return unsafe.Slice(unsafe.SliceData(buf), length) +} + +func (c *httpCApiImpl) HttpDrainBuffer(r unsafe.Pointer, bufferPtr uint64, length uint64) { + res := C.envoyGoFilterHttpDrainBuffer(r, C.uint64_t(bufferPtr), C.uint64_t(length)) handleCApiStatus(res) } func (c *httpCApiImpl) HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, value string, action api.BufferAction) { - sHeader := (*reflect.StringHeader)(unsafe.Pointer(&value)) + c.httpSetBufferHelper(r, bufferPtr, unsafe.Pointer(unsafe.StringData(value)), C.int(len(value)), action) +} + +func (c *httpCApiImpl) HttpSetBytesBufferHelper(r unsafe.Pointer, bufferPtr uint64, value []byte, action api.BufferAction) { + c.httpSetBufferHelper(r, bufferPtr, unsafe.Pointer(unsafe.SliceData(value)), C.int(len(value)), action) +} + +func (c *httpCApiImpl) httpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, data unsafe.Pointer, length C.int, action api.BufferAction) { var act C.bufferAction switch action { case api.SetBuffer: @@ -204,7 +234,7 @@ func (c *httpCApiImpl) HttpSetBufferHelper(r unsafe.Pointer, bufferPtr uint64, v case api.PrependBuffer: act = C.Prepend } - res := C.envoyGoFilterHttpSetBufferHelper(r, C.ulonglong(bufferPtr), unsafe.Pointer(sHeader.Data), C.int(sHeader.Len), act) + res := C.envoyGoFilterHttpSetBufferHelper(r, C.uint64_t(bufferPtr), data, length, act) handleCApiStatus(res) } @@ -223,10 +253,7 @@ func (c *httpCApiImpl) HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint // we have to make sure the all strings is not using before reusing, // but strings may be alive beyond the request life. buf := make([]byte, bytes) - sHeader := (*reflect.SliceHeader)(unsafe.Pointer(&strs)) - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - - res := C.envoyGoFilterHttpCopyTrailers(r, unsafe.Pointer(sHeader.Data), unsafe.Pointer(bHeader.Data)) + res := C.envoyGoFilterHttpCopyTrailers(r, unsafe.Pointer(unsafe.SliceData(strs)), unsafe.Pointer(unsafe.SliceData(buf))) handleCApiStatus(res) m := make(map[string][]string, num) @@ -240,58 +267,66 @@ func (c *httpCApiImpl) HttpCopyTrailers(r unsafe.Pointer, num uint64, bytes uint m[key] = append(v, value) } } + runtime.KeepAlive(buf) return m } -func (c *httpCApiImpl) HttpSetTrailer(r unsafe.Pointer, key *string, value *string, add bool) { +func (c *httpCApiImpl) HttpSetTrailer(r unsafe.Pointer, key string, value string, add bool) { var act C.headerAction if add { act = C.HeaderAdd } else { act = C.HeaderSet } - res := C.envoyGoFilterHttpSetTrailer(r, unsafe.Pointer(key), unsafe.Pointer(value), act) + res := C.envoyGoFilterHttpSetTrailer(r, unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), + unsafe.Pointer(unsafe.StringData(value)), C.int(len(value)), act) handleCApiStatus(res) } -func (c *httpCApiImpl) HttpRemoveTrailer(r unsafe.Pointer, key *string) { - res := C.envoyGoFilterHttpRemoveTrailer(r, unsafe.Pointer(key)) +func (c *httpCApiImpl) HttpRemoveTrailer(r unsafe.Pointer, key string) { + res := C.envoyGoFilterHttpRemoveTrailer(r, unsafe.Pointer(unsafe.StringData(key)), C.int(len(key))) handleCApiStatus(res) } func (c *httpCApiImpl) HttpGetStringValue(rr unsafe.Pointer, id int) (string, bool) { r := (*httpRequest)(rr) - var value string // add a lock to protect filter->req_->strValue field in the Envoy side, from being writing concurrency, // since there might be multiple concurrency goroutines invoking this API on the Go side. r.mutex.Lock() defer r.mutex.Unlock() - res := C.envoyGoFilterHttpGetStringValue(unsafe.Pointer(r.req), C.int(id), unsafe.Pointer(&value)) + + var valueData C.uint64_t + var valueLen C.int + res := C.envoyGoFilterHttpGetStringValue(unsafe.Pointer(r.req), C.int(id), &valueData, &valueLen) if res == C.CAPIValueNotFound { return "", false } handleCApiStatus(res) + value := unsafe.String((*byte)(unsafe.Pointer(uintptr(valueData))), int(valueLen)) // copy the memory from c to Go. return strings.Clone(value), true } func (c *httpCApiImpl) HttpGetIntegerValue(r unsafe.Pointer, id int) (uint64, bool) { - var value uint64 - res := C.envoyGoFilterHttpGetIntegerValue(r, C.int(id), unsafe.Pointer(&value)) + var value C.uint64_t + res := C.envoyGoFilterHttpGetIntegerValue(r, C.int(id), &value) if res == C.CAPIValueNotFound { return 0, false } handleCApiStatus(res) - return value, true + return uint64(value), true } func (c *httpCApiImpl) HttpGetDynamicMetadata(rr unsafe.Pointer, filterName string) map[string]interface{} { r := (*httpRequest)(rr) - var buf []byte r.mutex.Lock() defer r.mutex.Unlock() r.sema.Add(1) - res := C.envoyGoFilterHttpGetDynamicMetadata(unsafe.Pointer(r.req), unsafe.Pointer(&filterName), unsafe.Pointer(&buf)) + + var valueData C.uint64_t + var valueLen C.int + res := C.envoyGoFilterHttpGetDynamicMetadata(unsafe.Pointer(r.req), + unsafe.Pointer(unsafe.StringData(filterName)), C.int(len(filterName)), &valueData, &valueLen) if res == C.CAPIYield { atomic.AddInt32(&r.waitingOnEnvoy, 1) r.sema.Wait() @@ -299,6 +334,7 @@ func (c *httpCApiImpl) HttpGetDynamicMetadata(rr unsafe.Pointer, filterName stri r.sema.Done() handleCApiStatus(res) } + buf := unsafe.Slice((*byte)(unsafe.Pointer(uintptr(valueData))), int(valueLen)) // copy the memory from c to Go. var meta structpb.Struct proto.Unmarshal(buf, &meta) @@ -314,12 +350,15 @@ func (c *httpCApiImpl) HttpSetDynamicMetadata(r unsafe.Pointer, filterName strin if err != nil { panic(err) } - res := C.envoyGoFilterHttpSetDynamicMetadata(r, unsafe.Pointer(&filterName), unsafe.Pointer(&key), unsafe.Pointer(&buf)) + res := C.envoyGoFilterHttpSetDynamicMetadata(r, + unsafe.Pointer(unsafe.StringData(filterName)), C.int(len(filterName)), + unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), + unsafe.Pointer(unsafe.SliceData(buf)), C.int(len(buf))) handleCApiStatus(res) } func (c *httpCApiImpl) HttpLog(level api.LogType, message string) { - C.envoyGoFilterLog(C.uint32_t(level), unsafe.Pointer(&message)) + C.envoyGoFilterLog(C.uint32_t(level), unsafe.Pointer(unsafe.StringData(message)), C.int(len(message))) } func (c *httpCApiImpl) HttpLogLevel() api.LogType { @@ -342,17 +381,22 @@ func SetHttpCAPI(api api.HttpCAPI) { } func (c *httpCApiImpl) HttpSetStringFilterState(r unsafe.Pointer, key string, value string, stateType api.StateType, lifeSpan api.LifeSpan, streamSharing api.StreamSharing) { - res := C.envoyGoFilterHttpSetStringFilterState(r, unsafe.Pointer(&key), unsafe.Pointer(&value), C.int(stateType), C.int(lifeSpan), C.int(streamSharing)) + res := C.envoyGoFilterHttpSetStringFilterState(r, + unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), + unsafe.Pointer(unsafe.StringData(value)), C.int(len(value)), + C.int(stateType), C.int(lifeSpan), C.int(streamSharing)) handleCApiStatus(res) } func (c *httpCApiImpl) HttpGetStringFilterState(rr unsafe.Pointer, key string) string { r := (*httpRequest)(rr) - var value string + var valueData C.uint64_t + var valueLen C.int r.mutex.Lock() defer r.mutex.Unlock() r.sema.Add(1) - res := C.envoyGoFilterHttpGetStringFilterState(unsafe.Pointer(r.req), unsafe.Pointer(&key), unsafe.Pointer(&value)) + res := C.envoyGoFilterHttpGetStringFilterState(unsafe.Pointer(r.req), + unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), &valueData, &valueLen) if res == C.CAPIYield { atomic.AddInt32(&r.waitingOnEnvoy, 1) r.sema.Wait() @@ -361,18 +405,20 @@ func (c *httpCApiImpl) HttpGetStringFilterState(rr unsafe.Pointer, key string) s handleCApiStatus(res) } + value := unsafe.String((*byte)(unsafe.Pointer(uintptr(valueData))), int(valueLen)) return strings.Clone(value) } func (c *httpCApiImpl) HttpGetStringProperty(rr unsafe.Pointer, key string) (string, error) { r := (*httpRequest)(rr) - var value string - var rc int + var valueData C.uint64_t + var valueLen C.int + var rc C.int r.mutex.Lock() defer r.mutex.Unlock() r.sema.Add(1) - res := C.envoyGoFilterHttpGetStringProperty(unsafe.Pointer(r.req), unsafe.Pointer(&key), - unsafe.Pointer(&value), (*C.int)(unsafe.Pointer(&rc))) + res := C.envoyGoFilterHttpGetStringProperty(unsafe.Pointer(r.req), + unsafe.Pointer(unsafe.StringData(key)), C.int(len(key)), &valueData, &valueLen, &rc) if res == C.CAPIYield { atomic.AddInt32(&r.waitingOnEnvoy, 1) r.sema.Wait() @@ -383,18 +429,18 @@ func (c *httpCApiImpl) HttpGetStringProperty(rr unsafe.Pointer, key string) (str } if res == C.CAPIOK { + value := unsafe.String((*byte)(unsafe.Pointer(uintptr(valueData))), int(valueLen)) return strings.Clone(value), nil } - return "", errors.New(capiStatusToStr(res)) + return "", capiStatusToErr(res) } func (c *httpCApiImpl) HttpDefineMetric(cfg unsafe.Pointer, metricType api.MetricType, name string) uint32 { - var value uint32 - - res := C.envoyGoFilterHttpDefineMetric(unsafe.Pointer(cfg), C.uint32_t(metricType), unsafe.Pointer(&name), unsafe.Pointer(&value)) + var value C.uint32_t + res := C.envoyGoFilterHttpDefineMetric(cfg, C.uint32_t(metricType), unsafe.Pointer(unsafe.StringData(name)), C.int(len(name)), &value) handleCApiStatus(res) - return value + return uint32(value) } func (c *httpCApiImpl) HttpIncrementMetric(cc unsafe.Pointer, metricId uint32, offset int64) { @@ -405,10 +451,10 @@ func (c *httpCApiImpl) HttpIncrementMetric(cc unsafe.Pointer, metricId uint32, o func (c *httpCApiImpl) HttpGetMetric(cc unsafe.Pointer, metricId uint32) uint64 { cfg := (*httpConfig)(cc) - var value uint64 - res := C.envoyGoFilterHttpGetMetric(unsafe.Pointer(cfg.config), C.uint32_t(metricId), unsafe.Pointer(&value)) + var value C.uint64_t + res := C.envoyGoFilterHttpGetMetric(unsafe.Pointer(cfg.config), C.uint32_t(metricId), &value) handleCApiStatus(res) - return value + return uint64(value) } func (c *httpCApiImpl) HttpRecordMetric(cc unsafe.Pointer, metricId uint32, value uint64) { diff --git a/contrib/golang/filters/http/source/go/pkg/http/cgo_go122.go b/contrib/golang/filters/http/source/go/pkg/http/cgo_go122.go deleted file mode 100644 index 8e8bec7a0e66..000000000000 --- a/contrib/golang/filters/http/source/go/pkg/http/cgo_go122.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build go1.22 - -package http - -/* -// This is a performance optimization. -// The following noescape and nocallback directives are used to -// prevent the Go compiler from allocating function parameters on the heap. - -#cgo noescape envoyGoFilterHttpCopyHeaders -#cgo nocallback envoyGoFilterHttpCopyHeaders -#cgo noescape envoyGoFilterHttpSendPanicReply -#cgo nocallback envoyGoFilterHttpSendPanicReply -#cgo noescape envoyGoFilterHttpGetHeader -#cgo nocallback envoyGoFilterHttpGetHeader -#cgo noescape envoyGoFilterHttpSetHeaderHelper -#cgo nocallback envoyGoFilterHttpSetHeaderHelper -#cgo noescape envoyGoFilterHttpRemoveHeader -#cgo nocallback envoyGoFilterHttpRemoveHeader -#cgo noescape envoyGoFilterHttpGetBuffer -#cgo nocallback envoyGoFilterHttpGetBuffer -#cgo noescape envoyGoFilterHttpSetBufferHelper -#cgo nocallback envoyGoFilterHttpSetBufferHelper -#cgo noescape envoyGoFilterHttpCopyTrailers -#cgo nocallback envoyGoFilterHttpCopyTrailers -#cgo noescape envoyGoFilterHttpSetTrailer -#cgo nocallback envoyGoFilterHttpSetTrailer -#cgo noescape envoyGoFilterHttpRemoveTrailer -#cgo nocallback envoyGoFilterHttpRemoveTrailer -#cgo noescape envoyGoFilterHttpGetStringValue -#cgo nocallback envoyGoFilterHttpGetStringValue -#cgo noescape envoyGoFilterHttpGetIntegerValue -#cgo nocallback envoyGoFilterHttpGetIntegerValue -#cgo noescape envoyGoFilterHttpGetDynamicMetadata -#cgo nocallback envoyGoFilterHttpGetDynamicMetadata -#cgo noescape envoyGoFilterHttpSetDynamicMetadata -#cgo nocallback envoyGoFilterHttpSetDynamicMetadata -#cgo noescape envoyGoFilterLog -#cgo nocallback envoyGoFilterLog -#cgo noescape envoyGoFilterHttpSetStringFilterState -#cgo nocallback envoyGoFilterHttpSetStringFilterState -#cgo noescape envoyGoFilterHttpGetStringFilterState -#cgo nocallback envoyGoFilterHttpGetStringFilterState -#cgo noescape envoyGoFilterHttpGetStringProperty -#cgo nocallback envoyGoFilterHttpGetStringProperty -#cgo noescape envoyGoFilterHttpDefineMetric -#cgo nocallback envoyGoFilterHttpDefineMetric -#cgo noescape envoyGoFilterHttpIncrementMetric -#cgo nocallback envoyGoFilterHttpIncrementMetric -#cgo noescape envoyGoFilterHttpGetMetric -#cgo nocallback envoyGoFilterHttpGetMetric - -*/ -import "C" diff --git a/contrib/golang/filters/http/source/go/pkg/http/config.go b/contrib/golang/filters/http/source/go/pkg/http/config.go index 2cb3a3f7ccdf..99e231e98520 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/config.go +++ b/contrib/golang/filters/http/source/go/pkg/http/config.go @@ -36,6 +36,7 @@ import ( "runtime" "sync" "sync/atomic" + "time" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -46,7 +47,10 @@ import ( var ( configNumGenerator uint64 - configCache = &sync.Map{} // uint64 -> *anypb.Any + configCache = &sync.Map{} // uint64 -> config(interface{}) + // From get a cached merged_config_id_ in getMergedConfigId on the C++ side, + // to get the merged config by the id on the Go side, 2 seconds should be long enough. + delayDeleteTime = time.Second * 2 // 2s ) func configFinalize(c *httpConfig) { @@ -65,11 +69,6 @@ func createConfig(c *C.httpConfig) *httpConfig { //export envoyGoFilterNewHttpPluginConfig func envoyGoFilterNewHttpPluginConfig(c *C.httpConfig) uint64 { - if !api.CgoCheckDisabled() { - cAPI.HttpLog(api.Error, "The Envoy Golang filter requires the `GODEBUG=cgocheck=0` environment variable set.") - return 0 - } - buf := utils.BytesToSlice(uint64(c.config_ptr), uint64(c.config_len)) var any anypb.Any proto.Unmarshal(buf, &any) @@ -100,8 +99,19 @@ func envoyGoFilterNewHttpPluginConfig(c *C.httpConfig) uint64 { } //export envoyGoFilterDestroyHttpPluginConfig -func envoyGoFilterDestroyHttpPluginConfig(id uint64) { - configCache.Delete(id) +func envoyGoFilterDestroyHttpPluginConfig(id uint64, needDelay int) { + if needDelay == 1 { + // there is a concurrency race in the c++ side: + // 1. when A envoy worker thread is using the cached merged_config_id_ and it will call into Go after some time. + // 2. while B envoy worker thread may update the merged_config_id_ in getMergedConfigId, that will delete the id. + // so, we delay deleting the id in the Go side. + time.AfterFunc(delayDeleteTime, func() { + configCache.Delete(id) + }) + } else { + // there is no race for non-merged config. + configCache.Delete(id) + } } //export envoyGoFilterMergeHttpPluginConfig @@ -125,7 +135,8 @@ func envoyGoFilterMergeHttpPluginConfig(namePtr, nameLen, parentId, childId uint return configNum } else { - // child override parent by default + // child override parent by default. + // It's safe to reuse the childId, since the merged config have the same life time with the child config. return childId } } diff --git a/contrib/golang/filters/http/source/go/pkg/http/filter.go b/contrib/golang/filters/http/source/go/pkg/http/filter.go index e33f099f2469..c9b1d3424c75 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/filter.go +++ b/contrib/golang/filters/http/source/go/pkg/http/filter.go @@ -114,7 +114,7 @@ func (r *httpRequest) Continue(status api.StatusType) { cAPI.HttpContinue(unsafe.Pointer(r.req), uint64(status)) } -func (r *httpRequest) SendLocalReply(responseCode int, bodyText string, headers map[string]string, grpcStatus int64, details string) { +func (r *httpRequest) SendLocalReply(responseCode int, bodyText string, headers map[string][]string, grpcStatus int64, details string) { cAPI.HttpSendLocalReply(unsafe.Pointer(r.req), responseCode, bodyText, headers, grpcStatus, details) } diff --git a/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go b/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go index 9986b68fb66c..ce8c65c88f66 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go +++ b/contrib/golang/filters/http/source/go/pkg/http/filtermanager.go @@ -44,7 +44,7 @@ func RegisterHttpFilterConfigFactoryAndParser(name string, factory api.StreamFil func getOrCreateHttpFilterFactory(name string, configId uint64) api.StreamFilterFactory { config, ok := configCache.Load(configId) if !ok { - panic(fmt.Sprintf("get config failed, plugin: %s, configId: %d", name, configId)) + panic(fmt.Sprintf("config not found, plugin: %s, configId: %d", name, configId)) } if v, ok := httpFilterConfigFactoryAndParser.Load(name); ok { diff --git a/contrib/golang/filters/http/source/go/pkg/http/shim.go b/contrib/golang/filters/http/source/go/pkg/http/shim.go index ad8e10679a28..45d4cf1d4fb6 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/shim.go +++ b/contrib/golang/filters/http/source/go/pkg/http/shim.go @@ -169,6 +169,11 @@ func envoyGoFilterOnHttpHeader(r *C.httpRequest, endStream, headerNum, headerByt } status = f.EncodeTrailers(header) } + + if endStream == 1 && (status == api.StopAndBuffer || status == api.StopAndBufferWatermark) { + panic("received wait data status when there is no data, please fix the returned status") + } + return uint64(status) } @@ -242,6 +247,11 @@ func envoyGoFilterOnHttpDestroy(r *C.httpRequest, reason uint64) { f := req.httpFilter f.OnDestroy(v) + // Break circular references between httpRequest and StreamFilter, + // since Finalizers don't work with circular references, + // otherwise, it will leads to memory leaking. + req.httpFilter = nil + Requests.DeleteReq(r) } diff --git a/contrib/golang/filters/http/source/go/pkg/http/type.go b/contrib/golang/filters/http/source/go/pkg/http/type.go index d433f579ddb6..8dd8bad8a366 100644 --- a/contrib/golang/filters/http/source/go/pkg/http/type.go +++ b/contrib/golang/filters/http/source/go/pkg/http/type.go @@ -19,6 +19,7 @@ package http import ( "strconv" + "strings" "sync" "unsafe" @@ -31,10 +32,6 @@ const ( errFilterDestroyed = "golang filter has been destroyed" errNotInGo = "not proccessing Go" errInvalidPhase = "invalid phase, maybe headers/buffer already continued" - - errInternalFailure = "internal failure" - errValueNotFound = "value not found" - errSerializationFailure = "serialization failure" ) // api.HeaderMap @@ -46,11 +43,6 @@ type headerMapImpl struct { mutex sync.Mutex } -// ByteSize return size of HeaderMap -func (h *headerMapImpl) ByteSize() uint64 { - return h.headerBytes -} - type requestOrResponseHeaderMapImpl struct { headerMapImpl } @@ -62,12 +54,12 @@ func (h *requestOrResponseHeaderMapImpl) initHeaders() { } func (h *requestOrResponseHeaderMapImpl) GetRaw(key string) string { - var value string - cAPI.HttpGetHeader(unsafe.Pointer(h.request.req), &key, &value) - return value + // GetRaw is case-sensitive + return cAPI.HttpGetHeader(unsafe.Pointer(h.request.req), key) } func (h *requestOrResponseHeaderMapImpl) Get(key string) (string, bool) { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initHeaders() @@ -79,6 +71,7 @@ func (h *requestOrResponseHeaderMapImpl) Get(key string) (string, bool) { } func (h *requestOrResponseHeaderMapImpl) Values(key string) []string { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initHeaders() @@ -90,6 +83,7 @@ func (h *requestOrResponseHeaderMapImpl) Values(key string) []string { } func (h *requestOrResponseHeaderMapImpl) Set(key, value string) { + key = strings.ToLower(key) // Get all header values first before setting a value, since the set operation may not take affects immediately // when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread. // Otherwise, we may get outdated values in a following Get call. @@ -99,10 +93,11 @@ func (h *requestOrResponseHeaderMapImpl) Set(key, value string) { if h.headers != nil { h.headers[key] = []string{value} } - cAPI.HttpSetHeader(unsafe.Pointer(h.request.req), &key, &value, false) + cAPI.HttpSetHeader(unsafe.Pointer(h.request.req), key, value, false) } func (h *requestOrResponseHeaderMapImpl) Add(key, value string) { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initHeaders() @@ -113,10 +108,11 @@ func (h *requestOrResponseHeaderMapImpl) Add(key, value string) { h.headers[key] = []string{value} } } - cAPI.HttpSetHeader(unsafe.Pointer(h.request.req), &key, &value, true) + cAPI.HttpSetHeader(unsafe.Pointer(h.request.req), key, value, true) } func (h *requestOrResponseHeaderMapImpl) Del(key string) { + key = strings.ToLower(key) // Get all header values first before removing a key, since the del operation may not take affects immediately // when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread. // Otherwise, we may get outdated values in a following Get call. @@ -124,7 +120,7 @@ func (h *requestOrResponseHeaderMapImpl) Del(key string) { defer h.mutex.Unlock() h.initHeaders() delete(h.headers, key) - cAPI.HttpRemoveHeader(unsafe.Pointer(h.request.req), &key) + cAPI.HttpRemoveHeader(unsafe.Pointer(h.request.req), key) } func (h *requestOrResponseHeaderMapImpl) Range(f func(key, value string) bool) { @@ -166,11 +162,6 @@ type requestHeaderMapImpl struct { var _ api.RequestHeaderMap = (*requestHeaderMapImpl)(nil) -func (h *requestHeaderMapImpl) Protocol() string { - v, _ := h.Get(":protocol") - return v -} - func (h *requestHeaderMapImpl) Scheme() string { v, _ := h.Get(":scheme") return v @@ -217,12 +208,11 @@ func (h *requestOrResponseTrailerMapImpl) initTrailers() { } func (h *requestOrResponseTrailerMapImpl) GetRaw(key string) string { - var value string - cAPI.HttpGetHeader(unsafe.Pointer(h.request.req), &key, &value) - return value + return cAPI.HttpGetHeader(unsafe.Pointer(h.request.req), key) } func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initTrailers() @@ -234,6 +224,7 @@ func (h *requestOrResponseTrailerMapImpl) Get(key string) (string, bool) { } func (h *requestOrResponseTrailerMapImpl) Values(key string) []string { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initTrailers() @@ -245,6 +236,7 @@ func (h *requestOrResponseTrailerMapImpl) Values(key string) []string { } func (h *requestOrResponseTrailerMapImpl) Set(key, value string) { + key = strings.ToLower(key) // Get all header values first before setting a value, since the set operation may not take affects immediately // when it's invoked in a Go thread, instead, it will post a callback to run in the envoy worker thread. // Otherwise, we may get outdated values in a following Get call. @@ -255,10 +247,11 @@ func (h *requestOrResponseTrailerMapImpl) Set(key, value string) { h.headers[key] = []string{value} } - cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), &key, &value, false) + cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), key, value, false) } func (h *requestOrResponseTrailerMapImpl) Add(key, value string) { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initTrailers() @@ -269,15 +262,16 @@ func (h *requestOrResponseTrailerMapImpl) Add(key, value string) { h.headers[key] = []string{value} } } - cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), &key, &value, true) + cAPI.HttpSetTrailer(unsafe.Pointer(h.request.req), key, value, true) } func (h *requestOrResponseTrailerMapImpl) Del(key string) { + key = strings.ToLower(key) h.mutex.Lock() defer h.mutex.Unlock() h.initTrailers() delete(h.headers, key) - cAPI.HttpRemoveTrailer(unsafe.Pointer(h.request.req), &key) + cAPI.HttpRemoveTrailer(unsafe.Pointer(h.request.req), key) } func (h *requestOrResponseTrailerMapImpl) Range(f func(key, value string) bool) { @@ -331,23 +325,28 @@ type httpBuffer struct { request *httpRequest envoyBufferInstance uint64 length uint64 - value string + value []byte } var _ api.BufferInstance = (*httpBuffer)(nil) func (b *httpBuffer) Write(p []byte) (n int, err error) { - cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, string(p), api.AppendBuffer) - return len(p), nil + cAPI.HttpSetBytesBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, p, api.AppendBuffer) + n = len(p) + b.length += uint64(n) + return n, nil } func (b *httpBuffer) WriteString(s string) (n int, err error) { cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, s, api.AppendBuffer) - return len(s), nil + n = len(s) + b.length += uint64(n) + return n, nil } func (b *httpBuffer) WriteByte(p byte) error { cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, string(p), api.AppendBuffer) + b.length++ return nil } @@ -364,25 +363,32 @@ func (b *httpBuffer) WriteUint32(p uint32) error { } func (b *httpBuffer) WriteUint64(p uint64) error { - s := strconv.FormatUint(uint64(p), 10) + s := strconv.FormatUint(p, 10) _, err := b.WriteString(s) return err } -func (b *httpBuffer) Peek(n int) []byte { - panic("implement me") -} - func (b *httpBuffer) Bytes() []byte { if b.length == 0 { return nil } - cAPI.HttpGetBuffer(unsafe.Pointer(b.request.req), b.envoyBufferInstance, &b.value, b.length) - return []byte(b.value) + b.value = cAPI.HttpGetBuffer(unsafe.Pointer(b.request.req), b.envoyBufferInstance, b.length) + return b.value } func (b *httpBuffer) Drain(offset int) { - panic("implement me") + if offset <= 0 || b.length == 0 { + return + } + + size := uint64(offset) + if size > b.length { + size = b.length + } + + cAPI.HttpDrainBuffer(unsafe.Pointer(b.request.req), b.envoyBufferInstance, size) + + b.length -= size } func (b *httpBuffer) Len() int { @@ -390,43 +396,47 @@ func (b *httpBuffer) Len() int { } func (b *httpBuffer) Reset() { - panic("implement me") + b.Drain(b.Len()) } func (b *httpBuffer) String() string { if b.length == 0 { return "" } - cAPI.HttpGetBuffer(unsafe.Pointer(b.request.req), b.envoyBufferInstance, &b.value, b.length) - return b.value + b.value = cAPI.HttpGetBuffer(unsafe.Pointer(b.request.req), b.envoyBufferInstance, b.length) + return string(b.value) } func (b *httpBuffer) Append(data []byte) error { - cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, string(data), api.AppendBuffer) - return nil + _, err := b.Write(data) + return err } func (b *httpBuffer) Prepend(data []byte) error { - cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, string(data), api.PrependBuffer) + cAPI.HttpSetBytesBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, data, api.PrependBuffer) + b.length += uint64(len(data)) return nil } func (b *httpBuffer) AppendString(s string) error { - cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, s, api.AppendBuffer) - return nil + _, err := b.WriteString(s) + return err } func (b *httpBuffer) PrependString(s string) error { cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, s, api.PrependBuffer) + b.length += uint64(len(s)) return nil } func (b *httpBuffer) Set(data []byte) error { - cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, string(data), api.SetBuffer) + cAPI.HttpSetBytesBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, data, api.SetBuffer) + b.length = uint64(len(data)) return nil } func (b *httpBuffer) SetString(s string) error { cAPI.HttpSetBufferHelper(unsafe.Pointer(b.request.req), b.envoyBufferInstance, s, api.SetBuffer) + b.length = uint64(len(s)) return nil } diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index 2e26e762fcdc..306336fc6000 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -211,11 +211,10 @@ void Filter::onDestroy() { } // access_log is executed before the log of the stream filter -void Filter::log(const Http::RequestHeaderMap* headers, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&, - Envoy::AccessLog::AccessLogType type) { +void Filter::log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo&) { // `log` may be called multiple times with different log type - switch (type) { + switch (log_context.accessLogType()) { case Envoy::AccessLog::AccessLogType::DownstreamStart: case Envoy::AccessLog::AccessLogType::DownstreamPeriodic: case Envoy::AccessLog::AccessLogType::DownstreamEnd: { @@ -226,12 +225,12 @@ void Filter::log(const Http::RequestHeaderMap* headers, const Http::ResponseHead initRequest(state); request_headers_ = static_cast( - const_cast(headers)); + const_cast(&log_context.requestHeaders())); } state.enterLog(); req_->phase = static_cast(state.phase()); - dynamic_lib_->envoyGoFilterOnHttpLog(req_, int(type)); + dynamic_lib_->envoyGoFilterOnHttpLog(req_, int(log_context.accessLogType())); state.leaveLog(); } break; default: @@ -599,7 +598,7 @@ CAPIStatus Filter::continueStatus(GolangStatus status) { return CAPIStatus::CAPIOK; } -CAPIStatus Filter::getHeader(absl::string_view key, GoString* go_value) { +CAPIStatus Filter::getHeader(absl::string_view key, uint64_t* value_data, int* value_len) { Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -619,8 +618,8 @@ CAPIStatus Filter::getHeader(absl::string_view key, GoString* go_value) { if (!result.empty()) { auto str = result[0]->value().getStringView(); - go_value->p = str.data(); - go_value->n = str.length(); + *value_data = reinterpret_cast(str.data()); + *value_len = str.length(); } return CAPIStatus::CAPIOK; } @@ -807,6 +806,27 @@ CAPIStatus Filter::copyBuffer(Buffer::Instance* buffer, char* data) { return CAPIStatus::CAPIOK; } +CAPIStatus Filter::drainBuffer(Buffer::Instance* buffer, uint64_t length) { + // lock until this function return since it may running in a Go thread. + Thread::LockGuard lock(mutex_); + if (has_destroyed_) { + ENVOY_LOG(debug, "golang filter has been destroyed"); + return CAPIStatus::CAPIFilterIsDestroy; + } + auto& state = getProcessorState(); + if (!state.isProcessingInGo()) { + ENVOY_LOG(debug, "golang filter is not processing Go"); + return CAPIStatus::CAPINotInGo; + } + if (!state.doDataList.checkExisting(buffer)) { + ENVOY_LOG(debug, "invoking cgo api at invalid phase: {}", __func__); + return CAPIStatus::CAPIInvalidPhase; + } + + buffer->drain(length); + return CAPIStatus::CAPIOK; +} + CAPIStatus Filter::setBufferHelper(Buffer::Instance* buffer, absl::string_view& value, bufferAction action) { // lock until this function return since it may running in a Go thread. @@ -989,7 +1009,7 @@ CAPIStatus Filter::getIntegerValue(int id, uint64_t* value) { return CAPIStatus::CAPIOK; } -CAPIStatus Filter::getStringValue(int id, GoString* value_str) { +CAPIStatus Filter::getStringValue(int id, uint64_t* value_data, int* value_len) { // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { @@ -1060,12 +1080,13 @@ CAPIStatus Filter::getStringValue(int id, GoString* value_str) { RELEASE_ASSERT(false, absl::StrCat("invalid string value id: ", id)); } - value_str->p = req_->strValue.data(); - value_str->n = req_->strValue.length(); + *value_data = reinterpret_cast(req_->strValue.data()); + *value_len = req_->strValue.length(); return CAPIStatus::CAPIOK; } -CAPIStatus Filter::getDynamicMetadata(const std::string& filter_name, GoSlice* buf_slice) { +CAPIStatus Filter::getDynamicMetadata(const std::string& filter_name, uint64_t* buf_data, + int* buf_len) { Thread::LockGuard lock(mutex_); if (has_destroyed_) { ENVOY_LOG(debug, "golang filter has been destroyed"); @@ -1081,10 +1102,10 @@ CAPIStatus Filter::getDynamicMetadata(const std::string& filter_name, GoSlice* b if (!state.isThreadSafe()) { auto weak_ptr = weak_from_this(); ENVOY_LOG(debug, "golang filter getDynamicMetadata posting request to dispatcher"); - state.getDispatcher().post([this, &state, weak_ptr, filter_name, buf_slice] { + state.getDispatcher().post([this, &state, weak_ptr, filter_name, buf_data, buf_len] { ENVOY_LOG(debug, "golang filter getDynamicMetadata request in worker thread"); if (!weak_ptr.expired() && !hasDestroyed()) { - populateSliceWithMetadata(state, filter_name, buf_slice); + populateSliceWithMetadata(state, filter_name, buf_data, buf_len); dynamic_lib_->envoyGoRequestSemaDec(req_); } else { ENVOY_LOG(info, "golang filter has gone or destroyed in getDynamicMetadata"); @@ -1093,21 +1114,20 @@ CAPIStatus Filter::getDynamicMetadata(const std::string& filter_name, GoSlice* b return CAPIStatus::CAPIYield; } else { ENVOY_LOG(debug, "golang filter getDynamicMetadata replying directly"); - populateSliceWithMetadata(state, filter_name, buf_slice); + populateSliceWithMetadata(state, filter_name, buf_data, buf_len); } return CAPIStatus::CAPIOK; } void Filter::populateSliceWithMetadata(ProcessorState& state, const std::string& filter_name, - GoSlice* buf_slice) { + uint64_t* buf_data, int* buf_len) { const auto& metadata = state.streamInfo().dynamicMetadata().filter_metadata(); const auto filter_it = metadata.find(filter_name); if (filter_it != metadata.end()) { filter_it->second.SerializeToString(&req_->strValue); - buf_slice->data = req_->strValue.data(); - buf_slice->len = req_->strValue.length(); - buf_slice->cap = req_->strValue.length(); + *buf_data = reinterpret_cast(req_->strValue.data()); + *buf_len = req_->strValue.length(); } } @@ -1199,7 +1219,8 @@ CAPIStatus Filter::setStringFilterState(absl::string_view key, absl::string_view return CAPIStatus::CAPIOK; } -CAPIStatus Filter::getStringFilterState(absl::string_view key, GoString* value_str) { +CAPIStatus Filter::getStringFilterState(absl::string_view key, uint64_t* value_data, + int* value_len) { // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { @@ -1218,20 +1239,20 @@ CAPIStatus Filter::getStringFilterState(absl::string_view key, GoString* value_s state.streamInfo().filterState()->getDataReadOnly(key); if (go_filter_state) { req_->strValue = go_filter_state->value(); - value_str->p = req_->strValue.data(); - value_str->n = req_->strValue.length(); + *value_data = reinterpret_cast(req_->strValue.data()); + *value_len = req_->strValue.length(); } } else { auto key_str = std::string(key); auto weak_ptr = weak_from_this(); - state.getDispatcher().post([this, &state, weak_ptr, key_str, value_str] { + state.getDispatcher().post([this, &state, weak_ptr, key_str, value_data, value_len] { if (!weak_ptr.expired() && !hasDestroyed()) { auto go_filter_state = state.streamInfo().filterState()->getDataReadOnly(key_str); if (go_filter_state) { req_->strValue = go_filter_state->value(); - value_str->p = req_->strValue.data(); - value_str->n = req_->strValue.length(); + *value_data = reinterpret_cast(req_->strValue.data()); + *value_len = req_->strValue.length(); } dynamic_lib_->envoyGoRequestSemaDec(req_); } else { @@ -1243,7 +1264,8 @@ CAPIStatus Filter::getStringFilterState(absl::string_view key, GoString* value_s return CAPIStatus::CAPIOK; } -CAPIStatus Filter::getStringProperty(absl::string_view path, GoString* value_str, int* rc) { +CAPIStatus Filter::getStringProperty(absl::string_view path, uint64_t* value_data, int* value_len, + int* rc) { // lock until this function return since it may running in a Go thread. Thread::LockGuard lock(mutex_); if (has_destroyed_) { @@ -1265,13 +1287,13 @@ CAPIStatus Filter::getStringProperty(absl::string_view path, GoString* value_str } if (state.isThreadSafe()) { - return getStringPropertyCommon(path, value_str, state); + return getStringPropertyCommon(path, value_data, value_len, state); } auto weak_ptr = weak_from_this(); - state.getDispatcher().post([this, &state, weak_ptr, path, value_str, rc] { + state.getDispatcher().post([this, &state, weak_ptr, path, value_data, value_len, rc] { if (!weak_ptr.expired() && !hasDestroyed()) { - *rc = getStringPropertyCommon(path, value_str, state); + *rc = getStringPropertyCommon(path, value_data, value_len, state); dynamic_lib_->envoyGoRequestSemaDec(req_); } else { ENVOY_LOG(info, "golang filter has gone or destroyed in getStringProperty"); @@ -1280,13 +1302,13 @@ CAPIStatus Filter::getStringProperty(absl::string_view path, GoString* value_str return CAPIStatus::CAPIYield; } -CAPIStatus Filter::getStringPropertyCommon(absl::string_view path, GoString* value_str, - ProcessorState& state) { +CAPIStatus Filter::getStringPropertyCommon(absl::string_view path, uint64_t* value_data, + int* value_len, ProcessorState& state) { activation_info_ = &state.streamInfo(); CAPIStatus status = getStringPropertyInternal(path, &req_->strValue); if (status == CAPIStatus::CAPIOK) { - value_str->p = req_->strValue.data(); - value_str->n = req_->strValue.length(); + *value_data = reinterpret_cast(req_->strValue.data()); + *value_len = req_->strValue.length(); } return status; } @@ -1499,7 +1521,7 @@ void FilterConfig::newGoPluginConfig() { FilterConfig::~FilterConfig() { if (config_id_ > 0) { - dso_lib_->envoyGoFilterDestroyHttpPluginConfig(config_id_); + dso_lib_->envoyGoFilterDestroyHttpPluginConfig(config_id_, 0); } } @@ -1656,10 +1678,10 @@ RoutePluginConfig::RoutePluginConfig( RoutePluginConfig::~RoutePluginConfig() { absl::WriterMutexLock lock(&mutex_); if (config_id_ > 0) { - dso_lib_->envoyGoFilterDestroyHttpPluginConfig(config_id_); + dso_lib_->envoyGoFilterDestroyHttpPluginConfig(config_id_, 0); } - if (merged_config_id_ > 0) { - dso_lib_->envoyGoFilterDestroyHttpPluginConfig(merged_config_id_); + if (merged_config_id_ > 0 && config_id_ != merged_config_id_) { + dso_lib_->envoyGoFilterDestroyHttpPluginConfig(merged_config_id_, 0); } } @@ -1698,7 +1720,13 @@ uint64_t RoutePluginConfig::getMergedConfigId(uint64_t parent_id) { return merged_config_id_; } // upper level config changed, merged_config_id_ is outdated. - dso_lib_->envoyGoFilterDestroyHttpPluginConfig(merged_config_id_); + // there is a concurrency race: + // 1. when A envoy worker thread is using the cached merged_config_id_ and it will call into Go + // after some time. + // 2. while B envoy worker thread may update the merged_config_id_ in getMergedConfigId, that + // will delete the id. + // so, we delay deleting the id in the Go side. + dso_lib_->envoyGoFilterDestroyHttpPluginConfig(merged_config_id_, 1); } if (config_id_ == 0) { diff --git a/contrib/golang/filters/http/source/golang_filter.h b/contrib/golang/filters/http/source/golang_filter.h index 7c40320d995d..cb342c437327 100644 --- a/contrib/golang/filters/http/source/golang_filter.h +++ b/contrib/golang/filters/http/source/golang_filter.h @@ -204,11 +204,8 @@ class Filter : public Http::StreamFilter, } // AccessLog::Instance - void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - Envoy::AccessLog::AccessLogType access_log_type) override; + void log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& info) override; void onStreamComplete() override {} @@ -220,25 +217,27 @@ class Filter : public Http::StreamFilter, CAPIStatus sendPanicReply(absl::string_view details); - CAPIStatus getHeader(absl::string_view key, GoString* go_value); + CAPIStatus getHeader(absl::string_view key, uint64_t* value_data, int* value_len); CAPIStatus copyHeaders(GoString* go_strs, char* go_buf); CAPIStatus setHeader(absl::string_view key, absl::string_view value, headerAction act); CAPIStatus removeHeader(absl::string_view key); CAPIStatus copyBuffer(Buffer::Instance* buffer, char* data); + CAPIStatus drainBuffer(Buffer::Instance* buffer, uint64_t length); CAPIStatus setBufferHelper(Buffer::Instance* buffer, absl::string_view& value, bufferAction action); CAPIStatus copyTrailers(GoString* go_strs, char* go_buf); CAPIStatus setTrailer(absl::string_view key, absl::string_view value, headerAction act); CAPIStatus removeTrailer(absl::string_view key); - CAPIStatus getStringValue(int id, GoString* value_str); + CAPIStatus getStringValue(int id, uint64_t* value_data, int* value_len); CAPIStatus getIntegerValue(int id, uint64_t* value); - CAPIStatus getDynamicMetadata(const std::string& filter_name, GoSlice* buf_slice); + CAPIStatus getDynamicMetadata(const std::string& filter_name, uint64_t* buf_data, int* buf_len); CAPIStatus setDynamicMetadata(std::string filter_name, std::string key, absl::string_view buf); CAPIStatus setStringFilterState(absl::string_view key, absl::string_view value, int state_type, int life_span, int stream_sharing); - CAPIStatus getStringFilterState(absl::string_view key, GoString* value_str); - CAPIStatus getStringProperty(absl::string_view path, GoString* value_str, GoInt32* rc); + CAPIStatus getStringFilterState(absl::string_view key, uint64_t* value_data, int* value_len); + CAPIStatus getStringProperty(absl::string_view path, uint64_t* value_data, int* value_len, + GoInt32* rc); private: bool hasDestroyed() { @@ -273,9 +272,9 @@ class Filter : public Http::StreamFilter, const absl::string_view& buf); void populateSliceWithMetadata(ProcessorState& state, const std::string& filter_name, - GoSlice* buf_slice); + uint64_t* buf_data, int* buf_len); - CAPIStatus getStringPropertyCommon(absl::string_view path, GoString* value_str, + CAPIStatus getStringPropertyCommon(absl::string_view path, uint64_t* value_data, int* value_len, ProcessorState& state); CAPIStatus getStringPropertyInternal(absl::string_view path, std::string* result); absl::optional findValue(absl::string_view name, diff --git a/contrib/golang/filters/http/source/processor_state.cc b/contrib/golang/filters/http/source/processor_state.cc index e1b8e69ba557..028f3c516854 100644 --- a/contrib/golang/filters/http/source/processor_state.cc +++ b/contrib/golang/filters/http/source/processor_state.cc @@ -47,7 +47,7 @@ bool BufferList::checkExisting(Buffer::Instance* data) { }; // headers_ should set to nullptr when return true. -bool ProcessorState::handleHeaderGolangStatus(const GolangStatus status) { +bool ProcessorState::handleHeaderGolangStatus(GolangStatus status) { ENVOY_LOG(debug, "golang filter handle header status, state: {}, phase: {}, status: {}", stateStr(), phaseStr(), int(status)); diff --git a/contrib/golang/filters/http/test/BUILD b/contrib/golang/filters/http/test/BUILD index 7c6519b150cd..d6df96579249 100644 --- a/contrib/golang/filters/http/test/BUILD +++ b/contrib/golang/filters/http/test/BUILD @@ -16,7 +16,6 @@ envoy_cc_test( data = [ "//contrib/golang/filters/http/test/test_data/passthrough:filter.so", ], - env = {"GODEBUG": "cgocheck=0"}, deps = [ "//contrib/golang/filters/http/source:config", "//test/mocks/server:factory_context_mocks", @@ -31,7 +30,6 @@ envoy_cc_test( "//contrib/golang/filters/http/test/test_data/passthrough:filter.so", "//contrib/golang/filters/http/test/test_data/routeconfig:filter.so", ], - env = {"GODEBUG": "cgocheck=0"}, deps = [ "//contrib/golang/filters/http/source:golang_filter_lib", "//source/common/stream_info:stream_info_lib", @@ -53,17 +51,19 @@ envoy_cc_test( srcs = ["golang_integration_test.cc"], data = [ "//contrib/golang/filters/http/test/test_data/access_log:filter.so", + "//contrib/golang/filters/http/test/test_data/action:filter.so", "//contrib/golang/filters/http/test/test_data/basic:filter.so", + "//contrib/golang/filters/http/test/test_data/buffer:filter.so", "//contrib/golang/filters/http/test/test_data/echo:filter.so", "//contrib/golang/filters/http/test/test_data/metric:filter.so", "//contrib/golang/filters/http/test/test_data/passthrough:filter.so", "//contrib/golang/filters/http/test/test_data/property:filter.so", "//contrib/golang/filters/http/test/test_data/routeconfig:filter.so", ], - env = {"GODEBUG": "cgocheck=0"}, deps = [ "//contrib/golang/filters/http/source:config", "//source/exe:main_common_lib", + "//source/extensions/filters/http/lua:config", "//test/config:v2_link_hacks", "//test/integration:http_integration_lib", "//test/test_common:utility_lib", diff --git a/contrib/golang/filters/http/test/config_test.cc b/contrib/golang/filters/http/test/config_test.cc index d85792ee35b1..6bef239ec22d 100644 --- a/contrib/golang/filters/http/test/config_test.cc +++ b/contrib/golang/filters/http/test/config_test.cc @@ -28,8 +28,11 @@ std::string genSoPath(std::string name) { TEST(GolangFilterConfigTest, InvalidateEmptyConfig) { NiceMock context; EXPECT_THROW_WITH_REGEX( - GolangFilterConfig().createFilterFactoryFromProto( - envoy::extensions::filters::http::golang::v3alpha::Config(), "stats", context), + GolangFilterConfig() + .createFilterFactoryFromProto(envoy::extensions::filters::http::golang::v3alpha::Config(), + "stats", context) + .status() + .IgnoreError(), Envoy::ProtoValidationException, "ConfigValidationError.LibraryId: value length must be at least 1 characters"); } @@ -54,7 +57,8 @@ TEST(GolangFilterConfigTest, GolangFilterWithValidConfig) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; GolangFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); EXPECT_CALL(filter_callback, addAccessLogHandler(_)); @@ -77,7 +81,8 @@ TEST(GolangFilterConfigTest, GolangFilterWithNilPluginConfig) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; GolangFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); EXPECT_CALL(filter_callback, addAccessLogHandler(_)); diff --git a/contrib/golang/filters/http/test/golang_integration_test.cc b/contrib/golang/filters/http/test/golang_integration_test.cc index 0606d9a995dd..56e568caa596 100644 --- a/contrib/golang/filters/http/test/golang_integration_test.cc +++ b/contrib/golang/filters/http/test/golang_integration_test.cc @@ -65,8 +65,8 @@ class RetrieveDynamicMetadataFilterConfig RetrieveDynamicMetadataFilterConfig() : Extensions::HttpFilters::Common::EmptyHttpFilterConfig("validate-dynamic-metadata") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamEncoderFilter(std::make_shared<::Envoy::RetrieveDynamicMetadataFilter>()); }; @@ -177,7 +177,7 @@ name: golang [so_id]( envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { - // for testing http filter level config, a new virtualhost wihtout per route config + // for testing http filter level config, a new virtualhost without per route config auto vh = hcm.mutable_route_config()->add_virtual_hosts(); vh->add_domains("filter-level.com"); vh->set_name("filter-level.com"); @@ -186,7 +186,7 @@ name: golang rt->mutable_route()->set_cluster("cluster_0"); // virtualhost level per route config - const std::string key = "envoy.filters.http.golang"; + const std::string key = "golang"; const auto yaml_fmt = R"EOF( "@type": type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.ConfigsPerRoute @@ -500,7 +500,7 @@ name: golang EXPECT_EQ(header_0_existing, !response->headers().get(Http::LowerCaseString("x-test-header-0")).empty()); - if (set_header != "") { + if (!set_header.empty()) { EXPECT_EQ("test-value", getHeader(response->headers(), set_header)); } cleanup(); @@ -551,6 +551,15 @@ name: golang // verify content-type EXPECT_EQ("text/html", getHeader(response->headers(), "content-type")); + // verify two values + auto values = response->headers().get(Http::LowerCaseString("x-two-values")); + if (values.size() == 2) { + EXPECT_EQ("foo", values[0]->value().getStringView()); + EXPECT_EQ("bar", values[1]->value().getStringView()); + } else { + EXPECT_EQ(values.size(), 2); + } + cleanup(); } @@ -565,7 +574,7 @@ name: golang auto encoder_decoder = codec_client_->startRequest(request_headers); Http::RequestEncoder& request_encoder = encoder_decoder.first; auto response = std::move(encoder_decoder.second); - // 100 + 200 > 150, excced buffer limit. + // 100 + 200 > 150, exceed buffer limit. codec_client_->sendData(request_encoder, std::string(100, '-'), false); codec_client_->sendData(request_encoder, std::string(200, '-'), true); @@ -662,13 +671,74 @@ name: golang cleanup(); } + void testActionWithoutData(std::string query) { + initializeBasicFilter(ACTION, "test.com"); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test?" + query}, + {":scheme", "http"}, + {":authority", "test.com"}}; + + auto encoder_decoder = codec_client_->startRequest(request_headers, true); + auto response = std::move(encoder_decoder.second); + + if (query.find("encodeHeadersRet") != std::string::npos) { + waitForNextUpstreamRequest(); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + upstream_request_->encodeHeaders(response_headers, true); + } + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("500", response->headers().getStatusValue()); + + EXPECT_EQ(1, test_server_->counter("http.config_test.golang.panic_error")->value()); + + cleanup(); + } + + void testBufferApi(std::string query) { + initializeBasicFilter(BUFFER, "test.com"); + + auto path = std::string("/test?") + query; + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", path}, + {":scheme", "http"}, + {":authority", "test.com"}, + }; + + auto encoder_decoder = codec_client_->startRequest(request_headers); + Http::RequestEncoder& request_encoder = encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + std::string data = ""; + for (int i = 0; i < 10; i++) { + data += "12345"; + } + codec_client_->sendData(request_encoder, data, true); + + waitForNextUpstreamRequest(); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + upstream_request_->encodeHeaders(response_headers, false); + Buffer::OwnedImpl response_data("goodbye"); + upstream_request_->encodeData(response_data, true); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_EQ("200", response->headers().getStatusValue()); + cleanup(); + } + const std::string ECHO{"echo"}; const std::string BASIC{"basic"}; const std::string PASSTHROUGH{"passthrough"}; + const std::string BUFFER{"buffer"}; const std::string ROUTECONFIG{"routeconfig"}; const std::string PROPERTY{"property"}; const std::string ACCESSLOG{"access_log"}; const std::string METRIC{"metric"}; + const std::string ACTION{"action"}; }; INSTANTIATE_TEST_SUITE_P(IpVersions, GolangIntegrationTest, @@ -728,10 +798,10 @@ TEST_P(GolangIntegrationTest, Passthrough) { ASSERT_TRUE(response->waitForEndStream()); - // check status for pasthrough + // check status for passthrough EXPECT_EQ("200", response->headers().getStatusValue()); - // check body for pasthrough + // check body for passthrough auto body = absl::StrFormat("%s%s", good, bye); EXPECT_EQ(body, response->body()); @@ -754,6 +824,14 @@ TEST_P(GolangIntegrationTest, PluginNotFound) { cleanup(); } +TEST_P(GolangIntegrationTest, BufferDrain) { testBufferApi("Drain"); } + +TEST_P(GolangIntegrationTest, BufferReset) { testBufferApi("Reset"); } + +TEST_P(GolangIntegrationTest, BufferResetAfterDrain) { testBufferApi("ResetAfterDrain"); } + +TEST_P(GolangIntegrationTest, BufferLen) { testBufferApi("Len"); } + TEST_P(GolangIntegrationTest, Property) { initializePropertyConfig(PROPERTY, genSoPath(PROPERTY), PROPERTY); initialize(); @@ -857,6 +935,7 @@ TEST_P(GolangIntegrationTest, AccessLogDownstreamStart) { EXPECT_TRUE(response->complete()); EXPECT_EQ("r;r2", getHeader(upstream_request_->headers(), "referers")); + EXPECT_EQ("true", getHeader(upstream_request_->headers(), "canRunAsynclyForDownstreamStart")); cleanup(); } @@ -887,11 +966,12 @@ TEST_P(GolangIntegrationTest, AccessLogDownstreamPeriodic) { EXPECT_TRUE(response->complete()); EXPECT_EQ("r", getHeader(upstream_request_->headers(), "referers")); + EXPECT_EQ("true", getHeader(upstream_request_->headers(), "canRunAsynclyForDownstreamPeriodic")); cleanup(); } -// Mertic API testing +// Metric API testing TEST_P(GolangIntegrationTest, Metric) { testMetric("/test"); } // Metric API testing in async mode. @@ -954,7 +1034,7 @@ TEST_P(GolangIntegrationTest, LocalReply_DecodeData_Async) { } // The next filter(lua filter) send local reply after Go filter continue in decode header phase. -// Go filter will terimiate when lua filter send local reply. +// Go filter will terminate when lua filter send local reply. TEST_P(GolangIntegrationTest, LuaRespondAfterGoHeaderContinue) { // put lua filter after golang filter // golang filter => lua filter. @@ -1067,4 +1147,36 @@ TEST_P(GolangIntegrationTest, DynamicMetadata_Async_Sleep) { testDynamicMetadata("/test?dymeta=1&async=1&sleep=1"); } +TEST_P(GolangIntegrationTest, DecodeHeadersWithoutData_StopAndBuffer) { + testActionWithoutData("decodeHeadersRet=StopAndBuffer"); +} + +TEST_P(GolangIntegrationTest, DecodeHeadersWithoutData_StopAndBufferWatermark) { + testActionWithoutData("decodeHeadersRet=StopAndBufferWatermark"); +} + +TEST_P(GolangIntegrationTest, DecodeHeadersWithoutData_StopAndBuffer_Async) { + testActionWithoutData("decodeHeadersRet=StopAndBuffer&aysnc=1"); +} + +TEST_P(GolangIntegrationTest, DecodeHeadersWithoutData_StopAndBufferWatermark_Async) { + testActionWithoutData("decodeHeadersRet=StopAndBufferWatermark&aysnc=1"); +} + +TEST_P(GolangIntegrationTest, EncodeHeadersWithoutData_StopAndBuffer) { + testActionWithoutData("encodeHeadersRet=StopAndBuffer"); +} + +TEST_P(GolangIntegrationTest, EncodeHeadersWithoutData_StopAndBufferWatermark) { + testActionWithoutData("encodeHeadersRet=StopAndBufferWatermark"); +} + +TEST_P(GolangIntegrationTest, EncodeHeadersWithoutData_StopAndBuffer_Async) { + testActionWithoutData("encodeHeadersRet=StopAndBuffer&aysnc=1"); +} + +TEST_P(GolangIntegrationTest, EncodeHeadersWithoutData_StopAndBufferWatermark_Async) { + testActionWithoutData("encodeHeadersRet=StopAndBufferWatermark&aysnc=1"); +} + } // namespace Envoy diff --git a/contrib/golang/filters/http/test/test_data/access_log/filter.go b/contrib/golang/filters/http/test/test_data/access_log/filter.go index 7d5fa51aa267..750e19215b87 100644 --- a/contrib/golang/filters/http/test/test_data/access_log/filter.go +++ b/contrib/golang/filters/http/test/test_data/access_log/filter.go @@ -17,6 +17,10 @@ var ( respSize string canRunAsyncly bool + canRunAsynclyForDownstreamStart bool + + canRunAsynclyForDownstreamPeriodic bool + referers = []string{} ) @@ -47,6 +51,8 @@ func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api. header.Set("respCode", respCode) header.Set("respSize", respSize) header.Set("canRunAsyncly", strconv.FormatBool(canRunAsyncly)) + header.Set("canRunAsynclyForDownstreamStart", strconv.FormatBool(canRunAsynclyForDownstreamStart)) + header.Set("canRunAsynclyForDownstreamPeriodic", strconv.FormatBool(canRunAsynclyForDownstreamPeriodic)) header.Set("referers", strings.Join(referers, ";")) @@ -68,6 +74,13 @@ func (f *filter) OnLogDownstreamStart() { } referers = append(referers, referer) + + wg.Add(1) + go func() { + time.Sleep(1 * time.Millisecond) + canRunAsynclyForDownstreamStart = true + wg.Done() + }() } func (f *filter) OnLogDownstreamPeriodic() { @@ -78,6 +91,13 @@ func (f *filter) OnLogDownstreamPeriodic() { } referers = append(referers, referer) + + wg.Add(1) + go func() { + time.Sleep(1 * time.Millisecond) + canRunAsynclyForDownstreamPeriodic = true + wg.Done() + }() } func (f *filter) OnLog() { diff --git a/contrib/golang/filters/http/test/test_data/access_log/go.mod b/contrib/golang/filters/http/test/test_data/access_log/go.mod index 5cedc2cba33b..01693aa92cd2 100644 --- a/contrib/golang/filters/http/test/test_data/access_log/go.mod +++ b/contrib/golang/filters/http/test/test_data/access_log/go.mod @@ -1,6 +1,6 @@ module example.com/access_log -go 1.18 +go 1.20 require github.com/envoyproxy/envoy v1.24.0 diff --git a/contrib/golang/filters/http/test/test_data/action/BUILD b/contrib/golang/filters/http/test/test_data/action/BUILD new file mode 100644 index 000000000000..daaf13cce009 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/action/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") + +licenses(["notice"]) # Apache 2 + +go_binary( + name = "filter.so", + srcs = [ + "config.go", + "filter.go", + ], + out = "filter.so", + cgo = True, + importpath = "github.com/envoyproxy/envoy/contrib/golang/filters/http/test/test_data/action", + linkmode = "c-shared", + visibility = ["//visibility:public"], + deps = [ + "//contrib/golang/common/go/api", + "//contrib/golang/filters/http/source/go/pkg/http", + "@com_github_cncf_xds_go//xds/type/v3:type", + "@org_golang_google_protobuf//types/known/anypb", + "@org_golang_google_protobuf//types/known/structpb", + ], +) diff --git a/contrib/golang/filters/http/test/test_data/action/config.go b/contrib/golang/filters/http/test/test_data/action/config.go new file mode 100644 index 000000000000..f02a961673f1 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/action/config.go @@ -0,0 +1,26 @@ +package main + +import ( + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http" +) + +const Name = "action" + +func init() { + http.RegisterHttpFilterConfigFactoryAndParser(Name, ConfigFactory, nil) +} + +type config struct { + decodeHeadersRet api.StatusType +} + +func ConfigFactory(c interface{}) api.StreamFilterFactory { + return func(callbacks api.FilterCallbackHandler) api.StreamFilter { + return &filter{ + callbacks: callbacks, + } + } +} + +func main() {} diff --git a/contrib/golang/filters/http/test/test_data/action/filter.go b/contrib/golang/filters/http/test/test_data/action/filter.go new file mode 100644 index 000000000000..0ef080859cb8 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/action/filter.go @@ -0,0 +1,72 @@ +package main + +import ( + "net/url" + "strings" + + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" +) + +type filter struct { + api.PassThroughStreamFilter + + callbacks api.FilterCallbackHandler + query_params url.Values +} + +func parseQuery(path string) url.Values { + if idx := strings.Index(path, "?"); idx >= 0 { + query := path[idx+1:] + values, _ := url.ParseQuery(query) + return values + } + return make(url.Values) +} + +func getStatus(status string) api.StatusType { + switch status { + case "StopAndBuffer": + return api.StopAndBuffer + case "StopAndBufferWatermark": + return api.StopAndBufferWatermark + } + return api.Continue +} + +func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api.StatusType { + f.query_params = parseQuery(header.Path()) + + decodeHeadersRet := f.query_params.Get("decodeHeadersRet") + async := f.query_params.Get("async") + if decodeHeadersRet != "" { + if async != "" { + go func() { + defer f.callbacks.RecoverPanic() + f.callbacks.Continue(getStatus(decodeHeadersRet)) + }() + return api.Running + } + + return getStatus(decodeHeadersRet) + } + + return api.Continue +} + +func (f *filter) EncodeHeaders(header api.ResponseHeaderMap, endStream bool) api.StatusType { + encodeHeadersRet := f.query_params.Get("encodeHeadersRet") + async := f.query_params.Get("async") + if encodeHeadersRet != "" { + if async != "" { + go func() { + defer f.callbacks.RecoverPanic() + f.callbacks.Continue(getStatus(encodeHeadersRet)) + }() + return api.Running + } + + return getStatus(encodeHeadersRet) + } + + return api.Continue +} diff --git a/contrib/golang/filters/http/test/test_data/action/go.mod b/contrib/golang/filters/http/test/test_data/action/go.mod new file mode 100644 index 000000000000..9a07becf6234 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/action/go.mod @@ -0,0 +1,19 @@ +module example.com/action + +go 1.20 + +require ( + github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 + github.com/envoyproxy/envoy v1.24.0 +) + +require github.com/google/go-cmp v0.5.9 // indirect + +require ( + github.com/envoyproxy/protoc-gen-validate v0.9.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/protobuf v1.31.0 +) + +replace github.com/envoyproxy/envoy => ../../../../../../../ diff --git a/contrib/golang/filters/http/test/test_data/basic/filter.go b/contrib/golang/filters/http/test/test_data/basic/filter.go index 7adbf65ed67a..9990be5b4b77 100644 --- a/contrib/golang/filters/http/test/test_data/basic/filter.go +++ b/contrib/golang/filters/http/test/test_data/basic/filter.go @@ -16,7 +16,6 @@ type filter struct { callbacks api.FilterCallbackHandler req_body_length uint64 query_params url.Values - protocol string scheme string method string path string @@ -55,7 +54,6 @@ func (f *filter) initRequest(header api.RequestHeaderMap) { f.req_body_length = 0 - f.protocol = header.Protocol() f.scheme = header.Scheme() f.method = header.Method() f.path = header.Path() @@ -82,14 +80,16 @@ func (f *filter) initRequest(header api.RequestHeaderMap) { func (f *filter) fail(msg string, a ...any) api.StatusType { body := fmt.Sprintf(msg, a...) + f.callbacks.Log(api.Error, fmt.Sprintf("test failed: %s", body)) f.callbacks.SendLocalReply(500, body, nil, 0, "") return api.LocalReply } func (f *filter) sendLocalReply(phase string) api.StatusType { - headers := map[string]string{ - "Content-type": "text/html", - "test-phase": phase, + headers := map[string][]string{ + "Content-type": {"text/html"}, + "test-phase": {phase}, + "x-two-values": {"foo", "bar"}, } body := fmt.Sprintf("forbidden from go in %s\r\n", phase) f.callbacks.SendLocalReply(403, body, headers, 0, "") @@ -187,6 +187,30 @@ func (f *filter) decodeHeaders(header api.RequestHeaderMap, endStream bool) api. return f.fail("Values return unexpected data %v", hdrs) } + if found { + upperCase, _ := header.Get("X-Test-Header-0") + if upperCase != origin { + return f.fail("Get should be case-insensitive") + } + upperCaseHdrs := header.Values("X-Test-Header-0") + if hdrs[0] != upperCaseHdrs[0] { + return f.fail("Values should be case-insensitive") + } + } + + header.Add("UpperCase", "header") + if hdr, _ := header.Get("uppercase"); hdr != "header" { + return f.fail("Add should be case-insensitive") + } + header.Set("UpperCase", "header") + if hdr, _ := header.Get("uppercase"); hdr != "header" { + return f.fail("Set should be case-insensitive") + } + header.Del("UpperCase") + if hdr, _ := header.Get("uppercase"); hdr != "" { + return f.fail("Del should be case-insensitive") + } + header.Add("existed-header", "bar") header.Add("newly-added-header", "foo") header.Add("newly-added-header", "bar") @@ -217,6 +241,11 @@ func (f *filter) decodeData(buffer api.BufferInstance, endStream bool) api.Statu f.req_body_length += uint64(buffer.Len()) if buffer.Len() != 0 { data := buffer.String() + if string(buffer.Bytes()) != data { + return f.sendLocalReply(fmt.Sprintf("data in bytes: %s vs data in string: %s", + string(buffer.Bytes()), data)) + } + buffer.SetString(strings.ToUpper(data)) buffer.AppendString("_append") buffer.PrependString("prepend_") @@ -251,6 +280,28 @@ func (f *filter) decodeTrailers(trailers api.RequestTrailerMap) api.StatusType { trailers.Add("x-test-trailer-2", "bar") } + upperCase, _ := trailers.Get("X-Test-Trailer-0") + if upperCase != "bar" { + return f.fail("Get should be case-insensitive") + } + upperCaseHdrs := trailers.Values("X-Test-Trailer-0") + if upperCaseHdrs[0] != "bar" { + return f.fail("Values should be case-insensitive") + } + + trailers.Add("UpperCase", "trailers") + if hdr, _ := trailers.Get("uppercase"); hdr != "trailers" { + return f.fail("Add should be case-insensitive") + } + trailers.Set("UpperCase", "trailers") + if hdr, _ := trailers.Get("uppercase"); hdr != "trailers" { + return f.fail("Set should be case-insensitive") + } + trailers.Del("UpperCase") + if hdr, _ := trailers.Get("uppercase"); hdr != "" { + return f.fail("Del should be case-insensitive") + } + if f.panic == "decode-trailer" { badcode() } diff --git a/contrib/golang/filters/http/test/test_data/basic/go.mod b/contrib/golang/filters/http/test/test_data/basic/go.mod index 709dbc31fbce..b70054d4863a 100644 --- a/contrib/golang/filters/http/test/test_data/basic/go.mod +++ b/contrib/golang/filters/http/test/test_data/basic/go.mod @@ -1,6 +1,6 @@ module example.com/basic -go 1.18 +go 1.20 require github.com/envoyproxy/envoy v1.24.0 diff --git a/contrib/golang/filters/http/test/test_data/buffer/BUILD b/contrib/golang/filters/http/test/test_data/buffer/BUILD new file mode 100644 index 000000000000..c072c6e72f57 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/buffer/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") + +licenses(["notice"]) # Apache 2 + +go_binary( + name = "filter.so", + srcs = [ + "config.go", + "filter.go", + ], + out = "filter.so", + cgo = True, + importpath = "github.com/envoyproxy/envoy/contrib/golang/filters/http/test/test_data/buffer", + linkmode = "c-shared", + visibility = ["//visibility:public"], + deps = [ + "//contrib/golang/common/go/api", + "//contrib/golang/filters/http/source/go/pkg/http", + "@com_github_cncf_xds_go//xds/type/v3:type", + "@org_golang_google_protobuf//types/known/anypb", + "@org_golang_google_protobuf//types/known/structpb", + ], +) diff --git a/contrib/golang/filters/http/test/test_data/buffer/config.go b/contrib/golang/filters/http/test/test_data/buffer/config.go new file mode 100644 index 000000000000..cb88f346652b --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/buffer/config.go @@ -0,0 +1,44 @@ +package main + +import ( + "google.golang.org/protobuf/types/known/anypb" + + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" + "github.com/envoyproxy/envoy/contrib/golang/filters/http/source/go/pkg/http" +) + +const Name = "buffer" + +func init() { + http.RegisterHttpFilterConfigFactoryAndParser(Name, ConfigFactory, &parser{}) +} + +type config struct { +} + +type parser struct { +} + +func (p *parser) Parse(any *anypb.Any, callbacks api.ConfigCallbackHandler) (interface{}, error) { + conf := &config{} + return conf, nil +} + +func (p *parser) Merge(parent interface{}, child interface{}) interface{} { + return child +} + +func ConfigFactory(c interface{}) api.StreamFilterFactory { + conf, ok := c.(*config) + if !ok { + panic("unexpected config type") + } + return func(callbacks api.FilterCallbackHandler) api.StreamFilter { + return &filter{ + callbacks: callbacks, + config: conf, + } + } +} + +func main() {} diff --git a/contrib/golang/filters/http/test/test_data/buffer/filter.go b/contrib/golang/filters/http/test/test_data/buffer/filter.go new file mode 100644 index 000000000000..9b0d9f6a8d41 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/buffer/filter.go @@ -0,0 +1,138 @@ +package main + +import ( + "fmt" + "reflect" + + "github.com/envoyproxy/envoy/contrib/golang/common/go/api" +) + +type filter struct { + api.PassThroughStreamFilter + + callbacks api.FilterCallbackHandler + path string + config *config + + failed bool +} + +func testReset(b api.BufferInstance) { + b.Reset() + + bs := b.Bytes() + if len(bs) > 0 { + panic(fmt.Sprintf("unexpected data: %s", string(bs))) + } +} + +func testDrain(b api.BufferInstance) { + b.Drain(40) + bs := b.Bytes() + if string(bs) != "1234512345" { + panic(fmt.Sprintf("unexpected data: %s", string(bs))) + } + + b.Drain(5) + bs = b.Bytes() + if string(bs) != "12345" { + panic(fmt.Sprintf("unexpected data: %s", string(bs))) + } + + b.Drain(10) + bs = b.Bytes() + if string(bs) != "" { + panic(fmt.Sprintf("unexpected data: %s", string(bs))) + } + + // drain when all data are drained + b.Drain(10) + bs = b.Bytes() + if string(bs) != "" { + panic(fmt.Sprintf("unexpected data: %s", string(bs))) + } + + // bad offset + for _, n := range []int{-1, 0} { + b.Drain(n) + } +} + +func testResetAfterDrain(b api.BufferInstance) { + b.Drain(40) + b.Reset() + bs := b.Bytes() + if string(bs) != "" { + panic(fmt.Sprintf("unexpected data: %s", string(bs))) + } +} + +func panicIfNotEqual(a, b any) { + if !reflect.DeepEqual(a, b) { + panic(fmt.Sprintf("expected %v, got %v", a, b)) + } +} + +func panicIfLenMismatch(b api.BufferInstance, size int) { + panicIfNotEqual(size, b.Len()) + panicIfNotEqual(len(b.Bytes()), b.Len()) +} + +func testLen(b api.BufferInstance) { + b.Set([]byte("12")) + panicIfLenMismatch(b, 2) + b.SetString("123") + panicIfLenMismatch(b, 3) + + b.Write([]byte("45")) + panicIfLenMismatch(b, 5) + b.WriteString("67") + panicIfLenMismatch(b, 7) + b.WriteByte('8') + panicIfLenMismatch(b, 8) + b.WriteUint16(90) + panicIfLenMismatch(b, 10) + b.WriteUint32(12) + panicIfLenMismatch(b, 12) + b.WriteUint64(12) + panicIfLenMismatch(b, 14) + + b.Drain(2) + panicIfLenMismatch(b, 12) + b.Write([]byte("45")) + panicIfLenMismatch(b, 14) + + b.Reset() + panicIfLenMismatch(b, 0) + + b.Append([]byte("12")) + panicIfLenMismatch(b, 2) + b.Prepend([]byte("0")) + panicIfLenMismatch(b, 3) + b.AppendString("345") + panicIfLenMismatch(b, 6) + b.PrependString("00") + panicIfLenMismatch(b, 8) +} + +func (f *filter) DecodeData(buffer api.BufferInstance, endStream bool) api.StatusType { + if endStream { + return api.Continue + } + // run once + + query, _ := f.callbacks.GetProperty("request.query") + switch query { + case "Reset": + testReset(buffer) + case "ResetAfterDrain": + testResetAfterDrain(buffer) + case "Drain": + testDrain(buffer) + case "Len": + testLen(buffer) + default: + panic(fmt.Sprintf("unknown case %s", query)) + } + return api.Continue +} diff --git a/contrib/golang/filters/http/test/test_data/buffer/go.mod b/contrib/golang/filters/http/test/test_data/buffer/go.mod new file mode 100644 index 000000000000..45b49e5b5f31 --- /dev/null +++ b/contrib/golang/filters/http/test/test_data/buffer/go.mod @@ -0,0 +1,10 @@ +module example.com/buffer + +go 1.20 + +require ( + github.com/envoyproxy/envoy v1.24.0 + google.golang.org/protobuf v1.31.0 +) + +replace github.com/envoyproxy/envoy => ../../../../../../../../ diff --git a/contrib/golang/filters/http/test/test_data/dummy/go.mod b/contrib/golang/filters/http/test/test_data/dummy/go.mod index 2e9286b62ba9..beaf9d59c743 100644 --- a/contrib/golang/filters/http/test/test_data/dummy/go.mod +++ b/contrib/golang/filters/http/test/test_data/dummy/go.mod @@ -1,6 +1,6 @@ module example.com/dummy -go 1.18 +go 1.20 require github.com/envoyproxy/envoy v1.24.0 diff --git a/contrib/golang/filters/http/test/test_data/echo/go.mod b/contrib/golang/filters/http/test/test_data/echo/go.mod index 8614081f88c5..7a258b6138a0 100644 --- a/contrib/golang/filters/http/test/test_data/echo/go.mod +++ b/contrib/golang/filters/http/test/test_data/echo/go.mod @@ -1,6 +1,6 @@ module example.com/echo -go 1.18 +go 1.20 require ( github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 diff --git a/contrib/golang/filters/http/test/test_data/metric/go.mod b/contrib/golang/filters/http/test/test_data/metric/go.mod index 709dbc31fbce..b70054d4863a 100644 --- a/contrib/golang/filters/http/test/test_data/metric/go.mod +++ b/contrib/golang/filters/http/test/test_data/metric/go.mod @@ -1,6 +1,6 @@ module example.com/basic -go 1.18 +go 1.20 require github.com/envoyproxy/envoy v1.24.0 diff --git a/contrib/golang/filters/http/test/test_data/passthrough/go.mod b/contrib/golang/filters/http/test/test_data/passthrough/go.mod index 3a42612f666c..d6c620393300 100644 --- a/contrib/golang/filters/http/test/test_data/passthrough/go.mod +++ b/contrib/golang/filters/http/test/test_data/passthrough/go.mod @@ -1,6 +1,6 @@ module example.com/passthrough -go 1.18 +go 1.20 require github.com/envoyproxy/envoy v1.24.0 diff --git a/contrib/golang/filters/http/test/test_data/property/filter.go b/contrib/golang/filters/http/test/test_data/property/filter.go index bd1934b66187..9f77dd5382f9 100644 --- a/contrib/golang/filters/http/test/test_data/property/filter.go +++ b/contrib/golang/filters/http/test/test_data/property/filter.go @@ -54,7 +54,7 @@ func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api. f.assertProperty("request.useragent", "ua") f.assertProperty("request.id", "xri") - f.assertProperty("request.duration", "value not found") // available only when the request is finished + f.assertProperty("request.duration", api.ErrValueNotFound.Error()) // available only when the request is finished f.assertProperty("source.address", f.callbacks.StreamInfo().DownstreamRemoteAddress()) f.assertProperty("destination.address", f.callbacks.StreamInfo().DownstreamLocalAddress()) @@ -63,7 +63,7 @@ func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api. f.assertProperty("xds.route_name", "test-route-name") // non-existed attribute - f.assertProperty("request.user_agent", "value not found") + f.assertProperty("request.user_agent", api.ErrValueNotFound.Error()) // access response attribute in the decode phase f.assertProperty("response.total_size", "0") @@ -74,7 +74,7 @@ func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api. ".", ".total_size", } { - f.assertProperty(attr, "value not found") + f.assertProperty(attr, api.ErrValueNotFound.Error()) } // unsupported value type for _, attr := range []string{ @@ -84,7 +84,14 @@ func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api. "request", "request.", } { - f.assertProperty(attr, "serialization failure") + f.assertProperty(attr, api.ErrSerializationFailure.Error()) + } + + // error handling + _, err := f.callbacks.GetProperty(".not_found") + if err != api.ErrValueNotFound { + f.callbacks.Log(api.Critical, "unexpected error "+err.Error()) + f.failed = true } return api.Continue } @@ -94,9 +101,8 @@ func (f *filter) EncodeHeaders(header api.ResponseHeaderMap, endStream bool) api f.assertProperty("xds.cluster_name", "cluster_0") f.assertProperty("xds.cluster_metadata", "") - // response.code is available only after the response has started to send code, _ := f.callbacks.StreamInfo().ResponseCode() - exp := "value not found" + exp := "" if code != 0 { exp = strconv.Itoa(int(code)) } diff --git a/contrib/golang/filters/http/test/test_data/property/go.mod b/contrib/golang/filters/http/test/test_data/property/go.mod index 4bcd808d5e6f..55ee2207f6dc 100644 --- a/contrib/golang/filters/http/test/test_data/property/go.mod +++ b/contrib/golang/filters/http/test/test_data/property/go.mod @@ -1,10 +1,10 @@ module example.com/property -go 1.18 +go 1.20 require ( - github.com/envoyproxy/envoy v1.24.0 - google.golang.org/protobuf v1.31.0 + github.com/envoyproxy/envoy v1.24.0 + google.golang.org/protobuf v1.32.0 ) replace github.com/envoyproxy/envoy => ../../../../../../../ diff --git a/contrib/golang/filters/http/test/test_data/routeconfig/go.mod b/contrib/golang/filters/http/test/test_data/routeconfig/go.mod index 2df147ea5ed1..b11b4ac5d871 100644 --- a/contrib/golang/filters/http/test/test_data/routeconfig/go.mod +++ b/contrib/golang/filters/http/test/test_data/routeconfig/go.mod @@ -1,6 +1,6 @@ module example.com/routeconfig -go 1.18 +go 1.20 require ( github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 diff --git a/contrib/golang/filters/network/source/cgo.cc b/contrib/golang/filters/network/source/cgo.cc index a26f0ea10553..eb1f5442542d 100644 --- a/contrib/golang/filters/network/source/cgo.cc +++ b/contrib/golang/filters/network/source/cgo.cc @@ -82,7 +82,7 @@ void envoyGoFilterDownstreamFinalize(void* f, int reason) { auto* wrapper = reinterpret_cast(f); FilterWeakPtr& weak_ptr = wrapper->filter_ptr_; if (FilterSharedPtr filter = weak_ptr.lock()) { - // make sure that the deconstructor is also executed by envoy wrk thread. + // make sure that the deconstructor is also executed by envoy work thread. filter->dispatcher()->post([wrapper] { delete wrapper; }); } else { // the Filter is already deleted, just release the wrapper. @@ -116,11 +116,11 @@ CAPIStatus envoyGoFilterDownstreamInfo(void* f, int info_type, void* ret) { // Upstream // -void* envoyGoFilterUpstreamConnect(void* library_id, void* addr, unsigned long long int connID) { +void* envoyGoFilterUpstreamConnect(void* library_id, void* addr, uint64_t conn_id) { std::string id = copyGoString(library_id); auto dynamic_lib = Dso::DsoManager::getDsoByID(id); UpstreamConnPtr conn_ptr = - std::make_shared(copyGoString(addr), dynamic_lib, connID); + std::make_shared(copyGoString(addr), dynamic_lib, conn_id); // the upstream connect wrapper will be deleted by envoyGoFilterUpstreamFinalize UpstreamConnWrapper* wrapper = new UpstreamConnWrapper(conn_ptr); conn_ptr->setWrapper(wrapper); @@ -157,7 +157,7 @@ void envoyGoFilterUpstreamFinalize(void* u, int reason) { UNREFERENCED_PARAMETER(reason); auto* wrapper = reinterpret_cast(u); UpstreamConnPtr& conn_ptr = wrapper->conn_ptr_; - // make sure that the deconstructor is also executed by envoy wrk thread + // make sure that the deconstructor is also executed by envoy work thread conn_ptr->dispatcher()->post([wrapper] { delete wrapper; }); } diff --git a/contrib/golang/filters/network/source/config.cc b/contrib/golang/filters/network/source/config.cc index 6e87b438dca1..b75177f3caf4 100644 --- a/contrib/golang/filters/network/source/config.cc +++ b/contrib/golang/filters/network/source/config.cc @@ -16,7 +16,7 @@ Network::FilterFactoryCb GolangConfigFactory::createFilterFactoryFromProtoTyped( Server::Configuration::FactoryContext& context) { is_terminal_filter_ = proto_config.is_terminal_filter(); - UpstreamConn::initThreadLocalStorage(context, context.threadLocal()); + UpstreamConn::initThreadLocalStorage(context, context.serverFactoryContext().threadLocal()); FilterConfigSharedPtr config = std::make_shared(proto_config); std::string config_str; diff --git a/contrib/golang/filters/network/source/go/pkg/network/capi.go b/contrib/golang/filters/network/source/go/pkg/network/capi.go index 6747e97322e7..bb884b563711 100644 --- a/contrib/golang/filters/network/source/go/pkg/network/capi.go +++ b/contrib/golang/filters/network/source/go/pkg/network/capi.go @@ -92,7 +92,7 @@ func (c *cgoApiImpl) SetFilterState(f unsafe.Pointer, key string, value string, } func (c *cgoApiImpl) UpstreamConnect(libraryID string, addr string, connID uint64) unsafe.Pointer { - return unsafe.Pointer(C.envoyGoFilterUpstreamConnect(unsafe.Pointer(&libraryID), unsafe.Pointer(&addr), C.ulonglong(connID))) + return unsafe.Pointer(C.envoyGoFilterUpstreamConnect(unsafe.Pointer(&libraryID), unsafe.Pointer(&addr), C.uint64_t(connID))) } func (c *cgoApiImpl) UpstreamWrite(f unsafe.Pointer, bufferPtr unsafe.Pointer, bufferLen int, endStream int) { diff --git a/contrib/golang/filters/network/source/go/pkg/network/shim.go b/contrib/golang/filters/network/source/go/pkg/network/shim.go index b9e55d620167..9bf620635e62 100644 --- a/contrib/golang/filters/network/source/go/pkg/network/shim.go +++ b/contrib/golang/filters/network/source/go/pkg/network/shim.go @@ -63,14 +63,25 @@ var ( libraryID string ) +// wrap the UpstreamFilter to ensure that the runtime.finalizer can be triggered +// regardless of whether there is a circular reference in the UpstreamFilter. +type upstreamConnWrapper struct { + api.UpstreamFilter + finalizer *int +} + func CreateUpstreamConn(addr string, filter api.UpstreamFilter) { + conn := &upstreamConnWrapper{ + UpstreamFilter: filter, + finalizer: new(int), + } connID := atomic.AddUint64(&upstreamConnIDGenerator, 1) - _ = UpstreamFilters.StoreFilterByConnID(connID, filter) + _ = UpstreamFilters.StoreFilterByConnID(connID, conn) h := cgoAPI.UpstreamConnect(libraryID, addr, connID) // NP: make sure filter will be deleted. - runtime.SetFinalizer(filter, func(f api.UpstreamFilter) { + runtime.SetFinalizer(conn.finalizer, func(_ *int) { cgoAPI.UpstreamFinalize(unsafe.Pointer(uintptr(h)), api.NormalFinalize) }) } diff --git a/contrib/golang/filters/network/source/golang.cc b/contrib/golang/filters/network/source/golang.cc index 87e0be72d2d3..a91899627308 100644 --- a/contrib/golang/filters/network/source/golang.cc +++ b/contrib/golang/filters/network/source/golang.cc @@ -107,9 +107,6 @@ Network::FilterStatus Filter::onWrite(Buffer::Instance& data, bool end_stream) { auto ret = dynamic_lib_->envoyGoFilterOnDownstreamWrite( wrapper_, data.length(), reinterpret_cast(slices), slice_num, end_stream); - // TODO: do not drain buffer by default - data.drain(data.length()); - delete[] slices; return Network::FilterStatus(ret); diff --git a/contrib/golang/filters/network/source/golang.h b/contrib/golang/filters/network/source/golang.h index 4118fbaf88a0..1b181efbff01 100644 --- a/contrib/golang/filters/network/source/golang.h +++ b/contrib/golang/filters/network/source/golang.h @@ -76,7 +76,9 @@ class Filter : public Network::Filter, void close(Network::ConnectionCloseType close_type); Event::Dispatcher* dispatcher() { return dispatcher_; } - Upstream::ClusterManager& clusterManager() { return context_.clusterManager(); } + Upstream::ClusterManager& clusterManager() { + return context_.serverFactoryContext().clusterManager(); + } std::string getLocalAddrStr() const { return local_addr_; } std::string getRemoteAddrStr() const { return addr_; }; diff --git a/contrib/golang/filters/network/source/upstream.cc b/contrib/golang/filters/network/source/upstream.cc index 5081eb5f8e2d..b8d0a11384d8 100644 --- a/contrib/golang/filters/network/source/upstream.cc +++ b/contrib/golang/filters/network/source/upstream.cc @@ -22,16 +22,18 @@ void UpstreamConn::initThreadLocalStorage(Server::Configuration::FactoryContext& std::call_once(store.init_once_, [&context, &tls, &store]() { // should be the singleton for use by the entire server. ClusterManagerContainer& cluster_manager_container = clusterManagerContainer(); - cluster_manager_container.cluster_manager_ = &context.clusterManager(); + cluster_manager_container.cluster_manager_ = &context.serverFactoryContext().clusterManager(); SlotPtrContainer& slot_ptr_container = slotPtrContainer(); slot_ptr_container.slot_ = tls.allocateSlot(); - Thread::ThreadId main_thread_id = context.api().threadFactory().currentThreadId(); + Thread::ThreadId main_thread_id = + context.serverFactoryContext().api().threadFactory().currentThreadId(); slot_ptr_container.slot_->set( [&context, main_thread_id, &store](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { - if (context.api().threadFactory().currentThreadId() == main_thread_id) { + if (context.serverFactoryContext().api().threadFactory().currentThreadId() == + main_thread_id) { return nullptr; } @@ -46,8 +48,8 @@ void UpstreamConn::initThreadLocalStorage(Server::Configuration::FactoryContext& } UpstreamConn::UpstreamConn(std::string addr, Dso::NetworkFilterDsoPtr dynamic_lib, - unsigned long long int goConnID, Event::Dispatcher* dispatcher) - : dynamic_lib_(dynamic_lib), goConnID_(goConnID), dispatcher_(dispatcher), addr_(addr) { + unsigned long long int go_conn_id, Event::Dispatcher* dispatcher) + : dynamic_lib_(dynamic_lib), go_conn_id_(go_conn_id), dispatcher_(dispatcher), addr_(addr) { if (dispatcher_ == nullptr) { DispatcherStore& store = dispatcherStore(); Thread::LockGuard guard(store.lock_); @@ -58,8 +60,8 @@ UpstreamConn::UpstreamConn(std::string addr, Dso::NetworkFilterDsoPtr dynamic_li stream_info_ = std::make_unique( dispatcher_->timeSource(), nullptr, StreamInfo::FilterState::LifeSpan::FilterChain); stream_info_->filterState()->setData( - Network::DestinationAddress::key(), - std::make_shared( + "envoy.network.transport_socket.original_dst_address", + std::make_shared( Network::Utility::parseInternetAddressAndPort(addr, false)), StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::FilterChain, StreamInfo::StreamSharingMayImpactPooling::None); @@ -129,7 +131,7 @@ void UpstreamConn::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn, conn_->addUpstreamCallbacks(*this); remote_addr_ = conn_->connection().connectionInfoProvider().directRemoteAddress()->asString(); - dynamic_lib_->envoyGoFilterOnUpstreamConnectionReady(wrapper_, goConnID_); + dynamic_lib_->envoyGoFilterOnUpstreamConnectionReady(wrapper_, go_conn_id_); } void UpstreamConn::onPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason, @@ -142,7 +144,7 @@ void UpstreamConn::onPoolFailure(Tcp::ConnectionPool::PoolFailureReason reason, } dynamic_lib_->envoyGoFilterOnUpstreamConnectionFailure(wrapper_, static_cast(reason), - goConnID_); + go_conn_id_); } void UpstreamConn::onEvent(Network::ConnectionEvent event) { diff --git a/contrib/golang/filters/network/source/upstream.h b/contrib/golang/filters/network/source/upstream.h index d278c3e18c5e..8ee09d3f89c2 100644 --- a/contrib/golang/filters/network/source/upstream.h +++ b/contrib/golang/filters/network/source/upstream.h @@ -36,7 +36,7 @@ class UpstreamConn : public Tcp::ConnectionPool::Callbacks, Logger::Loggable { public: UpstreamConn(std::string addr, Dso::NetworkFilterDsoPtr dynamic_lib, - unsigned long long int goConnID, Event::Dispatcher* dispatcher = nullptr); + unsigned long long int go_conn_id, Event::Dispatcher* dispatcher = nullptr); ~UpstreamConn() override { if (handler_) { handler_->cancel(Tcp::ConnectionPool::CancelPolicy::Default); @@ -95,7 +95,7 @@ class UpstreamConn : public Tcp::ConnectionPool::Callbacks, } Dso::NetworkFilterDsoPtr dynamic_lib_{nullptr}; - unsigned long long int goConnID_{0}; + unsigned long long int go_conn_id_{0}; UpstreamConnWrapper* wrapper_{nullptr}; Event::Dispatcher* dispatcher_{nullptr}; std::unique_ptr stream_info_{nullptr}; diff --git a/contrib/golang/filters/network/test/config_test.cc b/contrib/golang/filters/network/test/config_test.cc index e60d86f4a550..0078412b1388 100644 --- a/contrib/golang/filters/network/test/config_test.cc +++ b/contrib/golang/filters/network/test/config_test.cc @@ -31,8 +31,10 @@ class GolangFilterConfigTestBase { void testConfig(envoy::extensions::filters::network::golang::v3alpha::Config& config) { EXPECT_CALL(slot_allocator_, allocateSlot()) .WillRepeatedly(Invoke(&slot_allocator_, &ThreadLocal::MockInstance::allocateSlotMock)); - ON_CALL(context_, threadLocal()).WillByDefault(ReturnRef(slot_allocator_)); - ON_CALL(context_.api_, threadFactory()).WillByDefault(ReturnRef(thread_factory_)); + ON_CALL(context_.server_factory_context_, threadLocal()) + .WillByDefault(ReturnRef(slot_allocator_)); + ON_CALL(context_.server_factory_context_.api_, threadFactory()) + .WillByDefault(ReturnRef(thread_factory_)); Network::FilterFactoryCb cb; EXPECT_NO_THROW({ cb = factory_.createFilterFactoryFromProto(config, context_); }); diff --git a/contrib/golang/filters/network/test/upstream_test.cc b/contrib/golang/filters/network/test/upstream_test.cc index 539cfb46987b..36a379b60212 100644 --- a/contrib/golang/filters/network/test/upstream_test.cc +++ b/contrib/golang/filters/network/test/upstream_test.cc @@ -37,9 +37,10 @@ class UpstreamConnTest : public testing::Test { void initialize() { EXPECT_CALL(slot_allocator_, allocateSlot()) .WillRepeatedly(Invoke(&slot_allocator_, &ThreadLocal::MockInstance::allocateSlotMock)); - context_.cluster_manager_.initializeClusters({"plainText"}, {}); - context_.cluster_manager_.initializeThreadLocalClusters({"plainText"}); - ON_CALL(context_.api_, threadFactory()).WillByDefault(ReturnRef(thread_factory_)); + context_.server_factory_context_.cluster_manager_.initializeClusters({"plainText"}, {}); + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"plainText"}); + ON_CALL(context_.server_factory_context_.api_, threadFactory()) + .WillByDefault(ReturnRef(thread_factory_)); UpstreamConn::initThreadLocalStorage(context_, slot_allocator_); dso_ = std::make_shared(); upConn_ = std::make_shared(addr_, dso_, 0, &dispatcher_); @@ -60,27 +61,33 @@ TEST_F(UpstreamConnTest, ConnectUpstream) { initialize(); const auto* dst_addr = - upConn_->requestStreamInfo()->filterState().getDataReadOnly( - Network::DestinationAddress::key()); + upConn_->requestStreamInfo()->filterState().getDataReadOnly( + "envoy.network.transport_socket.original_dst_address"); EXPECT_EQ(dst_addr->address()->asString(), addr_); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); EXPECT_CALL(*dso_.get(), envoyGoFilterOnUpstreamConnectionReady(_, _)); upConn_->connect(); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure, true); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure, true); return nullptr; })); EXPECT_CALL(*dso_.get(), diff --git a/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier/capi_impl.go b/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier/capi_impl.go index bb8e62765d6a..c2bb44732236 100644 --- a/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier/capi_impl.go +++ b/contrib/golang/router/cluster_specifier/source/go/pkg/cluster_specifier/capi_impl.go @@ -20,7 +20,6 @@ package cluster_specifier /* // ref https://github.com/golang/go/issues/25832 -#cgo CFLAGS: -I../../../../../../common/go/api -I../api #cgo linux LDFLAGS: -Wl,-unresolved-symbols=ignore-all #cgo darwin LDFLAGS: -Wl,-undefined,dynamic_lookup @@ -28,7 +27,6 @@ package cluster_specifier #include #include "api.h" - */ import "C" import ( diff --git a/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod b/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod index b4e13bffc8ab..f36b3e42a285 100644 --- a/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod +++ b/contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod @@ -3,19 +3,17 @@ module example.com/routeconfig go 1.18 require ( - github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 - github.com/envoyproxy/envoy/contrib/golang v1.24.0 + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 + github.com/envoyproxy/envoy v1.28.0 ) +require github.com/google/go-cmp v0.5.9 // indirect + require ( - github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect - github.com/golang/protobuf v1.5.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect - google.golang.org/grpc v1.25.1 // indirect - google.golang.org/protobuf v1.28.1 + github.com/envoyproxy/protoc-gen-validate v0.10.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/protobuf v1.31.0 ) -replace github.com/envoyproxy/envoy/contrib/golang => ../../../../../ +replace github.com/envoyproxy/envoy => ../../../../../../../ diff --git a/contrib/hyperscan/matching/input_matchers/source/BUILD b/contrib/hyperscan/matching/input_matchers/source/BUILD index 5342badb28d1..f41023076bcd 100644 --- a/contrib/hyperscan/matching/input_matchers/source/BUILD +++ b/contrib/hyperscan/matching/input_matchers/source/BUILD @@ -11,6 +11,7 @@ load( ) load( "//contrib:all_contrib_extensions.bzl", + "envoy_contrib_linux_aarch64_constraints", "envoy_contrib_linux_x86_64_constraints", ) @@ -41,8 +42,34 @@ envoy_cmake( ], ) +envoy_cmake( + name = "vectorscan", + build_data = ["@org_boost//:header"], + cache_entries = { + "BOOST_ROOT": "$EXT_BUILD_ROOT/external/org_boost", + # "BUILD_SVE2_BITPERM": "on", + # "BUILD_SVE2": "on", + # "BUILD_SVE": "on", + "BUILD_EXAMPLES": "off", + "BUILD_SHARED_LIBS": "off", + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_INSTALL_LIBDIR": "lib", + "FAT_RUNTIME": "off", + "SQLITE_SKIP_CHECK": "on", + "RAGEL": "$EXT_BUILD_DEPS/ragel/bin/ragel", + }, + default_cache_entries = {}, + lib_source = "@io_vectorscan//:all", + out_static_libs = ["libhs.a"], + tags = ["skip_on_windows"], + target_compatible_with = envoy_contrib_linux_aarch64_constraints(), + deps = [ + envoy_external_dep_path("ragel"), + ], +) + envoy_cc_library( - name = "matcher_lib", + name = "hyperscan_matcher_lib", srcs = ["matcher.cc"], hdrs = ["matcher.h"], deps = [ @@ -53,12 +80,25 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "vectorscan_matcher_lib", + srcs = ["matcher.cc"], + hdrs = ["matcher.h"], + deps = [ + ":vectorscan", + "//envoy/common:regex_interface", + "//envoy/matcher:matcher_interface", + "//envoy/thread_local:thread_local_interface", + ], +) + envoy_cc_contrib_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], defines = select({ "//bazel:linux_x86_64": [], + "//bazel:linux_aarch64": [], "//conditions:default": [ "HYPERSCAN_DISABLED=1", ], @@ -71,7 +111,10 @@ envoy_cc_contrib_extension( "@envoy_api//contrib/envoy/extensions/matching/input_matchers/hyperscan/v3alpha:pkg_cc_proto", ] + select({ "//bazel:linux_x86_64": [ - ":matcher_lib", + ":hyperscan_matcher_lib", + ], + "//bazel:linux_aarch64": [ + ":vectorscan_matcher_lib", ], "//conditions:default": [ ], diff --git a/contrib/hyperscan/matching/input_matchers/test/BUILD b/contrib/hyperscan/matching/input_matchers/test/BUILD index 9957b2a660d6..aa1ebb140fbe 100644 --- a/contrib/hyperscan/matching/input_matchers/test/BUILD +++ b/contrib/hyperscan/matching/input_matchers/test/BUILD @@ -22,12 +22,18 @@ envoy_cc_test( name = "matcher_test", srcs = ["matcher_test.cc"], deps = [ - "//contrib/hyperscan/matching/input_matchers/source:matcher_lib", "//source/common/thread_local:thread_local_lib", "//test/mocks/event:event_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/test_common:utility_lib", - ], + ] + select({ + "//bazel:linux_x86_64": [ + "//contrib/hyperscan/matching/input_matchers/source:hyperscan_matcher_lib", + ], + "//bazel:linux_aarch64": [ + "//contrib/hyperscan/matching/input_matchers/source:vectorscan_matcher_lib", + ], + }), ) envoy_cc_benchmark_binary( @@ -35,11 +41,17 @@ envoy_cc_benchmark_binary( srcs = ["hyperscan_speed_test.cc"], external_deps = ["benchmark"], deps = [ - "//contrib/hyperscan/matching/input_matchers/source:hyperscan", "//source/common/common:assert_lib", "//source/common/common:utility_lib", "@com_googlesource_code_re2//:re2", - ], + ] + select({ + "//bazel:linux_x86_64": [ + "//contrib/hyperscan/matching/input_matchers/source:hyperscan", + ], + "//bazel:linux_aarch64": [ + "//contrib/hyperscan/matching/input_matchers/source:vectorscan", + ], + }), ) envoy_cc_benchmark_binary( @@ -47,11 +59,17 @@ envoy_cc_benchmark_binary( srcs = ["matcher_speed_test.cc"], external_deps = ["benchmark"], deps = [ - "//contrib/hyperscan/matching/input_matchers/source:matcher_lib", "//source/common/common:assert_lib", "//source/common/common:regex_lib", "//source/common/common:utility_lib", "//source/common/thread_local:thread_local_lib", "//test/mocks/event:event_mocks", - ], + ] + select({ + "//bazel:linux_x86_64": [ + "//contrib/hyperscan/matching/input_matchers/source:hyperscan_matcher_lib", + ], + "//bazel:linux_aarch64": [ + "//contrib/hyperscan/matching/input_matchers/source:vectorscan_matcher_lib", + ], + }), ) diff --git a/contrib/hyperscan/regex_engines/source/BUILD b/contrib/hyperscan/regex_engines/source/BUILD index 2882492ccdfa..f4555dc97131 100644 --- a/contrib/hyperscan/regex_engines/source/BUILD +++ b/contrib/hyperscan/regex_engines/source/BUILD @@ -14,9 +14,15 @@ envoy_cc_library( srcs = ["regex.cc"], hdrs = ["regex.h"], deps = [ - "//contrib/hyperscan/matching/input_matchers/source:matcher_lib", "//envoy/common:regex_interface", - ], + ] + select({ + "//bazel:linux_x86_64": [ + "//contrib/hyperscan/matching/input_matchers/source:hyperscan_matcher_lib", + ], + "//bazel:linux_aarch64": [ + "//contrib/hyperscan/matching/input_matchers/source:vectorscan_matcher_lib", + ], + }), ) envoy_cc_contrib_extension( @@ -25,6 +31,7 @@ envoy_cc_contrib_extension( hdrs = ["config.h"], defines = select({ "//bazel:linux_x86_64": [], + "//bazel:linux_aarch64": [], "//conditions:default": [ "HYPERSCAN_DISABLED=1", ], @@ -36,6 +43,9 @@ envoy_cc_contrib_extension( "//bazel:linux_x86_64": [ ":regex_lib", ], + "//bazel:linux_aarch64": [ + ":regex_lib", + ], "//conditions:default": [ ], }), diff --git a/contrib/kafka/filters/network/source/BUILD b/contrib/kafka/filters/network/source/BUILD index ec50a777c50d..a7e075125bfe 100644 --- a/contrib/kafka/filters/network/source/BUILD +++ b/contrib/kafka/filters/network/source/BUILD @@ -2,7 +2,6 @@ load("@base_pip3//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") load( "//bazel:envoy_build_system.bzl", - "envoy_cc_contrib_extension", "envoy_cc_library", "envoy_contrib_package", ) @@ -11,39 +10,7 @@ licenses(["notice"]) # Apache 2 envoy_contrib_package() -# Kafka network filter. -# Broker filter public docs: https://envoyproxy.io/docs/envoy/latest/configuration/listeners/network_filters/kafka_broker_filter - -envoy_cc_contrib_extension( - name = "kafka_broker_config_lib", - srcs = ["broker/config.cc"], - hdrs = ["broker/config.h"], - deps = [ - ":kafka_broker_filter_lib", - "//source/extensions/filters/network:well_known_names", - "//source/extensions/filters/network/common:factory_base_lib", - "@envoy_api//contrib/envoy/extensions/filters/network/kafka_broker/v3:pkg_cc_proto", - ], -) - -envoy_cc_library( - name = "kafka_broker_filter_lib", - srcs = ["broker/filter.cc"], - hdrs = [ - "broker/filter.h", - "external/request_metrics.h", - "external/response_metrics.h", - ], - deps = [ - ":kafka_request_codec_lib", - ":kafka_response_codec_lib", - "//envoy/buffer:buffer_interface", - "//envoy/network:connection_interface", - "//envoy/network:filter_interface", - "//source/common/common:assert_lib", - "//source/common/common:minimal_logger_lib", - ], -) +# Common code for Kafka filters (Kafka type abstractions, protocol, metrics, etc.). envoy_cc_library( name = "abstract_codec_lib", @@ -201,6 +168,16 @@ py_library( srcs = ["protocol/generator.py"], ) +envoy_cc_library( + name = "kafka_metrics_lib", + hdrs = [ + "external/request_metrics.h", + "external/response_metrics.h", + ], + deps = [ + ], +) + envoy_cc_library( name = "parser_lib", hdrs = ["parser.h"], diff --git a/contrib/kafka/filters/network/source/broker/BUILD b/contrib/kafka/filters/network/source/broker/BUILD new file mode 100644 index 000000000000..4a8194148b97 --- /dev/null +++ b/contrib/kafka/filters/network/source/broker/BUILD @@ -0,0 +1,73 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_contrib_extension", + "envoy_cc_library", + "envoy_contrib_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +# Kafka-broker network filter. +# Broker filter public docs: https://envoyproxy.io/docs/envoy/latest/configuration/listeners/network_filters/kafka_broker_filter + +envoy_cc_contrib_extension( + name = "config_lib", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":filter_config_lib", + ":filter_lib", + "//source/extensions/filters/network:well_known_names", + "//source/extensions/filters/network/common:factory_base_lib", + "@envoy_api//contrib/envoy/extensions/filters/network/kafka_broker/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "filter_config_lib", + srcs = [ + "filter_config.cc", + ], + hdrs = [ + "filter_config.h", + ], + deps = [ + "//source/common/common:assert_lib", + "@envoy_api//contrib/envoy/extensions/filters/network/kafka_broker/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "filter_lib", + srcs = ["filter.cc"], + hdrs = [ + "filter.h", + ], + deps = [ + ":filter_config_lib", + ":rewriter_lib", + "//contrib/kafka/filters/network/source:kafka_metrics_lib", + "//contrib/kafka/filters/network/source:kafka_request_codec_lib", + "//contrib/kafka/filters/network/source:kafka_response_codec_lib", + "//envoy/buffer:buffer_interface", + "//envoy/network:connection_interface", + "//envoy/network:filter_interface", + "//source/common/common:minimal_logger_lib", + ], +) + +envoy_cc_library( + name = "rewriter_lib", + srcs = ["rewriter.cc"], + hdrs = [ + "rewriter.h", + ], + deps = [ + ":filter_config_lib", + "//contrib/kafka/filters/network/source:kafka_response_codec_lib", + "//envoy/buffer:buffer_interface", + "//source/common/common:minimal_logger_lib", + ], +) diff --git a/contrib/kafka/filters/network/source/broker/config.cc b/contrib/kafka/filters/network/source/broker/config.cc index 459ce5fd20d8..110605792862 100644 --- a/contrib/kafka/filters/network/source/broker/config.cc +++ b/contrib/kafka/filters/network/source/broker/config.cc @@ -5,6 +5,7 @@ #include "envoy/stats/scope.h" #include "contrib/kafka/filters/network/source/broker/filter.h" +#include "contrib/kafka/filters/network/source/broker/filter_config.h" namespace Envoy { namespace Extensions { @@ -15,13 +16,11 @@ namespace Broker { Network::FilterFactoryCb KafkaConfigFactory::createFilterFactoryFromProtoTyped( const KafkaBrokerProtoConfig& proto_config, Server::Configuration::FactoryContext& context) { - ASSERT(!proto_config.stat_prefix().empty()); - - const std::string& stat_prefix = proto_config.stat_prefix(); - - return [&context, stat_prefix](Network::FilterManager& filter_manager) -> void { - Network::FilterSharedPtr filter = - std::make_shared(context.scope(), context.timeSource(), stat_prefix); + const BrokerFilterConfigSharedPtr filter_config = + std::make_shared(proto_config); + return [&context, filter_config](Network::FilterManager& filter_manager) -> void { + Network::FilterSharedPtr filter = std::make_shared( + context.scope(), context.serverFactoryContext().timeSource(), *filter_config); filter_manager.addFilter(filter); }; } diff --git a/contrib/kafka/filters/network/source/broker/filter.cc b/contrib/kafka/filters/network/source/broker/filter.cc index 855226780ebd..616ac6560364 100644 --- a/contrib/kafka/filters/network/source/broker/filter.cc +++ b/contrib/kafka/filters/network/source/broker/filter.cc @@ -70,19 +70,23 @@ absl::flat_hash_map& KafkaMetricsFacadeImpl::getRequestA } KafkaBrokerFilter::KafkaBrokerFilter(Stats::Scope& scope, TimeSource& time_source, - const std::string& stat_prefix) - : KafkaBrokerFilter{ - std::make_shared(scope, time_source, stat_prefix)} {}; - -KafkaBrokerFilter::KafkaBrokerFilter(const KafkaMetricsFacadeSharedPtr& metrics) - : metrics_{metrics}, response_decoder_{new ResponseDecoder({metrics})}, + const BrokerFilterConfig& filter_config) + : KafkaBrokerFilter{filter_config, std::make_shared( + scope, time_source, filter_config.stat_prefix())} {}; + +KafkaBrokerFilter::KafkaBrokerFilter(const BrokerFilterConfig& filter_config, + const KafkaMetricsFacadeSharedPtr& metrics) + : metrics_{metrics}, response_rewriter_{createRewriter(filter_config)}, + response_decoder_{new ResponseDecoder({metrics, response_rewriter_})}, request_decoder_{ new RequestDecoder({std::make_shared(*response_decoder_), metrics})} {}; KafkaBrokerFilter::KafkaBrokerFilter(KafkaMetricsFacadeSharedPtr metrics, + ResponseRewriterSharedPtr response_rewriter, ResponseDecoderSharedPtr response_decoder, RequestDecoderSharedPtr request_decoder) - : metrics_{metrics}, response_decoder_{response_decoder}, request_decoder_{request_decoder} {}; + : metrics_{metrics}, response_rewriter_{response_rewriter}, response_decoder_{response_decoder}, + request_decoder_{request_decoder} {}; Network::FilterStatus KafkaBrokerFilter::onNewConnection() { return Network::FilterStatus::Continue; @@ -107,6 +111,7 @@ Network::FilterStatus KafkaBrokerFilter::onWrite(Buffer::Instance& data, bool) { ENVOY_LOG(trace, "data from Kafka broker [{} response bytes]", data.length()); try { response_decoder_->onData(data); + response_rewriter_->process(data); return Network::FilterStatus::Continue; } catch (const EnvoyException& e) { ENVOY_LOG(debug, "could not process data from Kafka broker: {}", e.what()); diff --git a/contrib/kafka/filters/network/source/broker/filter.h b/contrib/kafka/filters/network/source/broker/filter.h index 207115838000..60d88b3d2cfa 100644 --- a/contrib/kafka/filters/network/source/broker/filter.h +++ b/contrib/kafka/filters/network/source/broker/filter.h @@ -6,6 +6,8 @@ #include "source/common/common/logger.h" #include "absl/container/flat_hash_map.h" +#include "contrib/kafka/filters/network/source/broker/filter_config.h" +#include "contrib/kafka/filters/network/source/broker/rewriter.h" #include "contrib/kafka/filters/network/source/external/request_metrics.h" #include "contrib/kafka/filters/network/source/external/response_metrics.h" #include "contrib/kafka/filters/network/source/parser.h" @@ -111,7 +113,8 @@ class KafkaMetricsFacadeImpl : public KafkaMetricsFacade { /** * Implementation of Kafka broker-level filter. * Uses two decoders - request and response ones, that are connected using Forwarder instance. - * There's also a KafkaMetricsFacade, that is listening on codec events. + * KafkaMetricsFacade is listening for both request/response events to keep metrics. + * ResponseRewriter is listening for response events to capture and rewrite them if needed. * * +---------------------------------------------------+ * | | @@ -123,13 +126,18 @@ class KafkaMetricsFacadeImpl : public KafkaMetricsFacade { * | | v v v * +------+---+------+ +----+----+ +---------+---+----+ * |KafkaBrokerFilter| |Forwarder| |KafkaMetricsFacade| - * +----------+------+ +----+----+ +---------+--------+ - * | | ^ - * | | | - * | v | - * | +-------+-------+ | - * +---------->+ResponseDecoder+---------------+ - * +---------------+ + * +------+---+------+ +----+----+ +---------+--------+ + * | | | ^ + * | | | | + * | | v | + * | | +-------+-------+ | + * | +---------->+ResponseDecoder+---------------+ + * | +-------+-------+ + * | | + * | v + * | +-------+--------+ + * +-------------->+ResponseRewriter+ + * +----------------+ */ class KafkaBrokerFilter : public Network::Filter, private Logger::Loggable { public: @@ -138,12 +146,15 @@ class KafkaBrokerFilter : public Network::Filter, private Logger::Loggable extractRewriteRules(const KafkaBrokerProtoConfig& proto_config) { + if (proto_config.has_id_based_broker_address_rewrite_spec()) { + std::vector result; + const auto& spec = proto_config.id_based_broker_address_rewrite_spec(); + for (const auto& rule : spec.rules()) { + result.emplace_back(rule.id(), rule.host(), rule.port()); + } + return result; + } else { + return {}; + } +} + +BrokerFilterConfig::BrokerFilterConfig(const KafkaBrokerProtoConfig& proto_config) + : BrokerFilterConfig{proto_config.stat_prefix(), proto_config.force_response_rewrite(), + extractRewriteRules(proto_config)} {} + +BrokerFilterConfig::BrokerFilterConfig(const std::string& stat_prefix, + const bool force_response_rewrite, + const std::vector& broker_address_rewrite_rules) + : stat_prefix_{stat_prefix}, force_response_rewrite_{force_response_rewrite}, + broker_address_rewrite_rules_{broker_address_rewrite_rules} { + ASSERT(!stat_prefix_.empty()); +}; + +bool BrokerFilterConfig::needsResponseRewrite() const { + return force_response_rewrite_ || !broker_address_rewrite_rules_.empty(); +} + +absl::optional +BrokerFilterConfig::findBrokerAddressOverride(const uint32_t broker_id) const { + for (const auto& rule : broker_address_rewrite_rules_) { + if (rule.matches(broker_id)) { + const HostAndPort hp = {rule.host_, rule.port_}; + return {hp}; + } + } + return absl::nullopt; +} + +const std::string& BrokerFilterConfig::stat_prefix() const { return stat_prefix_; } + +} // namespace Broker +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/source/broker/filter_config.h b/contrib/kafka/filters/network/source/broker/filter_config.h new file mode 100644 index 000000000000..4d9526ba711c --- /dev/null +++ b/contrib/kafka/filters/network/source/broker/filter_config.h @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include "absl/types/optional.h" +#include "contrib/envoy/extensions/filters/network/kafka_broker/v3/kafka_broker.pb.h" +#include "contrib/envoy/extensions/filters/network/kafka_broker/v3/kafka_broker.pb.validate.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Broker { + +using KafkaBrokerProtoConfig = envoy::extensions::filters::network::kafka_broker::v3::KafkaBroker; + +using HostAndPort = std::pair; + +// Represents a rule matching a broker with given id. +struct RewriteRule { + + uint32_t id_; + + std::string host_; + uint32_t port_; + + RewriteRule(const uint32_t id, const std::string host, const uint32_t port) + : id_{id}, host_{host}, port_{port} {}; + + bool matches(const uint32_t broker_id) const { return id_ == broker_id; } +}; + +// Minor helper object that contains broker filter configuration. +class BrokerFilterConfig { +public: + virtual ~BrokerFilterConfig() = default; + + BrokerFilterConfig(const KafkaBrokerProtoConfig& proto_config); + + // Visible for testing. + BrokerFilterConfig(const std::string& stat_prefix, const bool force_response_rewrite, + const std::vector& broker_address_rewrite_rules); + + /** + * Returns the prefix for stats. + */ + virtual const std::string& stat_prefix() const; + + /** + * Whether this configuration means that rewrite should be happening. + */ + virtual bool needsResponseRewrite() const; + + /** + * Returns override address for a broker. + */ + virtual absl::optional findBrokerAddressOverride(const uint32_t broker_id) const; + +private: + std::string stat_prefix_; + bool force_response_rewrite_; + std::vector broker_address_rewrite_rules_; +}; + +using BrokerFilterConfigSharedPtr = std::shared_ptr; + +} // namespace Broker +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/source/broker/rewriter.cc b/contrib/kafka/filters/network/source/broker/rewriter.cc new file mode 100644 index 000000000000..e8a9de739b2d --- /dev/null +++ b/contrib/kafka/filters/network/source/broker/rewriter.cc @@ -0,0 +1,107 @@ +#include "contrib/kafka/filters/network/source/broker/rewriter.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Broker { + +// ResponseRewriterImpl. + +ResponseRewriterImpl::ResponseRewriterImpl(const BrokerFilterConfig& config) : config_{config} {}; + +void ResponseRewriterImpl::onMessage(AbstractResponseSharedPtr response) { + responses_to_rewrite_.push_back(response); +} + +void ResponseRewriterImpl::onFailedParse(ResponseMetadataSharedPtr) {} + +constexpr int16_t METADATA_API_KEY = 3; +constexpr int16_t FIND_COORDINATOR_API_KEY = 10; +constexpr int16_t DESCRIBE_CLUSTER_API_KEY = 60; + +template static T& extractResponseData(AbstractResponseSharedPtr& arg) { + using TSharedPtr = std::shared_ptr>; + TSharedPtr cast = std::dynamic_pointer_cast(arg); + if (nullptr == cast) { + throw new EnvoyException("bug: response class not matching response API key"); + } else { + return cast->data_; + } +} + +void ResponseRewriterImpl::process(Buffer::Instance& buffer) { + buffer.drain(buffer.length()); + ResponseEncoder encoder{buffer}; + ENVOY_LOG(trace, "emitting {} stored responses", responses_to_rewrite_.size()); + for (auto response : responses_to_rewrite_) { + switch (response->apiKey()) { + case METADATA_API_KEY: { + auto& mr = extractResponseData(response); + updateMetadataBrokerAddresses(mr); + break; + } + case FIND_COORDINATOR_API_KEY: { + auto& fcr = extractResponseData(response); + updateFindCoordinatorBrokerAddresses(fcr); + break; + } + case DESCRIBE_CLUSTER_API_KEY: { + auto& dcr = extractResponseData(response); + updateDescribeClusterBrokerAddresses(dcr); + break; + } + } + encoder.encode(*response); + } + responses_to_rewrite_.erase(responses_to_rewrite_.begin(), responses_to_rewrite_.end()); +} + +void ResponseRewriterImpl::updateMetadataBrokerAddresses(MetadataResponse& response) const { + for (MetadataResponseBroker& broker : response.brokers_) { + maybeUpdateHostAndPort(broker, &MetadataResponseBroker::node_id_); + } +} + +void ResponseRewriterImpl::updateFindCoordinatorBrokerAddresses( + FindCoordinatorResponse& response) const { + maybeUpdateHostAndPort(response, &FindCoordinatorResponse::node_id_); + for (Coordinator& coordinator : response.coordinators_) { + maybeUpdateHostAndPort(coordinator, &Coordinator::node_id_); + } +} + +void ResponseRewriterImpl::updateDescribeClusterBrokerAddresses( + DescribeClusterResponse& response) const { + for (DescribeClusterBroker& broker : response.brokers_) { + maybeUpdateHostAndPort(broker, &DescribeClusterBroker::broker_id_); + } +} + +size_t ResponseRewriterImpl::getStoredResponseCountForTest() const { + return responses_to_rewrite_.size(); +} + +// DoNothingRewriter. + +void DoNothingRewriter::onMessage(AbstractResponseSharedPtr) {} + +void DoNothingRewriter::onFailedParse(ResponseMetadataSharedPtr) {} + +void DoNothingRewriter::process(Buffer::Instance&) {} + +// Factory method. + +ResponseRewriterSharedPtr createRewriter(const BrokerFilterConfig& config) { + if (config.needsResponseRewrite()) { + return std::make_shared(config); + } else { + return std::make_shared(); + } +} + +} // namespace Broker +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/source/broker/rewriter.h b/contrib/kafka/filters/network/source/broker/rewriter.h new file mode 100644 index 000000000000..82f4613c4c0e --- /dev/null +++ b/contrib/kafka/filters/network/source/broker/rewriter.h @@ -0,0 +1,108 @@ +#pragma once + +#include + +#include "envoy/buffer/buffer.h" + +#include "source/common/common/logger.h" + +#include "contrib/kafka/filters/network/source/broker/filter_config.h" +#include "contrib/kafka/filters/network/source/external/responses.h" +#include "contrib/kafka/filters/network/source/response_codec.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Broker { + +/** + * Responsible for modifying any outbound requests. + */ +class ResponseRewriter : public ResponseCallback { +public: + virtual ~ResponseRewriter() = default; + + /** + * Performs any desired payload changes. + * @param buffer buffer with the original data from upstream + */ + virtual void process(Buffer::Instance& buffer) PURE; +}; + +using ResponseRewriterSharedPtr = std::shared_ptr; + +/** + * Uses captured response objects instead of original data. + * Entry point for any response payload changes. + */ +class ResponseRewriterImpl : public ResponseRewriter, private Logger::Loggable { +public: + ResponseRewriterImpl(const BrokerFilterConfig& config); + + // ResponseCallback + void onMessage(AbstractResponseSharedPtr response) override; + void onFailedParse(ResponseMetadataSharedPtr parse_failure) override; + + // ResponseRewriter + void process(Buffer::Instance& buffer) override; + + /** + * Mutates response according to config. + */ + void updateMetadataBrokerAddresses(MetadataResponse& response) const; + + /** + * Mutates response according to config. + */ + void updateFindCoordinatorBrokerAddresses(FindCoordinatorResponse& response) const; + + /** + * Mutates response according to config. + */ + void updateDescribeClusterBrokerAddresses(DescribeClusterResponse& response) const; + + size_t getStoredResponseCountForTest() const; + +private: + // Helper function to update various response structures. + // Pointer-to-member used to handle varying field names across the structs. + template void maybeUpdateHostAndPort(T& arg, const int32_t T::*node_id_field) const { + const int32_t node_id = arg.*node_id_field; + const absl::optional hostAndPort = config_.findBrokerAddressOverride(node_id); + if (hostAndPort) { + ENVOY_LOG(trace, "Changing broker [{}] from {}:{} to {}:{}", node_id, arg.host_, arg.port_, + hostAndPort->first, hostAndPort->second); + arg.host_ = hostAndPort->first; + arg.port_ = hostAndPort->second; + } + } + + const BrokerFilterConfig& config_; + std::vector responses_to_rewrite_; +}; + +/** + * Does nothing, letting the data from upstream pass without any changes. + * It allows us to avoid the unnecessary deserialization-then-serialization steps. + */ +class DoNothingRewriter : public ResponseRewriter { +public: + // ResponseCallback + void onMessage(AbstractResponseSharedPtr response) override; + void onFailedParse(ResponseMetadataSharedPtr parse_failure) override; + + // ResponseRewriter + void process(Buffer::Instance& buffer) override; +}; + +/** + * Factory method that creates a rewriter depending on configuration. + */ +ResponseRewriterSharedPtr createRewriter(const BrokerFilterConfig& config); + +} // namespace Broker +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/source/kafka_response.h b/contrib/kafka/filters/network/source/kafka_response.h index f135f5cacb74..f89985947616 100644 --- a/contrib/kafka/filters/network/source/kafka_response.h +++ b/contrib/kafka/filters/network/source/kafka_response.h @@ -93,6 +93,11 @@ class AbstractResponse { */ virtual uint32_t encode(Buffer::Instance& dst) const PURE; + /** + * Convenience method for response's API key. + */ + int16_t apiKey() const { return metadata_.api_key_; } + /** * Response's metadata. */ @@ -141,7 +146,7 @@ template class Response : public AbstractResponse { return metadata_ == rhs.metadata_ && data_ == rhs.data_; }; - const Data data_; + Data data_; }; } // namespace Kafka diff --git a/contrib/kafka/filters/network/source/mesh/config.cc b/contrib/kafka/filters/network/source/mesh/config.cc index d2c2501afb37..801eff53ad9e 100644 --- a/contrib/kafka/filters/network/source/mesh/config.cc +++ b/contrib/kafka/filters/network/source/mesh/config.cc @@ -32,15 +32,18 @@ Network::FilterFactoryCb KafkaMeshConfigFactory::createFilterFactoryFromProtoTyp const UpstreamKafkaConfigurationSharedPtr configuration = std::make_shared(config); + auto& server_context = context.serverFactoryContext(); + // Shared upstream facade (connects us to upstream Kafka clusters). const UpstreamKafkaFacadeSharedPtr upstream_kafka_facade = - std::make_shared(*configuration, context.threadLocal(), - context.api().threadFactory()); + std::make_shared(*configuration, server_context.threadLocal(), + server_context.api().threadFactory()); // Manager for consumers shared across downstream connections // (connects us to upstream Kafka clusters). const RecordCallbackProcessorSharedPtr shared_consumer_manager = - std::make_shared(*configuration, context.api().threadFactory()); + std::make_shared(*configuration, + server_context.api().threadFactory()); return [configuration, upstream_kafka_facade, shared_consumer_manager](Network::FilterManager& filter_manager) -> void { diff --git a/contrib/kafka/filters/network/source/mesh/outbound_record.h b/contrib/kafka/filters/network/source/mesh/outbound_record.h index a56baa5d2fb5..2e2ab6fd3d49 100644 --- a/contrib/kafka/filters/network/source/mesh/outbound_record.h +++ b/contrib/kafka/filters/network/source/mesh/outbound_record.h @@ -26,13 +26,12 @@ struct OutboundRecord { const std::vector
headers_; // These fields will get updated when delivery to upstream Kafka cluster finishes. - int16_t error_code_; - uint32_t saved_offset_; + int16_t error_code_{0}; + uint32_t saved_offset_{0}; OutboundRecord(const std::string& topic, const int32_t partition, const absl::string_view key, const absl::string_view value, const std::vector
& headers) - : topic_{topic}, partition_{partition}, key_{key}, value_{value}, headers_{headers}, - error_code_{0}, saved_offset_{0} {}; + : topic_{topic}, partition_{partition}, key_{key}, value_{value}, headers_{headers} {}; }; } // namespace Mesh diff --git a/contrib/kafka/filters/network/source/mesh/upstream_config.cc b/contrib/kafka/filters/network/source/mesh/upstream_config.cc index 5b2b1d451453..163e30ef308e 100644 --- a/contrib/kafka/filters/network/source/mesh/upstream_config.cc +++ b/contrib/kafka/filters/network/source/mesh/upstream_config.cc @@ -4,6 +4,7 @@ #include "source/common/common/assert.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" namespace Envoy { @@ -91,7 +92,7 @@ absl::optional UpstreamKafkaConfigurationImpl::computeClusterConfigForTopic(const std::string& topic) const { // We find the first matching prefix (this is why ordering is important). for (const auto& it : topic_prefix_to_cluster_config_) { - if (topic.rfind(it.first, 0) == 0) { + if (absl::StartsWith(topic, it.first)) { const ClusterConfig cluster_config = it.second; return absl::make_optional(cluster_config); } diff --git a/contrib/kafka/filters/network/source/mesh/upstream_kafka_client.h b/contrib/kafka/filters/network/source/mesh/upstream_kafka_client.h index f034f7da4f42..bb883e5c8b2e 100644 --- a/contrib/kafka/filters/network/source/mesh/upstream_kafka_client.h +++ b/contrib/kafka/filters/network/source/mesh/upstream_kafka_client.h @@ -69,7 +69,7 @@ class KafkaProducer { // Theoretically we do not need to do this and leave it all to destructor, but then closing N // producers would require doing that in sequence, while we can optimize it somewhat (so we just // wait for the slowest one). - // See https://github.com/edenhill/librdkafka/issues/2972 + // See https://github.com/confluentinc/librdkafka/issues/2972 virtual void markFinished() PURE; }; diff --git a/contrib/kafka/filters/network/source/mesh/upstream_kafka_consumer_impl.cc b/contrib/kafka/filters/network/source/mesh/upstream_kafka_consumer_impl.cc index a6ba7a3e1d71..0c0a6cd66463 100644 --- a/contrib/kafka/filters/network/source/mesh/upstream_kafka_consumer_impl.cc +++ b/contrib/kafka/filters/network/source/mesh/upstream_kafka_consumer_impl.cc @@ -133,7 +133,7 @@ std::vector RichKafkaConsumer::receiveRecordBatch() { // XXX (adam.kotwasinski) There could be something more present in the consumer, // and we could drain it (at least a little) in the next commits. - // See: https://github.com/edenhill/librdkafka/discussions/3897 + // See: https://github.com/confluentinc/librdkafka/discussions/3897 return {inbound_record}; } else { // Nothing extraordinary (timeout because there is nothing upstream), diff --git a/contrib/kafka/filters/network/source/protocol/complex_type_template.j2 b/contrib/kafka/filters/network/source/protocol/complex_type_template.j2 index 744fd8749889..a795c98f4fea 100644 --- a/contrib/kafka/filters/network/source/protocol/complex_type_template.j2 +++ b/contrib/kafka/filters/network/source/protocol/complex_type_template.j2 @@ -19,7 +19,7 @@ struct {{ complex_type.name }} { there are different Kafka versions that are actually composed of precisely the same fields). #} {% for field in complex_type.fields %} - const {{ field.field_declaration() }}_;{% endfor %} + {{ field.field_declaration() }}_;{% endfor %} {% for constructor in complex_type.compute_constructors() %} // constructor used in versions: {{ constructor['versions'] }} {{ constructor['full_declaration'] }}{% endfor %} diff --git a/contrib/kafka/filters/network/source/serialization.h b/contrib/kafka/filters/network/source/serialization.h index 711b45b3778c..9b714578b128 100644 --- a/contrib/kafka/filters/network/source/serialization.h +++ b/contrib/kafka/filters/network/source/serialization.h @@ -364,7 +364,7 @@ class StringDeserializer : public Deserializer { bool ready() const override { return ready_; } - std::string get() const override { return std::string(data_buf_.begin(), data_buf_.end()); } + std::string get() const override { return {data_buf_.begin(), data_buf_.end()}; } private: Int16Deserializer length_buf_; @@ -390,7 +390,7 @@ class CompactStringDeserializer : public Deserializer { bool ready() const override { return ready_; } - std::string get() const override { return std::string(data_buf_.begin(), data_buf_.end()); } + std::string get() const override { return {data_buf_.begin(), data_buf_.end()}; } private: VarUInt32Deserializer length_buf_; diff --git a/contrib/kafka/filters/network/test/BUILD b/contrib/kafka/filters/network/test/BUILD index f7bf15eba151..a06c076e904c 100644 --- a/contrib/kafka/filters/network/test/BUILD +++ b/contrib/kafka/filters/network/test/BUILD @@ -247,7 +247,7 @@ envoy_cc_test( srcs = ["metrics_integration_test.cc"], deps = [ ":message_utilities", - "//contrib/kafka/filters/network/source:kafka_broker_filter_lib", + "//contrib/kafka/filters/network/source:kafka_metrics_lib", "//test/common/stats:stat_test_utility_lib", ], ) diff --git a/contrib/kafka/filters/network/test/broker/BUILD b/contrib/kafka/filters/network/test/broker/BUILD index 1edda5875b4d..54fff0b86a9c 100644 --- a/contrib/kafka/filters/network/test/broker/BUILD +++ b/contrib/kafka/filters/network/test/broker/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_test", + "envoy_cc_test_library", "envoy_contrib_package", ) @@ -12,7 +13,7 @@ envoy_cc_test( name = "config_unit_test", srcs = ["config_unit_test.cc"], deps = [ - "//contrib/kafka/filters/network/source:kafka_broker_config_lib", + "//contrib/kafka/filters/network/source/broker:config_lib", "//test/mocks/server:factory_context_mocks", ], ) @@ -21,7 +22,7 @@ envoy_cc_test( name = "filter_unit_test", srcs = ["filter_unit_test.cc"], deps = [ - "//contrib/kafka/filters/network/source:kafka_broker_filter_lib", + "//contrib/kafka/filters/network/source/broker:filter_lib", "//envoy/event:timer_interface", "//test/mocks/network:network_mocks", "//test/mocks/stats:stats_mocks", @@ -32,10 +33,30 @@ envoy_cc_test( name = "filter_protocol_test", srcs = ["filter_protocol_test.cc"], deps = [ - "//contrib/kafka/filters/network/source:kafka_broker_filter_lib", + ":mock_filter_config_test_lib", + "//contrib/kafka/filters/network/source/broker:filter_lib", "//contrib/kafka/filters/network/test:buffer_based_test_lib", "//contrib/kafka/filters/network/test:message_utilities", "//test/common/stats:stat_test_utility_lib", "//test/test_common:test_time_lib", ], ) + +envoy_cc_test( + name = "rewriter_unit_test", + srcs = ["rewriter_unit_test.cc"], + deps = [ + ":mock_filter_config_test_lib", + "//contrib/kafka/filters/network/source/broker:rewriter_lib", + "//source/common/buffer:buffer_lib", + ], +) + +envoy_cc_test_library( + name = "mock_filter_config_test_lib", + srcs = [], + hdrs = ["mock_filter_config.h"], + deps = [ + "//contrib/kafka/filters/network/source/broker:filter_config_lib", + ], +) diff --git a/contrib/kafka/filters/network/test/broker/filter_protocol_test.cc b/contrib/kafka/filters/network/test/broker/filter_protocol_test.cc index 54c9e915ef24..8d7d9b583dd8 100644 --- a/contrib/kafka/filters/network/test/broker/filter_protocol_test.cc +++ b/contrib/kafka/filters/network/test/broker/filter_protocol_test.cc @@ -12,6 +12,7 @@ #include "contrib/kafka/filters/network/source/broker/filter.h" #include "contrib/kafka/filters/network/source/external/requests.h" #include "contrib/kafka/filters/network/source/external/responses.h" +#include "contrib/kafka/filters/network/test/broker/mock_filter_config.h" #include "contrib/kafka/filters/network/test/buffer_based_test.h" #include "contrib/kafka/filters/network/test/message_utilities.h" #include "gtest/gtest.h" @@ -35,7 +36,7 @@ class KafkaBrokerFilterProtocolTest : public testing::Test, Stats::TestUtil::TestStore store_; Stats::Scope& scope_{*store_.rootScope()}; Event::TestRealTimeSystem time_source_; - KafkaBrokerFilter testee_{scope_, time_source_, "prefix"}; + KafkaBrokerFilter testee_{scope_, time_source_, BrokerFilterConfig{"prefix", false, {}}}; Network::FilterStatus consumeRequestFromBuffer() { return testee_.onData(RequestB::buffer_, false); diff --git a/contrib/kafka/filters/network/test/broker/filter_unit_test.cc b/contrib/kafka/filters/network/test/broker/filter_unit_test.cc index a91316250db8..0b272e713dd0 100644 --- a/contrib/kafka/filters/network/test/broker/filter_unit_test.cc +++ b/contrib/kafka/filters/network/test/broker/filter_unit_test.cc @@ -4,6 +4,7 @@ #include "test/mocks/stats/mocks.h" #include "contrib/kafka/filters/network/source/broker/filter.h" +#include "contrib/kafka/filters/network/source/broker/filter_config.h" #include "contrib/kafka/filters/network/source/external/requests.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -32,6 +33,15 @@ class MockKafkaMetricsFacade : public KafkaMetricsFacade { using MockKafkaMetricsFacadeSharedPtr = std::shared_ptr; +class MockResponseRewriter : public ResponseRewriter { +public: + MOCK_METHOD(void, onMessage, (AbstractResponseSharedPtr)); + MOCK_METHOD(void, onFailedParse, (ResponseMetadataSharedPtr)); + MOCK_METHOD(void, process, (Buffer::Instance&)); +}; + +using MockResponseRewriterSharedPtr = std::shared_ptr; + class MockResponseDecoder : public ResponseDecoder { public: MockResponseDecoder() : ResponseDecoder{{}} {}; @@ -92,12 +102,13 @@ class MockResponse : public AbstractResponse { class KafkaBrokerFilterUnitTest : public testing::Test { protected: MockKafkaMetricsFacadeSharedPtr metrics_{std::make_shared()}; + MockResponseRewriterSharedPtr response_rewriter_{std::make_shared()}; MockResponseDecoderSharedPtr response_decoder_{std::make_shared()}; MockRequestDecoderSharedPtr request_decoder_{std::make_shared()}; NiceMock filter_callbacks_; - KafkaBrokerFilter testee_{metrics_, response_decoder_, request_decoder_}; + KafkaBrokerFilter testee_{metrics_, response_rewriter_, response_decoder_, request_decoder_}; void initialize() { testee_.initializeReadFilterCallbacks(filter_callbacks_); @@ -138,6 +149,7 @@ TEST_F(KafkaBrokerFilterUnitTest, ShouldAcceptDataSentByKafkaBroker) { // given Buffer::OwnedImpl data; EXPECT_CALL(*response_decoder_, onData(_)); + EXPECT_CALL(*response_rewriter_, process(_)); // when initialize(); diff --git a/contrib/kafka/filters/network/test/broker/integration_test/envoy_config_yaml.j2 b/contrib/kafka/filters/network/test/broker/integration_test/envoy_config_yaml.j2 index af945c5c61d7..b41da9415ea4 100644 --- a/contrib/kafka/filters/network/test/broker/integration_test/envoy_config_yaml.j2 +++ b/contrib/kafka/filters/network/test/broker/integration_test/envoy_config_yaml.j2 @@ -10,6 +10,12 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker stat_prefix: testfilter + id_based_broker_address_rewrite_spec: + rules: + - id: 0 + host: 127.0.0.1 + port: {{ data['kafka_envoy_port'] }} + # More ids go here if we add brokers to the test cluster. - name: tcp typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy diff --git a/contrib/kafka/filters/network/test/broker/integration_test/kafka_server_properties.j2 b/contrib/kafka/filters/network/test/broker/integration_test/kafka_server_properties.j2 index ce9bbdd057e4..ab8c02e9d43c 100644 --- a/contrib/kafka/filters/network/test/broker/integration_test/kafka_server_properties.j2 +++ b/contrib/kafka/filters/network/test/broker/integration_test/kafka_server_properties.j2 @@ -1,6 +1,6 @@ broker.id=0 listeners=PLAINTEXT://127.0.0.1:{{ data['kafka_real_port'] }} -advertised.listeners=PLAINTEXT://127.0.0.1:{{ data['kafka_envoy_port'] }} +advertised.listeners=PLAINTEXT://127.0.0.1:{{ data['kafka_real_port'] }} num.network.threads=3 num.io.threads=8 diff --git a/contrib/kafka/filters/network/test/broker/mock_filter_config.h b/contrib/kafka/filters/network/test/broker/mock_filter_config.h new file mode 100644 index 000000000000..f0996aab8346 --- /dev/null +++ b/contrib/kafka/filters/network/test/broker/mock_filter_config.h @@ -0,0 +1,24 @@ +#pragma once + +#include "contrib/kafka/filters/network/source/broker/filter_config.h" +#include "gmock/gmock.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Broker { + +class MockBrokerFilterConfig : public BrokerFilterConfig { +public: + MOCK_METHOD((const std::string&), stat_prefix, (), (const)); + MOCK_METHOD(bool, needsResponseRewrite, (), (const)); + MOCK_METHOD((absl::optional), findBrokerAddressOverride, (const uint32_t), (const)); + MockBrokerFilterConfig() : BrokerFilterConfig{"prefix", false, {}} {}; +}; + +} // namespace Broker +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/test/broker/rewriter_unit_test.cc b/contrib/kafka/filters/network/test/broker/rewriter_unit_test.cc new file mode 100644 index 000000000000..cf5195bd9e76 --- /dev/null +++ b/contrib/kafka/filters/network/test/broker/rewriter_unit_test.cc @@ -0,0 +1,175 @@ +#include "source/common/buffer/buffer_impl.h" + +#include "contrib/kafka/filters/network/source/broker/filter_config.h" +#include "contrib/kafka/filters/network/source/broker/rewriter.h" +#include "contrib/kafka/filters/network/source/external/responses.h" +#include "contrib/kafka/filters/network/test/broker/mock_filter_config.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Return; +using testing::Throw; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Broker { + +static void putBytesIntoBuffer(Buffer::Instance& buffer, const uint32_t size) { + std::vector data(size, 42); + absl::string_view sv = {data.data(), data.size()}; + buffer.add(sv); +} + +static Buffer::InstancePtr makeRandomBuffer(const uint32_t size) { + Buffer::InstancePtr result = std::make_unique(); + putBytesIntoBuffer(*result, size); + return result; +} + +class FakeResponse : public AbstractResponse { +public: + FakeResponse(const size_t size) : AbstractResponse{ResponseMetadata{-1, 0, 0}}, size_{size} {} + + uint32_t computeSize() const override { return size_; }; + + virtual uint32_t encode(Buffer::Instance& dst) const override { + putBytesIntoBuffer(dst, size_); + return size_; + }; + +private: + size_t size_; +}; + +TEST(ResponseRewriterImplUnitTest, ShouldRewriteBuffer) { + // given + ResponseRewriterImpl testee{MockBrokerFilterConfig{}}; + + auto response1 = std::make_shared(7); + auto response2 = std::make_shared(13); + auto response3 = std::make_shared(42); + + // when - 1 + testee.onMessage(response1); + testee.onMessage(response2); + testee.onMessage(response3); + + // then - 1 + ASSERT_EQ(testee.getStoredResponseCountForTest(), 3); + + // when - 2 + auto buffer = makeRandomBuffer(4242); + testee.process(*buffer); + + // then - 2 + ASSERT_EQ(testee.getStoredResponseCountForTest(), 0); + ASSERT_EQ(buffer->length(), (3 * 4) + 7 + 13 + 42); // 4 bytes for message length +} + +template +static void assertAddress(const T& arg, const std::string& host, const int32_t port) { + ASSERT_EQ(arg.host_, host); + ASSERT_EQ(arg.port_, port); +} + +TEST(ResponseRewriterImplUnitTest, ShouldRewriteMetadataResponse) { + // given + MetadataResponseBroker b1 = {13, "host1", 1111}; + MetadataResponseBroker b2 = {42, "host2", 2222}; + MetadataResponseBroker b3 = {77, "host3", 3333}; + std::vector brokers = {b1, b2, b3}; + MetadataResponse mr = {brokers, {}}; + + MockBrokerFilterConfig config; + absl::optional r1 = {{"nh1", 4444}}; + EXPECT_CALL(config, findBrokerAddressOverride(b1.node_id_)).WillOnce(Return(r1)); + absl::optional r2 = absl::nullopt; + EXPECT_CALL(config, findBrokerAddressOverride(b2.node_id_)).WillOnce(Return(r2)); + absl::optional r3 = {{"nh3", 6666}}; + EXPECT_CALL(config, findBrokerAddressOverride(b3.node_id_)).WillOnce(Return(r3)); + ResponseRewriterImpl testee{config}; + + // when + testee.updateMetadataBrokerAddresses(mr); + + // then + assertAddress(mr.brokers_[0], r1->first, r1->second); + assertAddress(mr.brokers_[1], b2.host_, b2.port_); + assertAddress(mr.brokers_[2], r3->first, r3->second); +} + +TEST(ResponseRewriterImplUnitTest, ShouldRewriteFindCoordinatorResponse) { + // given + + FindCoordinatorResponse fcr = {0, 13, "host1", 1111}; + Coordinator c1 = {"k1", 1, "ch1", 2222, 0, {}, {}}; + Coordinator c2 = {"k2", 2, "ch2", 3333, 0, {}, {}}; + Coordinator c3 = {"k3", 3, "ch3", 4444, 0, {}, {}}; + fcr.coordinators_ = {c1, c2, c3}; + + MockBrokerFilterConfig config; + absl::optional fcrhp = {{"nh1", 4444}}; + EXPECT_CALL(config, findBrokerAddressOverride(fcr.node_id_)).WillOnce(Return(fcrhp)); + absl::optional cr1 = {{"nh1", 4444}}; + EXPECT_CALL(config, findBrokerAddressOverride(c1.node_id_)).WillOnce(Return(cr1)); + absl::optional cr2 = absl::nullopt; + EXPECT_CALL(config, findBrokerAddressOverride(c2.node_id_)).WillOnce(Return(cr2)); + absl::optional cr3 = {{"nh3", 6666}}; + EXPECT_CALL(config, findBrokerAddressOverride(c3.node_id_)).WillOnce(Return(cr3)); + ResponseRewriterImpl testee{config}; + + // when + testee.updateFindCoordinatorBrokerAddresses(fcr); + + // then + assertAddress(fcr, fcrhp->first, fcrhp->second); + assertAddress(fcr.coordinators_[0], cr1->first, cr1->second); + assertAddress(fcr.coordinators_[1], c2.host_, c2.port_); + assertAddress(fcr.coordinators_[2], cr3->first, cr3->second); +} + +TEST(ResponseRewriterImplUnitTest, ShouldRewriteDescribeClusterResponse) { + // given + DescribeClusterBroker b1 = {13, "host1", 1111, absl::nullopt, {}}; + DescribeClusterBroker b2 = {42, "host2", 2222, absl::nullopt, {}}; + DescribeClusterBroker b3 = {77, "host3", 3333, absl::nullopt, {}}; + std::vector brokers = {b1, b2, b3}; + DescribeClusterResponse dcr = {0, 0, absl::nullopt, "", 0, brokers, 0, {}}; + + MockBrokerFilterConfig config; + absl::optional cr1 = {{"nh1", 4444}}; + EXPECT_CALL(config, findBrokerAddressOverride(b1.broker_id_)).WillOnce(Return(cr1)); + absl::optional cr2 = absl::nullopt; + EXPECT_CALL(config, findBrokerAddressOverride(b2.broker_id_)).WillOnce(Return(cr2)); + absl::optional cr3 = {{"nh3", 6666}}; + EXPECT_CALL(config, findBrokerAddressOverride(b3.broker_id_)).WillOnce(Return(cr3)); + ResponseRewriterImpl testee{config}; + + // when + testee.updateDescribeClusterBrokerAddresses(dcr); + + // then + assertAddress(dcr.brokers_[0], cr1->first, cr1->second); + assertAddress(dcr.brokers_[1], b2.host_, b2.port_); + assertAddress(dcr.brokers_[2], cr3->first, cr3->second); +} + +TEST(ResponseRewriterUnitTest, ShouldCreateProperRewriter) { + MockBrokerFilterConfig c1; + EXPECT_CALL(c1, needsResponseRewrite()).WillOnce(Return(true)); + ResponseRewriterSharedPtr r1 = createRewriter(c1); + ASSERT_NE(std::dynamic_pointer_cast(r1), nullptr); + + MockBrokerFilterConfig c2; + EXPECT_CALL(c2, needsResponseRewrite()).WillOnce(Return(false)); + ResponseRewriterSharedPtr r2 = createRewriter(c2); + ASSERT_NE(std::dynamic_pointer_cast(r2), nullptr); +} + +} // namespace Broker +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/test/mesh/command_handlers/fetch_record_converter_unit_test.cc b/contrib/kafka/filters/network/test/mesh/command_handlers/fetch_record_converter_unit_test.cc index 48e98bc9bfb6..fe2cc0a40b6e 100644 --- a/contrib/kafka/filters/network/test/mesh/command_handlers/fetch_record_converter_unit_test.cc +++ b/contrib/kafka/filters/network/test/mesh/command_handlers/fetch_record_converter_unit_test.cc @@ -77,7 +77,7 @@ TEST(FetchRecordConverterImpl, shouldProcessRecords) { const auto ptr = reinterpret_cast(data->data() + record_count_offset); const uint32_t record_count = be32toh(*ptr); ASSERT_EQ(record_count, 3); -} +} // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) // Here we check whether our manual implementation really works. // https://github.com/apache/kafka/blob/3.3.2/clients/src/main/java/org/apache/kafka/common/utils/Crc32C.java diff --git a/contrib/kafka/filters/network/test/mesh/config_unit_test.cc b/contrib/kafka/filters/network/test/mesh/config_unit_test.cc index 3ac2ad70a64e..500cc402b0dc 100644 --- a/contrib/kafka/filters/network/test/mesh/config_unit_test.cc +++ b/contrib/kafka/filters/network/test/mesh/config_unit_test.cc @@ -48,7 +48,8 @@ advertised_port: 19092 testing::NiceMock context; testing::NiceMock thread_factory; - ON_CALL(context.api_, threadFactory()).WillByDefault(ReturnRef(thread_factory)); + ON_CALL(context.server_factory_context_.api_, threadFactory()) + .WillByDefault(ReturnRef(thread_factory)); KafkaMeshConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); diff --git a/contrib/kafka/filters/network/test/mesh/integration_test/envoy_config_yaml.j2 b/contrib/kafka/filters/network/test/mesh/integration_test/envoy_config_yaml.j2 index fbb22d2af3a9..cb2cdeeee807 100644 --- a/contrib/kafka/filters/network/test/mesh/integration_test/envoy_config_yaml.j2 +++ b/contrib/kafka/filters/network/test/mesh/integration_test/envoy_config_yaml.j2 @@ -10,6 +10,7 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker stat_prefix: testfilter + force_response_rewrite: true - name: mesh typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.kafka_mesh.v3alpha.KafkaMesh diff --git a/contrib/mysql_proxy/filters/network/test/mysql_integration_test.cc b/contrib/mysql_proxy/filters/network/test/mysql_integration_test.cc index 4d0257153cfe..0b28e75cf952 100644 --- a/contrib/mysql_proxy/filters/network/test/mysql_integration_test.cc +++ b/contrib/mysql_proxy/filters/network/test/mysql_integration_test.cc @@ -110,7 +110,8 @@ TEST_P(MySQLIntegrationTest, MySQLLoginTest) { * - correct number of attempts * - no failures */ -TEST_P(MySQLIntegrationTest, MySQLUnitTestMultiClientsLoop) { +// TODO(https://github.com/envoyproxy/envoy/issues/30852) enable +TEST_P(MySQLIntegrationTest, DISABLED_MySQLUnitTestMultiClientsLoop) { int idx; std::string rcvd_data; diff --git a/contrib/postgres_proxy/filters/network/source/postgres_filter.cc b/contrib/postgres_proxy/filters/network/source/postgres_filter.cc index 190f23d1b2a1..5db7d1058e1c 100644 --- a/contrib/postgres_proxy/filters/network/source/postgres_filter.cc +++ b/contrib/postgres_proxy/filters/network/source/postgres_filter.cc @@ -46,6 +46,10 @@ void PostgresFilter::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& read_callbacks_ = &callbacks; } +void PostgresFilter::initializeWriteFilterCallbacks(Network::WriteFilterCallbacks& callbacks) { + write_callbacks_ = &callbacks; +} + // Network::WriteFilter Network::FilterStatus PostgresFilter::onWrite(Buffer::Instance& data, bool) { @@ -229,7 +233,7 @@ bool PostgresFilter::onSSLRequest() { } return true; }); - read_callbacks_->connection().write(buf, false); + write_callbacks_->injectWriteDataToFilterChain(buf, false); return false; } diff --git a/contrib/postgres_proxy/filters/network/source/postgres_filter.h b/contrib/postgres_proxy/filters/network/source/postgres_filter.h index f2c82f0a6076..c18bd4cf3817 100644 --- a/contrib/postgres_proxy/filters/network/source/postgres_filter.h +++ b/contrib/postgres_proxy/filters/network/source/postgres_filter.h @@ -102,6 +102,7 @@ class PostgresFilter : public Network::Filter, Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; Network::FilterStatus onNewConnection() override; void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override; + void initializeWriteFilterCallbacks(Network::WriteFilterCallbacks& callbacks) override; // Network::WriteFilter Network::FilterStatus onWrite(Buffer::Instance& data, bool end_stream) override; @@ -138,6 +139,7 @@ class PostgresFilter : public Network::Filter, private: Network::ReadFilterCallbacks* read_callbacks_{}; + Network::WriteFilterCallbacks* write_callbacks_{}; PostgresFilterConfigSharedPtr config_; Buffer::OwnedImpl frontend_buffer_; Buffer::OwnedImpl backend_buffer_; diff --git a/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc index 434c5018d9d9..6911a0c0facc 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc @@ -41,11 +41,12 @@ class PostgresFilterTest config_ = std::make_shared(config_options, scope_); filter_ = std::make_unique(config_); - filter_->initializeReadFilterCallbacks(filter_callbacks_); + filter_->initializeReadFilterCallbacks(read_callbacks_); + filter_->initializeWriteFilterCallbacks(write_callbacks_); } void setMetadata() { - EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); + EXPECT_CALL(read_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); EXPECT_CALL(connection_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); ON_CALL(stream_info_, setDynamicMetadata(NetworkFilterNames::get().PostgresProxy, _)) .WillByDefault(Invoke([this](const std::string&, const ProtobufWkt::Struct& obj) { @@ -60,7 +61,8 @@ class PostgresFilterTest std::string stat_prefix_{"test."}; std::unique_ptr filter_; PostgresFilterConfigSharedPtr config_; - NiceMock filter_callbacks_; + NiceMock read_callbacks_; + NiceMock write_callbacks_; NiceMock connection_; NiceMock stream_info_; @@ -364,11 +366,12 @@ TEST_F(PostgresFilterTest, QueryMessageMetadata) { // Decoder::Stopped. TEST_F(PostgresFilterTest, TerminateSSL) { filter_->getConfig()->terminate_ssl_ = true; - EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); + EXPECT_CALL(read_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); Network::Connection::BytesSentCb cb; EXPECT_CALL(connection_, addBytesSentCallback(_)).WillOnce(testing::SaveArg<0>(&cb)); Buffer::OwnedImpl buf; - EXPECT_CALL(connection_, write(_, false)).WillOnce(testing::SaveArg<0>(&buf)); + EXPECT_CALL(write_callbacks_, injectWriteDataToFilterChain(_, false)) + .WillOnce(testing::SaveArg<0>(&buf)); data_.writeBEInt(8); // 1234 in the most significant 16 bits and some code in the least significant 16 bits. data_.writeBEInt(80877103); // SSL code. @@ -398,7 +401,7 @@ TEST_F(PostgresFilterTest, TerminateSSL) { } TEST_F(PostgresFilterTest, UpstreamSSL) { - EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); + EXPECT_CALL(read_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); // Configure upstream SSL to be disabled. encryptUpstream must not be called. filter_->getConfig()->upstream_ssl_ = @@ -415,17 +418,17 @@ TEST_F(PostgresFilterTest, UpstreamSSL) { ASSERT_TRUE(filter_->shouldEncryptUpstream()); // Simulate that upstream server agreed for SSL and conversion of upstream Transport socket was // successful. - EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(true)); + EXPECT_CALL(read_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(true)); filter_->encryptUpstream(true, data_); ASSERT_EQ(1, filter_->getStats().sessions_upstream_ssl_success_.value()); // Simulate that upstream server agreed for SSL but conversion of upstream Transport socket // failed. - EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(false)); + EXPECT_CALL(read_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(false)); filter_->encryptUpstream(true, data_); ASSERT_EQ(1, filter_->getStats().sessions_upstream_ssl_failed_.value()); // Simulate that upstream server does not agree for SSL. Filter should close the connection to // downstream client. - EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).Times(0); + EXPECT_CALL(read_callbacks_, startUpstreamSecureTransport()).Times(0); EXPECT_CALL(connection_, close(_)); filter_->encryptUpstream(false, data_); ASSERT_EQ(2, filter_->getStats().sessions_upstream_ssl_failed_.value()); @@ -433,7 +436,7 @@ TEST_F(PostgresFilterTest, UpstreamSSL) { TEST_F(PostgresFilterTest, UpstreamSSLStats) { static_cast(filter_->getDecoder())->state(DecoderImpl::State::InitState); - EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); + EXPECT_CALL(read_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); filter_->getConfig()->upstream_ssl_ = envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::REQUIRE; @@ -443,7 +446,7 @@ TEST_F(PostgresFilterTest, UpstreamSSLStats) { Buffer::OwnedImpl upstream_data; upstream_data.add("S"); - EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(true)); + EXPECT_CALL(read_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(true)); ASSERT_THAT(Network::FilterStatus::StopIteration, filter_->onWrite(upstream_data, false)); createPostgresMsg(upstream_data, "C", "SELECT blah"); diff --git a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc index 4f1e3c829fb8..4e76d27eb247 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc @@ -223,9 +223,10 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, DownstreamSSLWrongConfigPostgresIntegration class UpstreamSSLBaseIntegrationTest : public PostgresBaseIntegrationTest { public: - UpstreamSSLBaseIntegrationTest(SSLConfig upstream_ssl_config) + UpstreamSSLBaseIntegrationTest(SSLConfig upstream_ssl_config, + SSLConfig downstream_ssl_config = NoDownstreamSSL) // Disable downstream SSL and attach synchronization filter. - : PostgresBaseIntegrationTest(NoDownstreamSSL, upstream_ssl_config, R"EOF( + : PostgresBaseIntegrationTest(downstream_ssl_config, upstream_ssl_config, R"EOF( - name: sync typed_config: "@type": type.googleapis.com/test.integration.postgres.SyncWriteFilterConfig @@ -483,6 +484,226 @@ TEST_P(UpstreamSSLRequirePostgresIntegrationTest, ServerDeniesSSLTest) { INSTANTIATE_TEST_SUITE_P(IpVersions, UpstreamSSLRequirePostgresIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); +// Base class for parameterized tests when upstream and downstream SSL is enabled. +class UpstreamAndDownstreamSSLIntegrationTest : public UpstreamSSLBaseIntegrationTest { +public: + UpstreamAndDownstreamSSLIntegrationTest() + : UpstreamSSLBaseIntegrationTest( + // Configure upstream SSL + std::make_tuple("upstream_ssl: REQUIRE", + R"EOF(transport_socket: + name: "starttls" + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.starttls.v3.UpstreamStartTlsConfig + tls_socket_config: + common_tls_context: {} + )EOF"), + // configure downstream SSL + std::make_tuple( + "terminate_ssl: true", + fmt::format( + R"EOF(transport_socket: + name: "starttls" + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.starttls.v3.StartTlsConfig + cleartext_socket_config: + tls_socket_config: + common_tls_context: + tls_certificates: + certificate_chain: + filename: {} + private_key: + filename: {} + )EOF", + TestEnvironment::runfilesPath("test/config/integration/certs/servercert.pem"), + TestEnvironment::runfilesPath( + "test/config/integration/certs/serverkey.pem")))) {} + + // Method changes IntegrationTcpClient's transport socket to TLS. + // Sending any traffic to newly attached TLS transport socket will trigger + // TLS handshake negotiation. + void enableTLSonTCPClient(const IntegrationTcpClientPtr& tcp_client) { + // Setup factory and context for tls transport socket. + // The tls transport socket will be inserted into fake_upstream when + // Envoy's upstream starttls transport socket is converted to secure mode. + std::unique_ptr tls_context_manager = + std::make_unique(timeSystem()); + + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext upstream_tls_context; + + NiceMock mock_factory_ctx; + ON_CALL(mock_factory_ctx.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + auto cfg = std::make_unique( + upstream_tls_context, mock_factory_ctx); + static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); + Network::UpstreamTransportSocketFactoryPtr tls_context = + Network::UpstreamTransportSocketFactoryPtr{ + new Extensions::TransportSockets::Tls::ClientSslSocketFactory( + std::move(cfg), *tls_context_manager, *(client_stats_store->rootScope()))}; + + Network::TransportSocketOptionsConstSharedPtr options; + + auto connection = dynamic_cast(tcp_client->connection()); + Network::TransportSocketPtr ts = tls_context->createTransportSocket( + options, connection->streamInfo().upstreamInfo()->upstreamHost()); + connection->transportSocket() = std::move(ts); + connection->transportSocket()->setTransportSocketCallbacks(*connection); + } +}; + +// Integration test when both downstream and upstream SSL is enabled. +// In this scenario test client establishes SSL connection to Envoy. Envoy de-crypts the traffic. +// The traffic is encrypted again when sent to upstream server. +// The test follows the following scenario: +// +// Test client Envoy Upstream +// ----- Can I use SSL? ------------> +// <------- Yes--------------------- +// <------- TLS handshake ----------> +// ------ Initial postgres msg -----> +// ------ Can I use SSL? ---> +// <------- Yes-------------- +// <------- TLS handshake---> +// --Initial postgres msg---> +// ------ close connection ---------> +// ------ close connection---> +// +TEST_P(UpstreamAndDownstreamSSLIntegrationTest, ServerAgreesForSSL) { + Buffer::OwnedImpl data; + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_)); + + // Send the startup message requesting SSL. + // Message is 8 bytes long. The first 4 bytes contain length (8) + // The next 8 bytes contain message code (RequestSSL=80877103) + data.writeBEInt(8); + data.writeBEInt(80877103); + + // Message will be processed by Postgres filter which + // is configured to accept SSL termination and confirm it + // by returning single byte 'S'. + ASSERT_TRUE(tcp_client->write(data.toString())); + data.drain(data.length()); + + tcp_client->waitForData("S", true); + + // Attach TLS to tcp_client. + enableTLSonTCPClient(tcp_client); + + // Write initial postgres message. This should trigger SSL negotiation between test TCP client and + // Envoy downstream transport socket. Postgres filter should ask the fake upstream server if it + // will accept encrypted connection. + Buffer::OwnedImpl upstream_data; + std::string rcvd; + createInitialPostgresRequest(data); + ASSERT_TRUE(tcp_client->write(data.toString())); + // Postgres filter should buffer the original message and negotiate SSL upstream. + // The first 4 bytes should be length on the message (8 bytes). + // The next 4 bytes should be SSL code. + ASSERT_TRUE(fake_upstream_connection_->waitForData(8, &rcvd)); + upstream_data.add(rcvd); + ASSERT_EQ(8, upstream_data.peekBEInt(0)); + ASSERT_EQ(80877103, upstream_data.peekBEInt(4)); + upstream_data.drain(upstream_data.length()); + + // Reply to Envoy with 'S' and attach TLS socket to upstream. + upstream_data.add("S"); + ASSERT_TRUE(fake_upstream_connection_->write(upstream_data.toString())); + fake_upstream_connection_->clearData(); + + config_factory_.recv_sync_.WaitForNotification(); + enableTLSOnFakeUpstream(); + config_factory_.proceed_sync_.Notify(); + + ASSERT_TRUE(fake_upstream_connection_->waitForData(data.length(), &rcvd)); + // Make sure that upstream received initial postgres request, which + // triggered upstream SSL negotiation and TLS handshake. + ASSERT_EQ(data.toString(), rcvd); + + data.drain(data.length()); + + tcp_client->close(); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_terminated_ssl", 1); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_success", 1); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_failed", 0); +} + +// Integration test when both downstream and upstream SSL is enabled. +// In this scenario test client establishes SSL connection to Envoy. Envoy de-crypts the traffic. +// The traffic is encrypted again when sent to upstream server, but the server +// rejects request for SSL. +// The test follows the following scenario: +// +// Test client Envoy Upstream +// ----- Can I use SSL? ------------> +// <------- Yes--------------------- +// <------- TLS handshake ----------> +// ------ Initial postgres msg -----> +// ------ Can I use SSL? ---> +// <------- No--------------- +// <----- close connection ---------- +// ------ close connection---> +TEST_P(UpstreamAndDownstreamSSLIntegrationTest, ServerRejectsSSL) { + Buffer::OwnedImpl data; + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_)); + + // Send the startup message requesting SSL. + // Message is 8 bytes long. The first 4 bytes contain length (8) + // The next 8 bytes contain message code (RequestSSL=80877103) + data.writeBEInt(8); + data.writeBEInt(80877103); + + // Message will be processed by Postgres filter which + // is configured to accept SSL termination and confirm it + // by returning single byte 'S'. + ASSERT_TRUE(tcp_client->write(data.toString())); + data.drain(data.length()); + + tcp_client->waitForData("S", true); + + // Attach TLS to tcp_client. + enableTLSonTCPClient(tcp_client); + + // Write initial postgres message. This should trigger SSL negotiation between test TCP client and + // Envoy downstream transport socket. Postgres filter should ask the fake upstream server if it + // will accept encrypted connection. + Buffer::OwnedImpl upstream_data; + std::string rcvd; + createInitialPostgresRequest(data); + ASSERT_TRUE(tcp_client->write(data.toString())); + // Postgres filter should buffer the original message and negotiate SSL upstream. + // The first 4 bytes should be length on the message (8 bytes). + // The next 4 bytes should be SSL code. + ASSERT_TRUE(fake_upstream_connection_->waitForData(8, &rcvd)); + upstream_data.add(rcvd); + ASSERT_EQ(8, upstream_data.peekBEInt(0)); + ASSERT_EQ(80877103, upstream_data.peekBEInt(4)); + upstream_data.drain(upstream_data.length()); + + // Reply to Envoy with 'E' (SSL not allowed). + upstream_data.add("E"); + ASSERT_TRUE(fake_upstream_connection_->write(upstream_data.toString())); + config_factory_.proceed_sync_.Notify(); + + data.drain(data.length()); + + // Envoy should close the connection to test client. + tcp_client->waitForDisconnect(); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_terminated_ssl", 1); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_success", 0); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_failed", 1); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, UpstreamAndDownstreamSSLIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); + } // namespace PostgresProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/qat/compression/qatzip/compressor/source/BUILD b/contrib/qat/compression/qatzip/compressor/source/BUILD new file mode 100644 index 000000000000..099bb2906651 --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/source/BUILD @@ -0,0 +1,80 @@ +load("@rules_foreign_cc//foreign_cc:configure.bzl", "configure_make") +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_contrib_extension", + "envoy_cc_library", + "envoy_contrib_package", +) +load( + "//contrib:all_contrib_extensions.bzl", + "envoy_contrib_linux_x86_64_constraints", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +configure_make( + name = "qatzip", + autogen = True, + configure_in_place = True, + configure_options = [ + "--enable-static", + "--disable-shared", + ], + env = select({ + "//bazel:clang_build": { + "CFLAGS": "-Wno-error=newline-eof -Wno-error=strict-prototypes -Wno-error=unused-but-set-variable", + }, + "//conditions:default": {}, + }), + lib_source = "@com_github_intel_qatzip//:all", + out_static_libs = [ + "libqatzip.a", + ], + tags = ["skip_on_windows"], + target_compatible_with = envoy_contrib_linux_x86_64_constraints(), + visibility = ["//visibility:public"], + deps = [ + "//bazel/foreign_cc:lz4", + "//bazel/foreign_cc:zlib", + "//contrib/qat:qatlib", + "@boringssl//:ssl", + ], + alwayslink = False, +) + +envoy_cc_library( + name = "compressor_lib", + srcs = ["qatzip_compressor_impl.cc"], + hdrs = ["qatzip_compressor_impl.h"], + deps = [ + ":qatzip", + "//envoy/compression/compressor:compressor_interface", + ], +) + +envoy_cc_contrib_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + defines = select({ + "//bazel:linux_x86_64": [ + ], + "//conditions:default": [ + "QAT_DISABLED=1", + ], + }), + deps = [ + "//envoy/compression/compressor:compressor_config_interface", + "//envoy/compression/compressor:compressor_factory_interface", + "//source/common/http:headers_lib", + "@envoy_api//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg_cc_proto", + ] + select({ + "//bazel:linux_x86_64": [ + ":compressor_lib", + ], + "//conditions:default": [ + ], + }), +) diff --git a/contrib/qat/compression/qatzip/compressor/source/config.cc b/contrib/qat/compression/qatzip/compressor/source/config.cc new file mode 100644 index 000000000000..48144e706b10 --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/source/config.cc @@ -0,0 +1,140 @@ +#include "contrib/qat/compression/qatzip/compressor/source/config.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Qatzip { +namespace Compressor { + +#ifndef QAT_DISABLED +namespace { + +// Default qatzip chunk size. +const uint32_t DefaultChunkSize = 4096; + +// Default qatzip stream buffer size. +const unsigned int DefaultStreamBufferSize = 128 * 1024; + +unsigned int hardwareBufferSizeEnum( + envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize + hardware_buffer_size) { + switch (hardware_buffer_size) { + case envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize:: + Qatzip_HardwareBufferSize_SZ_4K: + return 4 * 1024; + case envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize:: + Qatzip_HardwareBufferSize_SZ_8K: + return 8 * 1024; + case envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize:: + Qatzip_HardwareBufferSize_SZ_32K: + return 32 * 1024; + case envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize:: + Qatzip_HardwareBufferSize_SZ_64K: + return 64 * 1024; + case envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize:: + Qatzip_HardwareBufferSize_SZ_128K: + return 128 * 1024; + case envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip_HardwareBufferSize:: + Qatzip_HardwareBufferSize_SZ_512K: + return 512 * 1024; + default: + return 64 * 1024; + } +} + +unsigned int compressionLevelUint(Protobuf::uint32 compression_level) { + return compression_level > 0 ? compression_level : QZ_COMP_LEVEL_DEFAULT; +} + +unsigned int inputSizeThresholdUint(Protobuf::uint32 input_size_threshold) { + return input_size_threshold > 0 ? input_size_threshold : QZ_COMP_THRESHOLD_DEFAULT; +} + +unsigned int streamBufferSizeUint(Protobuf::uint32 stream_buffer_size) { + return stream_buffer_size > 0 ? stream_buffer_size : DefaultStreamBufferSize; +} + +} // namespace + +QatzipCompressorFactory::QatzipCompressorFactory( + const envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip& qatzip, + Server::Configuration::FactoryContext& context) + : chunk_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(qatzip, chunk_size, DefaultChunkSize)), + tls_slot_(context.serverFactoryContext().threadLocal().allocateSlot()) { + QzSessionParams_T params; + + int status = qzGetDefaults(¶ms); + RELEASE_ASSERT(status == QZ_OK, "failed to initialize hardware"); + params.comp_lvl = compressionLevelUint(qatzip.compression_level().value()); + params.hw_buff_sz = hardwareBufferSizeEnum(qatzip.hardware_buffer_size()); + params.strm_buff_sz = streamBufferSizeUint(qatzip.stream_buffer_size().value()); + params.input_sz_thrshold = inputSizeThresholdUint(qatzip.input_size_threshold().value()); + params.data_fmt = QZ_DEFLATE_RAW; + + tls_slot_->set([params](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(params); + }); +} + +Envoy::Compression::Compressor::CompressorPtr QatzipCompressorFactory::createCompressor() { + return std::make_unique( + tls_slot_->getTyped().getSession()); +} + +QatzipCompressorFactory::QatzipThreadLocal::QatzipThreadLocal(QzSessionParams_T params) + : params_(params), session_{}, initialized_(false) {} + +QatzipCompressorFactory::QatzipThreadLocal::~QatzipThreadLocal() { + if (initialized_) { + qzTeardownSession(&session_); + } +} + +QzSession_T* QatzipCompressorFactory::QatzipThreadLocal::getSession() { + // The session must be initialized only once in every worker thread. + if (!initialized_) { + + int status = qzInit(&session_, params_.sw_backup); + RELEASE_ASSERT(status == QZ_OK || status == QZ_DUPLICATE, "failed to initialize hardware"); + status = qzSetupSession(&session_, ¶ms_); + RELEASE_ASSERT(status == QZ_OK || status == QZ_DUPLICATE, "failed to setup session"); + initialized_ = true; + } + + return &session_; +} + +Envoy::Compression::Compressor::CompressorFactoryPtr +QatzipCompressorLibraryFactory::createCompressorFactoryFromProtoTyped( + const envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip& proto_config, + Server::Configuration::FactoryContext& context) { + return std::make_unique(proto_config, context); +} +#endif + +Envoy::Compression::Compressor::CompressorFactoryPtr +QatzipCompressorLibraryFactory::createCompressorFactoryFromProto( + const Protobuf::Message& proto_config, Server::Configuration::FactoryContext& context) { + + const envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip config = + MessageUtil::downcastAndValidate< + const envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip&>( + proto_config, context.messageValidationVisitor()); +#ifdef QAT_DISABLED + throw EnvoyException("X86_64 architecture is required for QAT."); +#else + return createCompressorFactoryFromProtoTyped(config, context); +#endif +} + +/** + * Static registration for the qatzip compressor library. @see NamedCompressorLibraryConfigFactory. + */ +REGISTER_FACTORY(QatzipCompressorLibraryFactory, + Envoy::Compression::Compressor::NamedCompressorLibraryConfigFactory); + +} // namespace Compressor +} // namespace Qatzip +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/qat/compression/qatzip/compressor/source/config.h b/contrib/qat/compression/qatzip/compressor/source/config.h new file mode 100644 index 000000000000..4df242bd8a1f --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/source/config.h @@ -0,0 +1,95 @@ +#pragma once + +#include "envoy/compression/compressor/config.h" +#include "envoy/compression/compressor/factory.h" +#include "envoy/thread_local/thread_local.h" + +#include "source/common/http/headers.h" + +#include "contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/qatzip.pb.h" +#include "contrib/envoy/extensions/compression/qatzip/compressor/v3alpha/qatzip.pb.validate.h" + +#ifndef QAT_DISABLED +#include "contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.h" +#endif + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Qatzip { +namespace Compressor { + +namespace { + +#ifndef QAT_DISABLED +const std::string& qatzipStatsPrefix() { CONSTRUCT_ON_FIRST_USE(std::string, "qatzip."); } +#endif + +const std::string& qatzipExtensionName() { + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.compression.qatzip.compressor"); +} + +} // namespace + +#ifndef QAT_DISABLED +class QatzipCompressorFactory : public Envoy::Compression::Compressor::CompressorFactory { +public: + QatzipCompressorFactory( + const envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip& qatzip, + Server::Configuration::FactoryContext& context); + + // Envoy::Compression::Compressor::CompressorFactory + Envoy::Compression::Compressor::CompressorPtr createCompressor() override; + const std::string& statsPrefix() const override { return qatzipStatsPrefix(); } + const std::string& contentEncoding() const override { + return Http::CustomHeaders::get().ContentEncodingValues.Gzip; + } + +private: + struct QatzipThreadLocal : public ThreadLocal::ThreadLocalObject { + QatzipThreadLocal(QzSessionParams_T params); + ~QatzipThreadLocal() override; + QzSession_T* getSession(); + + QzSessionParams_T params_; + QzSession_T session_; + bool initialized_; + }; + + const uint32_t chunk_size_; + ThreadLocal::SlotPtr tls_slot_; +}; +#endif + +class QatzipCompressorLibraryFactory + : public Envoy::Compression::Compressor::NamedCompressorLibraryConfigFactory { +public: + QatzipCompressorLibraryFactory() : name_{qatzipExtensionName()} {} + + Envoy::Compression::Compressor::CompressorFactoryPtr + createCompressorFactoryFromProto(const Protobuf::Message& proto_config, + Server::Configuration::FactoryContext& context) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return name_; } + +private: +#ifndef QAT_DISABLED + Envoy::Compression::Compressor::CompressorFactoryPtr createCompressorFactoryFromProtoTyped( + const envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip& config, + Server::Configuration::FactoryContext& context); +#endif + + const std::string name_; +}; + +DECLARE_FACTORY(QatzipCompressorLibraryFactory); + +} // namespace Compressor +} // namespace Qatzip +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.cc b/contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.cc new file mode 100644 index 000000000000..24b6a8797d76 --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.cc @@ -0,0 +1,82 @@ +#include "contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.h" + +#include + +#include "source/common/common/assert.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Qatzip { +namespace Compressor { + +QatzipCompressorImpl::QatzipCompressorImpl(QzSession_T* session) + : QatzipCompressorImpl(session, 4096) {} + +// TODO(rojkov): add lower limit to chunk_size in proto definition. +QatzipCompressorImpl::QatzipCompressorImpl(QzSession_T* session, size_t chunk_size) + : chunk_size_{chunk_size}, avail_in_{0}, avail_out_{chunk_size - 10}, + chunk_char_ptr_(new unsigned char[chunk_size]), session_{session}, stream_{}, input_len_(0) { + RELEASE_ASSERT(session_ != nullptr, + "QATzip compressor must be created with non-null QATzip session"); + static unsigned char gzheader[10] = {0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3}; + stream_.out = static_cast(mempcpy(chunk_char_ptr_.get(), gzheader, 10)); +} + +QatzipCompressorImpl::~QatzipCompressorImpl() { qzEndStream(session_, &stream_); } + +void QatzipCompressorImpl::compress(Buffer::Instance& buffer, + Envoy::Compression::Compressor::State state) { + + for (const Buffer::RawSlice& input_slice : buffer.getRawSlices()) { + avail_in_ = input_slice.len_; + stream_.in = static_cast(input_slice.mem_); + + while (avail_in_ > 0) { + process(buffer, 0); + } + + buffer.drain(input_slice.len_); + } + + if (state == Envoy::Compression::Compressor::State::Finish) { + do { + process(buffer, 1); + } while (stream_.pending_out > 0); + + const size_t n_output = chunk_size_ - avail_out_; + if (n_output > 0) { + buffer.add(static_cast(chunk_char_ptr_.get()), n_output); + } + buffer.writeLEInt(stream_.crc_32); + buffer.writeLEInt(input_len_); + } +} + +void QatzipCompressorImpl::process(Buffer::Instance& output_buffer, unsigned int last) { + stream_.in_sz = avail_in_; + stream_.out_sz = avail_out_; + auto status = qzCompressStream(session_, &stream_, last); + // NOTE: stream_.in_sz and stream_.out_sz have changed their semantics after the call + // to qzCompressStream(). Despite their name the new values are consumed input + // and produced output (not available buffer sizes). + avail_out_ -= stream_.out_sz; + avail_in_ -= stream_.in_sz; + input_len_ += stream_.in_sz; + stream_.in = stream_.in + stream_.in_sz; + stream_.out = stream_.out + stream_.out_sz; + RELEASE_ASSERT(status == QZ_OK, ""); + if (avail_out_ == 0) { + // The chunk is full, so copy it to the output buffer and reset context. + output_buffer.add(static_cast(chunk_char_ptr_.get()), chunk_size_); + chunk_char_ptr_ = std::make_unique(chunk_size_); + avail_out_ = chunk_size_; + stream_.out = chunk_char_ptr_.get(); + } +} + +} // namespace Compressor +} // namespace Qatzip +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.h b/contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.h new file mode 100644 index 000000000000..7b2e2ec23ba4 --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/source/qatzip_compressor_impl.h @@ -0,0 +1,51 @@ +#pragma once + +#include "envoy/compression/compressor/compressor.h" + +#define HAVE_QAT_HEADERS +#include "qatzip.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Qatzip { +namespace Compressor { + +/** + * Implementation of compressor's interface. + */ +class QatzipCompressorImpl : public Envoy::Compression::Compressor::Compressor { +public: + QatzipCompressorImpl(QzSession_T* session); + + /** + * Constructor that allows setting the size of compressor's output buffer. It + * should be called whenever a buffer size different than the 4096 bytes, normally set by the + * default constructor, is desired. + * @param chunk_size amount of memory reserved for the compressor output. + */ + QatzipCompressorImpl(QzSession_T* session, size_t chunk_size); + ~QatzipCompressorImpl() override; + + // Compressor + void compress(Buffer::Instance& buffer, Envoy::Compression::Compressor::State state) override; + +private: + void process(Buffer::Instance& output_buffer, unsigned int last); + + const size_t chunk_size_; + size_t avail_in_; + size_t avail_out_; + + std::unique_ptr chunk_char_ptr_; + QzSession_T* const session_; + QzStream_T stream_; + + uint32_t input_len_; +}; + +} // namespace Compressor +} // namespace Qatzip +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/qat/compression/qatzip/compressor/test/BUILD b/contrib/qat/compression/qatzip/compressor/test/BUILD new file mode 100644 index 000000000000..4c39ee53fc4a --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/test/BUILD @@ -0,0 +1,23 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_contrib_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +envoy_cc_test( + name = "qatzip_compressor_impl_test", + srcs = select({ + "//bazel:linux_x86_64": ["qatzip_compressor_impl_test.cc"], + "//conditions:default": [], + }), + deps = [ + "//contrib/qat/compression/qatzip/compressor/source:config", + "//source/extensions/compression/gzip/decompressor:zlib_decompressor_impl_lib", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/contrib/qat/compression/qatzip/compressor/test/qatzip_compressor_impl_test.cc b/contrib/qat/compression/qatzip/compressor/test/qatzip_compressor_impl_test.cc new file mode 100644 index 000000000000..7ff4e03b5b99 --- /dev/null +++ b/contrib/qat/compression/qatzip/compressor/test/qatzip_compressor_impl_test.cc @@ -0,0 +1,149 @@ +#include "source/common/buffer/buffer_impl.h" +#include "source/common/stats/isolated_store_impl.h" +#include "source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/utility.h" + +#include "contrib/qat/compression/qatzip/compressor/source/config.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Compression { +namespace Qatzip { +namespace Compressor { + +class QatzipCompressorImplTest : public ::testing::Test { +protected: + void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } + + void verifyWithDecompressor(Envoy::Compression::Compressor::CompressorPtr compressor, + int chunk_size) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + std::string original_text{}; + for (uint64_t i = 0; i < 10; i++) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size_ * i, i); + original_text.append(buffer.toString()); + ASSERT_EQ(default_input_size_ * i, buffer.length()); + compressor->compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + } + + compressor->compress(buffer, Envoy::Compression::Compressor::State::Finish); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + + Stats::IsolatedStoreImpl stats_store; + Compression::Gzip::Decompressor::ZlibDecompressorImpl decompressor(*stats_store.rootScope(), + "test.", chunk_size, 100); + // Window bits = 31 (15 for maximum window bits + 16 for gzip). + decompressor.init(31); + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + } + + Envoy::Compression::Compressor::CompressorFactoryPtr + createQatzipCompressorFactoryFromConfig(const std::string& json) { + envoy::extensions::compression::qatzip::compressor::v3alpha::Qatzip qatzip_config; + TestUtility::loadFromJson(json, qatzip_config); + + return qatzip_compressor_library_factory_.createCompressorFactoryFromProto(qatzip_config, + context_); + } + // A value which has an impact on the size of random input created for the tests that use + // verifyWithDecompressor. + static constexpr uint32_t default_input_size_{796}; + + QatzipCompressorLibraryFactory qatzip_compressor_library_factory_; + NiceMock context_; +}; + +class QatzipConfigTest + : public QatzipCompressorImplTest, + public ::testing::WithParamInterface> {}; + +// These tests should pass even if required hardware or setup steps required for qatzip are missing. +// Qatzip uses a sofware fallback in this case. +INSTANTIATE_TEST_SUITE_P(QatzipConfigTestInstantiation, QatzipConfigTest, + // First tuple has all default values. + ::testing::Values(std::make_tuple(1, "DEFAULT", 1024, 131072, 4096), + std::make_tuple(2, "DEFAULT", 128, 131072, 4096), + std::make_tuple(3, "DEFAULT", 524288, 131072, 4096), + std::make_tuple(4, "SZ_4K", 1024, 1024, 4096), + std::make_tuple(5, "SZ_8K", 1024, 2092032, 4096), + std::make_tuple(6, "SZ_32K", 1024, 131072, 65536), + std::make_tuple(7, "SZ_64K", 1024, 131072, 4096), + std::make_tuple(8, "SZ_128K", 1024, 131072, 4096), + std::make_tuple(9, "SZ_512K", 1024, 131072, 4096))); + +TEST_P(QatzipConfigTest, LoadConfigAndVerifyWithDecompressor) { + std::tuple config_value_tuple = GetParam(); + int chunk_size = std::get<4>(config_value_tuple); + std::string json{fmt::format(R"EOF({{ + "compression_level": {}, + "hardware_buffer_size": "{}", + "input_size_threshold": {}, + "stream_buffer_size": {}, + "chunk_size": {} +}})EOF", + std::get<0>(config_value_tuple), std::get<1>(config_value_tuple), + std::get<2>(config_value_tuple), std::get<3>(config_value_tuple), + chunk_size)}; + + Envoy::Compression::Compressor::CompressorFactoryPtr qatzip_compressor_factory = + createQatzipCompressorFactoryFromConfig(json); + + EXPECT_EQ("gzip", qatzip_compressor_factory->contentEncoding()); + EXPECT_EQ("qatzip.", qatzip_compressor_factory->statsPrefix()); + + verifyWithDecompressor(qatzip_compressor_factory->createCompressor(), chunk_size); +} + +class InvalidQatzipConfigTest + : public QatzipCompressorImplTest, + public ::testing::WithParamInterface> {}; + +// Tests with invalid qatzip configs. +INSTANTIATE_TEST_SUITE_P( + InvalidQatzipConfigTestInstantiation, InvalidQatzipConfigTest, + // This tuple has all default values: std::make_tuple(1, "DEFAULT", 1024, 131072, 4096). + ::testing::Values(std::make_tuple(0, "DEFAULT", 1024, 131072, 4096), + std::make_tuple(10, "DEFAULT", 1024, 131072, 4096), + std::make_tuple(1, "DEFAULT", 127, 131072, 4096), + std::make_tuple(1, "DEFAULT", 524289, 131072, 4096), + std::make_tuple(1, "DEFAULT", 1024, 1023, 4096), + std::make_tuple(1, "DEFAULT", 1024, 2092033, 4096), + std::make_tuple(1, "DEFAULT", 1024, 131072, 4095), + std::make_tuple(1, "DEFAULT", 1024, 131072, 65537))); + +TEST_P(InvalidQatzipConfigTest, LoadConfigWithInvalidValues) { + std::tuple config_value_tuple = GetParam(); + int chunk_size = std::get<4>(config_value_tuple); + std::string json{fmt::format(R"EOF({{ + "compression_level": {}, + "hardware_buffer_size": "{}", + "input_size_threshold": {}, + "stream_buffer_size": {}, + "chunk_size": {} +}})EOF", + std::get<0>(config_value_tuple), std::get<1>(config_value_tuple), + std::get<2>(config_value_tuple), std::get<3>(config_value_tuple), + chunk_size)}; + + EXPECT_THROW_WITH_REGEX(createQatzipCompressorFactoryFromConfig(json), EnvoyException, + "Proto constraint validation failed"); +} + +} // namespace Compressor +} // namespace Qatzip +} // namespace Compression +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/qat/private_key_providers/source/qat.cc b/contrib/qat/private_key_providers/source/qat.cc index 595ad63fe1ad..8a4998dce35c 100644 --- a/contrib/qat/private_key_providers/source/qat.cc +++ b/contrib/qat/private_key_providers/source/qat.cc @@ -374,9 +374,7 @@ static void decryptCb(void* callback_tag, CpaStatus status, void* data, CpaFlatB } } // namespace -QatContext::QatContext(QatHandle& handle) - : handle_(handle), last_status_(CPA_STATUS_RETRY), decrypted_data_length_(0), read_fd_(-1), - write_fd_(-1) {} +QatContext::QatContext(QatHandle& handle) : handle_(handle) {} QatContext::~QatContext() { if (read_fd_ >= 0) { diff --git a/contrib/qat/private_key_providers/source/qat.h b/contrib/qat/private_key_providers/source/qat.h index 038e75010888..78ce42e10111 100644 --- a/contrib/qat/private_key_providers/source/qat.h +++ b/contrib/qat/private_key_providers/source/qat.h @@ -131,12 +131,12 @@ class QatContext { CpaCyRsaDecryptOpData** dec_op_data, CpaFlatBuffer** output_buffer); QatHandle& handle_; - CpaStatus last_status_; + CpaStatus last_status_{CPA_STATUS_RETRY}; unsigned char decrypted_data_[QAT_BUFFER_SIZE]; - int decrypted_data_length_; + int decrypted_data_length_{0}; // Pipe for passing the message that the operation is completed. - int read_fd_; - int write_fd_; + int read_fd_{-1}; + int write_fd_{-1}; }; } // namespace Qat diff --git a/contrib/qat/private_key_providers/test/config_test.cc b/contrib/qat/private_key_providers/test/config_test.cc index 9cbbafada6fe..55542d230809 100644 --- a/contrib/qat/private_key_providers/test/config_test.cc +++ b/contrib/qat/private_key_providers/test/config_test.cc @@ -35,7 +35,8 @@ parsePrivateKeyProviderFromV3Yaml(const std::string& yaml_string) { class FakeSingletonManager : public Singleton::Manager { public: FakeSingletonManager(LibQatCryptoSharedPtr libqat) : libqat_(libqat) {} - Singleton::InstanceSharedPtr get(const std::string&, Singleton::SingletonFactoryCb) override { + Singleton::InstanceSharedPtr get(const std::string&, Singleton::SingletonFactoryCb, + bool) override { return std::make_shared(libqat_); } diff --git a/contrib/qat/private_key_providers/test/ops_test.cc b/contrib/qat/private_key_providers/test/ops_test.cc index 3f61f68c6367..9ea47b1d172e 100644 --- a/contrib/qat/private_key_providers/test/ops_test.cc +++ b/contrib/qat/private_key_providers/test/ops_test.cc @@ -44,7 +44,8 @@ class TestCallbacks : public Envoy::Ssl::PrivateKeyConnectionCallbacks { class FakeSingletonManager : public Singleton::Manager { public: FakeSingletonManager(LibQatCryptoSharedPtr libqat) : libqat_(libqat) {} - Singleton::InstanceSharedPtr get(const std::string&, Singleton::SingletonFactoryCb) override { + Singleton::InstanceSharedPtr get(const std::string&, Singleton::SingletonFactoryCb, + bool) override { return std::make_shared(libqat_); } diff --git a/contrib/rocketmq_proxy/filters/network/source/config.cc b/contrib/rocketmq_proxy/filters/network/source/config.cc index 25d630d6d123..4cfb969b14e2 100644 --- a/contrib/rocketmq_proxy/filters/network/source/config.cc +++ b/contrib/rocketmq_proxy/filters/network/source/config.cc @@ -24,7 +24,7 @@ Network::FilterFactoryCb RocketmqProxyFilterConfigFactory::createFilterFactoryFr std::shared_ptr filter_config = std::make_shared(proto_config, context); return [filter_config, &context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared( - *filter_config, context.mainThreadDispatcher().timeSource())); + *filter_config, context.serverFactoryContext().mainThreadDispatcher().timeSource())); }; } @@ -41,7 +41,7 @@ ConfigImpl::ConfigImpl(const RocketmqProxyConfig& config, TransientObjectLifeSpan)) {} std::string ConfigImpl::proxyAddress() { - const LocalInfo::LocalInfo& localInfo = context_.getServerFactoryContext().localInfo(); + const LocalInfo::LocalInfo& localInfo = context_.serverFactoryContext().localInfo(); Network::Address::InstanceConstSharedPtr address = localInfo.address(); if (address->type() == Network::Address::Type::Ip) { const std::string& ip = address->ip()->addressAsString(); diff --git a/contrib/rocketmq_proxy/filters/network/source/config.h b/contrib/rocketmq_proxy/filters/network/source/config.h index 978eb9ed5bfd..39e05132c6be 100644 --- a/contrib/rocketmq_proxy/filters/network/source/config.h +++ b/contrib/rocketmq_proxy/filters/network/source/config.h @@ -39,9 +39,11 @@ class ConfigImpl : public Config, public Router::Config, Logger::Loggable(context_.clusterManager()); + return std::make_unique(context_.serverFactoryContext().clusterManager()); } bool developMode() const override { return develop_mode_; } diff --git a/contrib/rocketmq_proxy/filters/network/source/router/router_impl.cc b/contrib/rocketmq_proxy/filters/network/source/router/router_impl.cc index b5b5a59d62d2..afab0c2e601a 100644 --- a/contrib/rocketmq_proxy/filters/network/source/router/router_impl.cc +++ b/contrib/rocketmq_proxy/filters/network/source/router/router_impl.cc @@ -16,7 +16,7 @@ namespace RocketmqProxy { namespace Router { RouterImpl::RouterImpl(Envoy::Upstream::ClusterManager& cluster_manager) - : cluster_manager_(cluster_manager), handle_(nullptr), active_message_(nullptr) {} + : cluster_manager_(cluster_manager) {} RouterImpl::~RouterImpl() { if (handle_) { diff --git a/contrib/rocketmq_proxy/filters/network/source/router/router_impl.h b/contrib/rocketmq_proxy/filters/network/source/router/router_impl.h index 1cb222f5d7f9..14ff5e5d54c3 100644 --- a/contrib/rocketmq_proxy/filters/network/source/router/router_impl.h +++ b/contrib/rocketmq_proxy/filters/network/source/router/router_impl.h @@ -62,9 +62,9 @@ class RouterImpl : public Router, public Logger::Loggable * * If there are connections which can be returned immediately, this handle is assigned as nullptr. */ - Tcp::ConnectionPool::Cancellable* handle_; + Tcp::ConnectionPool::Cancellable* handle_{nullptr}; Upstream::HostDescriptionConstSharedPtr upstream_host_; - ActiveMessage* active_message_; + ActiveMessage* active_message_{nullptr}; Upstream::ClusterInfoConstSharedPtr cluster_info_; UpstreamRequestPtr upstream_request_; const RouteEntry* route_entry_{}; diff --git a/contrib/rocketmq_proxy/filters/network/test/active_message_test.cc b/contrib/rocketmq_proxy/filters/network/test/active_message_test.cc index 0025859cd372..9ca46b392c6c 100644 --- a/contrib/rocketmq_proxy/filters/network/test/active_message_test.cc +++ b/contrib/rocketmq_proxy/filters/network/test/active_message_test.cc @@ -24,7 +24,8 @@ class ActiveMessageTest : public testing::Test { ActiveMessageTest() : stats_(RocketmqFilterStats::generateStats("test.", *store_.rootScope())), config_(rocketmq_proxy_config_, factory_context_), - connection_manager_(config_, factory_context_.mainThreadDispatcher().timeSource()) { + connection_manager_( + config_, factory_context_.serverFactoryContext().mainThreadDispatcher().timeSource()) { connection_manager_.initializeReadFilterCallbacks(filter_callbacks_); } diff --git a/contrib/rocketmq_proxy/filters/network/test/config_test.cc b/contrib/rocketmq_proxy/filters/network/test/config_test.cc index 62e5eb051632..10a36665fd5f 100644 --- a/contrib/rocketmq_proxy/filters/network/test/config_test.cc +++ b/contrib/rocketmq_proxy/filters/network/test/config_test.cc @@ -91,7 +91,7 @@ TEST_F(RocketmqFilterConfigTest, RocketmqProxyWithFullConfig) { TEST_F(RocketmqFilterConfigTest, ProxyAddress) { NiceMock context; Server::Configuration::MockServerFactoryContext factory_context; - EXPECT_CALL(context, getServerFactoryContext()).WillRepeatedly(ReturnRef(factory_context)); + EXPECT_CALL(context, serverFactoryContext()).WillRepeatedly(ReturnRef(factory_context)); LocalInfo::MockLocalInfo local_info; EXPECT_CALL(factory_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); @@ -116,7 +116,7 @@ TEST_F(RocketmqFilterConfigTest, ProxyAddress) { TEST_F(RocketmqFilterConfigTest, ProxyAddressWithDefaultPort) { NiceMock context; Server::Configuration::MockServerFactoryContext factory_context; - EXPECT_CALL(context, getServerFactoryContext()).WillRepeatedly(ReturnRef(factory_context)); + EXPECT_CALL(context, serverFactoryContext()).WillRepeatedly(ReturnRef(factory_context)); LocalInfo::MockLocalInfo local_info; EXPECT_CALL(factory_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); @@ -141,7 +141,7 @@ TEST_F(RocketmqFilterConfigTest, ProxyAddressWithDefaultPort) { TEST_F(RocketmqFilterConfigTest, ProxyAddressWithNonIpType) { NiceMock context; Server::Configuration::MockServerFactoryContext factory_context; - EXPECT_CALL(context, getServerFactoryContext()).WillRepeatedly(ReturnRef(factory_context)); + EXPECT_CALL(context, serverFactoryContext()).WillRepeatedly(ReturnRef(factory_context)); LocalInfo::MockLocalInfo local_info; EXPECT_CALL(factory_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); diff --git a/contrib/rocketmq_proxy/filters/network/test/conn_manager_test.cc b/contrib/rocketmq_proxy/filters/network/test/conn_manager_test.cc index 982dc69a7597..6cce152be326 100644 --- a/contrib/rocketmq_proxy/filters/network/test/conn_manager_test.cc +++ b/contrib/rocketmq_proxy/filters/network/test/conn_manager_test.cc @@ -56,10 +56,12 @@ class RocketmqConnectionManagerTest : public Event::TestUsingSimulatedTime, publ } config_ = std::make_unique(proto_config_, factory_context_, stats_); conn_manager_ = std::make_unique( - *config_, factory_context_.mainThreadDispatcher().timeSource()); + *config_, factory_context_.server_factory_context_.mainThreadDispatcher().timeSource()); conn_manager_->initializeReadFilterCallbacks(filter_callbacks_); conn_manager_->onNewConnection(); - current_ = factory_context_.mainThreadDispatcher().timeSource().monotonicTime(); + current_ = factory_context_.server_factory_context_.mainThreadDispatcher() + .timeSource() + .monotonicTime(); } void initializeCluster() { @@ -70,8 +72,10 @@ class RocketmqConnectionManagerTest : public Event::TestUsingSimulatedTime, publ Upstream::HostSetImpl::partitionHosts(std::make_shared(hosts), Upstream::HostsPerLocalityImpl::empty()), nullptr, hosts, {}, 100); - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); - ON_CALL(factory_context_.cluster_manager_.thread_local_cluster_, prioritySet()) + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); + ON_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + prioritySet()) .WillByDefault(ReturnRef(priority_set_)); } @@ -421,7 +425,7 @@ stat_prefix: test )EOF"; initializeFilter(yaml); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) .WillRepeatedly(Return(nullptr)); BufferUtility::fillRequestBuffer(buffer_, RequestCode::GetRouteInfoByTopic); @@ -450,7 +454,7 @@ develop_mode: true NiceMock ip; std::shared_ptr instance = std::make_shared("logical", "physical"); - EXPECT_CALL(factory_context_, getServerFactoryContext()) + EXPECT_CALL(factory_context_, serverFactoryContext()) .WillRepeatedly(ReturnRef(server_factory_context)); EXPECT_CALL(server_factory_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); EXPECT_CALL(local_info, address()).WillRepeatedly(Return(instance)); diff --git a/contrib/rocketmq_proxy/filters/network/test/router_test.cc b/contrib/rocketmq_proxy/filters/network/test/router_test.cc index e4128ee0d36c..c98239d258d2 100644 --- a/contrib/rocketmq_proxy/filters/network/test/router_test.cc +++ b/contrib/rocketmq_proxy/filters/network/test/router_test.cc @@ -23,16 +23,17 @@ class RocketmqRouterTestBase { RocketmqRouterTestBase() : config_(rocketmq_proxy_config_, context_), cluster_info_(std::make_shared()) { - context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); - conn_manager_ = - std::make_unique(config_, context_.mainThreadDispatcher().timeSource()); + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); + conn_manager_ = std::make_unique( + config_, context_.server_factory_context_.mainThreadDispatcher().timeSource()); conn_manager_->initializeReadFilterCallbacks(filter_callbacks_); } ~RocketmqRouterTestBase() { filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); } void initializeRouter() { - router_ = std::make_unique(context_.clusterManager()); + router_ = std::make_unique(context_.server_factory_context_.clusterManager()); EXPECT_EQ(nullptr, router_->downstreamConnection()); } @@ -99,16 +100,20 @@ class RocketmqRouterTestBase { void startRequest() { router_->sendRequestToUpstream(*active_message_); } void connectUpstream() { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); } void startRequestWithExistingConnection() { - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); router_->sendRequestToUpstream(*active_message_); @@ -167,8 +172,8 @@ TEST_F(RocketmqRouterTest, PoolRemoteConnectionFailure) { })); startRequest(); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - Tcp::ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(Tcp::ConnectionPool::PoolFailureReason::RemoteConnectionFailure); } TEST_F(RocketmqRouterTest, PoolTimeout) { @@ -183,8 +188,8 @@ TEST_F(RocketmqRouterTest, PoolTimeout) { EXPECT_CALL(*active_message_, onReset()); startRequest(); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - Tcp::ConnectionPool::PoolFailureReason::Timeout); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(Tcp::ConnectionPool::PoolFailureReason::Timeout); } TEST_F(RocketmqRouterTest, PoolLocalConnectionFailure) { @@ -199,8 +204,8 @@ TEST_F(RocketmqRouterTest, PoolLocalConnectionFailure) { EXPECT_CALL(*active_message_, onReset()); startRequest(); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - Tcp::ConnectionPool::PoolFailureReason::LocalConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(Tcp::ConnectionPool::PoolFailureReason::LocalConnectionFailure); } TEST_F(RocketmqRouterTest, PoolOverflowFailure) { @@ -215,8 +220,8 @@ TEST_F(RocketmqRouterTest, PoolOverflowFailure) { EXPECT_CALL(*active_message_, onReset()); startRequest(); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - Tcp::ConnectionPool::PoolFailureReason::Overflow); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(Tcp::ConnectionPool::PoolFailureReason::Overflow); } TEST_F(RocketmqRouterTest, ClusterMaintenanceMode) { @@ -228,7 +233,9 @@ TEST_F(RocketmqRouterTest, ClusterMaintenanceMode) { .WillOnce(Invoke([&](absl::string_view error_message) -> void { EXPECT_THAT(error_message, ContainsRegex(".*Cluster under maintenance*.")); })); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, maintenanceMode()) + EXPECT_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + maintenanceMode()) .WillOnce(Return(true)); EXPECT_CALL(*active_message_, onReset()); @@ -244,7 +251,8 @@ TEST_F(RocketmqRouterTest, NoHealthyHosts) { .WillOnce(Invoke([&](absl::string_view error_message) -> void { EXPECT_THAT(error_message, ContainsRegex(".*No host available*.")); })); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); EXPECT_CALL(*active_message_, onReset()); @@ -271,7 +279,8 @@ TEST_F(RocketmqRouterTest, NoCluster) { initSendMessageRequest(); EXPECT_CALL(*active_message_, onReset()); - EXPECT_CALL(context_.cluster_manager_, getThreadLocalCluster(_)).WillRepeatedly(Return(nullptr)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) + .WillRepeatedly(Return(nullptr)); startRequest(); } diff --git a/contrib/sip_proxy/filters/network/source/config.cc b/contrib/sip_proxy/filters/network/source/config.cc index d9a3b97e410f..36a1a0f31b9a 100644 --- a/contrib/sip_proxy/filters/network/source/config.cc +++ b/contrib/sip_proxy/filters/network/source/config.cc @@ -54,7 +54,7 @@ Network::FilterFactoryCb SipProxyFilterConfigFactory::createFilterFactoryFromPro Stats::ScopeSharedPtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.sip_cluster", cluster)); auto transaction_info_ptr = std::make_shared( - cluster, context.threadLocal(), + cluster, context.serverFactoryContext().threadLocal(), static_cast( PROTOBUF_GET_MS_OR_DEFAULT(proto_config.settings(), transaction_timeout, 32000))); transaction_info_ptr->init(); @@ -64,8 +64,9 @@ Network::FilterFactoryCb SipProxyFilterConfigFactory::createFilterFactoryFromPro return [filter_config, &context, transaction_infos](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared( - *filter_config, context.api().randomGenerator(), - context.mainThreadDispatcher().timeSource(), context, transaction_infos)); + *filter_config, context.serverFactoryContext().api().randomGenerator(), + context.serverFactoryContext().mainThreadDispatcher().timeSource(), context, + transaction_infos)); }; } diff --git a/contrib/sip_proxy/filters/network/source/metadata.cc b/contrib/sip_proxy/filters/network/source/metadata.cc index e9d6091846a8..c1ccbf2554a2 100644 --- a/contrib/sip_proxy/filters/network/source/metadata.cc +++ b/contrib/sip_proxy/filters/network/source/metadata.cc @@ -28,7 +28,7 @@ void SipHeader::parseHeader() { if (std::size_t found = header.find(": "); found != absl::string_view::npos) { header = header.substr(found + 2); } - if (std::size_t found = header.find("<"); found != absl::string_view::npos) { + if (std::size_t found = header.find('<'); found != absl::string_view::npos) { header = header.substr(found + 1); } @@ -40,7 +40,7 @@ void SipHeader::parseHeader() { str = header.substr(pos, found - pos); } - std::size_t value_pos = str.find("="); + std::size_t value_pos = str.find('='); if (value_pos == absl::string_view::npos) { // First as host if (isHost) { @@ -124,7 +124,7 @@ void MessageMetadata::addEPOperation( return; } - auto pos = header.find(">"); + auto pos = header.find('>'); if (pos == absl::string_view::npos) { // no url return; diff --git a/contrib/sip_proxy/filters/network/source/router/config.cc b/contrib/sip_proxy/filters/network/source/router/config.cc index 5637502f2d3f..1fa5e433ca40 100644 --- a/contrib/sip_proxy/filters/network/source/router/config.cc +++ b/contrib/sip_proxy/filters/network/source/router/config.cc @@ -22,7 +22,8 @@ SipFilters::FilterFactoryCb RouterFilterConfigFactory::createFilterFactoryFromPr return [config, &context, stat_prefix](SipFilters::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addDecoderFilter(std::make_shared(config, context.clusterManager(), context)); + callbacks.addDecoderFilter( + std::make_shared(config, context.serverFactoryContext().clusterManager(), context)); }; } diff --git a/contrib/sip_proxy/filters/network/source/tra/tra_impl.cc b/contrib/sip_proxy/filters/network/source/tra/tra_impl.cc index e0e08ead937d..86c68a071668 100644 --- a/contrib/sip_proxy/filters/network/source/tra/tra_impl.cc +++ b/contrib/sip_proxy/filters/network/source/tra/tra_impl.cc @@ -201,8 +201,10 @@ ClientPtr traClient(Event::Dispatcher& dispatcher, Server::Configuration::Factor // TODO(ramaraochavali): register client to singleton when GrpcClientImpl supports concurrent // requests. return std::make_unique( - context.clusterManager().grpcAsyncClientManager().getOrCreateRawAsyncClient( - grpc_service, context.scope(), true), + context.serverFactoryContext() + .clusterManager() + .grpcAsyncClientManager() + .getOrCreateRawAsyncClient(grpc_service, context.scope(), true), dispatcher, timeout); } diff --git a/contrib/sip_proxy/filters/network/test/config_test.cc b/contrib/sip_proxy/filters/network/test/config_test.cc index fc81f34f0a1d..803934b2844a 100644 --- a/contrib/sip_proxy/filters/network/test/config_test.cc +++ b/contrib/sip_proxy/filters/network/test/config_test.cc @@ -34,7 +34,7 @@ class SipFilterConfigTestBase { void testConfig(envoy::extensions::filters::network::sip_proxy::v3alpha::SipProxy& config) { Network::FilterFactoryCb cb; EXPECT_NO_THROW({ cb = factory_.createFilterFactoryFromProto(config, context_); }); - EXPECT_TRUE(factory_.isTerminalFilterByProto(config, context_.getServerFactoryContext())); + EXPECT_TRUE(factory_.isTerminalFilterByProto(config, context_.serverFactoryContext())); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); @@ -182,8 +182,9 @@ registration_affinity: true NiceMock context; const auto options = std::make_shared(config); - EXPECT_CALL(*context.cluster_manager_.thread_local_cluster_.cluster_.info_, - extensionProtocolOptions(_)) + EXPECT_CALL( + *context.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + extensionProtocolOptions(_)) .WillRepeatedly(Return(options)); EXPECT_EQ(true, options->sessionAffinity()); diff --git a/contrib/sip_proxy/filters/network/test/router_test.cc b/contrib/sip_proxy/filters/network/test/router_test.cc index 9964207138da..8e044715b4d2 100644 --- a/contrib/sip_proxy/filters/network/test/router_test.cc +++ b/contrib/sip_proxy/filters/network/test/router_test.cc @@ -99,8 +99,9 @@ class SipRouterTest : public testing::Test { } const auto options = std::make_shared(sip_protocol_options_config_); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, - extensionProtocolOptions(_)) + EXPECT_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + extensionProtocolOptions(_)) .WillRepeatedly(Return(options)); EXPECT_CALL(context_, getTransportSocketFactoryContext()) @@ -109,7 +110,8 @@ class SipRouterTest : public testing::Test { .WillRepeatedly(testing::ReturnRef(local_info_)); transaction_infos_ = std::make_shared(); - context_.cluster_manager_.initializeThreadLocalClusters({cluster_name_}); + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {cluster_name_}); StreamInfo::StreamInfoImpl stream_info{time_source_, nullptr}; SipFilterStats stat = SipFilterStats::generateStats("test.", *store_.rootScope()); @@ -130,7 +132,8 @@ class SipRouterTest : public testing::Test { route_ptr_.reset(route_); EXPECT_CALL(*router_filter_config_, stats()).WillRepeatedly(ReturnRef(router_stats_)); - router_ = std::make_unique(router_filter_config_, context_.clusterManager(), context_); + router_ = std::make_unique(router_filter_config_, + context_.server_factory_context_.clusterManager(), context_); EXPECT_EQ(nullptr, router_->downstreamConnection()); @@ -178,18 +181,21 @@ class SipRouterTest : public testing::Test { } void connectUpstream() { - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; })); conn_state_.reset(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); EXPECT_NE(nullptr, upstream_callbacks_); } @@ -402,7 +408,8 @@ TEST_F(SipRouterTest, NoTcpConnPool) { initializeRouter(); initializeTransaction(); initializeMetadata(MsgType::Request); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); try { startRequest(FilterStatus::Continue); @@ -423,7 +430,8 @@ TEST_F(SipRouterTest, NoTcpConnPoolEmptyDest) { metadata_->affinity().emplace_back("Route", "ep", "ep", false, false); metadata_->resetAffinityIteration(); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); try { startRequest(FilterStatus::Continue); @@ -508,7 +516,8 @@ TEST_F(SipRouterTest, CallNoCluster) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_, getThreadLocalCluster(Eq(cluster_name_))) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq(cluster_name_))) .WillOnce(Return(nullptr)); try { @@ -529,7 +538,9 @@ TEST_F(SipRouterTest, ClusterMaintenanceMode) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, maintenanceMode()) + EXPECT_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + maintenanceMode()) .WillOnce(Return(true)); try { @@ -551,7 +562,9 @@ TEST_F(SipRouterTest, NoHost) { EXPECT_CALL(route_entry_, clusterName()).WillOnce(ReturnRef(cluster_name_)); EXPECT_EQ(FilterStatus::Continue, router_->transportBegin(metadata_)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, host()) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + host()) .WillOnce(Return(nullptr)); EXPECT_EQ(FilterStatus::Continue, router_->messageBegin(metadata_)); destroyRouter(); @@ -602,12 +615,15 @@ TEST_F(SipRouterTest, CallWithExistingConnection) { metadata_->affinity().emplace_back("Route", "ep", "ep", false, false); metadata_->resetAffinityIteration(); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); EXPECT_EQ(FilterStatus::Continue, router_->messageBegin(metadata_)); @@ -635,12 +651,15 @@ TEST_F(SipRouterTest, CallWithExistingConnectionDefaultLoadBalance) { // initializeMetadata(MsgType::Request); metadata_->resetDestination(); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); EXPECT_EQ(FilterStatus::Continue, router_->messageBegin(metadata_)); @@ -653,8 +672,8 @@ TEST_F(SipRouterTest, PoolFailure) { initializeTransaction(); initializeMetadata(MsgType::Response); startRequest(); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); } TEST_F(SipRouterTest, NextAffinityAfterPoolFailure) { @@ -665,20 +684,23 @@ TEST_F(SipRouterTest, NextAffinityAfterPoolFailure) { startRequest(); metadata_->affinity().emplace_back("Route", "ep", "ep", false, false); metadata_->resetAffinityIteration(); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); } TEST_F(SipRouterTest, NewConnectionFailure) { initializeTrans(); initializeRouter(); initializeTransaction(); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); initializeMetadata(MsgType::Response); diff --git a/contrib/squash/filters/http/source/config.cc b/contrib/squash/filters/http/source/config.cc index e21f54dd6dc1..0e6798492258 100644 --- a/contrib/squash/filters/http/source/config.cc +++ b/contrib/squash/filters/http/source/config.cc @@ -17,13 +17,14 @@ namespace Squash { Http::FilterFactoryCb SquashFilterConfigFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::squash::v3::Squash& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); SquashFilterConfigSharedPtr config = std::make_shared( - SquashFilterConfig(proto_config, context.clusterManager())); + SquashFilterConfig(proto_config, server_context.clusterManager())); - return [&context, config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + return [&server_context, config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter( - std::make_shared(config, context.clusterManager())); + std::make_shared(config, server_context.clusterManager())); }; } diff --git a/contrib/squash/filters/http/source/squash_filter.cc b/contrib/squash/filters/http/source/squash_filter.cc index 8b29bb0efaca..a5671d96470a 100644 --- a/contrib/squash/filters/http/source/squash_filter.cc +++ b/contrib/squash/filters/http/source/squash_filter.cc @@ -125,13 +125,12 @@ std::string SquashFilterConfig::replaceEnv(const std::string& attachment_templat } SquashFilter::SquashFilter(SquashFilterConfigSharedPtr config, Upstream::ClusterManager& cm) - : config_(config), is_squashing_(false), attachment_poll_period_timer_(nullptr), - attachment_timeout_timer_(nullptr), in_flight_request_(nullptr), + : config_(config), attachment_poll_period_timer_(nullptr), attachment_timeout_timer_(nullptr), create_attachment_callback_(std::bind(&SquashFilter::onCreateAttachmentSuccess, this, _1), std::bind(&SquashFilter::onCreateAttachmentFailure, this, _1)), check_attachment_callback_(std::bind(&SquashFilter::onGetAttachmentSuccess, this, _1), std::bind(&SquashFilter::onGetAttachmentFailure, this, _1)), - cm_(cm), decoder_callbacks_(nullptr) {} + cm_(cm) {} SquashFilter::~SquashFilter() = default; diff --git a/contrib/squash/filters/http/source/squash_filter.h b/contrib/squash/filters/http/source/squash_filter.h index df1688210ac4..6a911572838e 100644 --- a/contrib/squash/filters/http/source/squash_filter.h +++ b/contrib/squash/filters/http/source/squash_filter.h @@ -113,7 +113,7 @@ class SquashFilter : public Http::StreamDecoderFilter, // Current state of the squash filter. If is_squashing_ is true, Hold the request while we // communicate with the squash server to attach a debugger. If it is false, let the request // pass-through. - bool is_squashing_; + bool is_squashing_{false}; // The API path of the created debug attachment (used for polling its state). std::string debug_attachment_path_; // A timer for polling the state of a debug attachment until it reaches a final state. @@ -122,7 +122,7 @@ class SquashFilter : public Http::StreamDecoderFilter, // filter iteration Event::TimerPtr attachment_timeout_timer_; // The current inflight request to the squash server. - Http::AsyncClient::Request* in_flight_request_; + Http::AsyncClient::Request* in_flight_request_{nullptr}; // Shims to get AsyncClient callbacks to specific methods, per API method. AsyncClientCallbackShim create_attachment_callback_; AsyncClientCallbackShim check_attachment_callback_; @@ -130,7 +130,7 @@ class SquashFilter : public Http::StreamDecoderFilter, // ClusterManager to send requests to squash server Upstream::ClusterManager& cm_; // Callbacks used to continue filter iteration. - Http::StreamDecoderFilterCallbacks* decoder_callbacks_; + Http::StreamDecoderFilterCallbacks* decoder_callbacks_{nullptr}; // Create debug attachment URL path. const static std::string POST_ATTACHMENT_PATH; diff --git a/contrib/squash/filters/http/test/config_test.cc b/contrib/squash/filters/http/test/config_test.cc index aac01c3117fe..3c8052c4438f 100644 --- a/contrib/squash/filters/http/test/config_test.cc +++ b/contrib/squash/filters/http/test/config_test.cc @@ -28,9 +28,10 @@ TEST(SquashFilterConfigFactoryTest, SquashFilterCorrectYaml) { envoy::extensions::filters::http::squash::v3::Squash proto_config; TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; - context.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); SquashFilterConfigFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); diff --git a/contrib/squash/filters/http/test/squash_filter_test.cc b/contrib/squash/filters/http/test/squash_filter_test.cc index 756e83e402fb..ee9eee759377 100644 --- a/contrib/squash/filters/http/test/squash_filter_test.cc +++ b/contrib/squash/filters/http/test/squash_filter_test.cc @@ -17,7 +17,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Eq; using testing::Invoke; using testing::NiceMock; using testing::Return; @@ -34,10 +33,10 @@ SquashFilterConfig constructSquashFilterConfigFromYaml( const std::string& yaml, NiceMock& context) { envoy::extensions::filters::http::squash::v3::Squash proto_config; TestUtility::loadFromYaml(yaml, proto_config); - return SquashFilterConfig(proto_config, context.cluster_manager_); + return {proto_config, context.server_factory_context_.cluster_manager_}; } -void EXPECT_JSON_EQ(const std::string& expected, const std::string& actual) { +void expectJsonEq(const std::string& expected, const std::string& actual) { ProtobufWkt::Struct actualjson; TestUtility::loadFromJson(actual, actualjson); @@ -60,11 +59,11 @@ TEST(SquashFilterConfigTest, V2ApiConversion) { )EOF"; NiceMock factory_context; - factory_context.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + factory_context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); const auto config = constructSquashFilterConfigFromYaml(yaml, factory_context); EXPECT_EQ("fake_cluster", config.clusterName()); - EXPECT_JSON_EQ("{\"a\":\"b\"}", config.attachmentJson()); + expectJsonEq("{\"a\":\"b\"}", config.attachmentJson()); EXPECT_EQ(std::chrono::milliseconds(1001), config.requestTimeout()); EXPECT_EQ(std::chrono::milliseconds(2002), config.attachmentPollPeriod()); EXPECT_EQ(std::chrono::milliseconds(3003), config.attachmentTimeout()); @@ -92,10 +91,10 @@ TEST(SquashFilterConfigTest, ParsesEnvironment) { const std::string expected_json = "{\"a\":\"\"}"; NiceMock factory_context; - factory_context.cluster_manager_.initializeClusters({"squash"}, {}); + factory_context.server_factory_context_.cluster_manager_.initializeClusters({"squash"}, {}); const auto config = constructSquashFilterConfigFromYaml(yaml, factory_context); - EXPECT_JSON_EQ(expected_json, config.attachmentJson()); + expectJsonEq(expected_json, config.attachmentJson()); } TEST(SquashFilterConfigTest, ParsesAndEscapesEnvironment) { @@ -110,9 +109,9 @@ TEST(SquashFilterConfigTest, ParsesAndEscapesEnvironment) { const std::string expected_json = "{\"a\":\"\\\"\"}"; NiceMock factory_context; - factory_context.cluster_manager_.initializeClusters({"squash"}, {}); + factory_context.server_factory_context_.cluster_manager_.initializeClusters({"squash"}, {}); const auto config = constructSquashFilterConfigFromYaml(yaml, factory_context); - EXPECT_JSON_EQ(expected_json, config.attachmentJson()); + expectJsonEq(expected_json, config.attachmentJson()); } TEST(SquashFilterConfigTest, TwoEnvironmentVariables) { @@ -128,9 +127,9 @@ TEST(SquashFilterConfigTest, TwoEnvironmentVariables) { const std::string expected_json = "{\"a\":\"1-2\"}"; NiceMock factory_context; - factory_context.cluster_manager_.initializeClusters({"squash"}, {}); + factory_context.server_factory_context_.cluster_manager_.initializeClusters({"squash"}, {}); auto config = constructSquashFilterConfigFromYaml(yaml, factory_context); - EXPECT_JSON_EQ(expected_json, config.attachmentJson()); + expectJsonEq(expected_json, config.attachmentJson()); } TEST(SquashFilterConfigTest, ParsesEnvironmentInComplexTemplate) { @@ -147,15 +146,16 @@ TEST(SquashFilterConfigTest, ParsesEnvironmentInComplexTemplate) { const std::string expected_json = R"EOF({"a":[{"e": "some-config-value"},{"c":"d"}]})EOF"; NiceMock factory_context; - factory_context.cluster_manager_.initializeClusters({"squash"}, {}); + factory_context.server_factory_context_.cluster_manager_.initializeClusters({"squash"}, {}); const auto config = constructSquashFilterConfigFromYaml(yaml, factory_context); - EXPECT_JSON_EQ(expected_json, config.attachmentJson()); + expectJsonEq(expected_json, config.attachmentJson()); } class SquashFilterTest : public testing::Test { public: SquashFilterTest() - : request_(&factory_context_.cluster_manager_.thread_local_cluster_.async_client_) {} + : request_(&factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_) {} protected: void SetUp() override {} @@ -163,11 +163,14 @@ class SquashFilterTest : public testing::Test { void initFilter() { envoy::extensions::filters::http::squash::v3::Squash p; p.set_cluster("squash"); - factory_context_.cluster_manager_.initializeClusters({"squash"}, {}); - factory_context_.cluster_manager_.initializeThreadLocalClusters({"squash"}); - config_ = std::make_shared(p, factory_context_.cluster_manager_); - - filter_ = std::make_shared(config_, factory_context_.cluster_manager_); + factory_context_.server_factory_context_.cluster_manager_.initializeClusters({"squash"}, {}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"squash"}); + config_ = std::make_shared( + p, factory_context_.server_factory_context_.cluster_manager_); + + filter_ = std::make_shared( + config_, factory_context_.server_factory_context_.cluster_manager_); filter_->setDecoderFilterCallbacks(filter_callbacks_); } @@ -181,9 +184,10 @@ class SquashFilterTest : public testing::Test { attachmentTimeout_timer_ = new NiceMock(&filter_callbacks_.dispatcher_); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, httpAsyncClient()) - .WillRepeatedly( - ReturnRef(factory_context_.cluster_manager_.thread_local_cluster_.async_client_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + httpAsyncClient()) + .WillRepeatedly(ReturnRef(factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.async_client_)); expectAsyncClientSend(); @@ -211,7 +215,8 @@ class SquashFilterTest : public testing::Test { } void expectAsyncClientSend() { - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.async_client_, + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_, send_(_, _, _)) .WillOnce(Invoke( [&](Envoy::Http::RequestMessagePtr&, Envoy::Http::AsyncClient::Callbacks& cb, @@ -260,10 +265,14 @@ class SquashFilterTest : public testing::Test { TEST_F(SquashFilterTest, DecodeHeaderContinuesOnClientFail) { initFilter(); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, httpAsyncClient()) - .WillOnce(ReturnRef(factory_context_.cluster_manager_.thread_local_cluster_.async_client_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + httpAsyncClient()) + .WillOnce(ReturnRef(factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.async_client_)); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.async_client_, send_(_, _, _)) + EXPECT_CALL( + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.async_client_, + send_(_, _, _)) .WillOnce(Invoke( [&](Envoy::Http::RequestMessagePtr&, Envoy::Http::AsyncClient::Callbacks& callbacks, const Http::AsyncClient::RequestOptions&) -> Envoy::Http::AsyncClient::Request* { @@ -300,7 +309,9 @@ TEST_F(SquashFilterTest, DecodeContinuesOnCreateAttachmentFail) { TEST_F(SquashFilterTest, DoesNothingWithNoHeader) { initFilter(); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, httpAsyncClient()).Times(0); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + httpAsyncClient()) + .Times(0); Http::TestRequestHeaderMapImpl headers{{":method", "GET"}, {":authority", "www.solo.io"}, @@ -376,7 +387,8 @@ TEST_F(SquashFilterTest, PollingAttachmentNoCluster) { completeGetStatusRequest("attaching"); // Expect the second get attachment request - ON_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("squash")) + ON_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("squash")) .WillByDefault(Return(nullptr)); EXPECT_CALL(filter_callbacks_.dispatcher_, pushTrackedObject(_)); EXPECT_CALL(filter_callbacks_.dispatcher_, popTrackedObject(_)); @@ -474,7 +486,9 @@ TEST_F(SquashFilterTest, TimerExpiresInline) { attachmentTimeout_timer_->invokeCallback(); })); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.async_client_, send_(_, _, _)) + EXPECT_CALL( + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.async_client_, + send_(_, _, _)) .WillOnce(Invoke([&](Envoy::Http::RequestMessagePtr&, Envoy::Http::AsyncClient::Callbacks&, const Http::AsyncClient::RequestOptions&) -> Envoy::Http::AsyncClient::Request* { return &request_; })); diff --git a/contrib/sxg/filters/http/source/config.cc b/contrib/sxg/filters/http/source/config.cc index ed977791713f..68b7d2292a72 100644 --- a/contrib/sxg/filters/http/source/config.cc +++ b/contrib/sxg/filters/http/source/config.cc @@ -40,7 +40,9 @@ Http::FilterFactoryCb FilterFactory::createFilterFactoryFromProtoTyped( const auto& certificate = proto_config.certificate(); const auto& private_key = proto_config.private_key(); - auto& cluster_manager = context.clusterManager(); + auto& server_context = context.serverFactoryContext(); + + auto& cluster_manager = server_context.clusterManager(); auto& secret_manager = cluster_manager.clusterManagerFactory().secretManager(); auto& transport_socket_factory = context.getTransportSocketFactoryContext(); auto secret_provider_certificate = @@ -55,9 +57,9 @@ Http::FilterFactoryCb FilterFactory::createFilterFactoryFromProtoTyped( } auto secret_reader = std::make_shared( - secret_provider_certificate, secret_provider_private_key, context.api()); - auto config = std::make_shared(proto_config, context.timeSource(), secret_reader, - stat_prefix, context.scope()); + secret_provider_certificate, secret_provider_private_key, server_context.api()); + auto config = std::make_shared(proto_config, server_context.timeSource(), + secret_reader, stat_prefix, context.scope()); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { const EncoderPtr encoder = std::make_unique(config); callbacks.addStreamFilter(std::make_shared(config, encoder)); diff --git a/contrib/sxg/filters/http/test/config_test.cc b/contrib/sxg/filters/http/test/config_test.cc index 3a1af48edd06..d5c65bb790ea 100644 --- a/contrib/sxg/filters/http/test/config_test.cc +++ b/contrib/sxg/filters/http/test/config_test.cc @@ -26,11 +26,12 @@ void expectCreateFilter(std::string yaml, bool is_sds_config) { FilterFactory factory; ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); TestUtility::loadFromYaml(yaml, *proto_config); - Server::Configuration::MockFactoryContext context; - context.cluster_manager_.initializeClusters({"foo"}, {}); + testing::NiceMock context; + context.server_factory_context_.cluster_manager_.initializeClusters({"foo"}, {}); // This returns non-nullptr for certificate and private_key. - auto& secret_manager = context.cluster_manager_.cluster_manager_factory_.secretManager(); + auto& secret_manager = + context.server_factory_context_.cluster_manager_.cluster_manager_factory_.secretManager(); if (is_sds_config) { ON_CALL(secret_manager, findOrCreateGenericSecretProvider(_, _, _, _)) .WillByDefault(Return(std::make_shared( @@ -41,13 +42,14 @@ void expectCreateFilter(std::string yaml, bool is_sds_config) { envoy::extensions::transport_sockets::tls::v3::GenericSecret()))); } EXPECT_CALL(context, messageValidationVisitor()); - EXPECT_CALL(context, clusterManager()); + EXPECT_CALL(context.server_factory_context_, clusterManager()); EXPECT_CALL(context, scope()); - EXPECT_CALL(context, timeSource()); - EXPECT_CALL(context, api()); + EXPECT_CALL(context.server_factory_context_, timeSource()); + EXPECT_CALL(context.server_factory_context_, api()); EXPECT_CALL(context, initManager()).Times(2); EXPECT_CALL(context, getTransportSocketFactoryContext()); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -70,14 +72,16 @@ validity_url: "/.sxg/validity.msg" TestUtility::loadFromYaml(yaml, *proto_config); NiceMock context; - auto& secret_manager = context.cluster_manager_.cluster_manager_factory_.secretManager(); + auto& secret_manager = + context.server_factory_context_.cluster_manager_.cluster_manager_factory_.secretManager(); ON_CALL(secret_manager, findStaticGenericSecretProvider( failed_secret_name == "private_key" ? "certificate" : "private_key")) .WillByDefault(Return(std::make_shared( envoy::extensions::transport_sockets::tls::v3::GenericSecret()))); - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(*proto_config, "stats", context), - EnvoyException, exception_message); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, exception_message); } } // namespace diff --git a/contrib/vcl/source/vcl_io_handle.cc b/contrib/vcl/source/vcl_io_handle.cc index 5a0c3b45c657..0855ca6cd82f 100644 --- a/contrib/vcl/source/vcl_io_handle.cc +++ b/contrib/vcl/source/vcl_io_handle.cc @@ -113,17 +113,13 @@ void vclEndptFromAddress(vppcom_endpt_t& endpt, Api::IoCallUint64Result vclCallResultToIoCallResult(const int32_t result) { if (result >= 0) { // Return nullptr as IoError upon success. - return Api::IoCallUint64Result( - result, Api::IoErrorPtr(nullptr, Envoy::Network::IoSocketError::deleteIoError)); + return {static_cast(result), Api::IoError::none()}; } RELEASE_ASSERT(result != VPPCOM_EINVAL, "Invalid argument passed in."); - return Api::IoCallUint64Result( - /*rc=*/0, (result == VPPCOM_EAGAIN - // EAGAIN is frequent enough that its memory allocation should be avoided. - ? Api::IoErrorPtr(Envoy::Network::IoSocketError::getIoSocketEagainInstance(), - Envoy::Network::IoSocketError::deleteIoError) - : Api::IoErrorPtr(new Envoy::Network::IoSocketError(-result), - Envoy::Network::IoSocketError::deleteIoError))); + return {/*rc=*/0, (result == VPPCOM_EAGAIN + // EAGAIN is frequent enough that its memory allocation should be avoided. + ? Envoy::Network::IoSocketError::getIoSocketEagainError() + : Envoy::Network::IoSocketError::create(-result))}; } } // namespace @@ -135,16 +131,24 @@ VclIoHandle::~VclIoHandle() { } Api::IoCallUint64Result VclIoHandle::close() { - VCL_LOG("closing sh {:x}", sh_); - RELEASE_ASSERT(VCL_SH_VALID(sh_), "sh must be valid"); + int wrk_index = vclWrkIndexOrRegister(); int rc = 0; - int wrk_index = vclWrkIndexOrRegister(); + VCL_LOG("closing sh {:x}", sh_); + + if (!VCL_SH_VALID(sh_)) { + ENVOY_LOG_MISC(info, "[{}] sh {:x} already closed is_listener {} isWrkListener{}", wrk_index, + sh_, is_listener_, isWrkListener()); + return {static_cast(rc), Api::IoError::none()}; + } if (is_listener_) { + ENVOY_LOG_MISC(info, "[{}] destroying listener sh {}", wrk_index, sh_); if (wrk_index) { - uint32_t sh = wrk_listener_->sh(); - RELEASE_ASSERT(wrk_index == vppcom_session_worker(sh), "listener close on wrong thread"); + if (wrk_listener_ != nullptr) { + uint32_t sh = wrk_listener_->sh(); + RELEASE_ASSERT(wrk_index == vppcom_session_worker(sh), "listener close on wrong thread"); + } clearChildWrkListener(); // sh_ not invalidated yet, waiting for destructor on main to call `vppcom_session_close` } else { @@ -157,8 +161,7 @@ Api::IoCallUint64Result VclIoHandle::close() { VCL_SET_SH_INVALID(sh_); } - return Api::IoCallUint64Result( - rc, Api::IoErrorPtr(nullptr, Envoy::Network::IoSocketError::deleteIoError)); + return {static_cast(rc), Api::IoError::none()}; } bool VclIoHandle::isOpen() const { return VCL_SH_VALID(sh_); } diff --git a/distribution/BUILD b/distribution/BUILD index b8d95a20e804..578f6de6f3b3 100644 --- a/distribution/BUILD +++ b/distribution/BUILD @@ -68,6 +68,7 @@ sh_binary( "$(location :distrotest.sh)", VERSION, "$(location :distros.yaml)", + "--rebuild", ], data = [ ":distros.yaml", @@ -96,6 +97,28 @@ label_flag( build_setting_default = "//distribution:custom/arm64/bin/release.tar.zst", ) +genrule( + name = "multi_arch_debs", + outs = ["multiarch-debs.tar.gz"], + # To ensure the debs tarball is not extracted and kept as a tarball, it is + # placed into a 2nd archive. + cmd = """ + tmpdir=$$(mktemp -d) \ + && tmpdir2=$$(mktemp -d) \ + && tar xf $(location :x64-packages) -C "$$tmpdir" \ + && tar xf $(location :arm64-packages) -C "$$tmpdir" \ + && rm "$${tmpdir}/signing.key" \ + && mv "$${tmpdir}/deb/"* "$${tmpdir}" \ + && rm -rf "$${tmpdir}/deb/" \ + && tar cf $$tmpdir2/debs.tar.gz -C "$${tmpdir}" . \ + && tar cf $@ -C "$${tmpdir2}" . \ + """, + tools = [ + ":arm64-packages", + ":x64-packages", + ], +) + genrule( name = "signed", outs = ["release.signed.tar.zst"], @@ -103,8 +126,7 @@ genrule( # Sign the packages VERSION=%s \ && $(location //tools/distribution:sign) \ - "deb.x64:$(location :x64-packages)" \ - "deb.arm64:$(location :arm64-packages)" \ + "bin:$(location :multi_arch_debs)" \ "x64:$(location :x64-release)" \ "arm64:$(location :arm64-release)" \ -m x64/envoy:bin/envoy-$${VERSION}-linux-x86_64 \ @@ -115,9 +137,8 @@ genrule( """ % VERSION, tags = ["no-remote"], tools = [ - ":arm64-packages", ":arm64-release", - ":x64-packages", + ":multi_arch_debs", ":x64-release", "//tools/distribution:sign", ], diff --git a/distribution/debian/packages.bzl b/distribution/debian/packages.bzl index ac1834e69021..19ecc8a3fa5b 100644 --- a/distribution/debian/packages.bzl +++ b/distribution/debian/packages.bzl @@ -10,7 +10,7 @@ def envoy_pkg_deb( description = "Envoy built for Debian/Ubuntu", preinst = "//distribution/debian:preinst", postinst = "//distribution/debian:postinst", - supported_distributions = "bullseye focal jammy", + supported_distributions = "bookworm bullseye focal jammy", architecture = select({ "//bazel:x86": "amd64", "//conditions:default": "arm64", @@ -49,7 +49,7 @@ def envoy_pkg_deb( output_group = "deb", ) -def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":envoy-bin-files", config = ":envoy-config"): +def envoy_pkg_debs(name, version, release_version, maintainer, bin_files, contrib_bin_files, config = ":envoy-config"): """Package the Envoy .debs with their .changes files. Packages are created for the version *and* the release version, eg @@ -57,7 +57,8 @@ def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":env - envoy_1.21.0_amd64.deb - envoy-1.21_1.21.0_amd64.deb - This way packages are available for both "envoy" and "envoy-1.21" in package managers. + This way packages are available for both "envoy" and "envoy-1.21" in package managers, and users can install either + a specifically versioned package, or the latest for that minor version. """ # generate deb data for all packages @@ -71,6 +72,17 @@ def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":env remap_paths = {"/copyright": "/usr/share/doc/envoy/copyright"}, ) + # generate deb data for all contrib packages + pkg_tar( + name = "contrib-deb-data", + srcs = [ + "//distribution/debian:copyright", + config, + contrib_bin_files, + ], + remap_paths = {"/copyright": "/usr/share/doc/envoy/copyright"}, + ) + # generate package for this patch version envoy_pkg_deb( name = "envoy", @@ -89,6 +101,24 @@ def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":env maintainer = maintainer, ) + # generate contrib package for this patch version + envoy_pkg_deb( + name = "envoy-contrib", + data = ":contrib-deb-data", + version = version, + maintainer = maintainer, + ) + + # generate contrib package for this minor version + envoy_pkg_deb( + name = "envoy-contrib-%s" % release_version, + data = ":contrib-deb-data", + version = version, + conflicts = ["envoy"], + provides = ["envoy"], + maintainer = maintainer, + ) + pkg_tar( name = name, srcs = [ @@ -96,6 +126,10 @@ def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":env "envoy.deb", "envoy-%s.changes" % release_version, "envoy-%s.deb" % release_version, + "envoy-contrib.changes", + "envoy-contrib.deb", + "envoy-contrib-%s.changes" % release_version, + "envoy-contrib-%s.deb" % release_version, ], extension = "tar", package_dir = "deb", diff --git a/distribution/distros.yaml b/distribution/distros.yaml index 6dc239ad27a1..40c54e657a50 100644 --- a/distribution/distros.yaml +++ b/distribution/distros.yaml @@ -2,6 +2,10 @@ debian_bullseye: image: debian:bullseye-slim ext: bullseye.changes +debian_bookworm: + image: debian:bookworm-slim + ext: bookworm.changes + ubuntu_focal: image: ubuntu:20.04 ext: focal.changes diff --git a/distribution/dockerhub/BUILD b/distribution/dockerhub/BUILD index cb48d42a20fd..599775efdf68 100644 --- a/distribution/dockerhub/BUILD +++ b/distribution/dockerhub/BUILD @@ -1,10 +1,13 @@ load("//bazel:envoy_build_system.bzl", "envoy_package") load("//tools/base:envoy_python.bzl", "envoy_gencontent") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + envoy_gencontent( name = "readme", srcs = ["@envoy_repo//:project"], diff --git a/distribution/packages.bzl b/distribution/packages.bzl index dacd54829fb8..4690bb7ee90f 100644 --- a/distribution/packages.bzl +++ b/distribution/packages.bzl @@ -12,6 +12,7 @@ def _release_version_for(version): def envoy_pkg_distros( name, envoy_bin = ":envoy-binary", + envoy_contrib_bin = ":envoy-contrib-binary", version = None, maintainer = None, config = "//configs:envoyproxy_io_proxy.yaml"): @@ -31,10 +32,19 @@ def envoy_pkg_distros( renames = {envoy_bin: "/usr/bin/envoy"}, ) + pkg_files( + name = "envoy-contrib-bin-files", + srcs = [envoy_contrib_bin], + attributes = pkg_attributes(mode = "0755"), + renames = {envoy_contrib_bin: "/usr/bin/envoy"}, + ) + # build debs envoy_pkg_debs( name = "debs", version = version, + bin_files = ":envoy-bin-files", + contrib_bin_files = ":envoy-contrib-bin-files", release_version = _release_version_for(version), maintainer = maintainer, ) @@ -43,9 +53,7 @@ def envoy_pkg_distros( pkg_tar( name = "distro_packages", extension = "tar", - deps = [ - ":debs", - ], + deps = [":debs"], ) # sign the packages diff --git a/docs/BUILD b/docs/BUILD index 555ce065e1d9..4d6f5a186b57 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -31,6 +31,10 @@ filegroup( # "Error: unable to read file: /etc/ssl/certs/ca-certificates.crt" "root/configuration/http/http_filters/_include/dns-cache-circuit-breaker.yaml", "root/configuration/other_features/_include/dlb.yaml", + "root/configuration/other_features/_include/hyperscan_matcher.yaml", + "root/configuration/other_features/_include/hyperscan_matcher_multiple.yaml", + "root/configuration/other_features/_include/hyperscan_regex_engine.yaml", + "root/configuration/other_features/_include/qatzip.yaml", "root/intro/arch_overview/security/_include/ssl.yaml", "root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml", "root/configuration/overview/_include/xds_api/oauth-sds-example.yaml", @@ -121,7 +125,7 @@ genrule( srcs = [":empty_extensions.json"], outs = ["empty_protos_rst.tar.gz"], cmd = """ - $(location //tools/protodoc:generate_empty) \\ + $(location //tools/protodoc:generate_empty) \ $(location empty_extensions.json) $@ """, tools = ["//tools/protodoc:generate_empty"], @@ -178,7 +182,7 @@ genrule( name = "version_histories", outs = ["version_histories.tar.gz"], cmd = """ - $(location //tools/docs:generate_version_histories) $@ + $(location //tools/docs:generate_version_histories) --path=$$(dirname $(location //:VERSION.txt)) $@ """, tools = [ ":versions.yaml", @@ -246,14 +250,18 @@ pkg_tar( genrule( name = "html_release", outs = ["html_release.tar.gz"], + # BUILD_SHA must be set in release builds + # The Envoy workspace will provide this on stamped builds. For external builds + # you must either pass an env var or pass it through the workspace's status. cmd = """ . $(location //bazel:volatile_env) \ + && _BUILD_SHA=$${BUILD_DOCS_SHA:-$${ENVOY_BUILD_SCM_REVISION:-$${{BUILD_SCM_REVISION}}} \ && $(location //tools/docs:sphinx_runner) \ $${SPHINX_RUNNER_ARGS:-} \ - --build_sha="$${BUILD_DOCS_SHA:-$${BUILD_SCM_REVISION}}" \ + --build_sha="$$_BUILD_SHA" \ --docs_tag="$${BUILD_DOCS_TAG:-}" \ --version_file=$(location //:VERSION.txt) \ - --descriptor_path=$(location @envoy_api//:v3_proto_set) \\ + --descriptor_path=$(location @envoy_api//:v3_proto_set) \ $(location rst) \ $@ """, @@ -274,6 +282,7 @@ genrule( cmd = """ $(location //tools/docs:sphinx_runner) \ $${SPHINX_RUNNER_ARGS:-} \ + --build_sha="$${BUILD_DOCS_SHA:-}" \ --version_file=$(location //:VERSION.txt) \ --descriptor_path=$(location @envoy_api//:v3_proto_set) \ $(location :rst) \ @@ -286,3 +295,8 @@ genrule( "@envoy_api//:v3_proto_set", ], ) + +alias( + name = "docs", + actual = ":html_release", +) diff --git a/docs/README.md b/docs/README.md index 923ae33a4bb8..070015e80dff 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,15 +8,23 @@ In both cases, the generated output can be found in `generated/docs`. If you have an [existing Envoy development environment](https://github.com/envoyproxy/envoy/tree/main/bazel#quick-start-bazel-build-for-developers), you should have the necessary dependencies and requirements and be able to build the documentation directly. +If using the Docker build container, you can run: + ```bash -./docs/build.sh +./ci/do_ci.sh docs ``` By default configuration examples are going to be validated during build. To disable validation, set `SPHINX_SKIP_CONFIG_VALIDATION` environment variable to `true`: ```bash -SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +SPHINX_SKIP_CONFIG_VALIDATION=true ./ci/do_ci.sh docs +``` + +If not using the Docker build container, you can run: + +```bash +bazel run --//tools/tarball:target=//docs:html //tools/tarball:unpack "$PWD"/generated/docs/ ``` ## Using the Docker build container to build the documentation @@ -27,7 +35,7 @@ image that is used in continuous integration. This can be done as follows: ``` -./ci/run_envoy_docker.sh 'docs/build.sh' +./ci/run_envoy_docker.sh './ci/do_ci.sh docs' ``` To use this method you will need a minimum of 4-5GB of disk space available to accommodate the build image. @@ -43,10 +51,9 @@ To do this: # How the Envoy website and docs are updated -1. The docs are published to [docs/envoy/latest](https://github.com/envoyproxy/envoy-website/tree/main/docs/envoy/latest) - on every commit to main. This process is handled by Azure Pipelines with the - [`publish.sh`](https://github.com/envoyproxy/envoy/blob/main/docs/publish.sh) script. -2. The docs are published to [docs/envoy](https://github.com/envoyproxy/envoy-website/tree/main/docs/envoy) - in a directory named after every tagged commit in this repo. Thus, on every tagged release there - are snapped docs. +The docs are published dynamically by Netlify on every commit to main. This process is handled by the +[envoy-website repo](https://github.com/envoyproxy/envoy-website) + +For tagged commits the docs are built statically by the [archive repo](https://github.com/envoyproxy/archive), +which in turn triggers a rebuild of the website. diff --git a/docs/build.sh b/docs/build.sh index 09af4f70d7fe..20089b3a2b6d 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,87 +1,5 @@ #!/usr/bin/env bash -# set SPHINX_SKIP_CONFIG_VALIDATION environment variable to true to skip -# validation of configuration examples - -set -e - - -if [[ ! $(command -v bazel) ]]; then - # shellcheck disable=SC2016 - echo 'ERROR: bazel must be installed and available in "$PATH" to build docs' >&2 - exit 1 -fi - -VERSION="$(cat VERSION.txt)" -MAIN_BRANCH="refs/heads/main" -DEV_VERSION_REGEX="-dev$" - -# default is to build html only -BUILD_TYPE=html - -if [[ "$VERSION" =~ $DEV_VERSION_REGEX ]]; then - if [[ "$AZP_BRANCH" == "$MAIN_BRANCH" ]]; then - # no need to build html, just rst - BUILD_TYPE=rst - fi -else - export BUILD_DOCS_TAG="v${VERSION}" - echo "BUILD AZP RELEASE BRANCH ${BUILD_DOCS_TAG}" - BAZEL_BUILD_OPTIONS+=("--action_env=BUILD_DOCS_TAG") -fi - -# This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files. -IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}" -IFS=" " read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTION_LIST:-}" - -# We want the binary at the end -BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel) - -if [[ "${AZP_BRANCH}" =~ ^refs/pull ]]; then - # For PRs use the unmerged PR commit in the version string. - # - # Staged/built docs still use the merged sha in the URL to distinguish builds - # - export BUILD_DOCS_SHA="${AZP_COMMIT_SHA}" - BAZEL_BUILD_OPTIONS+=("--action_env=BUILD_DOCS_SHA") -fi - -if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then - export SPHINX_RUNNER_ARGS="-v warn" - BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS") -fi - -# Building html/rst is determined by then needs of CI but can be overridden in dev. -if [[ "${BUILD_TYPE}" == "html" ]] || [[ -n "${DOCS_BUILD_HTML}" ]]; then - BUILD_HTML=1 - BUILD_HTML_TARGET="//docs:html" - BUILD_HTML_TARBALL="bazel-bin/docs/html.tar.gz" - if [[ -n "${AZP_BRANCH}" ]] || [[ -n "${DOCS_BUILD_RELEASE}" ]]; then - # CI build - use git sha - BUILD_HTML_TARGET="//docs:html_release" - BUILD_HTML_TARBALL="bazel-bin/docs/html_release.tar.gz" - fi -fi -if [[ "${BUILD_TYPE}" == "rst" ]] || [[ -n "${DOCS_BUILD_RST}" ]]; then - BUILD_RST=1 -fi - -# Build html/rst -if [[ -n "${BUILD_RST}" ]]; then - bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst -fi -if [[ -n "${BUILD_HTML}" ]]; then - bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" "$BUILD_HTML_TARGET" -fi - -[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs -rm -rf "${DOCS_OUTPUT_DIR}" -mkdir -p "${DOCS_OUTPUT_DIR}" - -# Save html/rst to output directory -if [[ -n "${BUILD_HTML}" ]]; then - tar -xzf "$BUILD_HTML_TARBALL" -C "$DOCS_OUTPUT_DIR" -fi -if [[ -n "${BUILD_RST}" ]]; then - cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz -fi +# shellcheck disable=SC2016 +echo 'This script has been removed. Please use `ci/do_ci.sh docs` instead' >&2 +exit 1 diff --git a/docs/inventories/v1.24/objects.inv b/docs/inventories/v1.24/objects.inv index 986bfcc604af..dc548419533f 100644 Binary files a/docs/inventories/v1.24/objects.inv and b/docs/inventories/v1.24/objects.inv differ diff --git a/docs/inventories/v1.25/objects.inv b/docs/inventories/v1.25/objects.inv index dd24b33b0039..5e31b50384a4 100644 Binary files a/docs/inventories/v1.25/objects.inv and b/docs/inventories/v1.25/objects.inv differ diff --git a/docs/inventories/v1.26/objects.inv b/docs/inventories/v1.26/objects.inv index c0b726818bb0..477894d1f6bc 100644 Binary files a/docs/inventories/v1.26/objects.inv and b/docs/inventories/v1.26/objects.inv differ diff --git a/docs/inventories/v1.27/objects.inv b/docs/inventories/v1.27/objects.inv new file mode 100644 index 000000000000..0b0acf5c7b42 Binary files /dev/null and b/docs/inventories/v1.27/objects.inv differ diff --git a/docs/inventories/v1.28/objects.inv b/docs/inventories/v1.28/objects.inv new file mode 100644 index 000000000000..c454862b2315 Binary files /dev/null and b/docs/inventories/v1.28/objects.inv differ diff --git a/docs/publish.sh b/docs/publish.sh deleted file mode 100755 index dee9cde4303e..000000000000 --- a/docs/publish.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# This pushes a prebuilt docs version to the website for published releases. - -set -e - -DOCS_DIR=generated/docs -CHECKOUT_DIR=envoy-docs -BUILD_SHA=$(git rev-parse HEAD) - -VERSION="$(cat VERSION.txt)" -PUBLISH_DIR="${CHECKOUT_DIR}/docs/envoy/v${VERSION}" -DOCS_MAIN_BRANCH="main" - -echo 'cloning' -git clone git@github.com:envoyproxy/envoy-website "${CHECKOUT_DIR}" -b "${DOCS_MAIN_BRANCH}" --depth 1 - -if [[ -e "$PUBLISH_DIR" ]]; then - # Defense against the unexpected. - echo 'Docs version already exists, not continuing!.' - exit 0 - # exit 1 -fi - -mkdir -p "$PUBLISH_DIR" -cp -r "$DOCS_DIR"/* "$PUBLISH_DIR" -cd "${CHECKOUT_DIR}" - -git config user.name "envoy-docs(Azure Pipelines)" -git config user.email envoy-docs@users.noreply.github.com - -git add . -git commit -m "docs envoy@$BUILD_SHA" -git push origin "${DOCS_MAIN_BRANCH}" diff --git a/docs/root/_configs/go/golang-with-per-route-config.yaml b/docs/root/_configs/go/golang-with-per-route-config.yaml index f44a43a907da..f91f8c9abcf9 100644 --- a/docs/root/_configs/go/golang-with-per-route-config.yaml +++ b/docs/root/_configs/go/golang-with-per-route-config.yaml @@ -20,6 +20,10 @@ static_resources: library_id: my-configurable-plugin-id library_path: "lib/my_configurable_plugin.so" plugin_name: my_configurable_plugin + plugin_config: + "@type": type.googleapis.com/xds.type.v3.TypedStruct + value: + foo: default_foo - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router diff --git a/docs/root/_configs/go/golang-with-per-virtualhost-config.yaml b/docs/root/_configs/go/golang-with-per-virtualhost-config.yaml index cfea9d10c68f..e7f1ec05f812 100644 --- a/docs/root/_configs/go/golang-with-per-virtualhost-config.yaml +++ b/docs/root/_configs/go/golang-with-per-virtualhost-config.yaml @@ -20,6 +20,10 @@ static_resources: library_id: my-configurable-plugin-id library_path: "lib/my_configurable_plugin.so" plugin_name: my_configurable_plugin + plugin_config: + "@type": type.googleapis.com/xds.type.v3.TypedStruct + value: + foo: default_foo - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router diff --git a/docs/root/_static/css/envoy.css b/docs/root/_static/css/envoy.css index c02a65b16df9..ae805c099312 100644 --- a/docs/root/_static/css/envoy.css +++ b/docs/root/_static/css/envoy.css @@ -46,6 +46,11 @@ table.docutils div.line-block { border: solid #eee 1px; } +/* restore margin bottom on aligned images */ +.rst-content img.align-center { + margin-bottom: 24px +} + /* suppress errs on pseudo-json code highlights */ .highlight-json .highlight .err { border: inherit; diff --git a/docs/root/api-v3/common_messages/common_messages.rst b/docs/root/api-v3/common_messages/common_messages.rst index 66d3389f738a..a1f3488a86d8 100644 --- a/docs/root/api-v3/common_messages/common_messages.rst +++ b/docs/root/api-v3/common_messages/common_messages.rst @@ -23,6 +23,7 @@ Common messages ../extensions/filters/common/dependency/v3/dependency.proto ../extensions/regex_engines/v3/google_re2.proto ../config/core/v3/grpc_method_list.proto + ../config/core/v3/http_service.proto ../config/core/v3/grpc_service.proto ../extensions/key_value/file_based/v3/config.proto ../config/common/key_value/v3/config.proto @@ -38,3 +39,4 @@ Common messages ../config/core/v3/socket_option.proto ../config/core/v3/substitution_format_string.proto ../config/core/v3/udp_socket_config.proto + ../extensions/filters/common/set_filter_state/v3/value.proto diff --git a/docs/root/api-v3/config/cluster_specifier/cluster_specifier.rst b/docs/root/api-v3/config/cluster_specifier/cluster_specifier.rst new file mode 100644 index 000000000000..6202887d3121 --- /dev/null +++ b/docs/root/api-v3/config/cluster_specifier/cluster_specifier.rst @@ -0,0 +1,8 @@ +Cluster specifier +================= + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../extensions/router/cluster_specifiers/*/v3/* diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index ca859c388e68..12d216da67c8 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -27,6 +27,7 @@ Extensions http/header_validators http/original_ip_detection http/stateful_session + geoip_provider/geoip_provider trace/trace internal_redirect/internal_redirect path/match/path_matcher @@ -44,3 +45,4 @@ Extensions wasm/wasm watchdog/watchdog load_balancing_policies/load_balancing_policies + cluster_specifier/cluster_specifier diff --git a/docs/root/api-v3/config/contrib/qat/qat.rst b/docs/root/api-v3/config/contrib/qat/qat.rst index 3aeab258356d..3e524d700b13 100644 --- a/docs/root/api-v3/config/contrib/qat/qat.rst +++ b/docs/root/api-v3/config/contrib/qat/qat.rst @@ -3,3 +3,4 @@ :maxdepth: 2 ../../../extensions/private_key_providers/qat/v3alpha/* + ../../../extensions/compression/qatzip/compressor/v3alpha/* diff --git a/docs/root/api-v3/config/filter/filter.rst b/docs/root/api-v3/config/filter/filter.rst index c076e80f6c28..c12835d917df 100644 --- a/docs/root/api-v3/config/filter/filter.rst +++ b/docs/root/api-v3/config/filter/filter.rst @@ -8,6 +8,7 @@ Filters listener/listener network/network udp/udp + udp/session http/http dubbo/dubbo thrift/thrift diff --git a/docs/root/api-v3/config/filter/udp/session.rst b/docs/root/api-v3/config/filter/udp/session.rst new file mode 100644 index 000000000000..d42591853d0d --- /dev/null +++ b/docs/root/api-v3/config/filter/udp/session.rst @@ -0,0 +1,8 @@ +UDP session filters +==================== + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../../extensions/filters/udp/udp_proxy/session/*/v3/* diff --git a/docs/root/api-v3/config/geoip_provider/geoip_provider.rst b/docs/root/api-v3/config/geoip_provider/geoip_provider.rst new file mode 100644 index 000000000000..2b9ad10b9f6a --- /dev/null +++ b/docs/root/api-v3/config/geoip_provider/geoip_provider.rst @@ -0,0 +1,10 @@ +.. _api-v3_config_geoip_providers: + +Geolocation providers +===================== + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../extensions/geoip_providers/*/v3/* diff --git a/docs/root/api-v3/config/trace/opentelemetry/resource_detectors.rst b/docs/root/api-v3/config/trace/opentelemetry/resource_detectors.rst new file mode 100644 index 000000000000..87790ac145ec --- /dev/null +++ b/docs/root/api-v3/config/trace/opentelemetry/resource_detectors.rst @@ -0,0 +1,10 @@ +OpenTelemetry Resource Detectors +================================ + +Resource detectors that can be configured with the OpenTelemetry Tracer: + +.. toctree:: + :glob: + :maxdepth: 3 + + ../../../extensions/tracers/opentelemetry/resource_detectors/v3/* diff --git a/docs/root/api-v3/config/trace/opentelemetry/samplers.rst b/docs/root/api-v3/config/trace/opentelemetry/samplers.rst new file mode 100644 index 000000000000..705155f640b9 --- /dev/null +++ b/docs/root/api-v3/config/trace/opentelemetry/samplers.rst @@ -0,0 +1,10 @@ +OpenTelemetry Samplers +====================== + +Samplers that can be configured with the OpenTelemetry Tracer: + +.. toctree:: + :glob: + :maxdepth: 3 + + ../../../extensions/tracers/opentelemetry/samplers/v3/* diff --git a/docs/root/api-v3/config/trace/trace.rst b/docs/root/api-v3/config/trace/trace.rst index 8f8d039a18d8..1bd09c1a1300 100644 --- a/docs/root/api-v3/config/trace/trace.rst +++ b/docs/root/api-v3/config/trace/trace.rst @@ -12,3 +12,5 @@ HTTP tracers :maxdepth: 2 v3/* + opentelemetry/resource_detectors + opentelemetry/samplers diff --git a/docs/root/configuration/advanced/advanced.rst b/docs/root/configuration/advanced/advanced.rst index d84ba2f53225..c330abbab617 100644 --- a/docs/root/configuration/advanced/advanced.rst +++ b/docs/root/configuration/advanced/advanced.rst @@ -6,3 +6,4 @@ Advanced well_known_dynamic_metadata well_known_filter_state + metadata_configurations diff --git a/docs/root/configuration/advanced/metadata_configurations.rst b/docs/root/configuration/advanced/metadata_configurations.rst new file mode 100644 index 000000000000..cbb8f473b790 --- /dev/null +++ b/docs/root/configuration/advanced/metadata_configurations.rst @@ -0,0 +1,19 @@ +.. _metadata_configurations: + +Metadata configurations +======================= + +Envoy utilizes :ref:`metadata ` to transport arbitrary untyped or typed +data from the control plane to Envoy. Metadata configurations can be applied to Listeners, clusters, routes, virtual hosts, +endpoints, and other elements. + + +Unlike other configurations, Envoy does not explicitly define the purpose of metadata configurations, which can be used for +stats, logging, or filter/extension behavior. Users can define the purpose of metadata configurations for their specific +use cases. Metadata configurations offer a flexible way to transport user-defined data from the control plane to Envoy without +modifying Envoy's core API or implementation. + + +For instance, users can add extra attributes to routes, such as the route owner or upstream service maintainer, to metadata. +They can then enable Envoy to log these attributes to the access log or report them to StatsD, among other possibilities. +Moreover, users can write a filter/extension to read these attributes and execute any specific logic. diff --git a/docs/root/configuration/advanced/well_known_dynamic_metadata.rst b/docs/root/configuration/advanced/well_known_dynamic_metadata.rst index 9d89ae70c28b..8b21caa3941a 100644 --- a/docs/root/configuration/advanced/well_known_dynamic_metadata.rst +++ b/docs/root/configuration/advanced/well_known_dynamic_metadata.rst @@ -31,6 +31,7 @@ The following Envoy filters can be configured to consume dynamic metadata emitte * :ref:`External Authorization Filter via the metadata context namespaces ` * :ref:`RateLimit Filter limit override ` +* :ref:`Original destination listener filter ` .. _shared_dynamic_metadata: diff --git a/docs/root/configuration/advanced/well_known_filter_state.rst b/docs/root/configuration/advanced/well_known_filter_state.rst index 98aca0ec75e8..640520bb10b5 100644 --- a/docs/root/configuration/advanced/well_known_filter_state.rst +++ b/docs/root/configuration/advanced/well_known_filter_state.rst @@ -5,32 +5,63 @@ Well Known Filter State Objects The following list of filter state objects are consumed by Envoy extensions: -.. list-table:: - :widths: auto - :header-rows: 1 - :stub-columns: 1 - - * - **Filter state key** - - **Purpose** - * - ``envoy.tcp_proxy.cluster`` - - | :ref:`TCP proxy ` dynamic cluster name selection - | on a per-connection basis. - | Accepts a cluster name as a constructor. - * - ``envoy.network.transport_socket.original_dst_address`` - - | :ref:`Original destination cluster ` dynamic address selection. - | Accepts an `IP:PORT` string as a constructor. - | Fields: - | - ``ip``: IP address value as a string; - | - ``port``: port value as a number. - * - ``envoy.upstream.dynamic_host`` - - | :ref:`Dynamic forward proxy ` - | upstream host override on a per-connection basis. - | Accepts a host string as a constructor. - * - ``envoy.upstream.dynamic_port`` - - | :ref:`Dynamic forward proxy ` - | upstream port override on a per-connection basis. - | Accepts a port number string as a constructor. +``envoy.network.upstream_server_name`` + Sets the transport socket option to override the `SNI `_ in + the upstream connections. Accepts a host name as a constructor, e.g. "lyft.com". +``envoy.network.application_protocols`` + Sets the transport socket option to override the `ALPN `_ list in the upstream connections. This setting takes precedence over the upstream cluster configuration. + Accepts a comma-separated list of protocols as a constructor, e.g. "h2,http/1.1". + +``envoy.network.upstream_subject_alt_names`` + Enables additional verification of the upstream peer certificate SAN names. Accepts a comma-separated list of SAN + names as a constructor. + +``envoy.tcp_proxy.cluster`` + :ref:`TCP proxy ` dynamic cluster name selection on a per-connection basis. Accepts + a cluster name as a constructor. + +``envoy.network.transport_socket.original_dst_address`` + :ref:`Original destination cluster ` dynamic address + selection. Accepts an `IP:PORT` string as a constructor. Fields: + + * ``ip``: IP address value as a string; + * ``port``: port value as a number. + +``envoy.filters.listener.original_dst.local_ip`` + :ref:`Original destination listener filter ` destination address selection for + the internal listeners. Accepts an `IP:PORT` string as a constructor. Fields: + + * ``ip``: IP address value as a string; + * ``port``: port value as a number. + +``envoy.filters.listener.original_dst.remote_ip`` + :ref:`Original destination listener filter ` source address selection for the + internal listeners. Accepts an `IP:PORT` string as a constructor. Fields: + + * ``ip``: IP address value as a string; + * ``port``: port value as a number. + +``envoy.upstream.dynamic_host`` + :ref:`Dynamic forward proxy ` upstream + host override on a per-connection basis. Accepts a host string as a constructor. + +``envoy.upstream.dynamic_port`` + :ref:`Dynamic forward proxy ` upstream + port override on a per-connection basis. Accepts a port number string as a constructor. + +``envoy.tcp_proxy.disable_tunneling`` + :ref:`TCP proxy tunneling override + ` to disable tunneling on a + per-connection bases. Accepts values "true" and "false". + +``envoy.filters.network.http_connection_manager.local_reply_owner`` + Shared filter status for logging which filter config name in the HTTP filter chain sent the local reply. + + +Filter state object fields +-------------------------- The filter state object fields can be used in the format strings. For example, the following format string references the port number in the original diff --git a/docs/root/configuration/http/cluster_specifier/cluster_specifier.rst b/docs/root/configuration/http/cluster_specifier/cluster_specifier.rst index 54e3d14fc1ef..c3c799abf131 100644 --- a/docs/root/configuration/http/cluster_specifier/cluster_specifier.rst +++ b/docs/root/configuration/http/cluster_specifier/cluster_specifier.rst @@ -7,3 +7,4 @@ HTTP cluster specifier :maxdepth: 2 golang + lua diff --git a/docs/root/configuration/http/cluster_specifier/lua.rst b/docs/root/configuration/http/cluster_specifier/lua.rst new file mode 100644 index 000000000000..0cab42c25cd0 --- /dev/null +++ b/docs/root/configuration/http/cluster_specifier/lua.rst @@ -0,0 +1,92 @@ +.. _config_http_cluster_specifier_lua: + +Lua cluster specifier +===================== + +Overview +-------- + +The HTTP Lua cluster specifier allows `Lua `_ scripts to select router cluster +during the request flows. `LuaJIT `_ is used as the runtime. Because of this, the +supported Lua version is mostly 5.1 with some 5.2 features. See the `LuaJIT documentation +`_ for more details. + +Configuration +------------- + +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.router.cluster_specifiers.lua.v3.LuaConfig``. +* :ref:`v3 API reference ` + +A simple example of configuring Lua cluster specifier is as follow: + +.. code-block:: yaml + + routes: + - match: + prefix: "/" + route: + inline_cluster_specifier_plugin: + extension: + name: envoy.router.cluster_specifier_plugin.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.router.cluster_specifiers.lua.v3.LuaConfig + source_code: + inline_string: | + function envoy_on_route(route_handle) + local header_value = route_handle:headers():get("header_key") + if header_value == "fake" then + return "fake_service" + end + return "web_service" + end + default_cluster: web_service + +Lua script defined in ``source_code`` will be executed to select router cluster, and just as cluster specifier +plugin in C++, Lua script can also select router cluster based on request headers. If Lua script execute failure, +``default_cluster`` will be used. + +Complete example +---------------- + +A complete example using Docker is available in :repo:`/examples/lua-cluster-specifier`. + +Route handle API +---------------- + +When Envoy loads the script in the configuration, it looks for a global function that the script defines: + +.. code-block:: lua + + function envoy_on_route(route_handle) + end + +During the route path, Envoy will run *envoy_on_route* as a coroutine, passing a handle to the route API. + +The following methods on the stream handle are supported: + +headers() +^^^^^^^^^ + +.. code-block:: lua + + local headers = handle:headers() + +Returns the stream's headers. The headers can be used to match to select a specific cluster. + +Returns a :ref:`header object `. + +.. _config_lua_cluster_specifier_header_wrapper: + +Header object API +----------------- + +get() +^^^^^ + +.. code-block:: lua + + headers:get(key) + +Gets a header. *key* is a string that supplies the header key. Returns a string that is the header +value or nil if there is no such header. If there are multiple headers in the same case-insensitive +key, their values will be combined with a *,* separator and returned as a string. diff --git a/docs/root/configuration/http/http_filters/_include/aws_credentials.rst b/docs/root/configuration/http/http_filters/_include/aws_credentials.rst index 83fd1aea5bd0..2253db26c2e5 100644 --- a/docs/root/configuration/http/http_filters/_include/aws_credentials.rst +++ b/docs/root/configuration/http/http_filters/_include/aws_credentials.rst @@ -11,6 +11,26 @@ secret access key (the session token is optional). the file ``~/.aws/credentials`` and profile ``default`` are used. The fields ``aws_access_key_id``, ``aws_secret_access_key``, and ``aws_session_token`` defined for the profile in the credentials file are used. These credentials are cached for 1 hour. -3. Either EC2 instance metadata or ECS task metadata. For EC2 instance metadata, the fields ``AccessKeyId``, ``SecretAccessKey``, and +3. From `AssumeRoleWithWebIdentity `_ API call + towards AWS Security Token Service using ``WebIdentityToken`` read from a file pointed by ``AWS_WEB_IDENTITY_TOKEN_FILE`` environment + variable and role arn read from ``AWS_ROLE_ARN`` environment variable. The credentials are extracted from the fields ``AccessKeyId``, + ``SecretAccessKey``, and ``SessionToken`` are used, and credentials are cached for 1 hour or until they expire (according to the field + ``Expiration``). To enable this credentials provider set ``envoy.reloadable_features.use_http_client_to_fetch_aws_credentials`` to ``true`` + so that it can use http async client to fetch the credentials. This provider is not compatible with :ref:`Grpc Credentials AWS AwsIamConfig + ` plugin which can only support deprecated libcurl credentials fetcher + , see https://github.com/envoyproxy/envoy/pull/30626. To fetch the credentials a static cluster is required with the name + ``sts_token_service_internal`` pointing towards regional AWS Security Token Service. The static internal cluster will still be added even + if initially ``envoy.reloadable_features.use_http_client_to_fetch_aws_credentials`` is not set so that subsequently if the reloadable feature + is set to ``true`` the cluster config is available to fetch the credentials. + +4. Either EC2 instance metadata or ECS task metadata. For EC2 instance metadata, the fields ``AccessKeyId``, ``SecretAccessKey``, and ``Token`` are used, and credentials are cached for 1 hour. For ECS task metadata, the fields ``AccessKeyId``, ``SecretAccessKey``, and - ``Token`` are used, and credentials are cached for 1 hour or until they expire (according to the field ``Expiration``). + ``Token`` are used, and credentials are cached for 1 hour or until they expire (according to the field ``Expiration``). Note that the + latest update on AWS credentials provider utility provides an option to use http async client functionality instead of libcurl to fetch the + credentials. This behavior can be changed by setting ``envoy.reloadable_features.use_http_client_to_fetch_aws_credentials`` to ``true``. + The usage of libcurl is on the deprecation path and will be removed soon. To fetch the credentials from either EC2 instance + metadata or ECS task metadata a static cluster is required pointing towards the credentials provider. The static cluster name has to be + ``ec2_instance_metadata_server_internal`` for fetching from EC2 instance metadata or ``ecs_task_metadata_server_internal`` for fetching + from ECS task metadata. If these clusters are not provided in the bootstrap configuration then either of these will be added by default. + The static internal cluster will still be added even if initially ``envoy.reloadable_features.use_http_client_to_fetch_aws_credentials`` is + not set so that subsequently if the reloadable feature is set to ``true`` the cluster config is available to fetch the credentials. diff --git a/docs/root/configuration/http/http_filters/_include/compressor-filter-request-response.yaml b/docs/root/configuration/http/http_filters/_include/compressor-filter-request-response.yaml new file mode 100644 index 000000000000..3d1667c133ad --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/compressor-filter-request-response.yaml @@ -0,0 +1,80 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: app + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: service + http_filters: + # This filter is only enabled for responses. + - name: envoy.filters.http.compressor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor + request_direction_config: + common_config: + enabled: + default_value: false + runtime_key: request_compressor_enabled + compressor_library: + name: for_response + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip + memory_level: 3 + window_bits: 10 + compression_level: BEST_COMPRESSION + compression_strategy: DEFAULT_STRATEGY + # This filter is only enabled for requests. + - name: envoy.filters.http.compressor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor + response_direction_config: + common_config: + enabled: + default_value: false + runtime_key: response_compressor_enabled + request_direction_config: + common_config: + enabled: + default_value: true + runtime_key: request_compressor_enabled + compressor_library: + name: for_request + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip + memory_level: 9 + window_bits: 15 + compression_level: BEST_SPEED + compression_strategy: DEFAULT_STRATEGY + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: service + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service + port_value: 8000 diff --git a/docs/root/configuration/http/http_filters/_include/compressor-filter.yaml b/docs/root/configuration/http/http_filters/_include/compressor-filter.yaml new file mode 100644 index 000000000000..7c3a2215a0e4 --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/compressor-filter.yaml @@ -0,0 +1,72 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + typed_per_filter_config: + envoy.filters.http.compression: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute + disabled: true + routes: + - match: { prefix: "/static" } + route: { cluster: service } + typed_per_filter_config: + envoy.filters.http.compression: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute + overrides: + response_direction_config: + - match: { prefix: "/" } + route: { cluster: service } + http_filters: + - name: envoy.filters.http.compressor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor + response_direction_config: + common_config: + min_content_length: 100 + content_type: + - text/html + - application/json + disable_on_etag_header: true + request_direction_config: + common_config: + enabled: + default_value: false + runtime_key: request_compressor_enabled + compressor_library: + name: text_optimized + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip + memory_level: 3 + window_bits: 10 + compression_level: BEST_COMPRESSION + compression_strategy: DEFAULT_STRATEGY + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: service + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service + port_value: 8000 diff --git a/docs/root/configuration/http/http_filters/basic_auth_filter.rst b/docs/root/configuration/http/http_filters/basic_auth_filter.rst new file mode 100644 index 000000000000..da8b160fd054 --- /dev/null +++ b/docs/root/configuration/http/http_filters/basic_auth_filter.rst @@ -0,0 +1,46 @@ +.. _config_http_filters_basic_auth: + +Basic Auth +========== + +This HTTP filter can be used to authenticate user credentials in the HTTP Authentication header defined +in `RFC7617 `. + +The filter will extract the username and password from the HTTP Authentication header and verify them +against the configured username and password list. + +If the username and password are valid, the request will be forwared to the next filter in the filter chains. +If they're invalid or not provided in the HTTP request, the request will be denied with a 401 Unauthorized response. + +Configuration +------------- + +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth``. +* :ref:`v3 API reference ` + +``users`` is a list of username-password pairs used to verify user credentials in the "Authorization" header. + The value needs to be the `htpasswd ` format. + + +An example configuration of the filter may look like the following: + +.. code-block:: yaml + + users: + inline_string: |- + user1:{SHA}hashed_user1_password + user2:{SHA}hashed_user2_password + +Note that only SHA format is currently supported. Other formats may be added in the future. + +Statistics +---------- + +The HTTP basic auth filter outputs statistics in the ``http..basic_auth.`` namespace. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + allowed, Counter, Total number of allowed requests + denied, Counter, Total number of denied requests diff --git a/docs/root/configuration/http/http_filters/cdn_loop_filter.rst b/docs/root/configuration/http/http_filters/cdn_loop_filter.rst index 5e36eccd68ce..0ea3e1f1778f 100644 --- a/docs/root/configuration/http/http_filters/cdn_loop_filter.rst +++ b/docs/root/configuration/http/http_filters/cdn_loop_filter.rst @@ -7,7 +7,7 @@ The CDN-Loop header filter participates in the cross-CDN loop detection protocol 8586 `_. The CDN-Loop header filter performs two actions. First, the filter checks to see how many times a particular CDN identifier has appeared in the CDN-Loop header. Next, if the check passes, the filter then appends the CDN identifier to the -CDN-Loop header and passes the request to the next upstream filter. If the check fails, the filter +CDN-Loop header and passes the request to the next upstream HTTP filter. If the check fails, the filter stops processing on the request and returns an error response. RFC 8586 is particular in how the CDN-Loop header should be modified. As such: diff --git a/docs/root/configuration/http/http_filters/checksum_filter.rst b/docs/root/configuration/http/http_filters/checksum_filter.rst index f689c1b4cdfc..b6fce635759e 100644 --- a/docs/root/configuration/http/http_filters/checksum_filter.rst +++ b/docs/root/configuration/http/http_filters/checksum_filter.rst @@ -1,5 +1,5 @@ -.. _config_http_filters_language: +.. _config_http_filters_checksum: Checksum ======== diff --git a/docs/root/configuration/http/http_filters/composite_filter.rst b/docs/root/configuration/http/http_filters/composite_filter.rst index 669865dbde62..af8efc2288ea 100644 --- a/docs/root/configuration/http/http_filters/composite_filter.rst +++ b/docs/root/configuration/http/http_filters/composite_filter.rst @@ -3,11 +3,6 @@ Composite Filter ================ -.. attention:: - - The composite filter is in alpha and is currently under active development. - Capabilities will be expanded over time and the configuration structures are likely to change. - The composite filter allows delegating filter actions to a filter specified by a :ref:`match result `. The purpose of this is to allow different filters or filter configurations to be selected based on the incoming request, allowing for more dynamic diff --git a/docs/root/configuration/http/http_filters/compressor_filter.rst b/docs/root/configuration/http/http_filters/compressor_filter.rst index 9c3c0dba9a31..9bcc66fb4796 100644 --- a/docs/root/configuration/http/http_filters/compressor_filter.rst +++ b/docs/root/configuration/http/http_filters/compressor_filter.rst @@ -26,32 +26,11 @@ compression only. Other compression libraries can be supported as extensions. An example configuration of the filter may look like the following: -.. code-block:: yaml - - http_filters: - - name: envoy.filters.http.compressor - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor - response_direction_config: - common_config: - min_content_length: 100 - content_type: - - text/html - - application/json - disable_on_etag_header: true - request_direction_config: - common_config: - enabled: - default_value: false - runtime_key: request_compressor_enabled - compressor_library: - name: text_optimized - typed_config: - "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip - memory_level: 3 - window_bits: 10 - compression_level: BEST_COMPRESSION - compression_strategy: DEFAULT_STRATEGY +.. literalinclude:: _include/compressor-filter.yaml + :language: yaml + :linenos: + :lines: 33-56 + :caption: :download:`compressor-filter.yaml <_include/compressor-filter.yaml>` By *default* request compression is disabled, but when enabled it will be *skipped* if: @@ -132,27 +111,11 @@ Per-Route Configuration Response compression can be enabled and disabled on individual virtual hosts and routes. For example, to disable response compression for a particular virtual host, but enable response compression for its ``/static`` route: -.. code-block:: yaml - - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - typed_per_filter_config: - envoy.filters.http.compression: - "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute - disabled: true - routes: - - match: { prefix: "/static" } - route: { cluster: some_service } - typed_per_filter_config: - envoy.filters.http.compression: - "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.CompressorPerRoute - overrides: - response_direction_config: - - match: { prefix: "/" } - route: { cluster: some_service } +.. literalinclude:: _include/compressor-filter.yaml + :language: yaml + :linenos: + :lines: 14-32 + :caption: :download:`compressor-filter.yaml <_include/compressor-filter.yaml>` Using different compressors for requests and responses -------------------------------------------------------- @@ -160,48 +123,11 @@ Using different compressors for requests and responses If different compression libraries are desired for requests and responses, it is possible to install multiple compressor filters enabled only for requests or responses. For instance: -.. code-block:: yaml - - http_filters: - # This filter is only enabled for responses. - - name: envoy.filters.http.compressor - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor - request_direction_config: - common_config: - enabled: - default_value: false - runtime_key: request_compressor_enabled - compressor_library: - name: for_response - typed_config: - "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip - memory_level: 3 - window_bits: 10 - compression_level: BEST_COMPRESSION - compression_strategy: DEFAULT_STRATEGY - # This filter is only enabled for requests. - - name: envoy.filters.http.compressor - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor - response_direction_config: - common_config: - enabled: - default_value: false - runtime_key: response_compressor_enabled - request_direction_config: - common_config: - enabled: - default_value: true - runtime_key: request_compressor_enabled - compressor_library: - name: for_request - typed_config: - "@type": type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip - memory_level: 9 - window_bits: 15 - compression_level: BEST_SPEED - compression_strategy: DEFAULT_STRATEGY +.. literalinclude:: _include/compressor-filter-request-response.yaml + :language: yaml + :linenos: + :lines: 25-64 + :caption: :download:`compressor-filter-request-response.yaml <_include/compressor-filter-request-response.yaml>` .. _compressor-statistics: diff --git a/docs/root/configuration/http/http_filters/connect_grpc_bridge_filter.rst b/docs/root/configuration/http/http_filters/connect_grpc_bridge_filter.rst index 472734b4e2d9..314e071685e0 100644 --- a/docs/root/configuration/http/http_filters/connect_grpc_bridge_filter.rst +++ b/docs/root/configuration/http/http_filters/connect_grpc_bridge_filter.rst @@ -7,11 +7,11 @@ Connect-gRPC Bridge * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.connect_grpc_bridge.v3.FilterConfig``. * :ref:`v3 API reference ` -This filter enables a Buf Connect client to connect to a compliant gRPC server. -More information on the Buf Connect protocol can be found `here `_. +This filter enables a Connect RPC client to connect to a compliant gRPC server. +More information on the Connect protocol can be found `here `_. HTTP GET support ---------------- -This filter supports `Buf Connect HTTP GET requests `_. The +This filter supports `Connect HTTP GET requests `_. The ``Connect-Version-Query`` parameter must be specified in requests in order for them to be translated by this filter, which will be done automatically when using a Connect implementation. diff --git a/docs/root/configuration/http/http_filters/ext_authz_filter.rst b/docs/root/configuration/http/http_filters/ext_authz_filter.rst index a59cdc1b030d..b3fd8bdac8b3 100644 --- a/docs/root/configuration/http/http_filters/ext_authz_filter.rst +++ b/docs/root/configuration/http/http_filters/ext_authz_filter.rst @@ -182,7 +182,7 @@ Dynamic Metadata The External Authorization filter supports emitting dynamic metadata as an opaque ``google.protobuf.Struct``. When using a gRPC authorization server, dynamic metadata will be emitted only when the :ref:`CheckResponse -` contains a filled :ref:`dynamic_metadata +` contains a non-empty :ref:`dynamic_metadata ` field. When using an HTTP authorization server, dynamic metadata will be emitted only when there are response headers diff --git a/docs/root/configuration/http/http_filters/geoip_filter.rst b/docs/root/configuration/http/http_filters/geoip_filter.rst index 7c25cbe0114a..0c6c18d7ac15 100644 --- a/docs/root/configuration/http/http_filters/geoip_filter.rst +++ b/docs/root/configuration/http/http_filters/geoip_filter.rst @@ -4,15 +4,29 @@ IP Geolocation Filter ========================= This filter decorates HTTP requests with the geolocation data. Filter uses client address to lookup information (e.g., client's city, country) in the geolocation provider database. -Upon a successful lookup request will be enriched with the configured geolocation header and value from the database. +Upon a successful lookup request will be enriched with the configured geolocation header and the value from the database. In case the configured geolocation headers are present in the incoming request, they will be overriden by the filter. -Geolocation filter emits stats for the number of successful lookups and the number of total lookups. +Geolocation filter emits stats for the number of the successful lookups and the number of total lookups. +English language is used for the geolocation lookups, the result of the lookup will be UTF-8 encoded. +Please note that Geolocation filter and providers are not yet supported on Windows. Configuration ------------- * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip``. * :ref:`v3 API reference ` +.. _config_geoip_providers_maxmind: + +Geolocation Providers +--------------------- +Currently only `Maxmind `_ geolocation provider is supported. +This provider should be configured with the type URL ``type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig``. + +* :ref:`v3 API reference ` + +.. _config_geoip_providers_common: + +* :ref:`Common provider configuration ` Configuration example --------------------- @@ -22,25 +36,45 @@ Configuration example name: envoy.filters.http.geoip typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip - use_xff: true - xff_num_trusted_hops: 1 - geo_headers_to_add: - country: "x-geo-country" - region: "x-geo-region" + xff_config: + xff_num_trusted_hops: 1 provider: name: "envoy.geoip_providers.maxmind" + typed_config: + "@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + asn: "x-geo-asn" + city_db_path: "geoip/GeoLite2-City-Test.mmdb" + isp_db_path: "geoip/GeoLite2-ASN-Test.mmdb" + Statistics ----------- -Geolocation filter outputs statistics in the -``http..geoip..`` namespace. The :ref:`stat prefix +------------- + +Geolocation HTTP filter has a statistics tree rooted at ``http..``. The :ref:`stat prefix ` comes from the owning HTTP connection manager. .. csv-table:: - :header: Name, Type, Description - :widths: auto + :header: Name, Type, Description + :widths: 1, 1, 2 + + ``rq_total``, Counter, Total number of requests for which geolocation filter was invoked. + +Besides Geolocation filter level statisctics, there is statistics emitted by the :ref:`Maxmind geolocation provider ` +per geolocation database type (rooted at ``.maxmind.``). Database type can be one of `city_db `_, +`isp_db `_, `anon_db `_. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + ``.total``, Counter, Total number of lookups performed for a given geolocation database file. + ``.hit``, Counter, Total number of successful lookups (with non empty lookup result) performed for a given geolocation database file. + ``.lookup_error``, Counter, Total number of errors that occured during lookups for a given geolocation database file. - .hit, Counter, Number of successful lookups within geolocation database for a configured geolocation header. - .total, Counter, Number of total lookups within geolocation database for a configured geolocation header. diff --git a/docs/root/configuration/http/http_filters/golang_filter.rst b/docs/root/configuration/http/http_filters/golang_filter.rst index 8721ce9e838d..4c690f86f846 100644 --- a/docs/root/configuration/http/http_filters/golang_filter.rst +++ b/docs/root/configuration/http/http_filters/golang_filter.rst @@ -12,13 +12,6 @@ See the `Envoy's Golang extension proposal documentation `_ for more details on the filter's implementation. -.. warning:: - The Envoy Golang filter is designed to be run with the ``GODEBUG=cgocheck=0`` environment variable set. - - This disables the cgo pointer check. - - Failure to set this environment variable will cause Envoy to crash! - Developing a Go plugin ---------------------- diff --git a/docs/root/configuration/http/http_filters/grpc_http1_bridge_filter.rst b/docs/root/configuration/http/http_filters/grpc_http1_bridge_filter.rst index 22124c0ea8ea..71604c44544f 100644 --- a/docs/root/configuration/http/http_filters/grpc_http1_bridge_filter.rst +++ b/docs/root/configuration/http/http_filters/grpc_http1_bridge_filter.rst @@ -56,6 +56,13 @@ the size specified in the gRPC frame. The response body returned to the client will not contain the gRPC header frame for requests that are upgraded in this fashion, i.e. the body will contain only the encoded Protobuf. +Ignore query parameters +----------------------- + +Some client requests' URL path may contain query params, while gRPC upstream servers can not handle these requests, +and may return error such as "unknown method". In order to solve this kind of problems, the filter will automatically strip query parameters in request's URL path if +:ref:`ignore_query_parameters ` is set. + Statistics ---------- diff --git a/docs/root/configuration/http/http_filters/grpc_stats_filter.rst b/docs/root/configuration/http/http_filters/grpc_stats_filter.rst index 226cec74ca72..3e53a812e0ea 100644 --- a/docs/root/configuration/http/http_filters/grpc_stats_filter.rst +++ b/docs/root/configuration/http/http_filters/grpc_stats_filter.rst @@ -30,10 +30,10 @@ and :ref:`stats_for_all_methods `. -Buf Connect +Connect RPC ----------- -In addition to supporting gRPC, this filter also transparently supports telemetry for RPC calls using the `Buf Connect protocol `_. +In addition to supporting gRPC, this filter also transparently supports telemetry for RPC calls using the `Connect protocol `_. Connect calls will be counted in the same stats as equivalent gRPC calls. diff --git a/docs/root/configuration/http/http_filters/header_mutation_filter.rst b/docs/root/configuration/http/http_filters/header_mutation_filter.rst index 215a555096de..69f1fd47bf90 100644 --- a/docs/root/configuration/http/http_filters/header_mutation_filter.rst +++ b/docs/root/configuration/http/http_filters/header_mutation_filter.rst @@ -18,7 +18,7 @@ The filter provides complete control over the position and order of the header m the route cache is cleared by a filter executing after the header mutation filter. -In addition, this filter can be used as upstream filter and mutate the request headers after load balancing and host selection. +In addition, this filter can be used as upstream HTTP filter and mutate the request headers after load balancing and host selection. Please note that as an encoder filter, this filter follows the standard rules of when it will execute in situations such as local replies - response diff --git a/docs/root/configuration/http/http_filters/http_filters.rst b/docs/root/configuration/http/http_filters/http_filters.rst index 5eb7463916c1..eb1333ad0e03 100644 --- a/docs/root/configuration/http/http_filters/http_filters.rst +++ b/docs/root/configuration/http/http_filters/http_filters.rst @@ -11,6 +11,7 @@ HTTP filters aws_lambda_filter aws_request_signing_filter bandwidth_limit_filter + basic_auth_filter buffer_filter cache_filter cdn_loop_filter @@ -54,6 +55,7 @@ HTTP filters rate_limit_quota_filter rbac_filter router_filter + set_filter_state set_metadata_filter squash_filter stateful_session_filter diff --git a/docs/root/configuration/http/http_filters/jwt_authn_filter.rst b/docs/root/configuration/http/http_filters/jwt_authn_filter.rst index e84a23a31400..f8dd7e3a66cb 100644 --- a/docs/root/configuration/http/http_filters/jwt_authn_filter.rst +++ b/docs/root/configuration/http/http_filters/jwt_authn_filter.rst @@ -274,10 +274,13 @@ The field :ref:`claim_to_headers x-jwt-claim-nested-key: + x-jwt-tenants: diff --git a/docs/root/configuration/http/http_filters/lua_filter.rst b/docs/root/configuration/http/http_filters/lua_filter.rst index 925320330cf5..a152a26e76f3 100644 --- a/docs/root/configuration/http/http_filters/lua_filter.rst +++ b/docs/root/configuration/http/http_filters/lua_filter.rst @@ -791,6 +791,8 @@ downstreamLocalAddress() Returns the string representation of :repo:`downstream local address ` used by the current request. +.. _config_http_filters_lua_stream_info_downstream_direct_remote_address: + downstreamDirectRemoteAddress() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -801,6 +803,19 @@ downstreamDirectRemoteAddress() Returns the string representation of :repo:`downstream directly connected address ` used by the current request. This is equivalent to the address of the physical connection. +.. _config_http_filters_lua_stream_info_downstream_remote_address: + +downstreamRemoteAddress() +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: lua + + streamInfo:downstreamRemoteAddress() + +Returns the string representation of the downstream remote address for the current request. This may differ from +:ref:`downstreamDirectRemoteAddress() ` depending upon the setting of +:ref:`xff_num_trusted_hops `. + dynamicMetadata() ^^^^^^^^^^^^^^^^^ @@ -930,12 +945,22 @@ peerCertificateValidated() .. code-block:: lua - if downstreamSslConnection:peerCertificateVaidated() then - print("peer certificate is valiedated") + if downstreamSslConnection:peerCertificateValidated() then + print("peer certificate is validated") end Returns bool whether the peer certificate was validated. +.. warning:: + + Client certificate validation is not currently performed upon TLS session resumption. For a + resumed TLS session this method will return false, regardless of whether the peer certificate is + valid. + + The only known workaround for this issue is to disable TLS session resumption entirely, by + setting both :ref:`disable_stateless_session_resumption ` + and :ref:`disable_stateful_session_resumption ` on the DownstreamTlsContext. + uriSanLocalCertificate() ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/root/configuration/http/http_filters/oauth2_filter.rst b/docs/root/configuration/http/http_filters/oauth2_filter.rst index 3c1a3975affc..6e802e07e5ba 100644 --- a/docs/root/configuration/http/http_filters/oauth2_filter.rst +++ b/docs/root/configuration/http/http_filters/oauth2_filter.rst @@ -38,7 +38,7 @@ is set to true the filter will send over a cookie named ``BearerToken`` to the upstream. Additionally, the ``Authorization`` header will be populated with the same value. -The OAuth filer encodes URLs in query parameters using the +The OAuth filter encodes URLs in query parameters using the `URL encoding algorithm. `_ When receiving request redirected from the authorization service the Oauth filer decodes URLs from query parameters. @@ -246,6 +246,11 @@ sending the user to the configured auth endpoint. an interface for users to provide specific header matching criteria such that, when applicable, the OAuth flow is entirely skipped. When this occurs, the ``oauth_passthrough`` metric is incremented but ``success`` is not. +:ref:`use_refresh_token ` provides +possibility to update access token by using a refresh token. By default after expiration the user is always redirected to the authorization endpoint to log in again. +By enabling this flag a new access token is obtained using by a refresh token without redirecting the user to log in again. This requires the refresh token to be provided by authorization_endpoint when the user logs in. +If the attempt to get an access token by using a refresh token fails then the user is redirected to the authorization endpoint as usual. + Generally, allowlisting is inadvisable from a security standpoint. Statistics @@ -261,3 +266,5 @@ The OAuth2 filter outputs statistics in the ``.`` namespace. oauth_passthrough, Counter, Total request that matched a passthrough header. oauth_success, Counter, Total requests that were allowed. oauth_unauthorization_rq, Counter, Total unauthorized requests. + oauth_refreshtoken_success, Counter, Total successfull requests for update access token using by refresh token + oauth_refreshtoken_failure, Counter, Total failed requests for update access token using by refresh token diff --git a/docs/root/configuration/http/http_filters/rate_limit_filter.rst b/docs/root/configuration/http/http_filters/rate_limit_filter.rst index 5e62c2b49b6d..5aca3d31473d 100644 --- a/docs/root/configuration/http/http_filters/rate_limit_filter.rst +++ b/docs/root/configuration/http/http_filters/rate_limit_filter.rst @@ -158,7 +158,7 @@ no entry is added. Statistics ---------- -The rate limit filter outputs statistics in the ``cluster..ratelimit.`` namespace. +The rate limit filter outputs statistics in the ``cluster..ratelimit..`` namespace. 429 responses or the configured :ref:`rate_limited_status ` are emitted to the normal cluster :ref:`dynamic HTTP statistics `. diff --git a/docs/root/configuration/http/http_filters/rate_limit_quota_filter.rst b/docs/root/configuration/http/http_filters/rate_limit_quota_filter.rst index fd13ca50a274..c39e3b0db5c6 100644 --- a/docs/root/configuration/http/http_filters/rate_limit_quota_filter.rst +++ b/docs/root/configuration/http/http_filters/rate_limit_quota_filter.rst @@ -30,7 +30,7 @@ Bucket definitions can be overridden by the virtual host or route configurations Initially all Envoy's quota assignments are empty. The rate limit quota filter requests quota assignment from RLQS when the request matches to a bucket for the first time. The behavior of the filter while it waits for the initial assignment is determined by the ``no_assignment_behavior`` value. In this state, requests can either all be -immediately allowed, denied or enqueued until quota assignment is received. +immediately allowed, denied until quota assignment is received. A quota assignment may have an associated :ref:`time to live `. The RLQS is expected to update the assignment before the TTL runs out. If RLQS failed to update the assignment and its TTL @@ -40,12 +40,23 @@ has expired, the filter can be configured to continue using the last quota assig The rate limit quota filter reports the request load for each bucket to the RLQS with the configured ``reporting_interval``. The RLQS may rebalance quota assignments based on the request load that each Envoy receives and push new quota assignments to Envoy instances. -When the connection to RLQS server fails, the filter will fall back to either the +Failure modes +^^^^^^^^^^^^^ + +In case the connection to RLQS server fails, the filter will fall back to either the :ref:`no assignment behavior ` if it has not yet received a rate limit quota or to the :ref:`expired assignment behavior ` if connection could not be re-established by the time the existing quota expired. +In case the RLQS client doesn't receive the initial bucket assignment (for any reason, including +RLQS server connection failures) within predetermined [1]_ time, such buckets will eventually be +purged from memory. Subsequent requests matched into the bucket will re-initialize the bucket +in the "no assignment" state, restarting the reports. This is explained in more details at +:ref:`Rate Limit Quota Service (RLQS) `. + +.. [1] The exact time to wait for the initial assignment is chosen by the filter, and may vary based on the implementation. + Example 1 ^^^^^^^^^ diff --git a/docs/root/configuration/http/http_filters/set_filter_state.rst b/docs/root/configuration/http/http_filters/set_filter_state.rst new file mode 100644 index 000000000000..52dbe82c5a6a --- /dev/null +++ b/docs/root/configuration/http/http_filters/set_filter_state.rst @@ -0,0 +1,60 @@ +.. _config_http_filters_set_filter_state: + +Set-Filter-State HTTP Filter +============================ +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.set_filter_state.v3.Config``. +* :ref:`v3 API reference ` + +This filter is configured with a sequence of values to update the request +filter state using the request data. The filter state value can then be used +for routing, load balancing decisions, telemetry, etc. See :ref:`the well-known +filter state keys ` for the controls used by Envoy +extensions. + +.. warning:: + This filter allows overriding the behavior of other extensions and + significantly and indirectly altering the request processing logic. + + +Examples +-------- + +A sample filter configuration to capture the authority header from an HTTP +CONNECT request as the original destination cluster dynamic destination +address: + +.. validated-code-block:: yaml + :type-name: envoy.extensions.filters.http.set_filter_state.v3.Config + + on_request_headers: + - object_key: envoy.network.transport_socket.original_dst_address + format_string: + text_format_source: + inline_string: "%REQ(:AUTHORITY)%" + + +A sample filter configuration to capture the authority header and the remote +address from an HTTP CONNECT request as the original destination listener +filter addresses on the upstream :ref:`internal listener connection +`: + +.. validated-code-block:: yaml + :type-name: envoy.extensions.filters.http.set_filter_state.v3.Config + + on_request_headers: + - object_key: envoy.filters.listener.original_dst.local_ip + format_string: + text_format_source: + inline_string: "%REQ(:AUTHORITY)%" + shared_with_upstream: ONCE + - object_key: envoy.filters.listener.original_dst.remote_ip + format_string: + text_format_source: + inline_string: "%DOWNSTREAM_REMOTE_ADDRESS%" + shared_with_upstream: ONCE + + +Statistics +---------- + +Currently, this filter generates no statistics. diff --git a/docs/root/configuration/http/http_filters/set_metadata_filter.rst b/docs/root/configuration/http/http_filters/set_metadata_filter.rst index 7d0b3c1ba7bb..fde69d5a5031 100644 --- a/docs/root/configuration/http/http_filters/set_metadata_filter.rst +++ b/docs/root/configuration/http/http_filters/set_metadata_filter.rst @@ -8,13 +8,14 @@ Set Metadata This filters adds or updates dynamic metadata with static data. -Dynamic metadata values are updated with the following scheme. If a key -does not exists, it's just copied into the current metadata. If the key exists -but has a different type, it is replaced by the new value. Otherwise: +Dynamic metadata values are updated with the following rules. If a key does not exist, it is copied into the current metadata. If the key exists, then following rules will be used: - * for scalar values (null, string, number, boolean) are replaced with the new value - * for lists: new values are added to the current list - * for structures: recursively apply this scheme +* if :ref:`typed metadata value ` is used, it will overwrite existing values iff :ref:`allow_overwrite ` is set to true, otherwise nothing is done. +* if :ref:`untyped metadata value ` is used and ``allow_overwrite`` is set to true, or if deprecated :ref:`value ` field is used, the values are updated with the following scheme: + - existing value with different type: the existing value is replaced. + - scalar values (null, string, number, boolean): the existing value is replaced. + - lists: new values are appended to the current list. + - structures: recursively apply this scheme. For instance, if the namespace already contains this structure: @@ -50,4 +51,12 @@ After applying this filter, the namespace will contain: Statistics ---------- -Currently, this filter generates no statistics. +The ``set_metadata`` filter outputs statistics in the ``http..set_metadata.`` namespace. The :ref:`stat prefix +` comes from the +owning HTTP connection manager. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + overwrite_denied, Counter, Total number of denied attempts to overwrite an existing metadata value diff --git a/docs/root/configuration/http/http_filters/upstream_codec_filter.rst b/docs/root/configuration/http/http_filters/upstream_codec_filter.rst index 4db14174382d..457f4878ea74 100644 --- a/docs/root/configuration/http/http_filters/upstream_codec_filter.rst +++ b/docs/root/configuration/http/http_filters/upstream_codec_filter.rst @@ -3,7 +3,7 @@ Upstream Codec ============== -The upstream codec filter is the only supported terminal filter for upstream filters. +The upstream codec filter is the only supported terminal filter for upstream HTTP filters. It is responsible for encoding headers/body/data to the upstream codec, and decoding headers/body/data from the upstream codec, as well as updating stats and timing metrics. diff --git a/docs/root/configuration/listeners/listener_filters/original_dst_filter.rst b/docs/root/configuration/listeners/listener_filters/original_dst_filter.rst index 7911534a9bd2..73bf11e3ecd7 100644 --- a/docs/root/configuration/listeners/listener_filters/original_dst_filter.rst +++ b/docs/root/configuration/listeners/listener_filters/original_dst_filter.rst @@ -40,5 +40,28 @@ rather than the address at which the listener is listening at. Furthermore, :ref destination cluster ` may be used to forward HTTP requests or TCP connections to the restored destination address. +Internal listeners +------------------ + +Original destination listener filter reads the dynamic metadata and the filter +state objects on the user space sockets handled by the :ref:`internal listeners +` instead of the system socket options. + +Dynamic metadata for the destination address is expected to be placed into the +key `envoy.filters.listener.original_dst` under the field `local` and should +contain a string with an IP and a port address. In the absence of the dynamic +metadata, the filter state is consulted. + +The filter state objects consumed by this filter are: + +* `envoy.filters.listener.original_dst.local_ip` for the destination address +* `envoy.filters.listener.original_dst.source_ip` for the source address + +Note that :ref:`internal upstream transport +` should be used for passing the dynamic +metadata from an endpoint host to the socket metadata and/or the filter state +objects that are shared with the upstream connection through a user space +socket to the internal listeners. + * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.listener.original_dst.v3.OriginalDst``. * :ref:`v3 API reference ` diff --git a/docs/root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml b/docs/root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml index d11f74b213f7..03a96cdd75a9 100644 --- a/docs/root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml +++ b/docs/root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml @@ -16,7 +16,7 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.router.v3.Router codec_config: - name: http + name: dubbo typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.generic_proxy.codecs.dubbo.v3.DubboCodecConfig route_config: diff --git a/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml b/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml index 4d577e5d7717..4b079ef6362d 100644 --- a/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml +++ b/docs/root/configuration/listeners/network_filters/_include/zookeeper-filter-proxy.yaml @@ -11,6 +11,8 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy stat_prefix: zookeeper + enable_per_opcode_request_bytes_metrics: true + enable_per_opcode_response_bytes_metrics: true enable_latency_threshold_metrics: true default_latency_threshold: "0.1s" latency_threshold_overrides: diff --git a/docs/root/configuration/listeners/network_filters/ext_authz_filter.rst b/docs/root/configuration/listeners/network_filters/ext_authz_filter.rst index f9124ad71847..0d0625a614f7 100644 --- a/docs/root/configuration/listeners/network_filters/ext_authz_filter.rst +++ b/docs/root/configuration/listeners/network_filters/ext_authz_filter.rst @@ -107,5 +107,5 @@ Dynamic Metadata The External Authorization filter emits dynamic metadata as an opaque ``google.protobuf.Struct`` *only* when the gRPC authorization server returns a :ref:`CheckResponse -` with a filled :ref:`dynamic_metadata +` with a non-empty :ref:`dynamic_metadata ` field. diff --git a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst index 8d9f96258b8f..c938476b2148 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst @@ -7,9 +7,17 @@ The Apache Kafka broker filter decodes the client protocol for `Apache Kafka `_, both the requests and responses in the payload. The message versions in `Kafka 3.5.1 `_ are supported (apart from ConsumerGroupHeartbeat). -The filter attempts not to influence the communication between client and brokers, so the messages -that could not be decoded (due to Kafka client or broker running a newer version than supported by -this filter) are forwarded as-is. + +By default the filter attempts not to influence the communication between client and brokers, so +the messages that could not be decoded (due to Kafka client or broker running a newer version than +supported by this filter) are forwarded as-is. However this requires the upstream Kafka cluster to +be configured in proxy-aware fashion (see :ref:`config_network_filters_kafka_broker_config_no_mutation`). + +If configured to mutate the received traffic, Envoy broker filter can be used to proxy a Kafka broker +without any changes in the broker configuration. +This requires the broker filter to be provided with rewrite rules so the addresses advertised by +the Kafka brokers can be changed to the Envoy listener addresses +(see :ref:`config_network_filters_kafka_broker_config_with_mutation`). * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker``. * :ref:`v3 API reference ` @@ -23,21 +31,22 @@ this filter) are forwarded as-is. The kafka_broker filter is experimental and is currently under active development. Capabilities will be expanded over time and the configuration structures are likely to change. -.. _config_network_filters_kafka_broker_config: +.. _config_network_filters_kafka_broker_config_no_mutation: -Configuration -------------- +Configuration (no traffic mutation) +----------------------------------- -The Kafka Broker filter should be chained with the TCP proxy filter as shown -in the configuration snippet below: +The Kafka Broker filter can run without rewriting any requests / responses. + +The filter should be chained with the TCP proxy filter as shown in the snippet below: .. code-block:: yaml listeners: - address: socket_address: - address: 127.0.0.1 # Host that Kafka clients should connect to. - port_value: 19092 # Port that Kafka clients should connect to. + address: 127.0.0.1 # Host that Kafka clients should connect to (i.e. bootstrap.servers). + port_value: 19092 # Port that Kafka clients should connect to (i.e. bootstrap.servers). filter_chains: - filters: - name: envoy.filters.network.kafka_broker @@ -61,10 +70,11 @@ in the configuration snippet below: - endpoint: address: socket_address: - address: 127.0.0.1 # Kafka broker's host - port_value: 9092 # Kafka broker's port. + address: 127.0.0.1 # Kafka broker's host. + port_value: 9092 # Kafka broker's port. -The Kafka broker needs to advertise the Envoy listener port instead of its own. +The Kafka broker then needs to advertise the Envoy listener port instead of its own - +this makes the downstream clients make any new connections to Envoy only. .. code-block:: text @@ -76,6 +86,117 @@ The Kafka broker needs to advertise the Envoy listener port instead of its own. # (will make clients discovering this broker talk to it through Envoy). advertised.listeners=PLAINTEXT://127.0.0.1:19092 +.. _config_network_filters_kafka_broker_config_with_mutation: + +Configuration (with traffic mutation) +------------------------------------- + +The Kafka Broker filter can mutate the contents of received responses to enable easier proxying +of Kafka clusters. + +The below example shows a configuration for an Envoy instance that attempts to proxy brokers +in 2-node cluster: + +.. code-block:: yaml + + listeners: + - address: # This listener proxies broker 1. + socket_address: + address: envoy.example.org # Host that Kafka clients should connect to (i.e. bootstrap.servers). + port_value: 19092 # Port that Kafka clients should connect to (i.e. bootstrap.servers). + filter_chains: + - filters: + - name: envoy.filters.network.kafka_broker + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker + stat_prefix: exampleprefix1 + id_based_broker_address_rewrite_spec: &kafka_rewrite_spec + rules: + - id: 1 + host: envoy.example.org + port: 19092 + - id: 2 + host: envoy.example.org + port: 19093 + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp + cluster: broker1cluster + - address: # This listener proxies broker 2. + socket_address: + address: envoy.example.org # Host that Kafka clients should connect to (i.e. bootstrap.servers). + port_value: 19093 # Port that Kafka clients should connect to (i.e. bootstrap.servers). + filter_chains: + - filters: + - name: envoy.filters.network.kafka_broker + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker + stat_prefix: exampleprefix2 + id_based_broker_address_rewrite_spec: *kafka_rewrite_spec + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp + cluster: broker2cluster + + clusters: + - name: broker1cluster + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: broker1.example.org # Kafka broker's host for broker 1. + port_value: 9092 # Kafka broker's port for broker 1. + - name: broker2cluster + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: broker2.example.org # Kafka broker's host for broker 2. + port_value: 9092 # Kafka broker's port for broker 2. + +The address rewrite rules should cover all brokers present in the cluster - YAML blocks can be +used to avoid repetition. + +The responses that can be mutated are: + +* metadata (all partition discovery operations), +* find coordinator (used by consumer groups and transactions), +* describe cluster. + +.. _config_network_filters_kafka_broker_debugging: + +Debugging +--------- + +Java clients can see the hosts used if they set the log level of +`org.apache.kafka.clients.NetworkClient` to `debug` - only Envoy's listeners should be visible +in the logs. + +.. code-block:: text + + [DEBUG] [NetworkClient] Initiating connection to node localhost:19092 (id: -1 rack: null) using address localhost/127.0.0.1 + [DEBUG] [NetworkClient] Completed connection to node -1. Fetching API versions. + [DEBUG] [NetworkClient] Initiating connection to node localhost:19092 (id: 1 rack: null) using address localhost/127.0.0.1 + [DEBUG] [NetworkClient] Completed connection to node 1. Fetching API versions. + [DEBUG] [NetworkClient] Initiating connection to node localhost:19094 (id: 3 rack: null) using address localhost/127.0.0.1 + [DEBUG] [NetworkClient] Initiating connection to node localhost:19093 (id: 2 rack: null) using address localhost/127.0.0.1 + [DEBUG] [NetworkClient] Completed connection to node 2. Fetching API versions. + [DEBUG] [NetworkClient] Completed connection to node 3. Fetching API versions. + .. _config_network_filters_kafka_broker_stats: Statistics diff --git a/docs/root/configuration/listeners/network_filters/network_filters.rst b/docs/root/configuration/listeners/network_filters/network_filters.rst index 0e7777451d2a..baf13b94d08b 100644 --- a/docs/root/configuration/listeners/network_filters/network_filters.rst +++ b/docs/root/configuration/listeners/network_filters/network_filters.rst @@ -28,6 +28,7 @@ filters. rbac_filter redis_proxy_filter rocketmq_proxy_filter + set_filter_state sni_cluster_filter sni_dynamic_forward_proxy_filter tcp_proxy_filter diff --git a/docs/root/configuration/listeners/network_filters/set_filter_state.rst b/docs/root/configuration/listeners/network_filters/set_filter_state.rst new file mode 100644 index 000000000000..0858815fc567 --- /dev/null +++ b/docs/root/configuration/listeners/network_filters/set_filter_state.rst @@ -0,0 +1,37 @@ +.. _config_network_filters_set_filter_state: + +Set-Filter-State Network Filter +=============================== +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.network.set_filter_state.v3.Config``. +* :ref:`v3 API reference ` + +This filter is configured with a sequence of values to update the connection +filter state using the connection data. The filter state value can then be used +for routing, load balancing decisions, telemetry, etc. See :ref:`the well-known +filter state keys ` for the controls used by Envoy +extensions. + +.. warning:: + This filter allows overriding the behavior of other extensions and + significantly and indirectly altering the connection processing logic. + + +Examples +-------- + +A sample filter configuration that propagates the downstream SNI as the upstream SNI: + +.. validated-code-block:: yaml + :type-name: envoy.extensions.filters.http.set_filter_state.v3.Config + + on_new_connection: + - object_key: envoy.network.upstream_server_name + format_string: + text_format_source: + inline_string: "%REQUESTED_SERVER_NAME%" + + +Statistics +---------- + +Currently, this filter generates no statistics. diff --git a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst index f62255909101..e98f00795008 100644 --- a/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst @@ -9,6 +9,8 @@ responses and events in the payload. Most opcodes known in `ZooKeeper 3.5 `_ are supported. The unsupported ones are related to SASL authentication. +:ref:`v3 API reference ` + .. attention:: The zookeeper_proxy filter is experimental and is currently under active @@ -32,7 +34,13 @@ Statistics ---------- Every configured ZooKeeper proxy filter has statistics rooted at *.zookeeper.*. -*_resp_fast* and *_resp_slow* metrics will only be emitted when ``enable_latency_threshold_metrics`` is set to ``true``. + +*_resp_fast* and *_resp_slow* are per-opcode metrics and will only be emitted when :ref:`enable_latency_threshold_metrics ` is set to ``true``. + +*_rq_bytes* are per-opcode metrics and will only be emitted when :ref:`enable_per_opcode_request_bytes_metrics ` is set to ``true``. + +*_resp_bytes* are per-opcode metrics and will only be emitted when :ref:`enable_per_opcode_response_bytes_metrics ` is set to ``true``. + The following counters are available: .. csv-table:: @@ -41,6 +49,33 @@ The following counters are available: decoder_error, Counter, Number of times a message wasn't decoded request_bytes, Counter, Number of bytes in decoded request messages + connect_rq_bytes, Counter, Number of bytes in decoded connect request messages + ping_rq_bytes, Counter, Number of bytes in decoded ping request messages + auth_rq_bytes, Counter, Number of bytes in decoded auth request messages + getdata_rq_bytes, Counter, Number of bytes in decoded getdata request messages + create_rq_bytes, Counter, Number of bytes in decoded create request messages + create2_rq_bytes, Counter, Number of bytes in decoded create2 request messages + createcontainer_rq_bytes, Counter, Number of bytes in decoded createcontainer request messages + createttl_rq_bytes, Counter, Number of bytes in decoded createttl request messages + setdata_rq_bytes, Counter, Number of bytes in decoded setdata request messages + getchildren_rq_bytes, Counter, Number of bytes in decoded getchildren request messages + getchildren2_rq_bytes, Counter, Number of bytes in decoded getchildren2 request messages + delete_rq_bytes, Counter, Number of bytes in decoded delete request messages + exists_rq_bytes, Counter, Number of bytes in decoded exists request messages + getacl_rq_bytes, Counter, Number of bytes in decoded getacl request messages + setacl_rq_bytes, Counter, Number of bytes in decoded setacl request messages + sync_rq_bytes, Counter, Number of bytes in decoded sync request messages + check_rq_bytes, Counter, Number of bytes in decoded check request messages + multi_rq_bytes, Counter, Number of bytes in decoded multi request messages + reconfig_rq_bytes, Counter, Number of bytes in decoded reconfig request messages + setwatches_rq_bytes, Counter, Number of bytes in decoded setwatches request messages + setwatches2_rq_bytes, Counter, Number of bytes in decoded setwatches2 request messages + addwatch_rq_bytes, Counter, Number of bytes in decoded addwatch request messages + checkwatches_rq_bytes, Counter, Number of bytes in decoded checkwatches request messages + removewatches_rq_bytes, Counter, Number of bytes in decoded removewatches request messages + getephemerals_rq_bytes, Counter, Number of bytes in decoded getephemerals request messages + getallchildrennumber_rq_bytes, Counter, Number of bytes in decoded getallchildrennumber request messages + close_rq_bytes, Counter, Number of bytes in decoded close request messages connect_rq, Counter, Number of regular connect (non-readonly) requests connect_readonly_rq, Counter, Number of connect requests with the readonly flag set ping_rq, Counter, Number of ping requests @@ -70,6 +105,33 @@ The following counters are available: getallchildrennumber_rq, Counter, Number of getallchildrennumber requests close_rq, Counter, Number of close requests response_bytes, Counter, Number of bytes in decoded response messages + connect_resp_bytes, Counter, Number of bytes in decoded connect response messages + ping_resp_bytes, Counter, Number of bytes in decoded ping response messages + auth_resp_bytes, Counter, Number of bytes in decoded auth response messages + getdata_resp_bytes, Counter, Number of bytes in decoded getdata response messages + create_resp_bytes, Counter, Number of bytes in decoded create response messages + create2_resp_bytes, Counter, Number of bytes in decoded create2 response messages + createcontainer_resp_bytes, Counter, Number of bytes in decoded createcontainer response messages + createttl_resp_bytes, Counter, Number of bytes in decoded createttl response messages + setdata_resp_bytes, Counter, Number of bytes in decoded setdata response messages + getchildren_resp_bytes, Counter, Number of bytes in decoded getchildren response messages + getchildren2_resp_bytes, Counter, Number of bytes in decoded getchildren2 response messages + delete_resp_bytes, Counter, Number of bytes in decoded delete response messages + exists_resp_bytes, Counter, Number of bytes in decoded exists response messages + getacl_resp_bytes, Counter, Number of bytes in decoded getacl response messages + setacl_resp_bytes, Counter, Number of bytes in decoded setacl response messages + sync_resp_bytes, Counter, Number of bytes in decoded sync response messages + check_resp_bytes, Counter, Number of bytes in decoded check response messages + multi_resp_bytes, Counter, Number of bytes in decoded multi response messages + reconfig_resp_bytes, Counter, Number of bytes in decoded reconfig response messages + setwatches_resp_bytes, Counter, Number of bytes in decoded setwatches response messages + setwatches2_resp_bytes, Counter, Number of bytes in decoded setwatches2 response messages + addwatch_resp_bytes, Counter, Number of bytes in decoded addwatch response messages + checkwatches_resp_bytes, Counter, Number of bytes in decoded checkwatches response messages + removewatches_resp_bytes, Counter, Number of bytes in decoded removewatches response messages + getephemerals_resp_bytes, Counter, Number of bytes in decoded getephemerals response messages + getallchildrennumber_resp_bytes, Counter, Number of bytes in decoded getallchildrennumber response messages + close_resp_bytes, Counter, Number of bytes in decoded close response messages connect_resp, Counter, Number of connect responses ping_resp, Counter, Number of ping responses auth_resp, Counter, Number of auth responses diff --git a/docs/root/configuration/listeners/udp_filters/session_filters/dynamic_forward_proxy.rst b/docs/root/configuration/listeners/udp_filters/session_filters/dynamic_forward_proxy.rst new file mode 100644 index 000000000000..3c93d1fae433 --- /dev/null +++ b/docs/root/configuration/listeners/udp_filters/session_filters/dynamic_forward_proxy.rst @@ -0,0 +1,27 @@ +.. _config_udp_session_filters_dynamic_forward_proxy: + +Dynamic forward proxy +================================== + +Through the combination of a custom preceding filter that sets the ``envoy.upstream.dynamic_host`` and ``envoy.upstream.dynamic_port`` filter state +keys, and when used with the :ref:`dynamic forward proxy cluster `, +Envoy supports dynamic forward proxy for UDP flows. The implementation works just like the +:ref:`HTTP dynamic forward proxy `, but using the value in +UDP session's filter state as target host and port instead. + +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.session.dynamic_forward_proxy.v3.FilterConfig``. +* :ref:`v3 API reference ` + +.. _config_udp_session_filters_dynamic_forward_proxy_stats: + +Statistics +---------- + +Every configured filter has statistics rooted at *udp.session.dynamic_forward_proxy..* +with the following statistics: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + buffer_overflow, Counter, Number of datagrams dropped due to the filter buffer being overflowed diff --git a/docs/root/configuration/listeners/udp_filters/session_filters/http_capsule.rst b/docs/root/configuration/listeners/udp_filters/session_filters/http_capsule.rst new file mode 100644 index 000000000000..17defd1d0f7d --- /dev/null +++ b/docs/root/configuration/listeners/udp_filters/session_filters/http_capsule.rst @@ -0,0 +1,13 @@ +.. _config_udp_session_filters_http_capsule: + +HTTP Capsule filter +================================== + +Tunneling UDP datagrams over HTTP (see `Proxying UDP in HTTP `_) requires an encapsulation mechanism to preserve the boundaries of the original datagram. +This filter applies the `HTTP Datagrams and the Capsule Protocol `_ to downstream and upstream datagrams, so they are compliant with the Capsule Protocol. + +.. note:: + This filter must be used last in the UDP session filter chain, and should only be used when tunneling UDP over HTTP streams. + +* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3.FilterConfig``. +* :ref:`v3 API reference ` diff --git a/docs/root/configuration/listeners/udp_filters/udp_proxy.rst b/docs/root/configuration/listeners/udp_filters/udp_proxy.rst index 209db74c26a3..b03c746cfa2d 100644 --- a/docs/root/configuration/listeners/udp_filters/udp_proxy.rst +++ b/docs/root/configuration/listeners/udp_filters/udp_proxy.rst @@ -74,6 +74,8 @@ remaining datagrams to different clusters according to their source ports. :lines: 14-53 :caption: :download:`udp-proxy-router.yaml <_include/udp-proxy-router.yaml>` +.. _config_udp_listener_filters_udp_proxy_session_filters: + Session filters --------------- @@ -89,6 +91,27 @@ upstream, similar to network filters. Additionally, since :ref:`per packet load balancing ` require choosing the upstream host for each received datagram, session filters can't be used when this option is enabled. +Envoy has the following builtin UDP session filters. + +.. toctree:: + :maxdepth: 2 + + session_filters/http_capsule + session_filters/dynamic_forward_proxy + +.. _config_udp_listener_filters_udp_proxy_tunneling_over_http: + +UDP Tunneling over HTTP +----------------------- + +The UDP proxy filter can be used to tunnel raw UDP over HTTP requests. Refer to :ref:`HTTP upgrades ` for more information. + +UDP tunneling configuration can be used by setting :ref:`tunneling_config ` + +.. note:: + Since :ref:`per packet load balancing ` require + choosing the upstream host for each received datagram, tunneling can't be used when this option is enabled. + Example configuration --------------------- @@ -149,3 +172,6 @@ The UDP proxy filter also emits custom upstream cluster stats prefixed with sess_rx_errors, Counter, Number of datagram receive errors sess_tx_datagrams, Counter, Number of datagrams transmitted sess_tx_errors, Counter, Number of datagrams transmitted + sess_tunnel_success, Counter, Number of successfully established UDP tunnels + sess_tunnel_failure, Counter, Number of UDP tunnels failed to establish + sess_tunnel_buffer_overflow, Counter, Number of datagrams dropped due to full tunnel buffer diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index d1afd3ff715d..eb9a1056c885 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -157,7 +157,9 @@ The following command operators are supported: .. code-block:: none - %START_TIME(%Y/%m/%dT%H:%M:%S%z %s)% + %START_TIME(%Y/%m/%dT%H:%M:%S%z)% + + %START_TIME(%s)% # To include millisecond fraction of the second (.000 ... .999). E.g. 1527590590.528. %START_TIME(%s.%3f)% @@ -168,6 +170,14 @@ The following command operators are supported: In typed JSON logs, START_TIME is always rendered as a string. +.. _config_access_log_format_emit_time: + +%EMIT_TIME% + The time when log entry is emitted including milliseconds. + + EMIT_TIME can be customized using a `format string `_. + See :ref:`START_TIME ` for additional format specifiers and examples. + %REQUEST_HEADERS_BYTES% HTTP Uncompressed bytes of request headers. @@ -448,6 +458,15 @@ The following command operators are supported: Renders a numeric value in typed JSON logs. +%UPSTREAM_CONNECTION_POOL_READY_DURATION% + HTTP/TCP + Total duration in milliseconds from when the upstream request was created to when the connection pool is ready. + + UDP + Not implemented ("-"). + + Renders a numeric value in typed JSON logs. + .. _config_access_log_format_response_flags: %RESPONSE_FLAGS% / %RESPONSE_FLAGS_LONG% @@ -493,6 +512,7 @@ HTTP only **UpstreamMaxStreamDurationReached**, **UMSDR**, The upstream request reached max stream duration. **OverloadManagerTerminated**, **OM**, Overload Manager terminated the request. **DnsResolutionFailed**, **DF**, The request was terminated due to DNS resolution failure. + **DropOverload**, **DO**, The request was terminated in addition to 503 response code due to :ref:`drop_overloads`. UDP Not implemented ("-"). @@ -511,8 +531,11 @@ UDP TCP/UDP Not implemented ("-") +.. _config_access_log_format_upstream_host: + %UPSTREAM_HOST% - Upstream host URL (e.g., tcp://ip:port for TCP connections). + Upstream host URL (e.g., tcp://ip:port for TCP connections). Identical to the :ref:`UPSTREAM_REMOTE_ADDRESS + ` value. %UPSTREAM_CLUSTER% Upstream cluster to which the upstream host belongs to. :ref:`alt_stat_name @@ -530,9 +553,11 @@ UDP Local port of the upstream connection. IP addresses are the only address type with a port component. +.. _config_access_log_format_upstream_remote_address: + %UPSTREAM_REMOTE_ADDRESS% Remote address of the upstream connection. If the address is an IP address it includes both - address and port. + address and port. Identical to the :ref:`UPSTREAM_HOST ` value. %UPSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% Remote address of the upstream connection, without any port component. @@ -1133,6 +1158,10 @@ UDP * UpstreamPoolReady - When a new HTTP request is received by the HTTP Router filter. * UpstreamPeriodic - On any HTTP Router filter periodic log record. * UpstreamEnd - When an HTTP request is finished on the HTTP Router filter. + * UdpTunnelUpstreamConnected - When UDP Proxy filter has successfully established an upstream connection. + Note: It is only relevant for UDP tunneling over HTTP. + * UdpPeriodic - On any UDP Proxy filter periodic log record. + * UdpSessionEnd - When a UDP session is ended on UDP Proxy filter. %ENVIRONMENT(X):Z% Environment value of environment variable X. If no valid environment variable X, '-' symbol will be used. diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index e9ce33d4788e..7d45a1bbd758 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -213,11 +213,26 @@ again 600 seconds, then the minimum timer value would be :math:`10\% \cdot 600s Limiting Active Connections --------------------------- -Currently, the only supported way to limit the total number of active connections allowed across all -listeners is via specifying an integer through the runtime key -``overload.global_downstream_max_connections``. The connection limit is recommended to be less than +To limit the total number of active downstream connections allowed across all +listeners configure :ref:`downstream connections monitor ` in Overload Manager: + +.. code-block:: yaml + + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 1000 + +:ref:`Downstream connections monitor ` does not +support runtime updates for the configured value of :ref:`max_active_downstream_connections +` +One could also set this limit via specifying an integer through the runtime key +``overload.global_downstream_max_connections``, though this key is deprecated and will be removed in future. +The connection limit is recommended to be less than half of the system's file descriptor limit, to account for upstream connections, files, and other usage of file descriptors. + If the value is unspecified, there is no global limit on the number of active downstream connections and Envoy will emit a warning indicating this at startup. To disable the warning without setting a limit on the number of active downstream connections, the runtime value may be set to a very large diff --git a/docs/root/configuration/other_features/_include/hyperscan_matcher.yaml b/docs/root/configuration/other_features/_include/hyperscan_matcher.yaml new file mode 100644 index 000000000000..3bf3e069a336 --- /dev/null +++ b/docs/root/configuration/other_features/_include/hyperscan_matcher.yaml @@ -0,0 +1,62 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: app + domains: + - "*" + matcher: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: request-headers + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: :path + custom_match: + name: hyperscan + typed_config: + "@type": type.googleapis.com/envoy.extensions.matching.input_matchers.hyperscan.v3alpha.Hyperscan + regexes: + - regex: allowed.*path + on_match: + action: + name: route + typed_config: + "@type": type.googleapis.com/envoy.config.route.v3.Route + match: + prefix: / + route: + cluster: service-http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: service-http + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service-http + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 diff --git a/docs/root/configuration/other_features/_include/hyperscan_matcher_multiple.yaml b/docs/root/configuration/other_features/_include/hyperscan_matcher_multiple.yaml new file mode 100644 index 000000000000..01aca3e4e473 --- /dev/null +++ b/docs/root/configuration/other_features/_include/hyperscan_matcher_multiple.yaml @@ -0,0 +1,72 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: app + domains: + - "*" + matcher: + matcher_list: + matchers: + - predicate: + single_predicate: + input: + name: request-headers + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: :path + custom_match: + name: hyperscan + typed_config: + "@type": type.googleapis.com/envoy.extensions.matching.input_matchers.hyperscan.v3alpha.Hyperscan + # The following multiple patterns match input including allowed.*path and excluding + # den(y|ied). E.g., the path /allowed/path will be matched, while the path + # /allowed/denied/path will not be matched. + regexes: + - regex: allowed.*path + id: 1 + quiet: true + - regex: den(y|ied) + id: 2 + quiet: true + - regex: 1 & !2 + combination: true + on_match: + action: + name: route + typed_config: + "@type": type.googleapis.com/envoy.config.route.v3.Route + match: + prefix: / + route: + cluster: service-http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: service-http + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service-http + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 diff --git a/docs/root/configuration/other_features/_include/hyperscan_regex_engine.yaml b/docs/root/configuration/other_features/_include/hyperscan_regex_engine.yaml new file mode 100644 index 000000000000..4ec75aaa35bd --- /dev/null +++ b/docs/root/configuration/other_features/_include/hyperscan_regex_engine.yaml @@ -0,0 +1,48 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: app + domains: + - "*" + routes: + - match: + safe_regex: + regex: allowed.*path + route: + cluster: service-http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: service-http + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service-http + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 + +default_regex_engine: + name: envoy.regex_engines.hyperscan + typed_config: + "@type": type.googleapis.com/envoy.extensions.regex_engines.hyperscan.v3alpha.Hyperscan diff --git a/docs/root/configuration/other_features/_include/qatzip.yaml b/docs/root/configuration/other_features/_include/qatzip.yaml new file mode 100644 index 000000000000..ef436cbda13c --- /dev/null +++ b/docs/root/configuration/other_features/_include/qatzip.yaml @@ -0,0 +1,60 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: service + http_filters: + - name: envoy.filters.http.compressor + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.compressor.v3.Compressor + response_direction_config: + common_config: + min_content_length: 100 + content_type: + - application/octet-stream + compressor_library: + name: qatzip + typed_config: + "@type": type.googleapis.com/envoy.extensions.compression.qatzip.compressor.v3alpha.Qatzip + compression_level: 3 + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: service + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9901 diff --git a/docs/root/configuration/other_features/hyperscan.rst b/docs/root/configuration/other_features/hyperscan.rst new file mode 100644 index 000000000000..a745299a7f34 --- /dev/null +++ b/docs/root/configuration/other_features/hyperscan.rst @@ -0,0 +1,57 @@ +.. _config_hyperscan: + +Hyperscan +========= + +* :ref:`Matcher v3 API reference ` +* :ref:`Regex engine v3 API reference ` + +`Hyperscan `_ is a high-performance multiple regex matching library, which uses +hybrid automata techniques to allow simultaneous matching of large numbers of regular expressions and for the matching +of regular expressions across streams of data. Hyperscan supports the `pattern syntax +`_ used by PCRE. + +Hyperscan is only valid in the :ref:`contrib image `. + +Hyperscan can be used as a matcher of :ref:`generic matching `, or enabled as a regex +engine globally. + +As a matcher of generic matching +-------------------------------- + +Generic matching has been implemented in a few of components and extensions in Envoy, including +:ref:`filter chain matcher `, :ref:`route matcher +` and :ref:`RBAC matcher +`. Hyperscan matcher can be used in generic matcher as +a custom matcher in the following structure: + +.. literalinclude:: _include/hyperscan_matcher.yaml + :language: yaml + :linenos: + :lines: 30-35 + :caption: :download:`hyperscan_matcher.yaml <_include/hyperscan_matcher.yaml>` + +The behavior of regex matching in Hyperscan matchers can be configured, please refer to the :ref:`API reference +`. + +Hyperscan matcher also supports multiple pattern matching which allows matches to be reported for several patterns +simultaneously. Multiple pattern matching can be turned on in the following structure: + +.. literalinclude:: _include/hyperscan_matcher_multiple.yaml + :language: yaml + :linenos: + :lines: 30-45 + :emphasize-lines: 8-16 + :caption: :download:`hyperscan_matcher_multiple.yaml <_include/hyperscan_matcher_multiple.yaml>` + +As a regex engine +----------------- + +Hyperscan regex engine acts in the similar behavior with the default regex engine Google RE2 like it turns on UTF-8 +support by default. Hyperscan regex engine can be easily configured with the following configuration. + +.. literalinclude:: _include/hyperscan_regex_engine.yaml + :language: yaml + :linenos: + :lines: 45-48 + :caption: :download:`hyperscan_regex_engine.yaml <_include/hyperscan_regex_engine.yaml>` diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 86f9caac5d6c..5e9d05931314 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -36,6 +36,8 @@ Third, an upstream cluster must include an endpoint with an internal address ref envoy_internal_address: server_listener_name: internal_listener +.. _config_internal_upstream_transport: + Internal upstream transport --------------------------- diff --git a/docs/root/configuration/other_features/other_features.rst b/docs/root/configuration/other_features/other_features.rst index 1375d4c76fbb..b5ba3fd18aeb 100644 --- a/docs/root/configuration/other_features/other_features.rst +++ b/docs/root/configuration/other_features/other_features.rst @@ -5,8 +5,10 @@ Other features :maxdepth: 2 dlb + hyperscan internal_listener rate_limit vcl wasm wasm_service + qatzip diff --git a/docs/root/configuration/other_features/qatzip.rst b/docs/root/configuration/other_features/qatzip.rst new file mode 100644 index 000000000000..1bbe056c6288 --- /dev/null +++ b/docs/root/configuration/other_features/qatzip.rst @@ -0,0 +1,34 @@ +.. _config_qatzip: + +Qatzip Compressor +======================= + +* :ref:`v3 API reference ` + + +Qatzip compressor provides Envoy with faster hardware-accelerated gzip compression by integrating with `Intel® QuickAssist Technology (Intel® QAT) `_ through the qatlib and QATzip libraries. + +Example configuration +--------------------- + +An example for Qatzip compressor configuration is: + +.. literalinclude:: _include/qatzip.yaml + :language: yaml + + +How it works +------------ + +If enabled, the Qatzip compressor will: + +- attach Qat hardware +- create Threadlocal Qat session context for each worker thread + +When a new http request comes, one worker thread will process it using its Qat session context and send the data needed to be compressed to +Qat hardware. + +Installing and using QATzip +--------------------------- + +For information on how to build/install and use QATzip see `QATzip `_. diff --git a/docs/root/faq/debugging/how_to_dump_heap_profile_of_envoy.rst b/docs/root/faq/debugging/how_to_dump_heap_profile_of_envoy.rst index e104cb7ce4fb..4f5b60624fd8 100644 --- a/docs/root/faq/debugging/how_to_dump_heap_profile_of_envoy.rst +++ b/docs/root/faq/debugging/how_to_dump_heap_profile_of_envoy.rst @@ -26,7 +26,7 @@ And then you can analyze the outputted heap profile with pprof: .. code-block:: bash - $ pprof -http:localhost:9999 /heap/output/envoy.heap + $ pprof -http localhost:9999 /heap/output/envoy.heap .. note:: If you dump the heap profile in the production environment and analyze it in the local environment, please ensure diff --git a/docs/root/intro/arch_overview/advanced/attributes.rst b/docs/root/intro/arch_overview/advanced/attributes.rst index 4e191a82d19e..aabbf41ce368 100644 --- a/docs/root/intro/arch_overview/advanced/attributes.rst +++ b/docs/root/intro/arch_overview/advanced/attributes.rst @@ -120,6 +120,7 @@ RBAC): connection.uri_san_local_certificate, string, The first URI entry in the SAN field of the local certificate in the downstream TLS connection connection.uri_san_peer_certificate, string, The first URI entry in the SAN field of the peer certificate in the downstream TLS connection connection.sha256_peer_certificate_digest, SHA256 digest of the peer certificate in the downstream TLS connection if present + connection.transport_failure_reason, string, The transport failure reason e.g. certificate validation failed The following additional attributes are available upon the downstream connection termination: @@ -176,8 +177,11 @@ following attributes: :header: Attribute, Type, Description :widths: 1, 1, 4 + xds.node, :ref:`Node`, Local node description xds.cluster_name, string, Upstream cluster name xds.cluster_metadata, :ref:`Metadata`, Upstream cluster metadata + xds.listener_direction, int, Enumeration value of the :ref:`listener traffic direction` + xds.listener_metadata, :ref:`Metadata`, Listener metadata xds.route_name, string, Route name xds.route_metadata, :ref:`Metadata`, Route metadata xds.upstream_host_metadata, :ref:`Metadata`, Upstream host metadata @@ -196,14 +200,14 @@ In addition to all above, the following extra attributes are available to Wasm e plugin_name, string, Plugin name plugin_root_id, string, Plugin root ID plugin_vm_id, string, Plugin VM ID - node, :ref:`Node`, Local node description - cluster_name, string, Upstream cluster name - cluster_metadata, :ref:`Metadata`, Upstream cluster metadata - listener_direction, int, Enumeration value of the :ref:`listener traffic direction` - listener_metadata, :ref:`Metadata`, Listener metadata - route_name, string, Route name - route_metadata, :ref:`Metadata`, Route metadata - upstream_host_metadata, :ref:`Metadata`, Upstream host metadata + node, :ref:`Node`, Local node description. DEPRECATED: please use `xds` attributes. + cluster_name, string, Upstream cluster name. DEPRECATED: please use `xds` attributes. + cluster_metadata, :ref:`Metadata`, Upstream cluster metadata. DEPRECATED: please use `xds` attributes. + listener_direction, int, Enumeration value of the :ref:`listener traffic direction`. DEPRECATED: please use `xds` attributes. + listener_metadata, :ref:`Metadata`, Listener metadata. DEPRECATED: please use `xds` attributes. + route_name, string, Route name. DEPRECATED: please use `xds` attributes. + route_metadata, :ref:`Metadata`, Route metadata. DEPRECATED: please use `xds` attributes. + upstream_host_metadata, :ref:`Metadata`, Upstream host metadata. DEPRECATED: please use `xds` attributes. Path expressions ---------------- diff --git a/docs/root/intro/arch_overview/advanced/matching/matching.rst b/docs/root/intro/arch_overview/advanced/matching/matching.rst index c4b4e20eaf87..2d96243eb6b0 100644 --- a/docs/root/intro/arch_overview/advanced/matching/matching.rst +++ b/docs/root/intro/arch_overview/advanced/matching/matching.rst @@ -1,3 +1,5 @@ +.. _arch_overview_generic_matching: + Generic Matching ================ diff --git a/docs/root/intro/arch_overview/advanced/matching/matching_api.rst b/docs/root/intro/arch_overview/advanced/matching/matching_api.rst index 767261840667..e87f209818f0 100644 --- a/docs/root/intro/arch_overview/advanced/matching/matching_api.rst +++ b/docs/root/intro/arch_overview/advanced/matching/matching_api.rst @@ -3,11 +3,6 @@ Matching API ============ -.. attention:: - - The matching API is alpha and is currently under active development. - Capabilities will be expanded over time and the configuration structures are likely to change. - Envoy makes use of a :ref:`matching API ` to allow the various subsystems to express actions that should be performed based on incoming data. @@ -98,6 +93,8 @@ are available in some contexts: * CEL matching input matcher: :ref:`CEL input matcher `. +* Regex matching using :ref:`Hyperscan matcher `. + Matching actions ################ diff --git a/docs/root/intro/arch_overview/http/http3.rst b/docs/root/intro/arch_overview/http/http3.rst index a8d969c45aab..e87cd4dc6b7b 100644 --- a/docs/root/intro/arch_overview/http/http3.rst +++ b/docs/root/intro/arch_overview/http/http3.rst @@ -8,8 +8,8 @@ HTTP/3 overview While HTTP/3 **downstream support is deemed ready for production use**, improvements are ongoing, tracked in the `area-quic `_ tag. - HTTP/3 **upstream support is fine for locally controlled networks**, but is not ready for - general internet use, and is missing some key latency features. See details below. + HTTP/3 **upstream support is fine for locally controlled networks**, but is alpha for + general internet use - key features are implemented but have not been tested at scale. .. _arch_overview_http3_downstream: diff --git a/docs/root/intro/arch_overview/http/http_connection_management.rst b/docs/root/intro/arch_overview/http/http_connection_management.rst index 8633a5ee5f53..f5ff6df01b25 100644 --- a/docs/root/intro/arch_overview/http/http_connection_management.rst +++ b/docs/root/intro/arch_overview/http/http_connection_management.rst @@ -242,6 +242,10 @@ upstream will be modified by: #. Putting the fully qualified original request URL in the ``x-envoy-original-url`` header. #. Replacing the ``Authority``/``Host``, ``Scheme``, and ``Path`` headers with the values from the ``Location`` header. +#. Copying any headers listed in + :ref:`response_headers_to_copy` + from the redirect response into the headers that will be used in the + subsequent request. The altered request headers will then have a new route selected, be sent through a new filter chain, and then shipped upstream with all of the normal Envoy request sanitization taking place. @@ -252,6 +256,11 @@ and then shipped upstream with all of the normal Envoy request sanitization taki second route, even if they are the same, so be careful configuring header modification rules to avoid duplicating undesired header values. +.. warning:: + Note that no downstream filters will see the response that triggers the internal redirect. If there is a need + to pass data between the redirect response and the followup request, see + :ref:`response_headers_to_copy`. + A sample redirect flow might look like this: #. Client sends a ``GET`` request for http://foo.com/bar. diff --git a/docs/root/intro/arch_overview/http/http_filters.rst b/docs/root/intro/arch_overview/http/http_filters.rst index 034a2a655287..5486ff37402c 100644 --- a/docs/root/intro/arch_overview/http/http_filters.rst +++ b/docs/root/intro/arch_overview/http/http_filters.rst @@ -9,8 +9,8 @@ HTTP level filter stack within the connection manager. Filters can be written that operate on HTTP level messages without knowledge of the underlying physical protocol (HTTP/1.1, HTTP/2, etc.) or multiplexing capabilities. -HTTP filters can be downstream filters, associated with a given listener and doing stream processing on each -downstream request before routing, or upstream filters, associated with a given cluster and doing stream processing once per upstream request, after the router filter. +HTTP filters can be downstream HTTP filters, associated with a given listener and doing stream processing on each +downstream request before routing, or upstream HTTP filters, associated with a given cluster and doing stream processing once per upstream request, after the router filter. There are three types of HTTP level filters: @@ -83,7 +83,7 @@ During downstream HTTP filter chain processing, when ``decodeHeaders()`` is invo connection manager performs route resolution and sets a *cached route* pointing to an upstream cluster. -Downstream filters have the capability to directly mutate this *cached route* after route resolution, via the +Downstream HTTP filters have the capability to directly mutate this *cached route* after route resolution, via the ``setRoute`` callback and :repo:`DelegatingRoute ` mechanism. @@ -148,3 +148,70 @@ the filter config name. Use of per filter config map is filter specific. See the :ref:`HTTP filter documentation ` for if and how it is utilized for every filter. + +.. _arch_overview_http_filters_route_based_filter_chain: + +Route based filter chain +------------------------ + +There is support for having different filter chains for different routes. There are two different modes for this: + +* Disabling a filter in the filter chain for specific routes. +* Overriding a filter in the filter chain that is disabled by default and enabling it for specific + routes. + +By default, the filter chain is the same for all routes and all filters are enabled. However, a filter +can be disabled for specific routes by using the :ref:`FilterConfig ` +and setting the :ref:`disabled field ` in the +per filter config map in the route configuration. See the +:ref:`Route specific config ` section for more details. + +For example, given following http filter config: + +.. code-block:: yaml + + http_filters: + - name: buffer + typed_config: { ... } + - name: lua + typed_config: { ... } + +Both the ``buffer`` and ``lua`` filters are enabled by default. If we want to disable the ``buffer`` filter +for a specific route, we can set the per filter config map in the route configuration: + +.. code-block:: yaml + + typed_per_filter_config: + buffer: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + +In addition, we can set a filter to be disabled by default by setting the :ref:`disabled field +` +in the HttpFilter configuration and then enable it for specific routes if needed. + +For example, given following http filter config: + +.. code-block:: yaml + + http_filters: + - name: buffer + typed_config: { ... } + disabled: true + - name: lua + typed_config: { ... } + disabled: true + +Both the ``buffer`` and ``lua`` filters are disabled by default. If we want to enable one of them +for a specific route, we can set per filter config map in the route configuration: + +.. code-block:: yaml + + typed_per_filter_config: + lua: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute + name: my_lua_script + + +Legitimate route-specific configuration for filter (like the above ``lua`` filter) is valid way to +enable the filter for the route. diff --git a/docs/root/intro/arch_overview/http/upgrades.rst b/docs/root/intro/arch_overview/http/upgrades.rst index 3e06357f9f93..b6301de9caf1 100644 --- a/docs/root/intro/arch_overview/http/upgrades.rst +++ b/docs/root/intro/arch_overview/http/upgrades.rst @@ -137,51 +137,6 @@ will synthesize 200 response headers, and then forward the TCP data as the HTTP "upstream" TLS loopback connection to encrypt it, then have a TCP listener take the encrypted payload and send the ``CONNECT`` upstream. -``CONNECT-UDP`` support -^^^^^^^^^^^^^^^^^^^^^^^ -.. note:: - ``CONNECT-UDP`` is in an alpha status and may not be stable enough for use in production. - We recommend to use this feature with caution. - -``CONNECT-UDP`` (`RFC 9298 `_) allows HTTP clients -to create UDP tunnels through an HTTP proxy server. Unlike ``CONNECT``, which is limited to -tunneling TCP, ``CONNECT-UDP`` can be used to proxy UDP-based protocols such as HTTP/3. - -``CONNECT-UDP`` support is disabled by default in Envoy. Similar to ``CONNECT``, it can be enabled -through the :ref:`upgrade_configs ` -by setting the value to the special keyword ``CONNECT-UDP``. Like ``CONNECT``, ``CONNECT-UDP`` requests -are forwarded to the upstream by default. -:ref:`connect_config ` -must be set to terminate the requests and forward the payload as UDP datagrams to the target. - -Example Configuration ---------------------- - -The following example configuration makes Envoy forward ``CONNECT-UDP`` requests to the upstream. Note -that the :ref:`upgrade_configs ` -is set to ``CONNECT-UDP``. - -.. literalinclude:: /_configs/repo/proxy_connect_udp_http3_downstream.yaml - :language: yaml - :linenos: - :lines: 27-54 - :lineno-start: 27 - :emphasize-lines: 15-16, 25-26 - :caption: :download:`proxy_connect_udp_http3_downstream.yaml ` - -The following example configuration makes Envoy terminate ``CONNECT-UDP`` requests and send UDP payloads -to the target. As in this example, the -:ref:`connect_config ` -must be set to terminate ``CONNECT-UDP`` requests. - -.. literalinclude:: /_configs/repo/terminate_http3_connect_udp.yaml - :language: yaml - :linenos: - :lines: 26-57 - :lineno-start: 26 - :emphasize-lines: 15-16, 19-22, 29-30 - :caption: :download:`terminate_http3_connect_udp.yaml ` - .. _tunneling-tcp-over-http: Tunneling TCP over HTTP @@ -250,3 +205,94 @@ before starting to stream the downstream TCP data to the upstream. If you want to decapsulate a ``CONNECT`` request and also do HTTP processing on the decapsulated payload, the easiest way to accomplish it is to use :ref:`internal listeners `. + +``CONNECT-UDP`` support +^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: + ``CONNECT-UDP`` is in an alpha status and may not be stable enough for use in production. + We recommend to use this feature with caution. + +``CONNECT-UDP`` (`RFC 9298 `_) allows HTTP clients +to create UDP tunnels through an HTTP proxy server. Unlike ``CONNECT``, which is limited to +tunneling TCP, ``CONNECT-UDP`` can be used to proxy UDP-based protocols such as HTTP/3. + +``CONNECT-UDP`` support is disabled by default in Envoy. Similar to ``CONNECT``, it can be enabled +through the :ref:`upgrade_configs ` +by setting the value to the special keyword ``CONNECT-UDP``. Like ``CONNECT``, ``CONNECT-UDP`` requests +are forwarded to the upstream by default. +:ref:`connect_config ` +must be set to terminate the requests and forward the payload as UDP datagrams to the target. + +Example Configuration +--------------------- + +The following example configuration makes Envoy forward ``CONNECT-UDP`` requests to the upstream. Note +that the :ref:`upgrade_configs ` +is set to ``CONNECT-UDP``. + +.. literalinclude:: /_configs/repo/proxy_connect_udp_http3_downstream.yaml + :language: yaml + :linenos: + :lines: 27-54 + :lineno-start: 27 + :emphasize-lines: 15-16, 25-26 + :caption: :download:`proxy_connect_udp_http3_downstream.yaml ` + +The following example configuration makes Envoy terminate ``CONNECT-UDP`` requests and send UDP payloads +to the target. As in this example, the +:ref:`connect_config ` +must be set to terminate ``CONNECT-UDP`` requests. + +.. literalinclude:: /_configs/repo/terminate_http3_connect_udp.yaml + :language: yaml + :linenos: + :lines: 26-57 + :lineno-start: 26 + :emphasize-lines: 15-16, 19-22, 29-30 + :caption: :download:`terminate_http3_connect_udp.yaml ` + +.. _tunneling-udp-over-http: + +Tunneling UDP over HTTP +^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + Raw UDP tunneling is in an alpha status and may not be stable enough for use in production. + We recommend to use this feature with caution. + +Apart from ``CONNECT-UDP`` termination, as described in the section above, Envoy also has support for tunneling raw UDP over HTTP +``CONNECT`` or HTTP ``POST`` requests, by utilizing the UDP Proxy listener filter. By default, UDP tunneling is disabled, and can be +enabled by setting the configuration for :ref:`tunneling_config `. + +.. note:: + Currently, Envoy only supports UDP tunneling over HTTP/2 streams. + +By default, the ``tunneling_config`` will upgrade the connection to create HTTP/2 streams for each UDP session (a UDP session is identified by the datagrams 5-tuple), according +to the `Proxying UDP in HTTP RFC `_. Since this upgrade protocol requires an encapsulation mechanism to preserve the boundaries of the original datagram, +it's required to apply the :ref:`HTTP Capsule ` session filter. +The HTTP/2 streams will be multiplexed over the upstream connection. + +As opposed to TCP tunneling, where downstream flow control can be applied by alternately disabling the read from the connection socket, for UDP datagrams, this mechanism is not supported. +Therefore, when tunneling UDP and a new datagram is received from the downstream, it is either streamed upstream, if the upstream is ready or halted by the UDP Proxy. +In case the upstream is not ready (for example, when waiting for HTTP response headers), the datagram can either be dropped or buffered until the upstream is ready. +In such cases, by default, downstream datagrams will be dropped, unless :ref:`buffer_options ` +is set by the ``tunneling_config``. The default buffer limits are modest to try and prevent a lot of unwanted buffered memory, but can and should be adjusted per the required use-case. +When the upstream becomes ready, the UDP Proxy will first flush all the previously buffered datagrams. + +.. note:: + If ``POST`` is set, the upstream stream does not comply with the connect-udp RFC, and instead it will be a POST request. + The path used in the headers will be set from the post_path field, and the headers will not contain the target host and + target port, as required by the connect-udp protocol. This option should be used carefully. + +Example Configuration +--------------------- + +The following example configuration makes Envoy tunnel raw UDP datagrams over an upgraded ``CONNECT-UDP`` requests to the upstream. + +.. literalinclude:: /_configs/repo/raw_udp_tunneling_http2.yaml + :language: yaml + :linenos: + :lines: 32-53 + :lineno-start: 32 + :emphasize-lines: 4-4, 15-17 + :caption: :download:`raw_udp_tunneling_http2.yaml ` diff --git a/docs/root/intro/arch_overview/observability/access_logging.rst b/docs/root/intro/arch_overview/observability/access_logging.rst index db3957f4f8fc..0371c385f52d 100644 --- a/docs/root/intro/arch_overview/observability/access_logging.rst +++ b/docs/root/intro/arch_overview/observability/access_logging.rst @@ -17,16 +17,23 @@ logs`. The listener a HTTP request access logging and can be enabled separately and independently from filter access logs. -If access log is enabled, then by default it will be reported to the configured sinks at the end of a TCP -connection, or HTTP stream. It is possible to extend this behavior and report access logs periodically or at the -start of a TCP connection or HTTP stream. Reporting access logs right after upstream connection establishment -or new incoming HTTP request does not depend on periodic reporting, and the other way around. +If access log is enabled, then by default it will be reported to the configured sinks at the end of a UDP +session, TCP connection, or HTTP stream. It is possible to extend this behavior and report access logs +periodically or at the start of a UDP session, TCP connection, or HTTP stream. Reporting access logs right +upstream connection establishment or new incoming HTTP request does not depend on periodic reporting, and +the other way around. .. _arch_overview_access_log_start: Start of session access logs ---------------------------- +UDP Proxy +********* + +For UDP Proxy, when UDP tunneling over HTTP is configured, it is possible to enable an access log record once after a successful upstream tunnel connected by using +:ref:`access log flush interval ` + TCP Proxy ********* @@ -53,6 +60,12 @@ Note: In case that the HTTP request involves retries, a start of request upstrea Periodic access logs -------------------- +UDP Proxy +********* + +For UDP Proxy, it is possible to enable a prediodic access log by using +:ref:`access log flush interval ` + TCP Proxy ********* diff --git a/docs/root/intro/arch_overview/observability/tracing.rst b/docs/root/intro/arch_overview/observability/tracing.rst index e6f2730a80dc..db84daf8c1bc 100644 --- a/docs/root/intro/arch_overview/observability/tracing.rst +++ b/docs/root/intro/arch_overview/observability/tracing.rst @@ -39,9 +39,6 @@ initiated: * Randomly sampled via the :ref:`random_sampling ` runtime setting. -The router filter is also capable of creating a child span for egress calls via the -:ref:`start_child_span ` option. - .. _arch_overview_tracing_context_propagation: Trace context propagation @@ -163,3 +160,62 @@ Tracing providers have varying level of support for getting and setting baggage: * Lightstep (and any OpenTelemetry-compliant tracer) can read/write baggage * Zipkin support is not yet implemented * X-Ray and OpenCensus don't support baggage + +Different types of span +----------------------- + +As mentioned in the previous paragraph, a trace is composed of one or more spans, which may have different types. +Tracing systems such as SkyWalking, ZipKin, and OpenTelemetry, among others, offer the same or similar span types. +The most common types are CLIENT and SERVER. A CLIENT type span is generated by a client for a request that is +sent to a server, while a SERVER type span is generated by a server for a request that is received from a client. + + +A basic trace chain looks like the following snippet. Typically, the parent span of a server span should be a +client span. Every hop in the chain must ensure the correctness of the span type. + +.. code-block:: text + + -> [SERVER -> CLIENT] -> [SERVER -> CLIENT] -> ... + App A App B + + +Different modes of Envoy +------------------------ + +Because Envoy is widely used in the service mesh as sidecar, it is important to understand the different tracing +modes of Envoy. + + +In the first mode, Envoy is used as a sidecar. The sidecar and its associated application are treated as a single +hop in the trace chain. If a tracing system with typed spans is used, the ideal trace chain might look like the +following snippet. + +.. code-block:: text + + -> [[SERVER (inbound sidecar) -> App -> CLIENT (outbound sidecar)]] -> ... + App + + +As you can see, in the first mode, the inbound sidecar will always generate a SERVER span, and the outbound sidecar +will always generate a CLIENT span. The application will not generate any spans but will only propagate the trace context. + + +In the second mode, Envoy is used as a gateway. Or, Envoy can be used as a sidecar, but in this case, +the sidecar and its application are treated as separate hops in the trace chain. If a tracing system with typed spans +is used, the ideal trace chain might look like the following snippet. + +.. code-block:: text + + -> [SERVER -> CLIENT] -> [SERVER -> CLIENT] -> [SERVER -> CLIENT] -> [SERVER -> CLIENT] -> ... + Gateway Inbound Sidecar App Outbound Sidecar + + + +As you can see, in the second mode, Envoy will generate a SERVER span for downstream requests and a CLIENT span for +upstream requests. The application may also generate spans for its own work. + + +To enable this mode, please set +:ref:`spawn_upstream_span ` +to true explicitly. This tells the tracing provider to generate a CLIENT span for upstream requests and treat Envoy +as an independent hop in the trace chain. diff --git a/docs/root/intro/arch_overview/other_protocols/grpc.rst b/docs/root/intro/arch_overview/other_protocols/grpc.rst index 5ab2fabff78e..eee8e6e6b0d8 100644 --- a/docs/root/intro/arch_overview/other_protocols/grpc.rst +++ b/docs/root/intro/arch_overview/other_protocols/grpc.rst @@ -34,8 +34,8 @@ Envoy supports multiple gRPC bridges: * :ref:`grpc_http1_reverse_bridge filter ` which allows gRPC requests to be sent to Envoy and then translated to HTTP/1.1 when sent to the upstream. The response is then converted back into gRPC when sent to the downstream. This filter can also optionally manage the gRPC frame header, allowing the upstream to not have to be gRPC aware at all. -* :ref:`connect_grpc_bridge filter ` which allows Buf Connect requests to be sent to Envoy. - Envoy then translates the requests to gRPC to be sent to the upstream. The response is converted back into the Buf Connect protocol +* :ref:`connect_grpc_bridge filter ` which allows Connect requests to be sent to Envoy. + Envoy then translates the requests to gRPC to be sent to the upstream. The response is converted back into the Connect protocol to be sent back to the downstream. HTTP/1.1 requests will be upgraded to HTTP/2 or HTTP/3 when needed. .. _arch_overview_grpc_services: diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index d760a2b9b15d..1b38fd8c887a 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -208,6 +208,7 @@ For details on each command's usage see the official SRANDMEMBER, Set SREM, Set SSCAN, Set + WATCH, String ZADD, Sorted Set ZCARD, Sorted Set ZCOUNT, Sorted Set @@ -237,6 +238,7 @@ For details on each command's usage see the official DECRBY, String GET, String GETBIT, String + GETDEL, String GETRANGE, String GETSET, String INCR, String diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst index e99fe65b231c..3dac22ac4519 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst @@ -36,9 +36,10 @@ same or different weights. host which has the fewest active requests (`Mitzenmacher et al. `_ has shown that this approach is nearly as good as an O(N) full scan). This is also known as P2C (power of two - choices). The P2C load balancer has the property that a host with the highest number of active - requests in the cluster will never receive new requests. It will be allowed to drain until it is - less than or equal to all of the other hosts. + choices). The P2C load balancer has the property that host weights will decrease as the number of + active requests on those hosts increases. P2C selection is particularly useful for load + balancer implementations due to its resistance to + [herding behavior](https://en.wikipedia.org/wiki/Thundering_herd_problem). * *all weights not equal*: If two or more hosts in the cluster have different load balancing weights, the load balancer shifts into a mode where it uses a weighted round robin schedule in which weights are dynamically adjusted based on the host's request load at the time of selection. diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/slow_start.rst b/docs/root/intro/arch_overview/upstream/load_balancing/slow_start.rst index 56d05dafc825..efc308c85bbd 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/slow_start.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/slow_start.rst @@ -63,3 +63,4 @@ Once endpoints E1 and E2 exit slow start mode, their load balancing weight remai :align: center *Note* in case when multiple priorities are used with slow start and lower priority has just one endpoint A, during cross-priority spillover there will be no progressive increase of traffic to endpoint A, all traffic will shift at once. +Same applies to locality weighted loadbalancing, when slow start is enabled for the upstream cluster and traffic is routed cross zone to a zone with one endpoint A, there will be no progressive increase of traffic to endpoint A. diff --git a/docs/root/intro/arch_overview/upstream/upstream_filters.rst b/docs/root/intro/arch_overview/upstream/upstream_filters.rst index 4a2b4da0d3b3..46be46cd32a0 100644 --- a/docs/root/intro/arch_overview/upstream/upstream_filters.rst +++ b/docs/root/intro/arch_overview/upstream/upstream_filters.rst @@ -5,8 +5,8 @@ Upstream network filters Upstream clusters provide an ability to inject network level (L3/L4) filters. It should be noted that a network filter needs to -be registered in code as an upstream filter before usage. Currently, -there are no upstream filters available in Envoy out of the box. +be registered in code as an upstream network filter before usage. Currently, +there are no upstream network filters available in Envoy out of the box. The filters apply to the connection to the upstream hosts, using the same API presented by listeners for the downstream connections. The write-callbacks are invoked for any chunk of data sent to the upstream host, and the read-callbacks are invoked for data diff --git a/docs/root/intro/life_of_a_request.rst b/docs/root/intro/life_of_a_request.rst index a633cd01404c..34def64da576 100644 --- a/docs/root/intro/life_of_a_request.rst +++ b/docs/root/intro/life_of_a_request.rst @@ -200,7 +200,7 @@ A brief outline of the life cycle of a request and response using the example co capacity. 8. For each stream an :ref:`Upstream HTTP filter ` chain is created and runs. By default this only includes the CodecFilter, sending data to the appropriate codec, but if - the cluster is configured with an upstream filter chain, that filter chain will be created and run + the cluster is configured with an upstream HTTP filter chain, that filter chain will be created and run on each stream, which includes creating and running separate filter chains for retries and shadowed requests. 9. The upstream endpoint connection's HTTP/2 codec multiplexes and frames the request’s stream with @@ -210,7 +210,7 @@ A brief outline of the life cycle of a request and response using the example co 11. The request, consisting of headers, and optional body and trailers, is proxied upstream, and the response is proxied downstream. The response passes through the HTTP filters in the :ref:`opposite order ` from the request, starting at the - codec filter, traversing any upstream filters, then going through the router filter and passing + codec filter, traversing any upstream HTTP filters, then going through the router filter and passing through CustomFilter, before being sent downstream. 12. When the response is complete, the stream is destroyed. Post-request processing will update stats, write to the access log and finalize trace spans. @@ -461,13 +461,13 @@ stream allocated from the HTTP connection pool. It also is responsible for reque and affinity. The router filter is also responsible for the creation and running of the `Upstream HTTP filter ` -chain. By default, upstream filters will start running immediately after headers arrive at the router +chain. By default, upstream HTTP filters will start running immediately after headers arrive at the router filter, however C++ filters can pause until the upstream connection is established if they need to -inspect the upstream stream or connection. Upstream filter chains are by default configured via cluster -configuration, so for example a shadowed request can have a separate upstream filter chain for the primary -and shadowed clusters. Also as the upstream filter chain is upstream of the router filter, it is run per each +inspect the upstream stream or connection. Upstream HTTP filter chains are by default configured via cluster +configuration, so for example a shadowed request can have a separate upstream HTTP filter chain for the primary +and shadowed clusters. Also as the upstream HTTP filter chain is upstream of the router filter, it is run per each retry attempt allowing header manipulation per retry and including information about the upstream stream and -connection. Unlike downstream filters, upstream filters can not alter the route. +connection. Unlike downstream HTTP filters, upstream HTTP filters can not alter the route. 7. Load balancing ^^^^^^^^^^^^^^^^^ diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index bcc938406ce6..573cf799a971 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -307,16 +307,32 @@ modify different aspects of the server: Enable/disable logging levels for different loggers. - - To change the logging level across all loggers, set the query parameter as level=. - - To change a particular logger's level, set the query parameter like so, =. - - To change multiple logging levels at once, set the query parameter as paths==,=. - - To list the loggers, send a POST request to the /logging endpoint without a query parameter. + If the default component logger is used, the logger name should be exactlly the component name. - .. note:: + - To change the logging level across all loggers, set the query parameter as ``level=``. + - To change a particular logger's level, set the query parameter like so, ``=``. + - To change multiple logging levels at once, set the query parameter as ``paths=:,:``. + - To list the loggers, send a POST request to the ``/logging`` endpoint without a query parameter. - Generally only used during development. With ``--enable-fine-grain-logging`` being set, the logger is represented - by the path of the file it belongs to (to be specific, the path determined by ``__FILE__``), so the logger list - will show a list of file paths, and the specific path should be used as to change the log level. + If ``--enable-fine-grain-logging`` is set, the logger is represented by the path of the file it belongs to (to be specific, the path determined by ``__FILE__``), + so the logger list will show a list of file paths, and the specific path should be used as ```` to change the log level. + + We also added the file basename, glob ``*`` and ``?`` support for fine-grain loggers. For example, we have the following active loggers with trace level. + + .. code-block:: text + + source/server/admin/admin_filter.cc: 0 + source/common/event/dispatcher_impl.cc: 0 + source/common/network/tcp_listener_impl.cc: 0 + source/common/network/udp_listener_impl.cc: 0 + + - ``/logging?paths=source/common/event/dispatcher_impl.cc:debug`` will make the level of ``source/common/event/dispatcher_impl.cc`` be debug. + - ``/logging?admin_filter=info`` will make the level of ``source/server/admin/admin_filter.cc`` be info, and other unmatched loggers will be the default trace. + - ``/logging?paths=source/common*:warning`` will make the level of ``source/common/event/dispatcher_impl.cc:``, ``source/common/network/tcp_listener_impl.cc`` be warning. + Other unmatched loggers will be the default trace, e.g., `admin_filter.cc`, even it was updated to info from the previous post update. + - ``/logging?paths=???_listener_impl:info`` will make the level of ``source/common/network/tcp_listener_impl.cc``, ``source/common/network/udp_listener_impl.cc`` be info. + - ``/logging?paths=???_listener_impl:info,tcp_listener_impl:warning``, the level of ``source/common/network/tcp_listener_impl.cc`` will be info, since the first match will take effect. + - ``/logging?level=info`` will change the default verbosity level to info. All the unmatched loggers in the following update will be this default level. .. http:get:: /memory @@ -350,6 +366,10 @@ modify different aspects of the server: This behaviour and duration is configurable via server options or CLI (:option:`--drain-time-s` and :option:`--drain-strategy`). + .. http:post:: /drain_listeners?graceful&skip_exit + + When draining listeners, do not exit after the drain period. This must be used with `graceful`. + .. attention:: This operation directly stops the matched listeners on workers. Once listeners in a given diff --git a/docs/root/start/building/local_docker_build.rst b/docs/root/start/building/local_docker_build.rst index 86147f26fc2a..714437603dd1 100644 --- a/docs/root/start/building/local_docker_build.rst +++ b/docs/root/start/building/local_docker_build.rst @@ -7,6 +7,13 @@ Building an Envoy Docker image The following steps guide you through building your own Envoy binary, and putting that in a clean Ubuntu container. +.. tip:: + These instructions run commands in Docker using ``ci/run_envoy_docker.sh``. + + By default this will place bazel run files and any artefacts in ``/tmp/envoy-docker-build``. + + You can override this by setting the ``ENVOY_DOCKER_BUILD_DIR`` env var to a path of your choosing. + **Step 1: Build Envoy** Using ``envoyproxy/envoy-build`` you will compile Envoy. @@ -16,7 +23,7 @@ This image has all software needed to build Envoy. From your Envoy directory: $ pwd src/envoy - $ ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release' + $ ./ci/run_envoy_docker.sh './ci/do_ci.sh release' That command will take some time to run because it is compiling an Envoy binary and running tests. @@ -27,13 +34,11 @@ also build as follows: $ pwd src/envoy - $ ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release.server_only' - + $ ./ci/run_envoy_docker.sh './ci/do_ci.sh release.server_only' For more information on building and different build targets, please refer to :repo:`ci/README.md`. .. warning:: - These instructions for building Envoy use `envoyproxy/envoy-build-ubuntu `_ image. You will need 4-5GB of disk space to accommodate this image. @@ -42,17 +47,21 @@ For more information on building and different build targets, please refer to :r **Step 2: Build image with only Envoy binary** -In this step we'll build an image that only has the Envoy binary, and none -of the software used to build it.: +In this step we'll build the Envoy deployment images. + +.. note:: + The ``docker`` CI target expects a release tarball to have been built previously using one of the steps above. + +In order to build Docker inside the Envoy build image we need to set the env var ``ENVOY_DOCKER_IN_DOCKER`` .. code-block:: console $ pwd src/envoy/ - $ docker build -f ci/Dockerfile-envoy -t envoy . + $ ENVOY_DOCKER_IN_DOCKER=1 ./ci/run_envoy_docker.sh './ci/do_ci.sh docker' -Now you can use this ``envoy`` image to build the any of the sandboxes if you change -the ``FROM`` line in any Dockerfile. +Now you can use the Envoy image to build the any of the sandboxes by changing +the ``FROM`` line in a related Dockerfile. -This will be particularly useful if you are interested in modifying Envoy, and testing +This can be particularly useful if you are interested in modifying Envoy, and testing your changes. diff --git a/docs/root/start/sandboxes/dynamic-configuration-control-plane.rst b/docs/root/start/sandboxes/dynamic-configuration-control-plane.rst index 4bf6090c4e62..d0c0c908102d 100644 --- a/docs/root/start/sandboxes/dynamic-configuration-control-plane.rst +++ b/docs/root/start/sandboxes/dynamic-configuration-control-plane.rst @@ -202,6 +202,12 @@ is configured to proxy to ``service2``: :language: json :emphasize-lines: 3, 11, 19-20 +.. note:: + In this example we increment the version for simplicity. + + Any change to the version will trigger an update in Envoy, and ordering is not significant + (see :ref:`xDS protocol docs for further information about updates `). + .. seealso:: :ref:`Dynamic configuration (control plane) quick start guide ` diff --git a/docs/root/start/sandboxes/golang-http.rst b/docs/root/start/sandboxes/golang-http.rst index 89f483a07914..43680611d71b 100644 --- a/docs/root/start/sandboxes/golang-http.rst +++ b/docs/root/start/sandboxes/golang-http.rst @@ -20,12 +20,12 @@ It also shows how Go plugins can be independently configured at runtime. Step 1: Compile the go plugin library ************************************* -Change to the ``examples/golang`` directory and build the go plugin library. +Change to the ``examples/golang-http`` directory and build the go plugin library. .. code-block:: console $ pwd - envoy/examples/golang + envoy/examples/golang-http $ docker compose -f docker-compose-go.yaml run --rm go_plugin_compile The compiled library should now be in the ``lib`` folder. @@ -40,15 +40,6 @@ Step 2: Start all of our containers Start all the containers. -.. warning:: - The Envoy Golang filter is designed to be run with the ``GODEBUG=cgocheck=0`` environment variable set. - - This disables the cgo pointer check. - - Failure to set this environment variable will cause Envoy to crash! - - Here, we have set this environment variable in :repo:`Dockerfile ` - .. code-block:: console $ docker compose pull diff --git a/docs/root/start/sandboxes/index.rst b/docs/root/start/sandboxes/index.rst index d04bf9eb78e7..aa8ce6e7ca85 100644 --- a/docs/root/start/sandboxes/index.rst +++ b/docs/root/start/sandboxes/index.rst @@ -65,6 +65,7 @@ The following sandboxes are available: load_reporting_service locality_load_balancing local_ratelimit + lua-cluster-specifier lua mysql opentelemetry diff --git a/docs/root/start/sandboxes/lua-cluster-specifier.rst b/docs/root/start/sandboxes/lua-cluster-specifier.rst new file mode 100644 index 000000000000..a7e4c852c849 --- /dev/null +++ b/docs/root/start/sandboxes/lua-cluster-specifier.rst @@ -0,0 +1,71 @@ +.. _install_sandboxes_lua_cluster_specifier: + +Lua Cluster Specifier +===================== + +.. sidebar:: Requirements + + .. include:: _include/docker-env-setup-link.rst + + :ref:`curl ` + Used to make ``HTTP`` requests. + +In this example, we show how the `Lua `_ cluster specifier can be used with the +Envoy proxy. + +The example Envoy proxy configuration includes a Lua cluster specifier plugin that contains a function: + +- ``envoy_on_route(route_handle)`` + +:ref:`See here ` for an overview of Envoy's Lua cluster specifier +and documentation regarding the function. + +Step 1: Build the sandbox +************************* + +Change to the ``examples/lua-cluster-specifier`` directory. + +.. code-block:: console + + $ pwd + envoy/examples/lua-cluster-specifier + $ docker compose pull + $ docker compose up --build -d + $ docker compose ps + + Name Command State Ports + -------------------------------------------------------------------------------------------- + lua-cluster-specifier-proxy-1 /docker-entrypoint.sh /usr ... Up 10000/tcp, 0.0.0.0:8000->8000/tcp + lua-cluster-specifier-web_service-1 /bin/echo-server Up 0.0.0.0:8080->8080/tcp + +Step 2: Send a request to the normal service +******************************************** + +The output from the ``curl`` command below should return 200, since the lua code select the normal service. + +Terminal 1 + +.. code-block:: console + + $ curl -i localhost:8000/anything 2>&1 |grep 200 + HTTP/1.1 200 OK + +Step 3: Send a request to the fake service +****************************************** + +The output from the ``curl`` command below should return 503, since the lua code select the fake service. + +Terminal 1 + +.. code-block:: console + + $ curl -i localhost:8000/anything -H "header_key:fake" 2>&1 |grep 503 + HTTP/1.1 503 Service Unavailable + +.. seealso:: + + :ref:`Envoy Lua cluster specifier ` + Learn more about the Envoy Lua cluster specifier. + + `Lua `_ + The Lua programming language. diff --git a/docs/root/start/sandboxes/setup.rst b/docs/root/start/sandboxes/setup.rst index 57c34201a09f..19de3de6b066 100644 --- a/docs/root/start/sandboxes/setup.rst +++ b/docs/root/start/sandboxes/setup.rst @@ -107,6 +107,15 @@ Many of the examples use the `curl `_ utility to make ``HTTP`` Instructions for installing `curl `_ on many platforms and operating systems can be `found on the curl website `_. +.. _start_sandboxes_setup_envsubst: + +envsubst +~~~~~~~~ + +Some of the examples require the ``envsubst`` command to interpolate environment variables in templates. + +The command is a part of the GNU ‘gettext’ package, and is available through most package managers. + .. _start_sandboxes_setup_jq: jq @@ -118,6 +127,15 @@ whether it be ``HTTP`` response data, logs or statistics. Instructions for installing `jq `_ on many platforms and operating systems can be `found on the jq website `_. +.. _start_sandboxes_setup_mkpasswd: + +mkpasswd +~~~~~~~~ + +Some of the examples require the ``mkpasswd`` command to generate ~random tokens. + +The command is a part of the ‘whois’ package, and is available through most package managers. + .. _start_sandboxes_setup_netcat: netcat diff --git a/docs/versions.yaml b/docs/versions.yaml index 6fed90d94e79..06b7cca3c7d4 100644 --- a/docs/versions.yaml +++ b/docs/versions.yaml @@ -17,6 +17,8 @@ "1.21": 1.21.6 "1.22": 1.22.11 "1.23": 1.23.12 -"1.24": 1.24.10 -"1.25": 1.25.9 -"1.26": 1.26.4 +"1.24": 1.24.12 +"1.25": 1.25.11 +"1.26": 1.26.6 +"1.27": 1.27.2 +"1.28": 1.28.0 diff --git a/envoy/access_log/BUILD b/envoy/access_log/BUILD index 207c63e5a25e..925681617ba4 100644 --- a/envoy/access_log/BUILD +++ b/envoy/access_log/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( deps = [ "//envoy/config:typed_config_interface", "//envoy/filesystem:filesystem_interface", + "//envoy/formatter:http_formatter_context_interface", "//envoy/http:header_map_interface", "//envoy/stream_info:stream_info_interface", "//source/common/protobuf", diff --git a/envoy/access_log/access_log.h b/envoy/access_log/access_log.h index 6bc75b9c398c..460a18f96c49 100644 --- a/envoy/access_log/access_log.h +++ b/envoy/access_log/access_log.h @@ -6,6 +6,7 @@ #include "envoy/common/pure.h" #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/filesystem/filesystem.h" +#include "envoy/formatter/http_formatter_context.h" #include "envoy/http/header_map.h" #include "envoy/stream_info/stream_info.h" @@ -92,48 +93,15 @@ template class InstanceBase { template using InstanceBaseSharedPtr = std::shared_ptr>; /** - * Interface for access log filters. + * Interface for HTTP access log filters. */ -class Filter { -public: - virtual ~Filter() = default; - - /** - * Evaluate whether an access log should be written based on request and response data. - * @return TRUE if the log should be written. - */ - virtual bool evaluate(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - AccessLogType access_log_type) const PURE; -}; - +using Filter = FilterBase; using FilterPtr = std::unique_ptr; /** - * Abstract access logger for requests and connections. + * Abstract access logger for HTTP requests and TCP connections. */ -class Instance { -public: - virtual ~Instance() = default; - - /** - * Log a completed request. - * @param request_headers supplies the incoming request headers after filtering. - * @param response_headers supplies response headers. - * @param response_trailers supplies response trailers. - * @param stream_info supplies additional information about the request not - * contained in the request headers. - * @param access_log_type supplies additional information about the type of the - * log record, i.e the location in the code which recorded the log. - */ - virtual void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, AccessLogType access_log_type) PURE; -}; - +using Instance = InstanceBase; using InstanceSharedPtr = std::shared_ptr; } // namespace AccessLog diff --git a/envoy/access_log/access_log_config.h b/envoy/access_log/access_log_config.h index ee0810aa6e30..c29659b89f12 100644 --- a/envoy/access_log/access_log_config.h +++ b/envoy/access_log/access_log_config.h @@ -16,8 +16,7 @@ namespace AccessLog { */ template class ExtensionFilterFactoryBase : public Config::TypedFactory { public: - ExtensionFilterFactoryBase() - : category_(fmt::format("envoy.{}.access_loggers.extension_filters", Context::category())) {} + ExtensionFilterFactoryBase() : category_(categoryByType()) {} ~ExtensionFilterFactoryBase() override = default; @@ -26,27 +25,40 @@ template class ExtensionFilterFactoryBase : public Config::Typed * implementation is unable to produce a filter with the provided parameters, it should throw an * EnvoyException. The returned pointer should never be nullptr. * @param config supplies the custom configuration for this filter type. - * @param context supplies the server factory context. + * @param context supplies the factory context. * @return an instance of extension filter implementation from a config proto. */ virtual FilterBasePtr - createFilter(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) PURE; + createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, + Server::Configuration::FactoryContext& context) PURE; std::string category() const override { return category_; } private: + std::string categoryByType() { + if constexpr (std::is_same_v) { + // This is a special case for the HTTP formatter context to ensure backwards compatibility. + return "envoy.access_loggers.extension_filters"; + } else { + return fmt::format("envoy.{}.access_loggers.extension_filters", Context::category()); + } + } + const std::string category_; }; +/** + * Extension filter factory that reads from ExtensionFilter proto. + */ +using ExtensionFilterFactory = ExtensionFilterFactoryBase; + /** * Implemented for each AccessLog::Instance and registered via Registry::registerFactory or the * convenience class RegisterFactory. */ template class AccessLogInstanceFactoryBase : public Config::TypedFactory { public: - AccessLogInstanceFactoryBase() - : category_(fmt::format("envoy.{}.access_loggers", Context::category())) {} + AccessLogInstanceFactoryBase() : category_(categoryByType()) {} ~AccessLogInstanceFactoryBase() override = default; @@ -62,27 +74,24 @@ template class AccessLogInstanceFactoryBase : public Config::Typ virtual AccessLog::InstanceBaseSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterBasePtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) PURE; - - /** - * Create a particular AccessLog::Instance implementation from a config proto. If the - * implementation is unable to produce a factory with the provided parameters, it should throw an - * EnvoyException. The returned pointer should never be nullptr. - * @param config the custom configuration for this access log type. - * @param filter filter to determine whether a particular request should be logged. If no filter - * was specified in the configuration, argument will be nullptr. - * @param context general filter context through which persistent resources can be accessed. - */ - virtual AccessLog::InstanceBaseSharedPtr - createAccessLogInstance(const Protobuf::Message& config, - AccessLog::FilterBasePtr&& filter, - Server::Configuration::CommonFactoryContext& context) PURE; + Server::Configuration::FactoryContext& context) PURE; std::string category() const override { return category_; } private: + std::string categoryByType() { + if constexpr (std::is_same_v) { + // This is a special case for the HTTP formatter context to ensure backwards compatibility. + return "envoy.access_loggers"; + } else { + return fmt::format("envoy.{}.access_loggers", Context::category()); + } + } + const std::string category_; }; +using AccessLogInstanceFactory = AccessLogInstanceFactoryBase; + } // namespace AccessLog } // namespace Envoy diff --git a/envoy/api/io_error.h b/envoy/api/io_error.h index ed6669bcdc8c..86ca9b48fcb0 100644 --- a/envoy/api/io_error.h +++ b/envoy/api/io_error.h @@ -9,6 +9,11 @@ namespace Envoy { namespace Api { +class IoError; + +using IoErrorDeleterType = void (*)(IoError*); +using IoErrorPtr = std::unique_ptr; + /** * Base class for any I/O error. */ @@ -47,10 +52,20 @@ class IoError { virtual IoErrorCode getErrorCode() const PURE; virtual std::string getErrorDetails() const PURE; virtual int getSystemErrorCode() const PURE; -}; -using IoErrorDeleterType = void (*)(IoError*); -using IoErrorPtr = std::unique_ptr; + // Wrap an IoError* in unique_ptr with custom deleter that doesn't delete. + static IoErrorPtr reusedStatic(IoError* err) { + return {err, [](IoError*) {}}; + } + // Wrap an IoError* in unique_ptr with custom deleter. + static IoErrorPtr wrap(IoError* err) { + return {err, [](IoError* err) { delete err; }}; + } + // Use this non-error for the success case. + static IoErrorPtr none() { + return {nullptr, [](IoError*) {}}; + } +}; /** * Basic type for return result which has a return code and error code defined @@ -64,7 +79,7 @@ template struct IoCallResult { : return_value_(return_value), err_(std::move(err)) {} IoCallResult(IoCallResult&& result) noexcept - : return_value_(result.return_value_), err_(std::move(result.err_)) {} + : return_value_(std::move(result.return_value_)), err_(std::move(result.err_)) {} virtual ~IoCallResult() = default; diff --git a/envoy/buffer/buffer.h b/envoy/buffer/buffer.h index b94fef17fe50..bd53cade2a8d 100644 --- a/envoy/buffer/buffer.h +++ b/envoy/buffer/buffer.h @@ -647,7 +647,7 @@ class Reservation final { // The following are for use only by implementations of Buffer. Because c++ // doesn't allow inheritance of friendship, these are just trying to make // misuse easy to spot in a code review. - static Reservation bufferImplUseOnlyConstruct(Instance& buffer) { return Reservation(buffer); } + static Reservation bufferImplUseOnlyConstruct(Instance& buffer) { return {buffer}; } decltype(slices_)& bufferImplUseOnlySlices() { return slices_; } ReservationSlicesOwnerPtr& bufferImplUseOnlySlicesOwner() { return slices_owner_; } void bufferImplUseOnlySetLength(uint64_t length) { length_ = length; } @@ -708,9 +708,7 @@ class ReservationSingleSlice final { // The following are for use only by implementations of Buffer. Because c++ // doesn't allow inheritance of friendship, these are just trying to make // misuse easy to spot in a code review. - static ReservationSingleSlice bufferImplUseOnlyConstruct(Instance& buffer) { - return ReservationSingleSlice(buffer); - } + static ReservationSingleSlice bufferImplUseOnlyConstruct(Instance& buffer) { return {buffer}; } RawSlice& bufferImplUseOnlySlice() { return slice_; } ReservationSlicesOwnerPtr& bufferImplUseOnlySliceOwner() { return slice_owner_; } }; diff --git a/envoy/common/BUILD b/envoy/common/BUILD index fb37784cbdf3..f32828da2aa8 100644 --- a/envoy/common/BUILD +++ b/envoy/common/BUILD @@ -29,6 +29,9 @@ envoy_basic_cc_library( hdrs = [ "exception.h", ], + deps = [ + "//source/common/common:assert_lib", + ], ) envoy_basic_cc_library( diff --git a/envoy/common/exception.h b/envoy/common/exception.h index 13cf32bda7cd..48240a1d232b 100644 --- a/envoy/common/exception.h +++ b/envoy/common/exception.h @@ -3,7 +3,24 @@ #include #include +#include "source/common/common/assert.h" + namespace Envoy { + +// This is a workaround to allow an exceptionless Envoy Mobile build while we +// have not finished plumbing Satus/StatusOr<> based error handling, so +// hard-failing instead. See +// (https://github.com/envoyproxy/envoy-mobile/issues/176) +// for example error handling PRs. +// TODO(alyssawilk) finish up error handling and remove this. +#ifdef ENVOY_DISABLE_EXCEPTIONS +#define throwEnvoyExceptionOrPanic(x) PANIC(x) +#define throwExceptionOrPanic(x, y) PANIC(y) +#else +#define throwEnvoyExceptionOrPanic(x) throw EnvoyException(x) +#define throwExceptionOrPanic(y, x) throw y(x) +#endif + /** * Base class for all envoy exceptions. */ @@ -12,26 +29,35 @@ class EnvoyException : public std::runtime_error { EnvoyException(const std::string& message) : std::runtime_error(message) {} }; -#define THROW_IF_NOT_OK(status_fn) \ - { \ - absl::Status status = status_fn; \ - if (!status.ok()) { \ - throw EnvoyException(std::string(status.message())); \ +#define THROW_IF_NOT_OK_REF(status) \ + do { \ + if (!(status).ok()) { \ + throwEnvoyExceptionOrPanic(std::string((status).message())); \ } \ - } + } while (0) + +#define THROW_IF_NOT_OK(status_fn) \ + do { \ + const absl::Status status = (status_fn); \ + THROW_IF_NOT_OK_REF(status); \ + } while (0) // Simple macro to handle bridging functions which return absl::StatusOr, and // functions which throw errors. // -// The completely unnecessary throw action argument is just so 'throw' appears -// at the call site, so format checks about use of exceptions are triggered. -#define THROW_IF_STATUS_NOT_OK(variable, throw_action) \ - if (!variable.status().ok()) { \ - throw_action EnvoyException(std::string(variable.status().message())); \ - } +// The completely unnecessary throw_action argument was just so 'throw' appears +// at the call site, so format checks about use of exceptions would be triggered. +// This didn't work, so the variable is no longer used and is not duplicated in +// the macros above. +#define THROW_IF_STATUS_NOT_OK(variable, throw_action) THROW_IF_NOT_OK_REF(variable.status()); #define RETURN_IF_STATUS_NOT_OK(variable) \ if (!variable.status().ok()) { \ return variable.status(); \ } + +#define RETURN_IF_NOT_OK(variable) \ + if (!variable.ok()) { \ + return variable; \ + } } // namespace Envoy diff --git a/envoy/common/io/io_uring.h b/envoy/common/io/io_uring.h index 602d7dede52b..83903c512f76 100644 --- a/envoy/common/io/io_uring.h +++ b/envoy/common/io/io_uring.h @@ -133,6 +133,20 @@ class IoUring { */ virtual IoUringResult prepareClose(os_fd_t fd, Request* user_data) PURE; + /** + * Prepares a cancellation and puts it into the submission queue. + * Returns IoUringResult::Failed in case the submission queue is full already + * and IoUringResult::Ok otherwise. + */ + virtual IoUringResult prepareCancel(Request* cancelling_user_data, Request* user_data) PURE; + + /** + * Prepares a shutdown operation and puts it into the submission queue. + * Returns IoUringResult::Failed in case the submission queue is full already + * and IoUringResult::Ok otherwise. + */ + virtual IoUringResult prepareShutdown(os_fd_t fd, int how, Request* user_data) PURE; + /** * Submits the entries in the submission queue to the kernel using the * `io_uring_enter()` system call. @@ -166,6 +180,37 @@ class IoUring { using IoUringPtr = std::unique_ptr; class IoUringWorker; +/** + * The Status of IoUringSocket. + */ +enum IoUringSocketStatus { + Initialized, + ReadEnabled, + ReadDisabled, + RemoteClosed, + Closed, +}; + +/** + * A callback will be invoked when a close requested done on the socket. + */ +using IoUringSocketOnClosedCb = std::function; + +/** + * The data returned from the read request. + */ +struct ReadParam { + Buffer::Instance& buf_; + int32_t result_; +}; + +/** + * The data returned from the write request. + */ +struct WriteParam { + int32_t result_; +}; + /** * Abstract for each socket. */ @@ -183,6 +228,55 @@ class IoUringSocket { */ virtual os_fd_t fd() const PURE; + /** + * Close the socket. + * @param keep_fd_open indicates the file descriptor of the socket will be closed or not in the + * end. The value of `true` is used for destroy the IoUringSocket but keep the file descriptor + * open. This is used for migrating the IoUringSocket between worker threads. + * @param cb will be invoked when the close request is done. This is also used for migrating the + * IoUringSocket between worker threads. + */ + virtual void close(bool keep_fd_open, IoUringSocketOnClosedCb cb = nullptr) PURE; + + /** + * Enable the read on the socket. The socket will be begin to submit the read request and deliver + * read event when the request is done. This is used when the socket is listening on the file read + * event. + */ + virtual void enableRead() PURE; + + /** + * Disable the read on the socket. The socket stops to submit the read request, although the + * existing read request won't be canceled and no read event will be delivered. This is used when + * the socket isn't listening on the file read event. + */ + virtual void disableRead() PURE; + + /** + * Enable close event. This is used for the case the socket is listening on the file close event. + * Then a remote close is found by a read request will delievered as file close event. + */ + virtual void enableCloseEvent(bool enable) PURE; + + /** + * Write data to the socket. + * @param data is going to write. + */ + virtual void write(Buffer::Instance& data) PURE; + + /** + * Write data to the socket. + * @param slices includes the data to write. + * @param num_slice the number of slices. + */ + virtual uint64_t write(const Buffer::RawSlice* slices, uint64_t num_slice) PURE; + + /** + * Shutdown the socket. + * @param how is SHUT_RD, SHUT_WR and SHUT_RDWR. + */ + virtual void shutdown(int how) PURE; + /** * On accept request completed. * TODO (soulxu): wrap the raw result into a type. It can be `IoCallUint64Result`. @@ -251,6 +345,31 @@ class IoUringSocket { * @param type the request type of injected completion. */ virtual void injectCompletion(Request::RequestType type) PURE; + + /** + * Return the current status of IoUringSocket. + * @return the status. + */ + virtual IoUringSocketStatus getStatus() const PURE; + + /** + * Return the data get from the read request. + * @return Only return valid ReadParam when the callback is invoked with + * `Event::FileReadyType::Read`, otherwise `absl::nullopt` returned. + */ + virtual const OptRef& getReadParam() const PURE; + /** + * Return the data get from the write request. + * @return Only return valid WriteParam when the callback is invoked with + * `Event::FileReadyType::Write`, otherwise `absl::nullopt` returned. + */ + virtual const OptRef& getWriteParam() const PURE; + + /** + * Set the callback when file ready event triggered. + * @param cb the callback function. + */ + virtual void setFileReadyCb(Event::FileReadyCb cb) PURE; }; using IoUringSocketPtr = std::unique_ptr; @@ -260,12 +379,55 @@ using IoUringSocketPtr = std::unique_ptr; */ class IoUringWorker : public ThreadLocal::ThreadLocalObject { public: - virtual ~IoUringWorker() = default; + ~IoUringWorker() override = default; + + /** + * Add an server socket socket to the worker. + */ + virtual IoUringSocket& addServerSocket(os_fd_t fd, Event::FileReadyCb cb, + bool enable_close_event) PURE; + + /** + * Add an server socket from an existing socket from another thread. + */ + virtual IoUringSocket& addServerSocket(os_fd_t fd, Buffer::Instance& read_buf, + Event::FileReadyCb cb, bool enable_close_event) PURE; /** * Return the current thread's dispatcher. */ virtual Event::Dispatcher& dispatcher() PURE; + + /** + * Submit a read request for a socket. + */ + virtual Request* submitReadRequest(IoUringSocket& socket) PURE; + + /** + * Submit a write request for a socket. + */ + virtual Request* submitWriteRequest(IoUringSocket& socket, + const Buffer::RawSliceVector& slices) PURE; + + /** + * Submit a close request for a socket. + */ + virtual Request* submitCloseRequest(IoUringSocket& socket) PURE; + + /** + * Submit a cancel request for a socket. + */ + virtual Request* submitCancelRequest(IoUringSocket& socket, Request* request_to_cancel) PURE; + + /** + * Submit a shutdown request for a socket. + */ + virtual Request* submitShutdownRequest(IoUringSocket& socket, int how) PURE; + + /** + * Return the number of sockets in the worker. + */ + virtual uint32_t getNumOfSockets() const PURE; }; /** diff --git a/envoy/common/optref.h b/envoy/common/optref.h index 869720b6ae4c..2983bde1b11c 100644 --- a/envoy/common/optref.h +++ b/envoy/common/optref.h @@ -48,12 +48,12 @@ template struct OptRef { /** * Helper to convert a OptRef into a ref. Behavior if !has_value() is undefined. */ - T& ref() const { return *ptr_; } + T& ref() const { return *ptr_; } // NOLINT(clang-analyzer-core.uninitialized.UndefReturn) /** * Helper to dereference an OptRef. Behavior if !has_value() is undefined. */ - T& operator*() const { return *ptr_; } + T& operator*() const { return *ptr_; } // NOLINT(clang-analyzer-core.uninitialized.UndefReturn) /** * @return true if the object has a value. diff --git a/envoy/config/BUILD b/envoy/config/BUILD index 046c8a68c83f..6a24595f4087 100644 --- a/envoy/config/BUILD +++ b/envoy/config/BUILD @@ -43,7 +43,7 @@ envoy_cc_library( hdrs = ["context_provider.h"], deps = [ "//envoy/common:callback", - "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/core/v3:pkg_cc_proto", ], ) @@ -89,7 +89,7 @@ envoy_cc_library( ":custom_config_validators_interface", ":subscription_interface", "//envoy/common:backoff_strategy_interface", - "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/envoy/event/BUILD b/envoy/event/BUILD index e20ea5683318..7ed0de276a5c 100644 --- a/envoy/event/BUILD +++ b/envoy/event/BUILD @@ -39,6 +39,7 @@ envoy_cc_library( "//envoy/network:listener_interface", "//envoy/network:transport_socket_interface", "//envoy/server:watchdog_interface", + "//envoy/server/overload:thread_local_overload_state", "//envoy/thread:thread_interface", "@com_google_absl//absl/functional:any_invocable", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/envoy/event/dispatcher.h b/envoy/event/dispatcher.h index 7d568d279354..87632969029e 100644 --- a/envoy/event/dispatcher.h +++ b/envoy/event/dispatcher.h @@ -23,6 +23,7 @@ #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" #include "envoy/network/transport_socket.h" +#include "envoy/server/overload/thread_local_overload_state.h" #include "envoy/server/watchdog.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" @@ -227,29 +228,6 @@ class Dispatcher : public DispatcherBase, public ScopeTracker { */ virtual Filesystem::WatcherPtr createFilesystemWatcher() PURE; - /** - * Creates a listener on a specific port. - * @param socket supplies the socket to listen on. - * @param cb supplies the callbacks to invoke for listener events. - * @param runtime supplies the runtime for this server. - * @param listener_config configuration for the TCP listener to be created. - * @return Network::ListenerPtr a new listener that is owned by the caller. - */ - virtual Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, - Runtime::Loader& runtime, - const Network::ListenerConfig& listener_config) PURE; - - /** - * Creates a logical udp listener on a specific port. - * @param socket supplies the socket to listen on. - * @param cb supplies the udp listener callbacks to invoke for listener events. - * @param config provides the UDP socket configuration. - * @return Network::ListenerPtr a new listener that is owned by the caller. - */ - virtual Network::UdpListenerPtr - createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, - const envoy::config::core::v3::UdpSocketConfig& config) PURE; /** * Submits an item for deferred delete. @see DeferredDeletable. */ diff --git a/envoy/filesystem/BUILD b/envoy/filesystem/BUILD index 4ce775dd968b..3652ad67c197 100644 --- a/envoy/filesystem/BUILD +++ b/envoy/filesystem/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( "//envoy/api:io_error_interface", "//envoy/api:os_sys_calls_interface", "//envoy/common:time_interface", + "@com_google_absl//absl/status:statusor", ], ) diff --git a/envoy/filesystem/filesystem.h b/envoy/filesystem/filesystem.h index fca513f38a83..3126f6a0e6b3 100644 --- a/envoy/filesystem/filesystem.h +++ b/envoy/filesystem/filesystem.h @@ -10,6 +10,7 @@ #include "envoy/common/pure.h" #include "envoy/common/time.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -195,18 +196,17 @@ class Instance { virtual ssize_t fileSize(const std::string& path) PURE; /** - * @return full file content as a string. - * @throw EnvoyException if the file cannot be read. + * @return full file content as a string or an error if the file can not be read. * Be aware, this is not most highly performing file reading method. */ - virtual std::string fileReadToEnd(const std::string& path) PURE; + virtual absl::StatusOr fileReadToEnd(const std::string& path) PURE; /** * @path file path to split - * @return PathSplitResult containing the parent directory of the input path and the file name - * @note will throw an exception if path does not contain any path separator character + * @return PathSplitResult containing the parent directory of the input path and the file name or + * an error status. */ - virtual PathSplitResult splitPathFromFilename(absl::string_view path) PURE; + virtual absl::StatusOr splitPathFromFilename(absl::string_view path) PURE; /** * Determine if the path is on a list of paths Envoy will refuse to access. This @@ -241,6 +241,9 @@ struct DirectoryEntry { }; class DirectoryIteratorImpl; + +// Failures during this iteration will be silent; check status() after initialization +// and after each increment, if error-handling is desired. class DirectoryIterator { public: DirectoryIterator() : entry_({"", FileType::Other, absl::nullopt}) {} @@ -252,8 +255,11 @@ class DirectoryIterator { virtual DirectoryIteratorImpl& operator++() PURE; + const absl::Status& status() const { return status_; } + protected: DirectoryEntry entry_; + absl::Status status_; }; } // namespace Filesystem diff --git a/envoy/formatter/BUILD b/envoy/formatter/BUILD index 86bb31ad37b7..d2da478b19c9 100644 --- a/envoy/formatter/BUILD +++ b/envoy/formatter/BUILD @@ -8,6 +8,17 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "http_formatter_context_interface", + hdrs = [ + "http_formatter_context.h", + ], + deps = [ + "//envoy/http:header_map_interface", + "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "substitution_formatter_interface", hdrs = [ @@ -15,7 +26,7 @@ envoy_cc_library( "substitution_formatter_base.h", ], deps = [ - "//envoy/access_log:access_log_interface", + ":http_formatter_context_interface", "//envoy/config:typed_config_interface", "//envoy/http:header_map_interface", "//envoy/server:factory_context_interface", diff --git a/envoy/formatter/http_formatter_context.h b/envoy/formatter/http_formatter_context.h new file mode 100644 index 000000000000..9d6bd625bf21 --- /dev/null +++ b/envoy/formatter/http_formatter_context.h @@ -0,0 +1,130 @@ +#pragma once + +#include "envoy/data/accesslog/v3/accesslog.pb.h" +#include "envoy/http/header_map.h" + +namespace Envoy { +namespace Formatter { + +using AccessLogType = envoy::data::accesslog::v3::AccessLogType; + +/** + * HTTP specific substitution formatter context for HTTP access logs or formatters. + * TODO(wbpcode): maybe we should move this class to envoy/http folder and rename it + * for more general usage. + */ +class HttpFormatterContext { +public: + /** + * Constructor that uses the provided request/response headers, response trailers, local reply + * body, and access log type. Any of the parameters can be nullptr/empty. + * + * @param request_headers supplies the request headers. + * @param response_headers supplies the response headers. + * @param response_trailers supplies the response trailers. + * @param local_reply_body supplies the local reply body. + * @param log_type supplies the access log type. + */ + HttpFormatterContext(const Http::RequestHeaderMap* request_headers = nullptr, + const Http::ResponseHeaderMap* response_headers = nullptr, + const Http::ResponseTrailerMap* response_trailers = nullptr, + absl::string_view local_reply_body = {}, + AccessLogType log_type = AccessLogType::NotSet); + /** + * Set or overwrite the request headers. + * @param request_headers supplies the request headers. + */ + HttpFormatterContext& setRequestHeaders(const Http::RequestHeaderMap& request_headers) { + request_headers_ = &request_headers; + return *this; + } + /** + * Set or overwrite the response headers. + * @param response_headers supplies the response headers. + */ + HttpFormatterContext& setResponseHeaders(const Http::ResponseHeaderMap& response_headers) { + response_headers_ = &response_headers; + return *this; + } + + /** + * Set or overwrite the response trailers. + * @param response_trailers supplies the response trailers. + */ + HttpFormatterContext& setResponseTrailers(const Http::ResponseTrailerMap& response_trailers) { + response_trailers_ = &response_trailers; + return *this; + } + + /** + * Set or overwrite the local reply body. + * @param local_reply_body supplies the local reply body. + */ + HttpFormatterContext& setLocalReplyBody(absl::string_view local_reply_body) { + local_reply_body_ = local_reply_body; + return *this; + } + + /** + * Set or overwrite the access log type. + * @param log_type supplies the access log type. + */ + HttpFormatterContext& setAccessLogType(AccessLogType log_type) { + log_type_ = log_type; + return *this; + } + + /** + * @return const Http::RequestHeaderMap& the request headers. Empty request header map if no + * request headers are available. + */ + const Http::RequestHeaderMap& requestHeaders() const; + + /** + * @return false if no request headers are available. + */ + bool hasRequestHeaders() const { return request_headers_ != nullptr; } + + /** + * @return const Http::ResponseHeaderMap& the response headers. Empty respnose header map if + * no response headers are available. + */ + const Http::ResponseHeaderMap& responseHeaders() const; + + /** + * @return false if no response headers are available. + */ + bool hasResponseHeaders() const { return response_headers_ != nullptr; } + + /** + * @return const Http::ResponseTrailerMap& the response trailers. Empty response trailer map + * if no response trailers are available. + */ + const Http::ResponseTrailerMap& responseTrailers() const; + + /** + * @return false if no response trailers are available. + */ + bool hasResponseTrailers() const { return response_trailers_ != nullptr; } + + /** + * @return absl::string_view the local reply body. Empty if no local reply body. + */ + absl::string_view localReplyBody() const; + + /** + * @return AccessLog::AccessLogType the type of access log. NotSet if this is not used for + * access logging. + */ + AccessLogType accessLogType() const; + +private: + const Http::RequestHeaderMap* request_headers_{}; + const Http::ResponseHeaderMap* response_headers_{}; + const Http::ResponseTrailerMap* response_trailers_{}; + absl::string_view local_reply_body_{}; + AccessLogType log_type_{AccessLogType::NotSet}; +}; + +} // namespace Formatter +} // namespace Envoy diff --git a/envoy/formatter/substitution_formatter.h b/envoy/formatter/substitution_formatter.h index 01ffae286eee..abb244bf1d45 100644 --- a/envoy/formatter/substitution_formatter.h +++ b/envoy/formatter/substitution_formatter.h @@ -1,112 +1,12 @@ #pragma once +#include "envoy/formatter/http_formatter_context.h" #include "envoy/formatter/substitution_formatter_base.h" #include "envoy/http/header_map.h" namespace Envoy { namespace Formatter { -/** - * HTTP specific substitution formatter context for HTTP access logs or formatters. - */ -class HttpFormatterContext { -public: - /** - * Constructor that uses the provided request/response headers, response trailers, local reply - * body, and access log type. Any of the parameters can be nullptr/empty. - * - * @param request_headers supplies the request headers. - * @param response_headers supplies the response headers. - * @param response_trailers supplies the response trailers. - * @param local_reply_body supplies the local reply body. - * @param log_type supplies the access log type. - */ - HttpFormatterContext(const Http::RequestHeaderMap* request_headers = nullptr, - const Http::ResponseHeaderMap* response_headers = nullptr, - const Http::ResponseTrailerMap* response_trailers = nullptr, - absl::string_view local_reply_body = {}, - AccessLog::AccessLogType log_type = AccessLog::AccessLogType::NotSet); - /** - * Set or overwrite the request headers. - * @param request_headers supplies the request headers. - */ - HttpFormatterContext& setRequestHeaders(const Http::RequestHeaderMap& request_headers) { - request_headers_ = &request_headers; - return *this; - } - /** - * Set or overwrite the response headers. - * @param response_headers supplies the response headers. - */ - HttpFormatterContext& setResponseHeaders(const Http::ResponseHeaderMap& response_headers) { - response_headers_ = &response_headers; - return *this; - } - - /** - * Set or overwrite the response trailers. - * @param response_trailers supplies the response trailers. - */ - HttpFormatterContext& setResponseTrailers(const Http::ResponseTrailerMap& response_trailers) { - response_trailers_ = &response_trailers; - return *this; - } - - /** - * Set or overwrite the local reply body. - * @param local_reply_body supplies the local reply body. - */ - HttpFormatterContext& setLocalReplyBody(absl::string_view local_reply_body) { - local_reply_body_ = local_reply_body; - return *this; - } - - /** - * Set or overwrite the access log type. - * @param log_type supplies the access log type. - */ - HttpFormatterContext& setAccessLogType(AccessLog::AccessLogType log_type) { - log_type_ = log_type; - return *this; - } - - /** - * @return const Http::RequestHeaderMap& the request headers. Empty request header map if no - * request headers are available. - */ - const Http::RequestHeaderMap& requestHeaders() const; - - /** - * @return const Http::ResponseHeaderMap& the response headers. Empty respnose header map if - * no response headers are available. - */ - const Http::ResponseHeaderMap& responseHeaders() const; - - /** - * @return const Http::ResponseTrailerMap& the response trailers. Empty response trailer map - * if no response trailers are available. - */ - const Http::ResponseTrailerMap& responseTrailers() const; - - /** - * @return absl::string_view the local reply body. Empty if no local reply body. - */ - absl::string_view localReplyBody() const; - - /** - * @return AccessLog::AccessLogType the type of access log. NotSet if this is not used for - * access logging. - */ - AccessLog::AccessLogType accessLogType() const; - -private: - const Http::RequestHeaderMap* request_headers_{}; - const Http::ResponseHeaderMap* response_headers_{}; - const Http::ResponseTrailerMap* response_trailers_{}; - absl::string_view local_reply_body_{}; - AccessLog::AccessLogType log_type_{AccessLog::AccessLogType::NotSet}; -}; - using Formatter = FormatterBase; using FormatterPtr = std::unique_ptr; using FormatterConstSharedPtr = std::shared_ptr; @@ -136,7 +36,7 @@ template <> class CommandParserFactoryBase : public Config */ virtual CommandParserPtr createCommandParserFromProto(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) PURE; + Server::Configuration::GenericFactoryContext& context) PURE; std::string category() const override { return "envoy.formatter"; } }; diff --git a/envoy/formatter/substitution_formatter_base.h b/envoy/formatter/substitution_formatter_base.h index 2783c0d217cc..abc63f925459 100644 --- a/envoy/formatter/substitution_formatter_base.h +++ b/envoy/formatter/substitution_formatter_base.h @@ -102,7 +102,7 @@ template class CommandParserFactoryBase : public Config */ virtual CommandParserBasePtr createCommandParserFromProto(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) PURE; + Server::Configuration::GenericFactoryContext& context) PURE; std::string category() const override { return fmt::format("envoy.{}.formatters", FormatterContext::category()); diff --git a/envoy/geoip/BUILD b/envoy/geoip/BUILD new file mode 100644 index 000000000000..05d323b6a06b --- /dev/null +++ b/envoy/geoip/BUILD @@ -0,0 +1,28 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +# HTTP L7 filter that decorates request with geolocation data +# Public docs: https://envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/geoip_filter + +envoy_cc_extension( + name = "geoip_provider_driver_interface", + hdrs = [ + "geoip_provider_driver.h", + ], + tags = ["skip_on_windows"], + deps = [ + "//envoy/config:typed_config_interface", + "//envoy/network:address_interface", + "//envoy/protobuf:message_validator_interface", + "//envoy/server:factory_context_interface", + "//source/common/common:hash_lib", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/source/extensions/filters/http/geoip/geoip_provider_config.h b/envoy/geoip/geoip_provider_driver.h similarity index 55% rename from source/extensions/filters/http/geoip/geoip_provider_config.h rename to envoy/geoip/geoip_provider_driver.h index 5928b6b599c4..fe038c0ffe16 100644 --- a/source/extensions/filters/http/geoip/geoip_provider_config.h +++ b/envoy/geoip/geoip_provider_driver.h @@ -1,22 +1,18 @@ #pragma once #include "envoy/config/typed_config.h" -#include "envoy/extensions/filters/http/geoip/v3/geoip.pb.h" -#include "envoy/extensions/filters/http/geoip/v3/geoip.pb.validate.h" #include "envoy/network/address.h" #include "envoy/protobuf/message_validator.h" +#include "envoy/server/factory_context.h" #include "absl/container/flat_hash_set.h" namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Geoip { +namespace Geolocation { // Actual result of the lookup. Each entry in the map represents the geolocation header (entry key) -// for which the lookup was invoked mapped to a lookup result from the database. Entry value will be -// set to absl::nullopt when database lookup yields an empty result. -using LookupResult = const absl::flat_hash_map>; +// for which the lookup was invoked mapped to a lookup result from the database. +using LookupResult = const absl::flat_hash_map; // Async callbacks used for geolocation provider lookups. using LookupGeoHeadersCallback = std::function; @@ -24,20 +20,13 @@ using LookupGeoHeadersCallback = std::function; class LookupRequest { public: LookupRequest() = default; - LookupRequest(Network::Address::InstanceConstSharedPtr&& remote_address, - absl::flat_hash_set&& geo_headers, - absl::flat_hash_set&& geo_anon_headers) - : remote_address_(std::move(remote_address)), geo_headers_(std::move(geo_headers)), - geo_anon_headers_(std::move(geo_anon_headers)){}; + LookupRequest(Network::Address::InstanceConstSharedPtr&& remote_address) + : remote_address_(std::move(remote_address)){}; - absl::flat_hash_set geoHeaders() const { return geo_headers_; } - absl::flat_hash_set geoAnonHeaders() const { return geo_anon_headers_; } const Network::Address::InstanceConstSharedPtr remoteAddress() const { return remote_address_; } private: Network::Address::InstanceConstSharedPtr remote_address_; - absl::flat_hash_set geo_headers_; - absl::flat_hash_set geo_anon_headers_; }; class Driver { @@ -55,22 +44,6 @@ class Driver { using DriverSharedPtr = std::shared_ptr; -/** - * Context passed to geolocation providers to access server resources. - */ -class GeoipProviderFactoryContext { -public: - virtual ~GeoipProviderFactoryContext() = default; - - /** - * @return ProtobufMessage::ValidationVisitor& validation visitor for geolocation provider - * configuration messages. - */ - virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; -}; - -using GeoipProviderFactoryContextPtr = std::unique_ptr; - /** * Implemented by each geolocation provider and registered via Registry::registerFactory() or the * convenience class RegisterFactory. @@ -89,13 +62,12 @@ class GeoipProviderFactory : public Config::TypedFactory { * @param config supplies the proto configuration for the geolocation provider * @param context supplies the factory context */ - virtual DriverSharedPtr createGeoipProviderDriver(const Protobuf::Message& config, - GeoipProviderFactoryContextPtr& context) PURE; + virtual DriverSharedPtr + createGeoipProviderDriver(const Protobuf::Message& config, const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) PURE; std::string category() const override { return "envoy.geoip_providers"; } }; -} // namespace Geoip -} // namespace HttpFilters -} // namespace Extensions +} // namespace Geolocation } // namespace Envoy diff --git a/envoy/http/BUILD b/envoy/http/BUILD index 2a1c95cc5f03..ba992698b4ee 100644 --- a/envoy/http/BUILD +++ b/envoy/http/BUILD @@ -140,7 +140,6 @@ envoy_cc_library( deps = [ ":header_formatter_interface", "//envoy/common:union_string", - "//envoy/tracing:trace_context_interface", "//source/common/common:assert_lib", "//source/common/common:hash_lib", ], @@ -163,6 +162,7 @@ envoy_cc_library( envoy_cc_library( name = "query_params_interface", hdrs = ["query_params.h"], + external_deps = ["abseil_btree"], ) envoy_cc_library( @@ -216,6 +216,7 @@ envoy_cc_library( name = "header_evaluator", hdrs = ["header_evaluator.h"], deps = [ + "//envoy/formatter:substitution_formatter_interface", "//envoy/http:header_map_interface", "//envoy/stream_info:stream_info_interface", ], diff --git a/envoy/http/filter.h b/envoy/http/filter.h index 361eacc24474..e250b3ab6685 100644 --- a/envoy/http/filter.h +++ b/envoy/http/filter.h @@ -202,7 +202,8 @@ enum class LocalErrorStatus { ContinueAndResetStream, }; -// These are events that upstream filters can register for, via the addUpstreamCallbacks function. +// These are events that upstream HTTP filters can register for, via the addUpstreamCallbacks +// function. class UpstreamCallbacks { public: virtual ~UpstreamCallbacks() = default; @@ -214,7 +215,7 @@ class UpstreamCallbacks { virtual void onUpstreamConnectionEstablished() PURE; }; -// These are filter callbacks specific to upstream filters, accessible via +// These are filter callbacks specific to upstream HTTP filters, accessible via // StreamFilterCallbacks::upstreamCallbacks() class UpstreamStreamFilterCallbacks { public: @@ -426,14 +427,14 @@ class StreamFilterCallbacks { virtual Http1StreamEncoderOptionsOptRef http1StreamEncoderOptions() PURE; /** - * Return a handle to the upstream callbacks. This is valid for upstream filters, and nullopt for - * downstream filters. + * Return a handle to the upstream callbacks. This is valid for upstream HTTP filters, and nullopt + * for downstream HTTP filters. */ virtual OptRef upstreamCallbacks() PURE; /** - * Return a handle to the downstream callbacks. This is valid for downstream filters, and nullopt - * for upstream filters. + * Return a handle to the downstream callbacks. This is valid for downstream HTTP filters, and + * nullopt for upstream HTTP filters. */ virtual OptRef downstreamCallbacks() PURE; @@ -441,6 +442,37 @@ class StreamFilterCallbacks { * @return absl::string_view the name of the filter as configured in the filter chain. */ virtual absl::string_view filterConfigName() const PURE; + + /** + * The downstream request headers if present. + */ + virtual RequestHeaderMapOptRef requestHeaders() PURE; + + /** + * The downstream request trailers if present. + */ + virtual RequestTrailerMapOptRef requestTrailers() PURE; + + /** + * Retrieves a pointer to the continue headers if present. + */ + virtual ResponseHeaderMapOptRef informationalHeaders() PURE; + + /** + * Retrieves a pointer to the response headers if present. + * Note that response headers might be set multiple times (e.g. if a local reply is issued after + * headers have been received but before headers have been encoded), so it is not safe in general + * to assume that any set of headers will be valid for the duration of the stream. + */ + virtual ResponseHeaderMapOptRef responseHeaders() PURE; + + /** + * Retrieves a pointer to the last response trailers if present. + * Note that response headers might be set multiple times (e.g. if a local reply is issued after + * headers have been received but before headers have been encoded), so it is not safe in general + * to assume that any set of headers will be valid for the duration of the stream. + */ + virtual ResponseTrailerMapOptRef responseTrailers() PURE; }; class DecoderFilterWatermarkCallbacks { @@ -608,12 +640,6 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks, */ virtual void encode1xxHeaders(ResponseHeaderMapPtr&& headers) PURE; - /** - * Returns the headers provided to encode1xxHeaders. Returns absl::nullopt if - * no headers have been provided yet. - */ - virtual ResponseHeaderMapOptRef informationalHeaders() const PURE; - /** * Called with headers to be encoded, optionally indicating end of stream. * @@ -630,12 +656,6 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks, virtual void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) PURE; - /** - * Returns the headers provided to encodeHeaders. Returns absl::nullopt if no headers have been - * provided yet. - */ - virtual ResponseHeaderMapOptRef responseHeaders() const PURE; - /** * Called with data to be encoded, optionally indicating end of stream. * @param data supplies the data to be encoded. @@ -649,12 +669,6 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks, */ virtual void encodeTrailers(ResponseTrailerMapPtr&& trailers) PURE; - /** - * Returns the trailers provided to encodeTrailers. Returns absl::nullopt if no headers have been - * provided yet. - */ - virtual ResponseTrailerMapOptRef responseTrailers() const PURE; - /** * Called with metadata to be encoded. * @@ -744,13 +758,14 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks, * host list of the routed cluster, the host should be selected first. * @param host The override host address. */ - virtual void setUpstreamOverrideHost(absl::string_view host) PURE; + virtual void setUpstreamOverrideHost(Upstream::LoadBalancerContext::OverrideHost) PURE; /** * @return absl::optional optional override host for the upstream * load balancing. */ - virtual absl::optional upstreamOverrideHost() const PURE; + virtual absl::optional + upstreamOverrideHost() const PURE; }; /** diff --git a/envoy/http/filter_factory.h b/envoy/http/filter_factory.h index 71d6bc51f19f..ec0247017157 100644 --- a/envoy/http/filter_factory.h +++ b/envoy/http/filter_factory.h @@ -6,6 +6,7 @@ #include "envoy/common/pure.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" namespace Envoy { namespace Http { @@ -51,13 +52,14 @@ class FilterChainOptions { * * @param config_name the config name of the filter. * @return whether the filter should be disabled or enabled based on the config name. + * nullopt if no decision can be made explicitly for the filter. */ - virtual bool filterDisabled(absl::string_view config_name) const PURE; + virtual absl::optional filterDisabled(absl::string_view config_name) const PURE; }; class EmptyFilterChainOptions : public FilterChainOptions { public: - bool filterDisabled(absl::string_view) const override { return false; } + absl::optional filterDisabled(absl::string_view) const override { return {}; } }; /** diff --git a/envoy/http/header_evaluator.h b/envoy/http/header_evaluator.h index 2fde178e76f3..3bb73ad6ef76 100644 --- a/envoy/http/header_evaluator.h +++ b/envoy/http/header_evaluator.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/formatter/substitution_formatter.h" #include "envoy/http/header_map.h" #include "envoy/stream_info/stream_info.h" @@ -16,13 +17,11 @@ class HeaderEvaluator { * from the `bar` field in the stream_info, request headers or response headers. * * @param headers the target header map to be mutated. - * @param request_headers request headers to be used in the header manipulation. - * @param response_headers response headers to be used in the header manipulation. + * @param context context to format the header value. * @param stream_info the source of values that can be used in the evaluation. */ virtual void evaluateHeaders(Http::HeaderMap& headers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const PURE; }; } // namespace Http diff --git a/envoy/http/header_map.h b/envoy/http/header_map.h index c4eb4ea915ca..0e95b6db91dd 100644 --- a/envoy/http/header_map.h +++ b/envoy/http/header_map.h @@ -13,7 +13,6 @@ #include "envoy/common/pure.h" #include "envoy/common/union_string.h" #include "envoy/http/header_formatter.h" -#include "envoy/tracing/trace_context.h" #include "source/common/common/assert.h" #include "source/common/common/hash.h" @@ -728,8 +727,7 @@ class RequestOrResponseHeaderMap : public HeaderMap { // Request headers. class RequestHeaderMap : public RequestOrResponseHeaderMap, - public CustomInlineHeaderBase, - public Tracing::TraceContext { + public CustomInlineHeaderBase { public: INLINE_REQ_STRING_HEADERS(DEFINE_INLINE_STRING_HEADER) INLINE_REQ_NUMERIC_HEADERS(DEFINE_INLINE_NUMERIC_HEADER) diff --git a/envoy/http/query_params.h b/envoy/http/query_params.h index eac6741fc28d..03cac1771d19 100644 --- a/envoy/http/query_params.h +++ b/envoy/http/query_params.h @@ -6,17 +6,13 @@ #include "absl/container/btree_map.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "header_map.h" namespace Envoy { namespace Http { namespace Utility { -// TODO(jmarantz): this should probably be a proper class, with methods to serialize -// using proper formatting. Perhaps similar to -// https://github.com/apache/incubator-pagespeed-mod/blob/master/pagespeed/kernel/http/query_params.h - -using QueryParams = std::map; using QueryParamsVector = std::vector>; class QueryParamsMulti { @@ -27,10 +23,11 @@ class QueryParamsMulti { void remove(absl::string_view key); void add(absl::string_view key, absl::string_view value); void overwrite(absl::string_view key, absl::string_view value); - std::string toString(); - std::string replaceQueryString(const HeaderString& path); + std::string toString() const; + std::string replaceQueryString(const HeaderString& path) const; + absl::optional getFirstValue(absl::string_view key) const; - const absl::btree_map>& data() { return data_; } + const absl::btree_map>& data() const { return data_; } static QueryParamsMulti parseParameters(absl::string_view data, size_t start, bool decode_params); static QueryParamsMulti parseQueryString(absl::string_view url); diff --git a/envoy/http/stateful_session.h b/envoy/http/stateful_session.h index c5da82f3aeed..4b04735ca89f 100644 --- a/envoy/http/stateful_session.h +++ b/envoy/http/stateful_session.h @@ -67,11 +67,13 @@ class SessionStateFactoryConfig : public Envoy::Config::TypedFactory { * Creates a particular session state factory implementation. * * @param config supplies the configuration for the session state factory extension. + * @param context supplies the factory context. Please don't store the reference to + * the context as it is only valid during the call. * @return SessionStateFactorySharedPtr the session state factory. */ virtual SessionStateFactorySharedPtr createSessionStateFactory(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) PURE; + Server::Configuration::GenericFactoryContext& context) PURE; std::string category() const override { return "envoy.http.stateful_session"; } }; diff --git a/envoy/matcher/BUILD b/envoy/matcher/BUILD index 73d056e39351..604c1bf4a8b4 100644 --- a/envoy/matcher/BUILD +++ b/envoy/matcher/BUILD @@ -18,7 +18,7 @@ envoy_cc_library( deps = [ "//envoy/config:typed_config_interface", "//envoy/protobuf:message_validator_interface", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 8144f9eab260..3e7d51a07e90 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -51,6 +51,7 @@ envoy_cc_library( ":connection_balancer_interface", ":listen_socket_interface", ":listener_interface", + "//envoy/common:random_generator_interface", "//envoy/runtime:runtime_interface", "//envoy/ssl:context_interface", "//source/common/common:interval_value", diff --git a/envoy/network/connection.h b/envoy/network/connection.h index f5b6f5b885e0..340ce317c28f 100644 --- a/envoy/network/connection.h +++ b/envoy/network/connection.h @@ -287,7 +287,7 @@ class Connection : public Event::DeferredDeletable, virtual bool connecting() const PURE; /** - * Write data to the connection. Will iterate through downstream filters with the buffer if any + * Write data to the connection. Will iterate through network filters with the buffer if any * are installed. * @param data Supplies the data to write to the connection. * @param end_stream If true, this indicates that this is the last write to the connection. If diff --git a/envoy/network/connection_handler.h b/envoy/network/connection_handler.h index 1f5d8f2453c5..ed665428b630 100644 --- a/envoy/network/connection_handler.h +++ b/envoy/network/connection_handler.h @@ -3,6 +3,7 @@ #include #include +#include "envoy/common/random_generator.h" #include "envoy/network/address.h" #include "envoy/network/connection.h" #include "envoy/network/connection_balancer.h" @@ -10,6 +11,7 @@ #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" #include "envoy/runtime/runtime.h" +#include "envoy/server/overload/thread_local_overload_state.h" #include "envoy/ssl/context.h" #include "source/common/common/interval_value.h" @@ -65,9 +67,10 @@ class ConnectionHandler { * @param overridden_listener tag of the existing listener. nullopt if no previous listener. * @param config listener configuration options. * @param runtime the runtime for the server. + * @param random a random number generator. */ virtual void addListener(absl::optional overridden_listener, ListenerConfig& config, - Runtime::Loader& runtime) PURE; + Runtime::Loader& runtime, Random::RandomGenerator& random) PURE; /** * Remove listeners using the listener tag as a key. All connections owned by the removed @@ -217,6 +220,20 @@ class TcpConnectionHandler : public virtual ConnectionHandler { */ virtual BalancedConnectionHandlerOptRef getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; + + /** + * Creates a TCP listener on a specific port. + * @param socket supplies the socket to listen on. + * @param cb supplies the callbacks to invoke for listener events. + * @param runtime supplies the runtime for this server. + * @param listener_config configuration for the TCP listener to be created. + * @return Network::ListenerPtr a new listener that is owned by the caller. + */ + virtual Network::ListenerPtr + createListener(Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state) PURE; }; /** diff --git a/envoy/network/listener.h b/envoy/network/listener.h index 7d0454d88025..117fb0fa4871 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -133,6 +133,39 @@ class InternalListenerConfig { using InternalListenerConfigOptRef = OptRef; +/** + * Description of the listener. + */ +class ListenerInfo { +public: + virtual ~ListenerInfo() = default; + + /** + * @return const envoy::config::core::v3::Metadata& the config metadata associated with this + * listener. + */ + virtual const envoy::config::core::v3::Metadata& metadata() const PURE; + + /** + * @return const Envoy::Config::TypedMetadata& return the typed metadata provided in the config + * for this listener. + */ + virtual const Envoy::Config::TypedMetadata& typedMetadata() const PURE; + + /** + * @return envoy::config::core::v3::TrafficDirection the direction of the traffic relative to + * the local proxy. + */ + virtual envoy::config::core::v3::TrafficDirection direction() const PURE; + + /** + * @return whether the listener is a Quic listener. + */ + virtual bool isQuic() const PURE; +}; + +using ListenerInfoConstSharedPtr = std::shared_ptr; + /** * A configuration for an individual listener. */ @@ -206,6 +239,11 @@ class ListenerConfig { */ virtual const std::string& name() const PURE; + /** + * @return ListenerInfoConstSharedPtr& description of the listener. + */ + virtual const ListenerInfoConstSharedPtr& listenerInfo() const PURE; + /** * @return the UDP configuration for the listener IFF it is a UDP listener. */ @@ -216,11 +254,6 @@ class ListenerConfig { */ virtual InternalListenerConfigOptRef internalListenerConfig() PURE; - /** - * @return traffic direction of the listener. - */ - virtual envoy::config::core::v3::TrafficDirection direction() const PURE; - /** * @param address is used for query the address specific connection balancer. * @return the connection balancer for this listener. All listeners have a connection balancer, diff --git a/envoy/network/socket.h b/envoy/network/socket.h index b59f09871c80..b83d0f213047 100644 --- a/envoy/network/socket.h +++ b/envoy/network/socket.h @@ -176,6 +176,8 @@ class FilterChainInfo { virtual absl::string_view name() const PURE; }; +class ListenerInfo; + using FilterChainInfoConstSharedPtr = std::shared_ptr; /** @@ -253,6 +255,11 @@ class ConnectionInfoProvider { * @return the filter chain info provider backing this socket. */ virtual OptRef filterChainInfo() const PURE; + + /** + * @return the listener info backing this socket. + */ + virtual OptRef listenerInfo() const PURE; }; class ConnectionInfoSetter : public ConnectionInfoProvider { @@ -324,6 +331,11 @@ class ConnectionInfoSetter : public ConnectionInfoProvider { * @param filter_chain_info the filter chain info provider backing this socket. */ virtual void setFilterChainInfo(FilterChainInfoConstSharedPtr filter_chain_info) PURE; + + /** + * @param listener_info the listener info provider backing this socket. + */ + virtual void setListenerInfo(std::shared_ptr listener_info) PURE; }; using ConnectionInfoSetterSharedPtr = std::shared_ptr; diff --git a/envoy/router/BUILD b/envoy/router/BUILD index 28de863193e0..e3306bee4907 100644 --- a/envoy/router/BUILD +++ b/envoy/router/BUILD @@ -58,6 +58,20 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "router_filter_interface", + hdrs = ["router_filter_interface.h"], + external_deps = ["abseil_optional"], + deps = [ + "//envoy/http:filter_interface", + "//envoy/http:header_map_interface", + "//envoy/stats:stats_interface", + "//envoy/stream_info:stream_info_interface", + "//envoy/upstream:cluster_manager_interface", + "//envoy/upstream:host_description_interface", + ], +) + envoy_cc_library( name = "router_interface", hdrs = ["router.h"], diff --git a/envoy/router/route_config_provider_manager.h b/envoy/router/route_config_provider_manager.h index e83ad69225f1..67a184f2ba8e 100644 --- a/envoy/router/route_config_provider_manager.h +++ b/envoy/router/route_config_provider_manager.h @@ -25,7 +25,6 @@ namespace Router { class RouteConfigProviderManager { public: virtual ~RouteConfigProviderManager() = default; - using OptionalHttpFilters = absl::flat_hash_set; /** * Get a RouteConfigProviderPtr for a route from RDS. Ownership of the RouteConfigProvider is the @@ -34,7 +33,6 @@ class RouteConfigProviderManager { * the RouteConfigProvider. This method creates a RouteConfigProvider which may share the * underlying RDS subscription with the same (route_config_name, cluster). * @param rds supplies the proto configuration of an RDS-configured RouteConfigProvider. - * @param optional_http_filters a set of optional http filter names. * @param factory_context is the context to use for the route config provider. * @param stat_prefix supplies the stat_prefix to use for the provider stats. * @param init_manager the Init::Manager used to coordinate initialization of a the underlying RDS @@ -42,7 +40,6 @@ class RouteConfigProviderManager { */ virtual RouteConfigProviderSharedPtr createRdsRouteConfigProvider( const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, Init::Manager& init_manager) PURE; @@ -50,13 +47,11 @@ class RouteConfigProviderManager { * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for * getRdsRouteConfigProvider above. This method always create a new RouteConfigProvider. * @param route_config supplies the RouteConfiguration for this route - * @param optional_http_filters a set of optional http filter names. * @param factory_context is the context to use for the route config provider. * @param validator is the message validator for route config. */ virtual RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::config::route::v3::RouteConfiguration& route_config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) PURE; }; diff --git a/envoy/router/router.h b/envoy/router/router.h index a98907d64c96..299cb25c2960 100644 --- a/envoy/router/router.h +++ b/envoy/router/router.h @@ -334,6 +334,11 @@ class InternalRedirectPolicy { */ virtual std::vector predicates() const PURE; + /** + * @return a vector of response header names to preserve in the redirected request. + */ + virtual const std::vector& responseHeadersToCopy() const PURE; + /** * @return the maximum number of allowed internal redirects on this route. */ @@ -599,7 +604,7 @@ class VirtualCluster { static VirtualClusterStats generateStats(Stats::Scope& scope, const VirtualClusterStatNames& stat_names) { - return VirtualClusterStats(stat_names, scope); + return {stat_names, scope}; } }; @@ -677,6 +682,18 @@ class VirtualHost { virtual void traversePerFilterConfig( const std::string& filter_name, std::function cb) const PURE; + + /** + * @return const envoy::config::core::v3::Metadata& return the metadata provided in the config for + * this virtual host. + */ + virtual const envoy::config::core::v3::Metadata& metadata() const PURE; + + /** + * @return const Envoy::Config::TypedMetadata& return the typed metadata provided in the config + * for this virtual host. + */ + virtual const Envoy::Config::TypedMetadata& typedMetadata() const PURE; }; /** @@ -1193,9 +1210,10 @@ class Route { * Check if the filter is disabled for this route. * @param config_name supplies the name of the filter config in the HTTP filter chain. This name * may be different from the filter extension qualified name. - * @return true if the filter is disabled for this route, false otherwise. + * @return true if the filter is disabled for this route, false if the filter is enabled. + * nullopt if no decision can be made explicitly for the filter. */ - virtual bool filterDisabled(absl::string_view config_name) const PURE; + virtual absl::optional filterDisabled(absl::string_view config_name) const PURE; /** * This is a helper to get the route's per-filter config if it exists, up along the config @@ -1313,6 +1331,18 @@ class CommonConfig { * TODO(dio): To allow overrides at different levels (e.g. per-route, virtual host, etc). */ virtual uint32_t maxDirectResponseBodySizeBytes() const PURE; + + /** + * @return const envoy::config::core::v3::Metadata& return the metadata provided in the config for + * this route configuration. + */ + virtual const envoy::config::core::v3::Metadata& metadata() const PURE; + + /** + * @return const Envoy::Config::TypedMetadata& return the typed metadata provided in the config + * for this route configuration. + */ + virtual const Envoy::Config::TypedMetadata& typedMetadata() const PURE; }; /** @@ -1358,7 +1388,7 @@ class GenericConnectionPoolCallbacks; class GenericUpstream; /** - * An API for wrapping either an HTTP or a TCP connection pool. + * An API for wrapping either an HTTP, TCP, or UDP connection pool. * * The GenericConnPool exists to create a GenericUpstream handle via a call to * newStream resulting in an eventual call to onPoolReady @@ -1368,7 +1398,8 @@ class GenericConnPool { virtual ~GenericConnPool() = default; /** - * Called to create a new HTTP stream or TCP connection for "CONNECT streams". + * Called to create a new HTTP stream, TCP connection for CONNECT streams, or UDP socket for + * CONNECT-UDP streams. * * The implementation of the GenericConnPool will either call * GenericConnectionPoolCallbacks::onPoolReady @@ -1418,7 +1449,7 @@ class UpstreamToDownstream : public Http::ResponseDecoder, public Http::StreamCa }; /** - * An API for wrapping callbacks from either an HTTP or a TCP connection pool. + * An API for wrapping callbacks from either an HTTP, TCP, or UDP connection pool. * * Just like the connection pool callbacks, the GenericConnectionPoolCallbacks * will either call onPoolReady when a GenericUpstream is ready, or @@ -1464,7 +1495,7 @@ class GenericConnectionPoolCallbacks { }; /** - * An API for sending information to either a TCP or HTTP upstream. + * An API for sending information to either a TCP, UDP, or HTTP upstream. * * It is similar logically to RequestEncoder, only without the getStream interface. */ @@ -1537,7 +1568,7 @@ class GenericConnPoolFactory : public Envoy::Config::TypedFactory { virtual GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const PURE; }; diff --git a/envoy/router/router_filter_interface.h b/envoy/router/router_filter_interface.h new file mode 100644 index 000000000000..34c306afc860 --- /dev/null +++ b/envoy/router/router_filter_interface.h @@ -0,0 +1,158 @@ +#pragma once + +#include "envoy/http/filter.h" +#include "envoy/http/header_map.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" +#include "envoy/stream_info/stream_info.h" +#include "envoy/upstream/cluster_manager.h" +#include "envoy/upstream/host_description.h" + +namespace Envoy { +namespace Router { + +class UpstreamRequest; + +// This groups various per-stream timeouts conveniently together. +struct TimeoutData { + std::chrono::milliseconds global_timeout_{0}; + std::chrono::milliseconds per_try_timeout_{0}; + std::chrono::milliseconds per_try_idle_timeout_{0}; +}; + +// The interface the UpstreamRequest has to interact with the router filter. +class RouterFilterInterface { +public: + virtual ~RouterFilterInterface() = default; + + /** + * This will be called when upstream 1xx headers are ready to be processed by downstream code. + * @param headers contains the 1xx headers + * @param upstream_request inicates which UpstreamRequest the 1xx headers are from. + * + */ + virtual void onUpstream1xxHeaders(Http::ResponseHeaderMapPtr&& headers, + UpstreamRequest& upstream_request) PURE; + /** + * This will be called when upstream non-1xx headers are ready to be processed by downstream code. + * @param headers contains the headers + * @param upstream_request inicates which UpstreamRequest the headers are from. + * @param end_stream indicates if the response is complete. + * + */ + virtual void onUpstreamHeaders(uint64_t response_code, Http::ResponseHeaderMapPtr&& headers, + UpstreamRequest& upstream_request, bool end_stream) PURE; + /** + * This will be called when upstream data is ready to be processed by downstream code. + * @param data contains the data to process + * @param upstream_request inicates which UpstreamRequest the data is from. + * @param end_stream indicates if the response is complete. + * + */ + virtual void onUpstreamData(Buffer::Instance& data, UpstreamRequest& upstream_request, + bool end_stream) PURE; + /** + * This will be called when upstream trailers are ready to be processed by downstream code. + * @param trailers contains the trailers to process + * @param upstream_request inicates which UpstreamRequest the trailers are from. + * + */ + virtual void onUpstreamTrailers(Http::ResponseTrailerMapPtr&& trailers, + UpstreamRequest& upstream_request) PURE; + /** + * This will be called when upstream metadata is ready to be processed by downstream code. + * @param metadata contains the metadata to process + * @param upstream_request inicates which UpstreamRequest the metadata is from. + * + */ + virtual void onUpstreamMetadata(Http::MetadataMapPtr&& metadata_map) PURE; + + /** + * This will be called when an upstream reset is ready to be processed by downstream code. + * @param reset_reason indicates the reason for the reset. + * @param transport_failure optionally indicates any transport failure. + * @param upstream_request inicates which UpstreamRequest the reset is from. + * + */ + virtual void onUpstreamReset(Http::StreamResetReason reset_reason, + absl::string_view transport_failure, + UpstreamRequest& upstream_request) PURE; + + /** + * This will be called when an upstream host is selected. This is called both + * if the host can accomodate the stream and if the host is selected but unusable. + * @param host the host selected for the request + * @param pool_success indicates if the host can be used for the request. + */ + virtual void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host, + bool pool_success) PURE; + /* + * This will be called if a per-try timeout fails. + * @param upstream_request inicates which UpstreamRequest which timed out + */ + virtual void onPerTryTimeout(UpstreamRequest& upstream_request) PURE; + + /* + * This will be called if a per-try idle timeout fails. + * @param upstream_request inicates which UpstreamRequest which timed out + */ + virtual void onPerTryIdleTimeout(UpstreamRequest& upstream_request) PURE; + + /* + * This will be called if the max stream duration was reached. + * @param upstream_request inicates which UpstreamRequest which timed out + */ + virtual void onStreamMaxDurationReached(UpstreamRequest& upstream_request) PURE; + + /* + * @returns the Router filter's StreamDecoderFilterCallbacks. + */ + virtual Http::StreamDecoderFilterCallbacks* callbacks() PURE; + /* + * @returns the cluster for this stream. + */ + virtual Upstream::ClusterInfoConstSharedPtr cluster() PURE; + + /* + * @returns the FilterConfig for this stream + */ + virtual FilterConfig& config() PURE; + + /* + * @returns the various timeouts for this stream. + */ + virtual TimeoutData timeout() PURE; + + /* + * @returns the dynamic max stream duraration for this stream, if set. + */ + virtual absl::optional dynamicMaxStreamDuration() const PURE; + + /* + * @returns the request headers for the stream. + */ + virtual Http::RequestHeaderMap* downstreamHeaders() PURE; + + /* + * @returns the request trailers for the stream. + */ + virtual Http::RequestTrailerMap* downstreamTrailers() PURE; + + /* + * @returns true if the downstream response has started. + */ + virtual bool downstreamResponseStarted() const PURE; + + /* + * @returns true if end_stream has been sent from the upstream side to the downstream side. + */ + virtual bool downstreamEndStream() const PURE; + + /* + * @returns the number of attempts (e.g. retries) performed for this stream. + */ + virtual uint32_t attemptCount() const PURE; +}; + +} // namespace Router +} // namespace Envoy diff --git a/envoy/secret/secret_manager.h b/envoy/secret/secret_manager.h index 80713398a3f8..6bac464b5ce3 100644 --- a/envoy/secret/secret_manager.h +++ b/envoy/secret/secret_manager.h @@ -25,9 +25,9 @@ class SecretManager { /** * @param add a static secret from envoy::extensions::transport_sockets::tls::v3::Secret. - * @throw an EnvoyException if the secret is invalid or not supported, or there is duplicate. + * @return a status indicating if the function completed successfully. */ - virtual void + virtual absl::Status addStaticSecret(const envoy::extensions::transport_sockets::tls::v3::Secret& secret) PURE; /** diff --git a/envoy/server/BUILD b/envoy/server/BUILD index eb2bd7f99c3e..a412c4e164dd 100644 --- a/envoy/server/BUILD +++ b/envoy/server/BUILD @@ -8,16 +8,6 @@ licenses(["notice"]) # Apache 2 envoy_package() -envoy_cc_library( - name = "access_log_config_interface", - hdrs = ["access_log_config.h"], - deps = [ - ":filter_config_interface", - "//envoy/access_log:access_log_interface", - "//source/common/protobuf", - ], -) - envoy_cc_library( name = "admin_interface", hdrs = ["admin.h"], @@ -43,6 +33,7 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ "//envoy/http:context_interface", + "//envoy/stats:sink_interface", "//envoy/upstream:cluster_manager_interface", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], diff --git a/envoy/server/access_log_config.h b/envoy/server/access_log_config.h deleted file mode 100644 index 0d07554a7d8b..000000000000 --- a/envoy/server/access_log_config.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include - -#include "envoy/access_log/access_log.h" -#include "envoy/config/typed_config.h" -#include "envoy/server/filter_config.h" - -#include "source/common/protobuf/protobuf.h" - -namespace Envoy { -namespace Server { -namespace Configuration { - -/** - * Implemented for each AccessLog::Instance and registered via Registry::registerFactory or the - * convenience class RegisterFactory. - */ -class AccessLogInstanceFactory : public Config::TypedFactory { -public: - ~AccessLogInstanceFactory() override = default; - - /** - * Create a particular AccessLog::Instance implementation from a config proto. If the - * implementation is unable to produce a factory with the provided parameters, it should throw an - * EnvoyException. The returned pointer should never be nullptr. - * @param config the custom configuration for this access log type. - * @param filter filter to determine whether a particular request should be logged. If no filter - * was specified in the configuration, argument will be nullptr. - * @param context access log context through which persistent resources can be accessed. - */ - virtual AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - ListenerAccessLogFactoryContext& context) PURE; - - /** - * Create a particular AccessLog::Instance implementation from a config proto. If the - * implementation is unable to produce a factory with the provided parameters, it should throw an - * EnvoyException. The returned pointer should never be nullptr. - * @param config the custom configuration for this access log type. - * @param filter filter to determine whether a particular request should be logged. If no filter - * was specified in the configuration, argument will be nullptr. - * @param context general filter context through which persistent resources can be accessed. - */ - virtual AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, - AccessLog::FilterPtr&& filter, - CommonFactoryContext& context) PURE; - - std::string category() const override { return "envoy.access_loggers"; } -}; - -} // namespace Configuration -} // namespace Server -} // namespace Envoy diff --git a/envoy/server/admin.h b/envoy/server/admin.h index 2ec8d90dc28b..251b0cf47b44 100644 --- a/envoy/server/admin.h +++ b/envoy/server/admin.h @@ -62,7 +62,7 @@ class AdminStream { * * @param The query name/value map. */ - virtual Http::Utility::QueryParams queryParams() const PURE; + virtual Http::Utility::QueryParamsMulti queryParams() const PURE; }; /** @@ -233,15 +233,12 @@ class Admin { /** * Expose this Admin console as an HTTP server. * @param access_logs access_logs list of file loggers to write the HTTP request log to. - * @param address_out_path file path to write the listening socket's address to. * @param address network address to bind and listen on. - * @param listener_scope stats scope for the listener being started, + * @param socket_options socket options to apply to the listening socket. */ - virtual void startHttpListener(const std::list& access_logs, - const std::string& address_out_path, + virtual void startHttpListener(std::list access_logs, Network::Address::InstanceConstSharedPtr address, - const Network::Socket::OptionsSharedPtr& socket_options, - Stats::ScopeSharedPtr&& listener_scope) PURE; + Network::Socket::OptionsSharedPtr socket_options) PURE; /** * Executes an admin request with the specified query params. Note: this must diff --git a/envoy/server/factory_context.h b/envoy/server/factory_context.h index e665289120d1..8a139fbcb0c4 100644 --- a/envoy/server/factory_context.h +++ b/envoy/server/factory_context.h @@ -39,10 +39,13 @@ namespace Envoy { namespace Server { namespace Configuration { -// Shared factory context between server factories and cluster factories -class FactoryContextBase { +/** + * Common interface for downstream and upstream network filters to access server + * wide resources. This could be treated as limited form of server factory context. + */ +class CommonFactoryContext { public: - virtual ~FactoryContextBase() = default; + virtual ~CommonFactoryContext() = default; /** * @return Server::Options& the command-line options that Envoy was started with. @@ -80,6 +83,12 @@ class FactoryContextBase { */ virtual Singleton::Manager& singletonManager() PURE; + /** + * @return ProtobufMessage::ValidationContext& validation visitor for xDS and static configuration + * messages. + */ + virtual ProtobufMessage::ValidationContext& messageValidationContext() PURE; + /** * @return ProtobufMessage::ValidationVisitor& validation visitor for configuration messages. */ @@ -100,24 +109,12 @@ class FactoryContextBase { * used to allow runtime lockless updates to configuration, etc. across multiple threads. */ virtual ThreadLocal::SlotAllocator& threadLocal() PURE; -}; -/** - * Common interface for downstream and upstream network filters. - */ -class CommonFactoryContext : public FactoryContextBase { -public: /** * @return Upstream::ClusterManager& singleton for use by the entire server. */ virtual Upstream::ClusterManager& clusterManager() PURE; - /** - * @return ProtobufMessage::ValidationContext& validation visitor for xDS and static configuration - * messages. - */ - virtual ProtobufMessage::ValidationContext& messageValidationContext() PURE; - /** * @return TimeSource& a reference to the time source. */ @@ -132,16 +129,6 @@ class CommonFactoryContext : public FactoryContextBase { * @return ServerLifecycleNotifier& the lifecycle notifier for the server. */ virtual ServerLifecycleNotifier& lifecycleNotifier() PURE; - - /** - * @return the init manager of the cluster. This can be used for extensions that need - * to initialize after cluster manager init but before the server starts listening. - * All extensions should register themselves during configuration load. initialize() - * will be called on each registered target after cluster manager init but before the - * server starts listening. Once all targets have initialized and invoked their callbacks, - * the server will start listening. - */ - virtual Init::Manager& initManager() PURE; }; /** @@ -154,15 +141,36 @@ class ServerFactoryContext : public virtual CommonFactoryContext { ~ServerFactoryContext() override = default; /** - * @return the server-wide grpc context. + * @return Http::Context& the server-wide HTTP context. + */ + virtual Http::Context& httpContext() PURE; + + /** + * @return Grpc::Context& the server-wide grpc context. */ virtual Grpc::Context& grpcContext() PURE; /** - * @return Router::Context& a reference to the router context. + * @return Router::Context& the server-wide router context. */ virtual Router::Context& routerContext() PURE; + /** + * @return ProcessContextOptRef an optional reference to the + * process context. Will be unset when running in validation mode. + */ + virtual ProcessContextOptRef processContext() PURE; + + /** + * @return the init manager of the cluster. This can be used for extensions that need + * to initialize after cluster manager init but before the server starts listening. + * All extensions should register themselves during configuration load. initialize() + * will be called on each registered target after cluster manager init but before the + * server starts listening. Once all targets have initialized and invoked their callbacks, + * the server will start listening. + */ + virtual Init::Manager& initManager() PURE; + /** * @return DrainManager& the server-wide drain manager. */ @@ -177,33 +185,50 @@ class ServerFactoryContext : public virtual CommonFactoryContext { * @return envoy::config::bootstrap::v3::Bootstrap& the servers bootstrap configuration. */ virtual envoy::config::bootstrap::v3::Bootstrap& bootstrap() PURE; + + /** + * @return OverloadManager& the overload manager for the server. + */ + virtual OverloadManager& overloadManager() PURE; + + /** + * @return whether external healthchecks are currently failed or not. + */ + virtual bool healthCheckFailed() const PURE; }; /** - * Factory context for access loggers that need access to listener properties. - * This context is supplied to the access log factory when called with the listener context - * available, such as from downstream HTTP filters. - * NOTE: this interface is used in proprietary access loggers, please do not delete - * without reaching to Envoy maintainers first. + * Generic factory context for multiple scenarios. This context provides a server factory context + * reference and other resources. Note that except for server factory context, other resources are + * not guaranteed to be available for the entire server lifetime. For example, context powered by a + * listener is only available for the lifetime of the listener. */ -class ListenerAccessLogFactoryContext : public virtual CommonFactoryContext { +class GenericFactoryContext { public: + virtual ~GenericFactoryContext() = default; + /** - * @return Stats::Scope& the listener's stats scope. + * @return ServerFactoryContext which lifetime is no shorter than the server and provides + * access to the server's resources. */ - virtual Stats::Scope& listenerScope() PURE; + virtual ServerFactoryContext& serverFactoryContext() const PURE; /** - * @return const envoy::config::core::v3::Metadata& the config metadata associated with this - * listener. + * @return ProtobufMessage::ValidationVisitor& validation visitor for configuration messages. */ - virtual const envoy::config::core::v3::Metadata& listenerMetadata() const PURE; + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() const PURE; /** - * @return ProcessContextOptRef an optional reference to the - * process context. Will be unset when running in validation mode. + * @return Init::Manager& the init manager of the server/listener/cluster/etc, depending on the + * backend implementation. */ - virtual ProcessContextOptRef processContext() PURE; + virtual Init::Manager& initManager() PURE; + + /** + * @return Stats::Scope& the stats scope of the server/listener/cluster/etc, depending on the + * backend implementation. + */ + virtual Stats::Scope& scope() PURE; }; /** @@ -211,26 +236,20 @@ class ListenerAccessLogFactoryContext : public virtual CommonFactoryContext { * TODO(mattklein123): When we lock down visibility of the rest of the code, filters should only * access the rest of the server via interfaces exposed here. */ -class FactoryContext : public virtual ListenerAccessLogFactoryContext { +class FactoryContext : public virtual GenericFactoryContext { public: ~FactoryContext() override = default; /** - * @return ServerFactoryContext which lifetime is no shorter than the server. + * @return Stats::Scope& the listener's stats scope. */ - virtual ServerFactoryContext& getServerFactoryContext() const PURE; + virtual Stats::Scope& listenerScope() PURE; /** * @return TransportSocketFactoryContext which lifetime is no shorter than the server. */ virtual TransportSocketFactoryContext& getTransportSocketFactoryContext() const PURE; - /** - * @return envoy::config::core::v3::TrafficDirection the direction of the traffic relative to - * the local proxy. - */ - virtual envoy::config::core::v3::TrafficDirection direction() const PURE; - /** * @return const Network::DrainDecision& a drain decision that filters can use to determine if * they should be doing graceful closes on connections when possible. @@ -238,40 +257,9 @@ class FactoryContext : public virtual ListenerAccessLogFactoryContext { virtual const Network::DrainDecision& drainDecision() PURE; /** - * @return whether external healthchecks are currently failed or not. + * @return ListenerInfo description of the listener. */ - virtual bool healthCheckFailed() PURE; - - /** - * @return bool if these filters are created under the scope of a Quic listener. - */ - virtual bool isQuicListener() const PURE; - - /** - * @return const Envoy::Config::TypedMetadata& return the typed metadata provided in the config - * for this listener. - */ - virtual const Envoy::Config::TypedMetadata& listenerTypedMetadata() const PURE; - - /** - * @return OverloadManager& the overload manager for the server. - */ - virtual OverloadManager& overloadManager() PURE; - - /** - * @return Http::Context& a reference to the http context. - */ - virtual Http::Context& httpContext() PURE; - - /** - * @return Grpc::Context& a reference to the grpc context. - */ - virtual Grpc::Context& grpcContext() PURE; - - /** - * @return Router::Context& a reference to the router context. - */ - virtual Router::Context& routerContext() PURE; + virtual const Network::ListenerInfo& listenerInfo() const PURE; }; /** @@ -307,13 +295,7 @@ class FilterChainBaseAction : public Matcher::Action { * An implementation of FactoryContext. The life time should cover the lifetime of the filter chains * and connections. It can be used to create ListenerFilterChain. */ -class ListenerFactoryContext : public virtual FactoryContext { -public: - /** - * Give access to the listener configuration - */ - virtual const Network::ListenerConfig& listenerConfig() const PURE; -}; +class ListenerFactoryContext : public virtual FactoryContext {}; /** * FactoryContext for ProtocolOptionsFactory. @@ -321,7 +303,7 @@ class ListenerFactoryContext : public virtual FactoryContext { using ProtocolOptionsFactoryContext = Server::Configuration::TransportSocketFactoryContext; /** - * FactoryContext for upstream filters. + * FactoryContext for upstream HTTP filters. */ class UpstreamFactoryContext { public: @@ -330,7 +312,7 @@ class UpstreamFactoryContext { /** * @return ServerFactoryContext which lifetime is no shorter than the server. */ - virtual ServerFactoryContext& getServerFactoryContext() const PURE; + virtual ServerFactoryContext& serverFactoryContext() const PURE; /** * @return the init manager of the particular context. This can be used for extensions that need diff --git a/envoy/server/filter_config.h b/envoy/server/filter_config.h index 05dfc37ace76..aaffe9edd00b 100644 --- a/envoy/server/filter_config.h +++ b/envoy/server/filter_config.h @@ -279,9 +279,10 @@ class NamedHttpFilterConfigFactory : public virtual HttpFilterConfigFactoryBase * configuration. * @param stat_prefix prefix for stat logging * @param context supplies the filter's context. - * @return Http::FilterFactoryCb the factory creation function. + * @return absl::StatusOr the factory creation function or an error if + * creation fails. */ - virtual Http::FilterFactoryCb + virtual absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message& config, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) PURE; @@ -316,7 +317,7 @@ class UpstreamHttpFilterConfigFactory : public virtual HttpFilterConfigFactoryBa * @param context supplies the filter's context. * @return Http::FilterFactoryCb the factory creation function. */ - virtual Http::FilterFactoryCb + virtual absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message& config, const std::string& stat_prefix, Server::Configuration::UpstreamFactoryContext& context) PURE; }; diff --git a/envoy/server/instance.h b/envoy/server/instance.h index 5dcb2bf8cab6..683fef31b60d 100644 --- a/envoy/server/instance.h +++ b/envoy/server/instance.h @@ -46,6 +46,11 @@ class Instance { public: virtual ~Instance() = default; + /** + * Runs the server. + */ + virtual void run() PURE; + /** * @return OptRef the global HTTP admin endpoint for the server. */ diff --git a/envoy/server/listener_manager.h b/envoy/server/listener_manager.h index f3cfadc5a468..db6cefe41232 100644 --- a/envoy/server/listener_manager.h +++ b/envoy/server/listener_manager.h @@ -221,10 +221,10 @@ class ListenerManager { /** * Start all workers accepting new connections on all added listeners. - * @param guard_dog supplies the guard dog to use for thread watching. + * @param guard_dog supplies the optional guard dog to use for thread watching. * @param callback supplies the callback to complete server initialization. */ - virtual void startWorkers(GuardDog& guard_dog, std::function callback) PURE; + virtual void startWorkers(OptRef guard_dog, std::function callback) PURE; /** * Stop all listeners from accepting new connections without actually removing any of them. This diff --git a/envoy/server/overload/overload_manager.h b/envoy/server/overload/overload_manager.h index bc71798a5ac8..5833aaab9e88 100644 --- a/envoy/server/overload/overload_manager.h +++ b/envoy/server/overload/overload_manager.h @@ -99,6 +99,13 @@ class OverloadManager : public LoadShedPointProvider { * Get a factory for constructing scaled timer managers that respond to overload state. */ virtual Event::ScaledRangeTimerManagerFactory scaledTimerFactory() PURE; + + /** + * Stop the overload manager timer and wait for any pending resource updates to complete. + * After this returns, overload manager clients should not receive any more callbacks + * about overload state changes. + */ + virtual void stop() PURE; }; } // namespace Server diff --git a/envoy/server/overload/thread_local_overload_state.h b/envoy/server/overload/thread_local_overload_state.h index 35f36050945e..896cf3c4fe5e 100644 --- a/envoy/server/overload/thread_local_overload_state.h +++ b/envoy/server/overload/thread_local_overload_state.h @@ -86,13 +86,21 @@ class ThreadLocalOverloadState : public ThreadLocal::ThreadLocalObject { int64_t decrement) PURE; /** - * TODO(nezdolik) remove this method once downstream connection tracking is fully moved to - * overload manager. Checks if resource monitor is registered and resource usage tracking is + * Checks if resource monitor is registered and resource usage tracking is * enabled in overload manager. Returns true if resource monitor is registered, false otherwise. * @param name of resource monitor to check. */ virtual bool isResourceMonitorEnabled(OverloadProactiveResourceName resource_name) PURE; + + /** + * Returns the proactive resource owned by the overload manager. + * @param name of the proactive resource to retrieve. + */ + virtual ProactiveResourceMonitorOptRef + getProactiveResourceMonitorForTest(OverloadProactiveResourceName resource_name) PURE; }; +using ThreadLocalOverloadStateOptRef = OptRef; + } // namespace Server } // namespace Envoy diff --git a/envoy/server/proactive_resource_monitor.h b/envoy/server/proactive_resource_monitor.h index f4d69aa832b5..36ec5a6aac44 100644 --- a/envoy/server/proactive_resource_monitor.h +++ b/envoy/server/proactive_resource_monitor.h @@ -2,6 +2,7 @@ #include +#include "envoy/common/optref.h" #include "envoy/common/pure.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats.h" @@ -42,6 +43,7 @@ class ProactiveResourceMonitor { }; using ProactiveResourceMonitorPtr = std::unique_ptr; +using ProactiveResourceMonitorOptRef = OptRef; class ProactiveResource { public: @@ -68,7 +70,9 @@ class ProactiveResource { } } - int64_t currentResourceUsage() { return monitor_->currentResourceUsage(); } + ProactiveResourceMonitorOptRef getProactiveResourceMonitorForTest() { + return makeOptRefFromPtr(monitor_.get()); + }; private: const std::string name_; diff --git a/envoy/server/request_id_extension_config.h b/envoy/server/request_id_extension_config.h index cd94e3cb1d81..fa2288ae7f9f 100644 --- a/envoy/server/request_id_extension_config.h +++ b/envoy/server/request_id_extension_config.h @@ -24,8 +24,8 @@ class RequestIDExtensionFactory : public Envoy::Config::TypedFactory { * @param config the custom configuration for this request id extension type. * @param context general filter context through which persistent resources can be accessed. */ - virtual Http::RequestIDExtensionSharedPtr - createExtensionInstance(const Protobuf::Message& config, CommonFactoryContext& context) PURE; + virtual Http::RequestIDExtensionSharedPtr createExtensionInstance(const Protobuf::Message& config, + FactoryContext& context) PURE; std::string category() const override { return "envoy.request_id"; } }; diff --git a/envoy/server/worker.h b/envoy/server/worker.h index 5bc2ab5f712b..74e24d01f017 100644 --- a/envoy/server/worker.h +++ b/envoy/server/worker.h @@ -32,10 +32,11 @@ class Worker { * @param completion supplies the completion to call when the listener has been added (or not) on * the worker. * @param runtime, supplies the runtime for the server + * @param random, supplies a random number generator */ virtual void addListener(absl::optional overridden_listener, Network::ListenerConfig& listener, AddListenerCompletion completion, - Runtime::Loader& runtime) PURE; + Runtime::Loader& runtime, Random::RandomGenerator& random) PURE; /** * @return uint64_t the number of connections across all listeners that the worker owns. @@ -44,10 +45,10 @@ class Worker { /** * Start the worker thread. - * @param guard_dog supplies the guard dog to use for thread watching. + * @param guard_dog supplies the optional guard dog to use for thread watching. * @param cb a callback to run when the worker thread starts running. */ - virtual void start(GuardDog& guard_dog, const std::function& cb) PURE; + virtual void start(OptRef guard_dog, const std::function& cb) PURE; /** * Initialize stats for this worker's dispatcher, if available. The worker will output diff --git a/envoy/singleton/manager.h b/envoy/singleton/manager.h index 88800eda6273..e8d65b13ae42 100644 --- a/envoy/singleton/manager.h +++ b/envoy/singleton/manager.h @@ -67,30 +67,53 @@ class Manager { * This is a helper on top of get() that casts the object stored to the specified type. Since the * manager only stores pointers to the base interface, dynamic_cast provides some level of * protection via RTTI. + * @param name the unique name of the singleton instance. This should be provided by the macro + * SINGLETON_MANAGER_REGISTERED_NAME. + * @param cb supplies the singleton creation callback. This will only be called if the singleton + * does not already exist. + * @param pin supplies whether the singleton should be pinned. By default, the manager only stores + * a weak pointer. This allows a singleton to be cleaned up if it is not needed any more. All code + * that uses singletons must store the shared_ptr for as long as the singleton is needed. But if + * the pin is set to true, the singleton will be stored as a shared_ptr. This is useful if the + * users want to keep the singleton around for the lifetime of the server even if it is not used + * for a while. + * @return InstancePtr the singleton cast to the specified type. nullptr if the singleton does not + * exist. */ - template std::shared_ptr getTyped(const std::string& name, SingletonFactoryCb cb) { - return std::dynamic_pointer_cast(get(name, cb)); + template + std::shared_ptr getTyped(const std::string& name, SingletonFactoryCb cb, bool pin = false) { + return std::dynamic_pointer_cast(get(name, cb, pin)); } /** * This is a non-constructing getter. Use when the caller can deal with instances where * the singleton being accessed may not have been constructed previously. - * @return InstancePtr the singleton. nullptr if the singleton does not exist. + * @param name the unique name of singleton instance. This should be provided by the macro + * SINGLETON_MANAGER_REGISTERED_NAME. + * @return InstancePtr the singleton cast to the specified type. nullptr if the singleton does not + * exist. */ template std::shared_ptr getTyped(const std::string& name) { - return std::dynamic_pointer_cast(get(name, [] { return nullptr; })); + return std::dynamic_pointer_cast(get( + name, [] { return nullptr; }, false)); } /** * Get a singleton and create it if it does not exist. - * @param name supplies the singleton name. Must be registered via RegistrationImpl. - * @param singleton supplies the singleton creation callback. This will only be called if the - * singleton does not already exist. NOTE: The manager only stores a weak pointer. This - * allows a singleton to be cleaned up if it is not needed any more. All code that uses - * singletons must store the shared_ptr for as long as the singleton is needed. - * @return InstancePtr the singleton. + * @param name the unique name of the singleton instance. This should be provided by the macro + * SINGLETON_MANAGER_REGISTERED_NAME. + * @param cb supplies the singleton creation callback. This will only be called if the singleton + * does not already exist. + * @param pin supplies whether the singleton should be pinned. By default, the manager only stores + * a weak pointer. This allows a singleton to be cleaned up if it is not needed any more. All code + * that uses singletons must store the shared_ptr for as long as the singleton is needed. But if + * the pin is set to true, the singleton will be stored as a shared_ptr. This is useful if the + * users want to keep the singleton around for the lifetime of the server even if it is not used + * for a while. + * @return InstancePtr the singleton cast to the specified type. nullptr if the singleton does not + * exist. */ - virtual InstanceSharedPtr get(const std::string& name, SingletonFactoryCb) PURE; + virtual InstanceSharedPtr get(const std::string& name, SingletonFactoryCb cb, bool pin) PURE; }; using ManagerPtr = std::unique_ptr; diff --git a/envoy/ssl/context_config.h b/envoy/ssl/context_config.h index 68baffc37c7f..596c70df2fea 100644 --- a/envoy/ssl/context_config.h +++ b/envoy/ssl/context_config.h @@ -189,6 +189,11 @@ class ServerContextConfig : public virtual ContextConfig { */ virtual bool disableStatelessSessionResumption() const PURE; + /** + * @return True if stateful TLS session resumption is disabled, false otherwise. + */ + virtual bool disableStatefulSessionResumption() const PURE; + /** * @return True if we allow full scan certificates when there is no cert matching SNI during * downstream TLS handshake, false otherwise. diff --git a/envoy/stats/BUILD b/envoy/stats/BUILD index 4b6f64a76765..d707c449b019 100644 --- a/envoy/stats/BUILD +++ b/envoy/stats/BUILD @@ -14,6 +14,11 @@ envoy_cc_library( deps = ["//source/common/common:assert_lib"], ) +envoy_cc_library( + name = "tag_interface", + hdrs = ["tag.h"], +) + # TODO(jmarantz): atomize the build rules to match the include files. envoy_cc_library( name = "stats_interface", @@ -21,29 +26,38 @@ envoy_cc_library( "allocator.h", "histogram.h", "scope.h", - "sink.h", "stats.h", "stats_matcher.h", "store.h", - "tag.h", "tag_extractor.h", "tag_producer.h", ], external_deps = ["abseil_inlined_vector"], deps = [ ":refcount_ptr_interface", + ":tag_interface", "//envoy/common:interval_set_interface", "//envoy/common:optref_lib", "//envoy/common:time_interface", ], ) +envoy_cc_library( + name = "sink_interface", + hdrs = ["sink.h"], + deps = [ + ":primitive_stats_interface", + ":stats_interface", + ], +) + envoy_cc_library( name = "primitive_stats_interface", hdrs = [ "primitive_stats.h", ], deps = [ + ":tag_interface", "//source/common/common:assert_lib", "//source/common/common:non_copyable", ], diff --git a/envoy/stats/histogram.h b/envoy/stats/histogram.h index e8aa64cb7532..227ae3c01993 100644 --- a/envoy/stats/histogram.h +++ b/envoy/stats/histogram.h @@ -86,6 +86,12 @@ class HistogramStatistics { * Returns sum of all values during the period. */ virtual double sampleSum() const PURE; + + /** + * Returns the count of values which are out of the boundaries of the histogram bins. + * I.e., the count of values in the (bound_of_last_bucket, +inf) bucket. + */ + virtual uint64_t outOfBoundCount() const PURE; }; /** diff --git a/envoy/stats/primitive_stats.h b/envoy/stats/primitive_stats.h index d27375a21c00..be210b581b99 100644 --- a/envoy/stats/primitive_stats.h +++ b/envoy/stats/primitive_stats.h @@ -2,6 +2,8 @@ #include +#include "envoy/stats/tag.h" + #include "source/common/common/assert.h" #include "source/common/common/non_copyable.h" @@ -34,7 +36,7 @@ class PrimitiveCounter : NonCopyable { std::atomic pending_increment_{0}; }; -using PrimitiveCounterReference = std::reference_wrapper; +using PrimitiveCounterReference = std::reference_wrapper; /** * Primitive, low-memory-overhead gauge with increment and decrement capabilities. @@ -58,7 +60,52 @@ class PrimitiveGauge : NonCopyable { std::atomic value_{0}; }; -using PrimitiveGaugeReference = std::reference_wrapper; +using PrimitiveGaugeReference = std::reference_wrapper; + +class PrimitiveMetricMetadata { +public: + // Mirror some of the API for Stats::Metric for use in templates that + // accept either Counter/Gauge or PrimitiveCounterSnapshot/PrimitiveGaugeSnapshot. + const std::string& tagExtractedName() const { return tag_extracted_name_; } + const std::string& name() const { return name_; } + const Stats::TagVector& tags() const { return tags_; } + bool used() const { return true; } + bool hidden() const { return false; } + + void setName(std::string&& name) { name_ = std::move(name); } + void setTagExtractedName(std::string&& tag_extracted_name) { + tag_extracted_name_ = std::move(tag_extracted_name); + } + void setTags(const Stats::TagVector& tags) { tags_ = tags; } + +private: + std::string name_; + std::string tag_extracted_name_; + Stats::TagVector tags_; +}; + +class PrimitiveCounterSnapshot : public PrimitiveMetricMetadata { +public: + PrimitiveCounterSnapshot(PrimitiveCounter& counter) + : value_(counter.value()), delta_(counter.latch()) {} + + uint64_t value() const { return value_; } + uint64_t delta() const { return delta_; } + +private: + const uint64_t value_; + const uint64_t delta_; +}; + +class PrimitiveGaugeSnapshot : public PrimitiveMetricMetadata { +public: + PrimitiveGaugeSnapshot(PrimitiveGauge& gauge) : value_(gauge.value()) {} + + uint64_t value() const { return value_; } + +private: + const uint64_t value_; +}; } // namespace Stats } // namespace Envoy diff --git a/envoy/stats/refcount_ptr.h b/envoy/stats/refcount_ptr.h index b073a3a2f39a..664780b648a1 100644 --- a/envoy/stats/refcount_ptr.h +++ b/envoy/stats/refcount_ptr.h @@ -91,6 +91,7 @@ template class RefcountPtr { #endif bool operator==(const RefcountPtr& a) const { return ptr_ == a.ptr_; } bool operator!=(const RefcountPtr& a) const { return ptr_ != a.ptr_; } + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete) uint32_t use_count() const { return ptr_->use_count(); } void reset() { resetInternal(); @@ -115,7 +116,7 @@ template class RefcountPtr { // forth. #if __cplusplus < 202002L template static bool operator==(std::nullptr_t, const RefcountPtr& a) { - return a == nullptr; + return a == nullptr; // NOLINT(clang-analyzer-cplusplus.Move) } template static bool operator!=(std::nullptr_t, const RefcountPtr& a) { return a != nullptr; diff --git a/envoy/stats/sink.h b/envoy/stats/sink.h index b83a06241b65..a56ec2bb2757 100644 --- a/envoy/stats/sink.h +++ b/envoy/stats/sink.h @@ -6,6 +6,7 @@ #include "envoy/common/pure.h" #include "envoy/common/time.h" #include "envoy/stats/histogram.h" +#include "envoy/stats/primitive_stats.h" #include "envoy/stats/stats.h" namespace Envoy { @@ -42,6 +43,16 @@ class MetricSnapshot { */ virtual const std::vector>& textReadouts() PURE; + /** + * @return a snapshot of all host/endpoint-specific primitive counters. + */ + virtual const std::vector& hostCounters() PURE; + + /** + * @return a snapshot of all host/endpoint-specific primitive gauges. + */ + virtual const std::vector& hostGauges() PURE; + /** * @return the time in UTC since epoch when the snapshot was created. */ diff --git a/envoy/stats/stats.h b/envoy/stats/stats.h index 040dc392d1fb..ea4596f64aa5 100644 --- a/envoy/stats/stats.h +++ b/envoy/stats/stats.h @@ -26,6 +26,10 @@ class SymbolTable; /** * General interface for all stats objects. + * + * Note: some methods must match those in `PrimitiveMetricMetadata` because stats sinks + * use templates to handle either type. The interface is not used for size/performance + * reasons. */ class Metric : public RefcountInterface { public: diff --git a/envoy/stats/store.h b/envoy/stats/store.h index 5af35defd9c6..ff1c6a08c53e 100644 --- a/envoy/stats/store.h +++ b/envoy/stats/store.h @@ -193,6 +193,11 @@ class Store { */ virtual void extractAndAppendTags(absl::string_view name, StatNamePool& pool, StatNameTagVector& stat_tags) PURE; + + /** + * Returns the configured fixed tags (which don't depend on the name of the stat). + */ + virtual const TagVector& fixedTags() PURE; }; using StorePtr = std::unique_ptr; diff --git a/envoy/stats/tag_producer.h b/envoy/stats/tag_producer.h index 91a6d1da1818..02fe95501554 100644 --- a/envoy/stats/tag_producer.h +++ b/envoy/stats/tag_producer.h @@ -30,6 +30,8 @@ class TagProducer { * @param tags TagVector a set of Stats::Tag. */ virtual std::string produceTags(absl::string_view metric_name, TagVector& tags) const PURE; + + virtual const TagVector& fixedTags() const PURE; }; using TagProducerPtr = std::unique_ptr; diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index a4f2faca84ca..4091b6eeddc5 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -91,8 +91,10 @@ enum ResponseFlag { OverloadManager = 0x2000000, // DNS resolution failed. DnsResolutionFailed = 0x4000000, + // Drop certain percentage of overloaded traffic. + DropOverLoad = 0x8000000, // ATTENTION: MAKE SURE THIS REMAINS EQUAL TO THE LAST FLAG. - LastFlag = DnsResolutionFailed, + LastFlag = DropOverLoad, }; /** @@ -156,6 +158,8 @@ struct ResponseCodeDetailValues { const std::string ClusterNotFound = "cluster_not_found"; // The request was rejected by the router filter because the cluster was in maintenance mode. const std::string MaintenanceMode = "maintenance_mode"; + // The request was rejected by the router filter because the DROP_OVERLOAD configuration. + const std::string DropOverload = "drop_overload"; // The request was rejected by the router filter because there was no healthy upstream found. const std::string NoHealthyUpstream = "no_healthy_upstream"; // The request was forwarded upstream but the response timed out. @@ -341,6 +345,9 @@ class DownstreamTiming { absl::optional lastDownstreamAckReceived() const { return last_downstream_ack_received_; } + absl::optional lastDownstreamHeaderRxByteReceived() const { + return last_downstream_header_rx_byte_received_; + } void onLastDownstreamRxByteReceived(TimeSource& time_source) { ASSERT(!last_downstream_rx_byte_received_); @@ -362,6 +369,10 @@ class DownstreamTiming { ASSERT(!last_downstream_ack_received_); last_downstream_ack_received_ = time_source.monotonicTime(); } + void onLastDownstreamHeaderRxByteReceived(TimeSource& time_source) { + ASSERT(!last_downstream_header_rx_byte_received_); + last_downstream_header_rx_byte_received_ = time_source.monotonicTime(); + } private: absl::flat_hash_map timings_; @@ -375,6 +386,8 @@ class DownstreamTiming { absl::optional downstream_handshake_complete_; // The time the final ack was received from the client. absl::optional last_downstream_ack_received_; + // The time when the last header byte was received. + absl::optional last_downstream_header_rx_byte_received_; }; // Measure the number of bytes sent and received for a stream. @@ -693,6 +706,11 @@ class StreamInfo { */ virtual MonotonicTime startTimeMonotonic() const PURE; + /** + * @return returns the time source. + */ + virtual TimeSource& timeSource() const PURE; + /** * Sets the upstream information for this stream. */ diff --git a/envoy/tcp/async_tcp_client.h b/envoy/tcp/async_tcp_client.h index 684dc3aa47a7..cc369bb221ca 100644 --- a/envoy/tcp/async_tcp_client.h +++ b/envoy/tcp/async_tcp_client.h @@ -87,6 +87,11 @@ class AsyncTcpClient { * @return if the client connects to a peer host. */ virtual bool connected() PURE; + + /** + * @return the streamInfo of the current connection if there is any. + */ + virtual OptRef getStreamInfo() PURE; }; using AsyncTcpClientPtr = std::unique_ptr; diff --git a/envoy/tracing/BUILD b/envoy/tracing/BUILD index 29c2776504ba..3e2e9c9f5281 100644 --- a/envoy/tracing/BUILD +++ b/envoy/tracing/BUILD @@ -43,6 +43,10 @@ envoy_cc_library( envoy_cc_library( name = "trace_context_interface", hdrs = ["trace_context.h"], + deps = [ + "//envoy/common:optref_lib", + "//envoy/http:header_map_interface", + ], ) envoy_cc_library( diff --git a/envoy/tracing/custom_tag.h b/envoy/tracing/custom_tag.h index f930fe0527e2..51968026e8dd 100644 --- a/envoy/tracing/custom_tag.h +++ b/envoy/tracing/custom_tag.h @@ -16,7 +16,7 @@ namespace Tracing { * The context for the custom tag to obtain the tag value. */ struct CustomTagContext { - const TraceContext* trace_context; + const TraceContext& trace_context; const StreamInfo::StreamInfo& stream_info; }; diff --git a/envoy/tracing/trace_config.h b/envoy/tracing/trace_config.h index 8cf4b98b0491..c556aa592f38 100644 --- a/envoy/tracing/trace_config.h +++ b/envoy/tracing/trace_config.h @@ -21,6 +21,11 @@ class Config { */ virtual OperationName operationName() const PURE; + /** + * @return create separated child span for upstream request if true. + */ + virtual bool spawnUpstreamSpan() const PURE; + /** * @return custom tags to be attached to the active span. */ @@ -80,6 +85,11 @@ class ConnectionManagerTracingConfig : public TracingConfig { */ virtual OperationName operationName() const PURE; + /** + * @return create separated child span for upstream request if true. + */ + virtual bool spawnUpstreamSpan() const PURE; + /** * @return true if spans should be annotated with more detailed information. */ diff --git a/envoy/tracing/trace_context.h b/envoy/tracing/trace_context.h index f0fcd8d0207d..fb2bd7eafba7 100644 --- a/envoy/tracing/trace_context.h +++ b/envoy/tracing/trace_context.h @@ -3,7 +3,9 @@ #include #include +#include "envoy/common/optref.h" #include "envoy/common/pure.h" +#include "envoy/http/header_map.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -11,6 +13,8 @@ namespace Envoy { namespace Tracing { +class TraceContextHandler; + /** * Protocol-independent abstraction for traceable stream. It hides the differences between different * protocol and provides tracer driver with common methods for obtaining and setting the tracing @@ -66,7 +70,7 @@ class TraceContext { * @param key The context key of string view type. * @return The optional context value of string_view type. */ - virtual absl::optional getByKey(absl::string_view key) const PURE; + virtual absl::optional get(absl::string_view key) const PURE; /** * Set new tracing context key/value pair. @@ -74,25 +78,25 @@ class TraceContext { * @param key The context key of string view type. * @param val The context value of string view type. */ - virtual void setByKey(absl::string_view key, absl::string_view val) PURE; + virtual void set(absl::string_view key, absl::string_view val) PURE; /** - * Set new tracing context key/value pair. The key MUST point to data that will live beyond - * the lifetime of any traceable stream that using the string. + * Removes the following key and its associated values from the tracing + * context. * - * @param key The context key of string view type. - * @param val The context value of string view type. + * @param key The key to remove if it exists. */ - virtual void setByReferenceKey(absl::string_view key, absl::string_view val) PURE; + virtual void remove(absl::string_view key) PURE; + +private: + friend class TraceContextHandler; /** - * Set new tracing context key/value pair. Both key and val MUST point to data that will live - * beyond the lifetime of any traceable stream that using the string. - * - * @param key The context key of string view type. - * @param val The context value of string view type. + * Optional HTTP request headers map. This is valid for HTTP protocol or any protocol that + * that provides HTTP request headers. */ - virtual void setByReference(absl::string_view key, absl::string_view val) PURE; + virtual OptRef requestHeaders() { return {}; }; + virtual OptRef requestHeaders() const { return {}; }; }; } // namespace Tracing diff --git a/envoy/tracing/trace_driver.h b/envoy/tracing/trace_driver.h index 111500063e14..2b50c3dcd562 100644 --- a/envoy/tracing/trace_driver.h +++ b/envoy/tracing/trace_driver.h @@ -111,7 +111,7 @@ class Driver { virtual SpanPtr startSpan(const Config& config, TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) PURE; + Tracing::Decision tracing_decision) PURE; }; using DriverPtr = std::unique_ptr; diff --git a/envoy/tracing/tracer.h b/envoy/tracing/tracer.h index 43aa84907e67..9e2eaf65f8fd 100644 --- a/envoy/tracing/tracer.h +++ b/envoy/tracing/tracer.h @@ -20,7 +20,7 @@ class Tracer { virtual SpanPtr startSpan(const Config& config, TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, - const Tracing::Decision tracing_decision) PURE; + Tracing::Decision tracing_decision) PURE; }; using TracerSharedPtr = std::shared_ptr; diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index 538ced140360..7857015e7642 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -242,7 +242,7 @@ class ClusterManager { * The "initialized callback" set in the method above is invoked when secondary and * dynamically provisioned clusters have finished initializing. */ - virtual void + virtual absl::Status initializeSecondaryClusters(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) PURE; using ClusterInfoMap = absl::flat_hash_map>; @@ -422,10 +422,10 @@ class ClusterManager { virtual void drainConnections(DrainConnectionsHostPredicate predicate) PURE; /** - * Check if the cluster is active and statically configured, and if not, throw exception. + * Check if the cluster is active and statically configured, and if not, return an error * @param cluster, the cluster to check. */ - virtual void checkActiveStaticCluster(const std::string& cluster) PURE; + virtual absl::Status checkActiveStaticCluster(const std::string& cluster) PURE; /** * Allocates an on-demand CDS API provider from configuration proto or locator. diff --git a/envoy/upstream/host_description.h b/envoy/upstream/host_description.h index 5db12811c176..1272091e737c 100644 --- a/envoy/upstream/host_description.h +++ b/envoy/upstream/host_description.h @@ -46,12 +46,12 @@ struct HostStats { ALL_HOST_STATS(GENERATE_PRIMITIVE_COUNTER_STRUCT, GENERATE_PRIMITIVE_GAUGE_STRUCT); // Provide access to name,counter pairs. - std::vector> counters() const { + std::vector> counters() { return {ALL_HOST_STATS(PRIMITIVE_COUNTER_NAME_AND_REFERENCE, IGNORE_PRIMITIVE_GAUGE)}; } // Provide access to name,gauge pairs. - std::vector> gauges() const { + std::vector> gauges() { return {ALL_HOST_STATS(IGNORE_PRIMITIVE_COUNTER, PRIMITIVE_GAUGE_NAME_AND_REFERENCE)}; } }; diff --git a/envoy/upstream/load_balancer.h b/envoy/upstream/load_balancer.h index 0cf04779bb13..06b3f5c8f3d3 100644 --- a/envoy/upstream/load_balancer.h +++ b/envoy/upstream/load_balancer.h @@ -96,7 +96,7 @@ class LoadBalancerContext { */ virtual Network::TransportSocketOptionsConstSharedPtr upstreamTransportSocketOptions() const PURE; - using OverrideHost = absl::string_view; + using OverrideHost = std::pair; /** * Returns the host the load balancer should select directly. If the expected host exists and * the host can be selected directly, the load balancer can bypass the load balancing algorithm @@ -278,7 +278,7 @@ class TypedLoadBalancerFactory : public Config::TypedFactory { * @param visitor supplies the validation visitor that will be used to validate the embedded * Any proto message. */ - virtual LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config, + virtual LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, ProtobufMessage::ValidationVisitor& visitor) PURE; std::string category() const override { return "envoy.load_balancing_policies"; } diff --git a/envoy/upstream/thread_local_cluster.h b/envoy/upstream/thread_local_cluster.h index 7b356942c9a8..cb97bc55efaf 100644 --- a/envoy/upstream/thread_local_cluster.h +++ b/envoy/upstream/thread_local_cluster.h @@ -151,6 +151,16 @@ class ThreadLocalCluster { virtual Tcp::AsyncTcpClientPtr tcpAsyncClient(LoadBalancerContext* context, Tcp::AsyncTcpClientOptionsConstSharedPtr options) PURE; + + /** + * @return the thread local cluster drop_overload configuration. + */ + virtual UnitFloat dropOverload() const PURE; + + /** + * Set up the drop_overload value for the thread local cluster. + */ + virtual void setDropOverload(UnitFloat drop_overload) PURE; }; using ThreadLocalClusterOptRef = absl::optional>; diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index 85c3bafe5007..3a7c5f27a256 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -99,7 +99,7 @@ class UpstreamLocalAddressSelectorFactory : public Config::TypedFactory { * from cluster config. If the bind config from the cluster manager, the param * is empty. */ - virtual UpstreamLocalAddressSelectorConstSharedPtr + virtual absl::StatusOr createLocalAddressSelector(std::vector upstream_local_addresses, absl::optional cluster_name) const PURE; @@ -771,11 +771,14 @@ class PrioritySet { /** * All cluster load report stats. These are only use for EDS load reporting and not sent to the - * stats sink. See envoy.api.v2.endpoint.ClusterStats for the definition of upstream_rq_dropped. - * These are latched by LoadStatsReporter, independent of the normal stats sink flushing. + * stats sink. See envoy.config.endpoint.v3.ClusterStats for the definition of + * total_dropped_requests and dropped_requests, which correspond to the upstream_rq_dropped and + * upstream_rq_drop_overload counter here. These are latched by LoadStatsReporter, independent of + * the normal stats sink flushing. */ #define ALL_CLUSTER_LOAD_REPORT_STATS(COUNTER, GAUGE, HISTOGRAM, TEXT_READOUT, STATNAME) \ - COUNTER(upstream_rq_dropped) + COUNTER(upstream_rq_dropped) \ + COUNTER(upstream_rq_drop_overload) /** * Cluster circuit breakers gauges. Note that we do not generate a stats @@ -1018,6 +1021,9 @@ class ClusterInfo : public Http::FilterChainFactory { /** * @return the load balancer factory for this cluster if the load balancing type is * LOAD_BALANCING_POLICY_CONFIG. + * TODO(wbpcode): change the return type to return a reference after + * 'envoy_reloadable_features_convert_legacy_lb_config' is removed. The factory should never be + * nullptr when the load balancing type is LOAD_BALANCING_POLICY_CONFIG. */ virtual TypedLoadBalancerFactory* loadBalancerFactory() const PURE; @@ -1169,6 +1175,11 @@ class ClusterInfo : public Http::FilterChainFactory { */ virtual ClusterTimeoutBudgetStatsOptRef timeoutBudgetStats() const PURE; + /** + * @return true if this cluster should produce per-endpoint stats. + */ + virtual bool perEndpointStatsEnabled() const PURE; + /** * @return std::shared_ptr as upstream local address selector. */ @@ -1331,6 +1342,16 @@ class Cluster { * @return the const PrioritySet for the cluster. */ virtual const PrioritySet& prioritySet() const PURE; + + /** + * @return the cluster drop_overload configuration. + */ + virtual UnitFloat dropOverload() const PURE; + + /** + * Set up the drop_overload value for the cluster. + */ + virtual void setDropOverload(UnitFloat drop_overload) PURE; }; using ClusterSharedPtr = std::shared_ptr; diff --git a/examples/cache/requirements.txt b/examples/cache/requirements.txt index 9a7ff5a03d3d..c160459ca0cf 100644 --- a/examples/cache/requirements.txt +++ b/examples/cache/requirements.txt @@ -1,11 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements.in # pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ @@ -13,7 +15,10 @@ pyyaml==6.0.1 \ --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ @@ -21,9 +26,12 @@ pyyaml==6.0.1 \ --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ @@ -38,7 +46,9 @@ pyyaml==6.0.1 \ --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ diff --git a/examples/cache/service.py b/examples/cache/service.py index 37e18a370959..4e85ec595261 100644 --- a/examples/cache/service.py +++ b/examples/cache/service.py @@ -15,12 +15,11 @@ # Etag fun lifted from https://github.com/zhangkaizhao/aiohttp-etag -def _check_etag_header(request, response) -> bool: - computed_etag = response.headers.get("Etag", "") +def _check_etag_header(request, computed_etag) -> bool: # Find all weak and strong etag values from If-None-Match header # because RFC 7232 allows multiple etag values in a single header. etags = re.findall(r'\*|(?:W/)?"[^"]*"', request.headers.get("If-None-Match", "")) - if not computed_etag or not etags: + if not etags: return False match = False @@ -38,22 +37,14 @@ def val(x: str) -> str: return match -def _compute_etag(response) -> Optional[str]: - if hasattr(response, 'body'): - # The aiohttp.web.StreamResponse does not have ``body`` attribute. - body = response.body +def _compute_etag(body) -> str: + hasher = hashlib.sha1() + hasher.update(body.encode()) + return f'"{hasher.hexdigest()}"' - hasher = hashlib.sha1() - hasher.update(body) - return f'"{hasher.hexdigest()}"' - return None - - -def _set_etag_header(response) -> None: - etag = _compute_etag(response) - if etag is not None: - response.headers["Etag"] = etag +def _set_etag_header(response, computed_etag) -> None: + response.headers["Etag"] = computed_etag @routes.get("/service/{service_number}/{response_id}") @@ -65,17 +56,23 @@ async def get(request): if stored_response is None: raise web.HTTPNotFound(reason="No response found with the given id") + # Etag is computed for every response, which only depends on the response body in + # the yaml file (i.e. the appended date is not taken into account). + body = stored_response.get('body') + computed_etag = _compute_etag(body) + + if _check_etag_header(request, computed_etag): + return web.HTTPNotModified(headers={'ETag': computed_etag}) + request_date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - response = web.Response( - text=f"{stored_response.get('body')}\nResponse generated at: {request_date}\n") + response = web.Response(text=f"{body}\nResponse generated at: {request_date}\n") if stored_response.get('headers'): response.headers.update(stored_response.get('headers')) - _set_etag_header(response) + _set_etag_header(response, computed_etag) - return ( - _check_etag_header(request, response) if request.headers.get("If-None-Match") else response) + return response if __name__ == "__main__": diff --git a/examples/cache/verify.sh b/examples/cache/verify.sh index 004cdc2da213..b8993be6bb24 100755 --- a/examples/cache/verify.sh +++ b/examples/cache/verify.sh @@ -11,10 +11,18 @@ export CACHE_RESPONSES_YAML=./ci-responses.yaml check_validated() { # Get the date header and the response generation timestamp - local _dates dates + local _dates dates httpCode _dates=$(grep -oP '\d\d:\d\d:\d\d' <<< "$1") + httpCode=$(echo "$response" | head -n 1 | cut -d ' ' -f 2) while read -r line; do dates+=("$line"); done \ <<< "$_dates" + + # Make sure it succeeds + if [[ $httpCode != "200" ]]; then + echo "ERROR: HTTP response code should be 200, but it was $httpCode" >&2 + return 1 + fi + # Make sure they are different if [[ ${dates[0]} == "${dates[1]}" ]]; then echo "ERROR: validated responses should have a date AFTER the generation timestamp" >&2 diff --git a/examples/ext_authz/Dockerfile-opa b/examples/ext_authz/Dockerfile-opa index 1b0fa2add76f..9b69a5373a4c 100644 --- a/examples/ext_authz/Dockerfile-opa +++ b/examples/ext_authz/Dockerfile-opa @@ -1 +1 @@ -FROM openpolicyagent/opa:0.56.0-istio@sha256:b9c81ab586de93a52ff063ba92035fbca98b61ac40226a1aab482fb970b8c366 +FROM openpolicyagent/opa:0.59.0-istio@sha256:559c15eeea98c78b4b88779afc17b1db4a09858b59af19594eccaefc96fb29e3 diff --git a/examples/ext_authz/auth/grpc-service/go.mod b/examples/ext_authz/auth/grpc-service/go.mod index 813dc3437945..72ee6d16d88f 100644 --- a/examples/ext_authz/auth/grpc-service/go.mod +++ b/examples/ext_authz/auth/grpc-service/go.mod @@ -3,8 +3,8 @@ module github.com/envoyproxy/envoy/examples/ext_authz/auth/grpc-service go 1.14 require ( - github.com/envoyproxy/go-control-plane v0.11.1 + github.com/envoyproxy/go-control-plane v0.12.0 github.com/golang/protobuf v1.5.3 - google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 - google.golang.org/grpc v1.58.1 + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 + google.golang.org/grpc v1.60.1 ) diff --git a/examples/ext_authz/auth/grpc-service/go.sum b/examples/ext_authz/auth/grpc-service/go.sum index e6f05609bfb6..25019aec0e7c 100644 --- a/examples/ext_authz/auth/grpc-service/go.sum +++ b/examples/ext_authz/auth/grpc-service/go.sum @@ -38,6 +38,9 @@ cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -55,12 +58,15 @@ cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6l cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/aiplatform v1.50.0/go.mod h1:IRc2b8XAMTa9ZmfJV1BCCQbieWWvDnP1A8znyz5N7y4= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= @@ -121,14 +127,19 @@ cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3 cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= +cloud.google.com/go/baremetalsolution v1.2.0/go.mod h1:68wi9AwPYkEWIUT4SvSGS9UJwKzNpshjHsH4lzk8iOw= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= +cloud.google.com/go/batch v1.4.1/go.mod h1:KdBmDD61K0ovcxoRHGrN6GmOBWeAOyCgKD0Mugx4Fkk= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -143,6 +154,8 @@ cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= @@ -150,12 +163,14 @@ cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOA cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/billing v1.17.0/go.mod h1:Z9+vZXEq+HwH7bhJkyI4OQcR6TSbeMrjlpEjO2vzY64= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/binaryauthorization v1.7.0/go.mod h1:Zn+S6QqTMn6odcMU1zDZCJxPjU2tZPV1oDl45lWY154= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= @@ -165,16 +180,20 @@ cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/channel v1.17.0/go.mod h1:RpbhJsGi/lXWAUM1eF4IbQGbsfVlg2o8Iiy2/YLfVT0= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.14.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/clouddms v1.7.0/go.mod h1:MW1dC6SOtI/tPNCciTsXtsGNEM0i0OccykPvv3hiYeM= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= @@ -182,6 +201,7 @@ cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQky cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -200,6 +220,7 @@ cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IK cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -208,17 +229,21 @@ cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbT cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/container v1.26.0/go.mod h1:YJCmRet6+6jnYYRS000T6k0D0xUXQgBSaJ7VwI8FBj4= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/containeranalysis v0.11.0/go.mod h1:4n2e99ZwpGxpNcz+YsFT1dfOHPQFGcAC8FN2M2/ne/U= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= @@ -229,6 +254,8 @@ cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3 cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= @@ -252,9 +279,13 @@ cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0 cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= +cloud.google.com/go/dataproc/v2 v2.2.0/go.mod h1:lZR7AQtwZPvmINx5J87DSOOpTfof9LVZju6/Qo4lmcY= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= @@ -265,6 +296,8 @@ cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.14.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= @@ -272,11 +305,13 @@ cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2 cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= @@ -286,6 +321,8 @@ cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHih cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dialogflow v1.43.0/go.mod h1:pDUJdi4elL0MFmt1REMvFkdsUTYSHq+rTCS8wg0S3+M= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= @@ -297,6 +334,8 @@ cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.1/go.mod h1:LKs22aDHbJv7ufXuPypzRO7rG3ALLJxzdCXDPutw4Qc= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= @@ -316,6 +355,7 @@ cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEu cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= @@ -323,6 +363,8 @@ cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466d cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.12.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.13.0/go.mod h1:QojqqOh8IntInDUSTAh0c8ZsPYAr68Ma8c5DWOy8xb8= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= @@ -340,6 +382,8 @@ cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= +cloud.google.com/go/gkebackup v1.3.1/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= @@ -353,6 +397,7 @@ cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dP cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= @@ -371,12 +416,14 @@ cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCta cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/iap v1.9.0/go.mod h1:01OFxd1R+NFrg78S+hoPV5PxEzv22HXaNqUUlmNHFuY= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= @@ -395,18 +442,22 @@ cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63 cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/language v1.11.0/go.mod h1:uDx+pFDdAKTY8ehpWbiXyQdz8tDSYLJbQcXsCkjYyvQ= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= @@ -420,6 +471,8 @@ cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHn cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= +cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= @@ -436,11 +489,13 @@ cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSox cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/monitoring v1.16.0/go.mod h1:Ptp15HgAyM1fNICAojDMoNc/wUmn67mLHQfyqbw+poY= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= @@ -448,10 +503,12 @@ cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5Mp cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkconnectivity v1.13.0/go.mod h1:SAnGPes88pl7QRLUen2HmcBSE9AowVAcdug8c0RSBFk= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networkmanagement v1.9.0/go.mod h1:UTUaEU9YwbCAhhz3jEOHr+2/K/MrBk2XxOLS89LQzFw= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= @@ -464,10 +521,12 @@ cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vu cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/notebooks v1.10.0/go.mod h1:SOPYMZnttHxqot0SGSFSkRrwE29eqnKPBJFqgWmiK2k= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/optimization v1.5.0/go.mod h1:evo1OvTxeBRBu6ydPlrIRizKY/LJKo/drDMMRKqGEUU= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= @@ -499,6 +558,8 @@ cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LK cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= +cloud.google.com/go/policytroubleshooter v1.9.0/go.mod h1:+E2Lga7TycpeSTj2FsH4oXxTnrbHJGRlKhVZBLGgU64= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= @@ -513,6 +574,7 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= @@ -536,6 +598,7 @@ cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclC cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/recommender v1.11.0/go.mod h1:kPiRQhPyTJ9kyXPCG6u/dlPLbYfFlkwHNRwdzPVAoII= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= @@ -562,6 +625,7 @@ cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldR cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= @@ -601,6 +665,7 @@ cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UV cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= @@ -617,6 +682,7 @@ cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5 cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/spanner v1.49.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= @@ -624,6 +690,7 @@ cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSy cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -666,6 +733,8 @@ cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXN cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/translate v1.9.0/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= @@ -673,6 +742,8 @@ cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1t cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/video v1.20.0/go.mod h1:U3G3FTnsvAGqglq9LxgqzOiBc/Nt8zis8S+850N2DUM= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= @@ -696,6 +767,7 @@ cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= @@ -716,6 +788,7 @@ cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vf cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +cloud.google.com/go/workflows v1.12.0/go.mod h1:PYhSk2b6DhZ508tj8HXKaBh+OFe+xdl0dHF/tJdzPQM= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -779,8 +852,9 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= -github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= @@ -809,6 +883,7 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGw github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -888,11 +963,13 @@ github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -907,11 +984,13 @@ github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -930,6 +1009,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -958,6 +1038,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -983,6 +1064,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1003,6 +1085,7 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1018,6 +1101,7 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1138,8 +1222,10 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1171,6 +1257,7 @@ golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1189,6 +1276,7 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1269,8 +1357,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1283,6 +1372,7 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1300,8 +1390,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1450,6 +1541,7 @@ google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZ google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1457,6 +1549,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1591,12 +1684,19 @@ google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -1604,6 +1704,11 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go. google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= @@ -1611,8 +1716,13 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920183334-c177e329c48b/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1655,9 +1765,12 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= -google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1676,8 +1789,9 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/examples/golang-http/simple/config.go b/examples/golang-http/simple/config.go index 97b2cc164cad..effd79cb87be 100644 --- a/examples/golang-http/simple/config.go +++ b/examples/golang-http/simple/config.go @@ -25,6 +25,8 @@ type config struct { type parser struct { } +// Parse the filter configuration. We can call the ConfigCallbackHandler to control the filter's +// behavior func (p *parser) Parse(any *anypb.Any, callbacks api.ConfigCallbackHandler) (interface{}, error) { configStruct := &xds.TypedStruct{} if err := any.UnmarshalTo(configStruct); err != nil { @@ -45,6 +47,7 @@ func (p *parser) Parse(any *anypb.Any, callbacks api.ConfigCallbackHandler) (int return conf, nil } +// Merge configuration from the inherited parent configuration func (p *parser) Merge(parent interface{}, child interface{}) interface{} { parentConfig := parent.(*config) childConfig := child.(*config) diff --git a/examples/golang-http/simple/filter.go b/examples/golang-http/simple/filter.go index f4fb83f006b9..5eeddae31306 100644 --- a/examples/golang-http/simple/filter.go +++ b/examples/golang-http/simple/filter.go @@ -9,6 +9,8 @@ import ( var UpdateUpstreamBody = "upstream response body updated by the simple plugin" +// The callbacks in the filter, like `DecodeHeaders`, can be implemented on demand. +// Because api.PassThroughStreamFilter provides a default implementation. type filter struct { api.PassThroughStreamFilter @@ -20,10 +22,12 @@ type filter struct { func (f *filter) sendLocalReplyInternal() api.StatusType { body := fmt.Sprintf("%s, path: %s\r\n", f.config.echoBody, f.path) f.callbacks.SendLocalReply(200, body, nil, 0, "") + // Remember to return LocalReply when the request is replied locally return api.LocalReply } // Callbacks which are called in request path +// The endStream is true if the request doesn't have body func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api.StatusType { f.path, _ = header.Get(":path") api.LogDebugf("get path %s", f.path) @@ -32,60 +36,104 @@ func (f *filter) DecodeHeaders(header api.RequestHeaderMap, endStream bool) api. return f.sendLocalReplyInternal() } return api.Continue + /* + // If the code is time-consuming, to avoid blocking the Envoy, + // we need to run the code in a background goroutine + // and suspend & resume the filter + go func() { + defer f.callbacks.RecoverPanic() + // do time-consuming jobs + + // resume the filter + f.callbacks.Continue(status) + }() + + // suspend the filter + return api.Running + */ } -/* -The callbacks can be implemented on demand - +// DecodeData might be called multiple times during handling the request body. +// The endStream is true when handling the last piece of the body. func (f *filter) DecodeData(buffer api.BufferInstance, endStream bool) api.StatusType { + // support suspending & resuming the filter in a background goroutine return api.Continue } func (f *filter) DecodeTrailers(trailers api.RequestTrailerMap) api.StatusType { + // support suspending & resuming the filter in a background goroutine return api.Continue } -*/ +// Callbacks which are called in response path +// The endStream is true if the response doesn't have body func (f *filter) EncodeHeaders(header api.ResponseHeaderMap, endStream bool) api.StatusType { if f.path == "/update_upstream_response" { header.Set("Content-Length", strconv.Itoa(len(UpdateUpstreamBody))) } header.Set("Rsp-Header-From-Go", "bar-test") + // support suspending & resuming the filter in a background goroutine return api.Continue } -// Callbacks which are called in response path +// EncodeData might be called multiple times during handling the response body. +// The endStream is true when handling the last piece of the body. func (f *filter) EncodeData(buffer api.BufferInstance, endStream bool) api.StatusType { if f.path == "/update_upstream_response" { if endStream { buffer.SetString(UpdateUpstreamBody) } else { - // TODO implement buffer->Drain, buffer.SetString means buffer->Drain(buffer.Len()) - buffer.SetString("") + buffer.Reset() } } + // support suspending & resuming the filter in a background goroutine return api.Continue } -/* -The callbacks can be implemented on demand - func (f *filter) EncodeTrailers(trailers api.ResponseTrailerMap) api.StatusType { return api.Continue } +// OnLog is called when the HTTP stream is ended on HTTP Connection Manager filter. func (f *filter) OnLog() { - // Collect request info when it is ended + code, _ := f.callbacks.StreamInfo().ResponseCode() + respCode := strconv.Itoa(int(code)) + api.LogDebug(respCode) + + /* + // It's possible to kick off a goroutine here. + // But it's unsafe to access the f.callbacks because the FilterCallbackHandler + // may be already released when the goroutine is scheduled. + go func() { + defer func() { + if p := recover(); p != nil { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + fmt.Printf("http: panic serving: %v\n%s", p, buf) + } + }() + + // do time-consuming jobs + }() + */ } +// OnLogDownstreamStart is called when HTTP Connection Manager filter receives a new HTTP request +// (required the corresponding access log type is enabled) func (f *filter) OnLogDownstreamStart() { - // Collect request info when the request is started (required corresponding Envoy configuration) + // also support kicking off a goroutine here, like OnLog. } +// OnLogDownstreamPeriodic is called on any HTTP Connection Manager periodic log record +// (required the corresponding access log type is enabled) func (f *filter) OnLogDownstreamPeriodic() { - // Collect request info periodically (required corresponding Envoy configuration) + // also support kicking off a goroutine here, like OnLog. } func (f *filter) OnDestroy(reason api.DestroyReason) { + // One should not access f.callbacks here because the FilterCallbackHandler + // is released. But we can still access other Go fields in the filter f. + + // goroutine can be used everywhere. } -*/ diff --git a/examples/golang-http/simple/go.mod b/examples/golang-http/simple/go.mod index 2e96bd4b9d10..2862ede61690 100644 --- a/examples/golang-http/simple/go.mod +++ b/examples/golang-http/simple/go.mod @@ -1,7 +1,7 @@ module github.com/envoyproxy/envoy/examples/golang-http/simple // the version should >= 1.18 -go 1.18 +go 1.20 // NOTICE: these lines could be generated automatically by "go mod tidy" require ( diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index c25b1ad04486..55693b7dba09 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements.in # @@ -8,124 +8,228 @@ certifi==2023.7.22 \ --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 # via requests -charset-normalizer==2.0.6 \ - --hash=sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6 \ - --hash=sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f +charset-normalizer==3.3.0 \ + --hash=sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843 \ + --hash=sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786 \ + --hash=sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e \ + --hash=sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8 \ + --hash=sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4 \ + --hash=sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa \ + --hash=sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d \ + --hash=sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82 \ + --hash=sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7 \ + --hash=sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895 \ + --hash=sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d \ + --hash=sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a \ + --hash=sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382 \ + --hash=sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678 \ + --hash=sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b \ + --hash=sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e \ + --hash=sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741 \ + --hash=sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4 \ + --hash=sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596 \ + --hash=sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9 \ + --hash=sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69 \ + --hash=sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c \ + --hash=sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77 \ + --hash=sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13 \ + --hash=sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459 \ + --hash=sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e \ + --hash=sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7 \ + --hash=sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908 \ + --hash=sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a \ + --hash=sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f \ + --hash=sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8 \ + --hash=sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482 \ + --hash=sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d \ + --hash=sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d \ + --hash=sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545 \ + --hash=sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34 \ + --hash=sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86 \ + --hash=sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6 \ + --hash=sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe \ + --hash=sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e \ + --hash=sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc \ + --hash=sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7 \ + --hash=sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd \ + --hash=sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c \ + --hash=sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557 \ + --hash=sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a \ + --hash=sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89 \ + --hash=sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078 \ + --hash=sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e \ + --hash=sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4 \ + --hash=sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403 \ + --hash=sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0 \ + --hash=sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89 \ + --hash=sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115 \ + --hash=sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9 \ + --hash=sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05 \ + --hash=sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a \ + --hash=sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec \ + --hash=sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56 \ + --hash=sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38 \ + --hash=sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479 \ + --hash=sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c \ + --hash=sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e \ + --hash=sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd \ + --hash=sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186 \ + --hash=sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455 \ + --hash=sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c \ + --hash=sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65 \ + --hash=sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78 \ + --hash=sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287 \ + --hash=sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df \ + --hash=sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43 \ + --hash=sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1 \ + --hash=sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7 \ + --hash=sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989 \ + --hash=sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a \ + --hash=sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63 \ + --hash=sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884 \ + --hash=sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649 \ + --hash=sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810 \ + --hash=sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828 \ + --hash=sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4 \ + --hash=sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2 \ + --hash=sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd \ + --hash=sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5 \ + --hash=sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe \ + --hash=sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293 \ + --hash=sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e \ + --hash=sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e \ + --hash=sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8 # via requests -grpcio==1.58.0 \ - --hash=sha256:002f228d197fea12797a14e152447044e14fb4fdb2eb5d6cfa496f29ddbf79ef \ - --hash=sha256:039003a5e0ae7d41c86c768ef8b3ee2c558aa0a23cf04bf3c23567f37befa092 \ - --hash=sha256:09206106848462763f7f273ca93d2d2d4d26cab475089e0de830bb76be04e9e8 \ - --hash=sha256:128eb1f8e70676d05b1b0c8e6600320fc222b3f8c985a92224248b1367122188 \ - --hash=sha256:1c1c5238c6072470c7f1614bf7c774ffde6b346a100521de9ce791d1e4453afe \ - --hash=sha256:1ed979b273a81de36fc9c6716d9fb09dd3443efa18dcc8652501df11da9583e9 \ - --hash=sha256:201e550b7e2ede113b63e718e7ece93cef5b0fbf3c45e8fe4541a5a4305acd15 \ - --hash=sha256:212f38c6a156862098f6bdc9a79bf850760a751d259d8f8f249fc6d645105855 \ - --hash=sha256:24765a627eb4d9288ace32d5104161c3654128fe27f2808ecd6e9b0cfa7fc8b9 \ - --hash=sha256:24edec346e69e672daf12b2c88e95c6f737f3792d08866101d8c5f34370c54fd \ - --hash=sha256:2ef8d4a76d2c7d8065aba829f8d0bc0055495c998dce1964ca5b302d02514fb3 \ - --hash=sha256:2f85f87e2f087d9f632c085b37440a3169fda9cdde80cb84057c2fc292f8cbdf \ - --hash=sha256:3886b4d56bd4afeac518dbc05933926198aa967a7d1d237a318e6fbc47141577 \ - --hash=sha256:3e6bebf1dfdbeb22afd95650e4f019219fef3ab86d3fca8ebade52e4bc39389a \ - --hash=sha256:458899d2ebd55d5ca2350fd3826dfd8fcb11fe0f79828ae75e2b1e6051d50a29 \ - --hash=sha256:4891bbb4bba58acd1d620759b3be11245bfe715eb67a4864c8937b855b7ed7fa \ - --hash=sha256:4b12754af201bb993e6e2efd7812085ddaaef21d0a6f0ff128b97de1ef55aa4a \ - --hash=sha256:532410c51ccd851b706d1fbc00a87be0f5312bd6f8e5dbf89d4e99c7f79d7499 \ - --hash=sha256:5b23d75e5173faa3d1296a7bedffb25afd2fddb607ef292dfc651490c7b53c3d \ - --hash=sha256:62831d5e251dd7561d9d9e83a0b8655084b2a1f8ea91e4bd6b3cedfefd32c9d2 \ - --hash=sha256:652978551af02373a5a313e07bfef368f406b5929cf2d50fa7e4027f913dbdb4 \ - --hash=sha256:6801ff6652ecd2aae08ef994a3e49ff53de29e69e9cd0fd604a79ae4e545a95c \ - --hash=sha256:6cba491c638c76d3dc6c191d9c75041ca5b8f5c6de4b8327ecdcab527f130bb4 \ - --hash=sha256:7e473a7abad9af48e3ab5f3b5d237d18208024d28ead65a459bd720401bd2f8f \ - --hash=sha256:8774219e21b05f750eef8adc416e9431cf31b98f6ce9def288e4cea1548cbd22 \ - --hash=sha256:8f061722cad3f9aabb3fbb27f3484ec9d4667b7328d1a7800c3c691a98f16bb0 \ - --hash=sha256:92ae871a902cf19833328bd6498ec007b265aabf2fda845ab5bd10abcaf4c8c6 \ - --hash=sha256:9f13a171281ebb4d7b1ba9f06574bce2455dcd3f2f6d1fbe0fd0d84615c74045 \ - --hash=sha256:a2d67ff99e70e86b2be46c1017ae40b4840d09467d5455b2708de6d4c127e143 \ - --hash=sha256:b5e8db0aff0a4819946215f156bd722b6f6c8320eb8419567ffc74850c9fd205 \ - --hash=sha256:ba0af11938acf8cd4cf815c46156bcde36fa5850518120920d52620cc3ec1830 \ - --hash=sha256:bc325fed4d074367bebd465a20763586e5e1ed5b943e9d8bc7c162b1f44fd602 \ - --hash=sha256:bc7ffef430b80345729ff0a6825e9d96ac87efe39216e87ac58c6c4ef400de93 \ - --hash=sha256:cde11577d5b6fd73a00e6bfa3cf5f428f3f33c2d2878982369b5372bbc4acc60 \ - --hash=sha256:d4cef77ad2fed42b1ba9143465856d7e737279854e444925d5ba45fc1f3ba727 \ - --hash=sha256:d79b660681eb9bc66cc7cbf78d1b1b9e335ee56f6ea1755d34a31108b80bd3c8 \ - --hash=sha256:d81c2b2b24c32139dd2536972f1060678c6b9fbd106842a9fcdecf07b233eccd \ - --hash=sha256:dc72e04620d49d3007771c0e0348deb23ca341c0245d610605dddb4ac65a37cb \ - --hash=sha256:dcfba7befe3a55dab6fe1eb7fc9359dc0c7f7272b30a70ae0af5d5b063842f28 \ - --hash=sha256:e9f995a8a421405958ff30599b4d0eec244f28edc760de82f0412c71c61763d2 \ - --hash=sha256:eb6b92036ff312d5b4182fa72e8735d17aceca74d0d908a7f08e375456f03e07 \ - --hash=sha256:f0241f7eb0d2303a545136c59bc565a35c4fc3b924ccbd69cb482f4828d6f31c \ - --hash=sha256:fad9295fe02455d4f158ad72c90ef8b4bcaadfdb5efb5795f7ab0786ad67dd58 \ - --hash=sha256:fbcecb6aedd5c1891db1d70efbfbdc126c986645b5dd616a045c07d6bd2dfa86 \ - --hash=sha256:fe643af248442221db027da43ed43e53b73e11f40c9043738de9a2b4b6ca7697 +grpcio==1.60.0 \ + --hash=sha256:073f959c6f570797272f4ee9464a9997eaf1e98c27cb680225b82b53390d61e6 \ + --hash=sha256:0fd3b3968ffe7643144580f260f04d39d869fcc2cddb745deef078b09fd2b328 \ + --hash=sha256:1434ca77d6fed4ea312901122dc8da6c4389738bf5788f43efb19a838ac03ead \ + --hash=sha256:1c30bb23a41df95109db130a6cc1b974844300ae2e5d68dd4947aacba5985aa5 \ + --hash=sha256:20e7a4f7ded59097c84059d28230907cd97130fa74f4a8bfd1d8e5ba18c81491 \ + --hash=sha256:2199165a1affb666aa24adf0c97436686d0a61bc5fc113c037701fb7c7fceb96 \ + --hash=sha256:297eef542156d6b15174a1231c2493ea9ea54af8d016b8ca7d5d9cc65cfcc444 \ + --hash=sha256:2aef56e85901c2397bd557c5ba514f84de1f0ae5dd132f5d5fed042858115951 \ + --hash=sha256:30943b9530fe3620e3b195c03130396cd0ee3a0d10a66c1bee715d1819001eaf \ + --hash=sha256:3b36a2c6d4920ba88fa98075fdd58ff94ebeb8acc1215ae07d01a418af4c0253 \ + --hash=sha256:428d699c8553c27e98f4d29fdc0f0edc50e9a8a7590bfd294d2edb0da7be3629 \ + --hash=sha256:43e636dc2ce9ece583b3e2ca41df5c983f4302eabc6d5f9cd04f0562ee8ec1ae \ + --hash=sha256:452ca5b4afed30e7274445dd9b441a35ece656ec1600b77fff8c216fdf07df43 \ + --hash=sha256:467a7d31554892eed2aa6c2d47ded1079fc40ea0b9601d9f79204afa8902274b \ + --hash=sha256:4b44d7e39964e808b071714666a812049765b26b3ea48c4434a3b317bac82f14 \ + --hash=sha256:4c86343cf9ff7b2514dd229bdd88ebba760bd8973dac192ae687ff75e39ebfab \ + --hash=sha256:5208a57eae445ae84a219dfd8b56e04313445d146873117b5fa75f3245bc1390 \ + --hash=sha256:5ff21e000ff2f658430bde5288cb1ac440ff15c0d7d18b5fb222f941b46cb0d2 \ + --hash=sha256:675997222f2e2f22928fbba640824aebd43791116034f62006e19730715166c0 \ + --hash=sha256:676e4a44e740deaba0f4d95ba1d8c5c89a2fcc43d02c39f69450b1fa19d39590 \ + --hash=sha256:6e306b97966369b889985a562ede9d99180def39ad42c8014628dd3cc343f508 \ + --hash=sha256:6fd9584bf1bccdfff1512719316efa77be235469e1e3295dce64538c4773840b \ + --hash=sha256:705a68a973c4c76db5d369ed573fec3367d7d196673fa86614b33d8c8e9ebb08 \ + --hash=sha256:74d7d9fa97809c5b892449b28a65ec2bfa458a4735ddad46074f9f7d9550ad13 \ + --hash=sha256:77c8a317f0fd5a0a2be8ed5cbe5341537d5c00bb79b3bb27ba7c5378ba77dbca \ + --hash=sha256:79a050889eb8d57a93ed21d9585bb63fca881666fc709f5d9f7f9372f5e7fd03 \ + --hash=sha256:7db16dd4ea1b05ada504f08d0dca1cd9b926bed3770f50e715d087c6f00ad748 \ + --hash=sha256:83f2292ae292ed5a47cdcb9821039ca8e88902923198f2193f13959360c01860 \ + --hash=sha256:87c9224acba0ad8bacddf427a1c2772e17ce50b3042a789547af27099c5f751d \ + --hash=sha256:8a97a681e82bc11a42d4372fe57898d270a2707f36c45c6676e49ce0d5c41353 \ + --hash=sha256:9073513ec380434eb8d21970e1ab3161041de121f4018bbed3146839451a6d8e \ + --hash=sha256:90bdd76b3f04bdb21de5398b8a7c629676c81dfac290f5f19883857e9371d28c \ + --hash=sha256:91229d7203f1ef0ab420c9b53fe2ca5c1fbeb34f69b3bc1b5089466237a4a134 \ + --hash=sha256:92f88ca1b956eb8427a11bb8b4a0c0b2b03377235fc5102cb05e533b8693a415 \ + --hash=sha256:95ae3e8e2c1b9bf671817f86f155c5da7d49a2289c5cf27a319458c3e025c320 \ + --hash=sha256:9e30be89a75ee66aec7f9e60086fadb37ff8c0ba49a022887c28c134341f7179 \ + --hash=sha256:a48edde788b99214613e440fce495bbe2b1e142a7f214cce9e0832146c41e324 \ + --hash=sha256:a7152fa6e597c20cb97923407cf0934e14224af42c2b8d915f48bc3ad2d9ac18 \ + --hash=sha256:a9c7b71211f066908e518a2ef7a5e211670761651039f0d6a80d8d40054047df \ + --hash=sha256:b0571a5aef36ba9177e262dc88a9240c866d903a62799e44fd4aae3f9a2ec17e \ + --hash=sha256:b0fb2d4801546598ac5cd18e3ec79c1a9af8b8f2a86283c55a5337c5aeca4b1b \ + --hash=sha256:b10241250cb77657ab315270b064a6c7f1add58af94befa20687e7c8d8603ae6 \ + --hash=sha256:b87efe4a380887425bb15f220079aa8336276398dc33fce38c64d278164f963d \ + --hash=sha256:b98f43fcdb16172dec5f4b49f2fece4b16a99fd284d81c6bbac1b3b69fcbe0ff \ + --hash=sha256:c193109ca4070cdcaa6eff00fdb5a56233dc7610216d58fb81638f89f02e4968 \ + --hash=sha256:c826f93050c73e7769806f92e601e0efdb83ec8d7c76ddf45d514fee54e8e619 \ + --hash=sha256:d020cfa595d1f8f5c6b343530cd3ca16ae5aefdd1e832b777f9f0eb105f5b139 \ + --hash=sha256:d6a478581b1a1a8fdf3318ecb5f4d0cda41cacdffe2b527c23707c9c1b8fdb55 \ + --hash=sha256:de2ad69c9a094bf37c1102b5744c9aec6cf74d2b635558b779085d0263166454 \ + --hash=sha256:e278eafb406f7e1b1b637c2cf51d3ad45883bb5bd1ca56bc05e4fc135dfdaa65 \ + --hash=sha256:e381fe0c2aa6c03b056ad8f52f8efca7be29fb4d9ae2f8873520843b6039612a \ + --hash=sha256:e61e76020e0c332a98290323ecfec721c9544f5b739fab925b6e8cbe1944cf19 \ + --hash=sha256:f897c3b127532e6befdcf961c415c97f320d45614daf84deba0a54e64ea2457b \ + --hash=sha256:fb464479934778d7cc5baf463d959d361954d6533ad34c3a4f1d267e86ee25fd # via # -r requirements.in # grpcio-tools -grpcio-tools==1.58.0 \ - --hash=sha256:1086fe240c4c879b9721952b47d46996deb283c2d9355a8dc24a804811aacf70 \ - --hash=sha256:149fb48f53cb691a6328f68bed8e4036c730f7106b7f98e92c2c0403f0b9e93c \ - --hash=sha256:1852e798f31e5437ca7b37abc910e028b34732fb19364862cedb87b1dab66fad \ - --hash=sha256:1cb6e24194786687d4f23c64de1f0ce553af51de22746911bc37340f85f9783e \ - --hash=sha256:28eefebddec3d3adf19baca78f8b82a2287d358e1b1575ae018cdca8eacc6269 \ - --hash=sha256:2c2221123d010dc6231799e63a37f2f4786bf614ef65b23009c387cd20d8b193 \ - --hash=sha256:2ef8c696e9d78676cc3f583a92bbbf2c84e94e350f7ad22f150a52559f4599d1 \ - --hash=sha256:32d51e933c3565414dd0835f930bb28a1cdeba435d9d2c87fa3cf8b1d284db3c \ - --hash=sha256:343f572312039059a8797d6e29a7fc62196e73131ab01755660a9d48202267c1 \ - --hash=sha256:3f8904ac7fc3da2e874f00b3a986e8b7e004f499344a8e7eb213c26dfb025041 \ - --hash=sha256:43cc23908b63fcaefe690b10f68a2d8652c994b5b36ab77d2271d9608c895320 \ - --hash=sha256:453023120114c35d3d9d6717ea0820e5d5c140f51f9d0b621de4397ff854471b \ - --hash=sha256:46628247fbce86d18232eead24bd22ed0826c79f3fe2fc2fbdbde45971361049 \ - --hash=sha256:4882382631e6352819059278a5c878ce0b067008dd490911d16d5616e8a36d85 \ - --hash=sha256:4be49ed320b0ebcbc21d19ef555fbf229c1c452105522b728e1171ee2052078e \ - --hash=sha256:4ee26e9253a721fff355737649678535f76cf5d642aa3ac0cd937832559b90af \ - --hash=sha256:51587842a54e025a3d0d37afcf4ef2b7ac1def9a5d17448665cb424b53d6c287 \ - --hash=sha256:579c11a9f198847ed48dbc4f211c67fe96a73320b87c81f01b044b72e24a7d77 \ - --hash=sha256:60c874908f3b40f32f1bb0221f7b3ab65ecb53a4d0a9f0a394f031f1b292c177 \ - --hash=sha256:6997511e9d2979f7a2389479682dbb06823f21a904e8fb0a5c6baaf1b4b4a863 \ - --hash=sha256:6997df6e7c5cf4d3ddc764240c1ff6a04b45d70ec28913b38fbc6396ef743e12 \ - --hash=sha256:6ca2fc1dd8049d417a5034d944c9df05cee76f855b3e431627ab4292e7c01c47 \ - --hash=sha256:6ec43909095c630df3e479e77469bdad367067431f4af602f6ccb978a3b78afd \ - --hash=sha256:6f4d80ceb591e31ca4dceec747dbe56132e1392a0a9bb1c8fe001d1b5cac898a \ - --hash=sha256:6f7144aad9396d35fb1b80429600a970b559c2ad4d07020eeb180fe83cea2bee \ - --hash=sha256:7371d8ea80234b29affec145e25569523f549520ed7e53b2aa92bed412cdecfd \ - --hash=sha256:85ac28a9621e9b92a3fc416288c4ce45542db0b4c31b3e23031dd8e0a0ec5590 \ - --hash=sha256:88e8191d0dd789bebf42533808728f5ce75d2c51e2a72bdf20abe5b5e3fbec42 \ - --hash=sha256:8ad9d77f25514584b1ddc981d70c9e50dfcfc388aa5ba943eee67520c5267ed9 \ - --hash=sha256:8de0b701da479643f71fad71fe66885cddd89441ae16e2c724939b47742dc72e \ - --hash=sha256:9aeb5949e46558d21c51fd3ec3eeecc59c94dbca76c67c0a80d3da6b7437930c \ - --hash=sha256:a062ae3072a2a39a3c057f4d68b57b021f1dd2956cd09aab39709f6af494e1de \ - --hash=sha256:a3dbece2a121761499a659b799979d4b738586d1065439053de553773eee11ca \ - --hash=sha256:a7ae3dca059d5b358dd03fb63277428fa7d771605d4074a019138dd38d70719a \ - --hash=sha256:aadbd8393ae332e49731adb31e741f2e689989150569b7acc939f5ea43124e2d \ - --hash=sha256:ac65b8d6e3acaf88b815edf9af88ff844b6600ff3d2591c05ba4f655b45d5fb4 \ - --hash=sha256:b63f823ac991ff77104da614d2a2485a59d37d57830eb2e387a6e2a3edc7fa2b \ - --hash=sha256:b6c896f1df99c35cf062d4803c15663ff00a33ff09add28baa6e475cf6b5e258 \ - --hash=sha256:b6ea5578712cdb29b0ff60bfc6405bf0e8d681b9c71d106dd1cda54fe7fe4e55 \ - --hash=sha256:ba3d383e5ca93826038b70f326fce8e8d12dd9b2f64d363a3d612f7475f12dd2 \ - --hash=sha256:c29880f491581c83181c0a84a4d11402af2b13166a5266f64e246adf1da7aa66 \ - --hash=sha256:cd7acfbb43b7338a78cf4a67528d05530d574d92b7c829d185b78dfc451d158f \ - --hash=sha256:d84091a189d848d94645b7c48b61734c12ec03b0d46e5fc0049343a26989ac5c \ - --hash=sha256:df2788736bdf58abe7b0e4d6b1ff806f7686c98c5ad900da312252e3322d91c4 \ - --hash=sha256:eec3c93a08df11c80ef1c29a616bcbb0d83dbc6ea41b48306fcacc720416dfa7 +grpcio-tools==1.60.0 \ + --hash=sha256:081336d8258f1a56542aa8a7a5dec99a2b38d902e19fbdd744594783301b0210 \ + --hash=sha256:1748893efd05cf4a59a175d7fa1e4fbb652f4d84ccaa2109f7869a2be48ed25e \ + --hash=sha256:17a32b3da4fc0798cdcec0a9c974ac2a1e98298f151517bf9148294a3b1a5742 \ + --hash=sha256:18976684a931ca4bcba65c78afa778683aefaae310f353e198b1823bf09775a0 \ + --hash=sha256:1b93ae8ffd18e9af9a965ebca5fa521e89066267de7abdde20721edc04e42721 \ + --hash=sha256:1fbb9554466d560472f07d906bfc8dcaf52f365c2a407015185993e30372a886 \ + --hash=sha256:24c4ead4a03037beaeb8ef2c90d13d70101e35c9fae057337ed1a9144ef10b53 \ + --hash=sha256:2a8a758701f3ac07ed85f5a4284c6a9ddefcab7913a8e552497f919349e72438 \ + --hash=sha256:2dd01257e4feff986d256fa0bac9f56de59dc735eceeeb83de1c126e2e91f653 \ + --hash=sha256:2e00de389729ca8d8d1a63c2038703078a887ff738dc31be640b7da9c26d0d4f \ + --hash=sha256:2fb4cf74bfe1e707cf10bc9dd38a1ebaa145179453d150febb121c7e9cd749bf \ + --hash=sha256:2fd1671c52f96e79a2302c8b1c1f78b8a561664b8b3d6946f20d8f1cc6b4225a \ + --hash=sha256:321b18f42a70813545e416ddcb8bf20defa407a8114906711c9710a69596ceda \ + --hash=sha256:3456df087ea61a0972a5bc165aed132ed6ddcc63f5749e572f9fff84540bdbad \ + --hash=sha256:4041538f55aad5b3ae7e25ab314d7995d689e968bfc8aa169d939a3160b1e4c6 \ + --hash=sha256:559ce714fe212aaf4abbe1493c5bb8920def00cc77ce0d45266f4fd9d8b3166f \ + --hash=sha256:5a907a4f1ffba86501b2cdb8682346249ea032b922fc69a92f082ba045cca548 \ + --hash=sha256:5ce6bbd4936977ec1114f2903eb4342781960d521b0d82f73afedb9335251f6f \ + --hash=sha256:6170873b1e5b6580ebb99e87fb6e4ea4c48785b910bd7af838cc6e44b2bccb04 \ + --hash=sha256:6192184b1f99372ff1d9594bd4b12264e3ff26440daba7eb043726785200ff77 \ + --hash=sha256:6807b7a3f3e6e594566100bd7fe04a2c42ce6d5792652677f1aaf5aa5adaef3d \ + --hash=sha256:687f576d7ff6ce483bc9a196d1ceac45144e8733b953620a026daed8e450bc38 \ + --hash=sha256:74025fdd6d1cb7ba4b5d087995339e9a09f0c16cf15dfe56368b23e41ffeaf7a \ + --hash=sha256:7a5263a0f2ddb7b1cfb2349e392cfc4f318722e0f48f886393e06946875d40f3 \ + --hash=sha256:7a6fe752205caae534f29fba907e2f59ff79aa42c6205ce9a467e9406cbac68c \ + --hash=sha256:7c1cde49631732356cb916ee1710507967f19913565ed5f9991e6c9cb37e3887 \ + --hash=sha256:811abb9c4fb6679e0058dfa123fb065d97b158b71959c0e048e7972bbb82ba0f \ + --hash=sha256:857c5351e9dc33a019700e171163f94fcc7e3ae0f6d2b026b10fda1e3c008ef1 \ + --hash=sha256:87cf439178f3eb45c1a889b2e4a17cbb4c450230d92c18d9c57e11271e239c55 \ + --hash=sha256:9970d384fb0c084b00945ef57d98d57a8d32be106d8f0bd31387f7cbfe411b5b \ + --hash=sha256:9ee35234f1da8fba7ddbc544856ff588243f1128ea778d7a1da3039be829a134 \ + --hash=sha256:addc9b23d6ff729d9f83d4a2846292d4c84f5eb2ec38f08489a6a0d66ac2b91e \ + --hash=sha256:b22b1299b666eebd5752ba7719da536075eae3053abcf2898b65f763c314d9da \ + --hash=sha256:b8f7a5094adb49e85db13ea3df5d99a976c2bdfd83b0ba26af20ebb742ac6786 \ + --hash=sha256:b96981f3a31b85074b73d97c8234a5ed9053d65a36b18f4a9c45a2120a5b7a0a \ + --hash=sha256:bbf0ed772d2ae7e8e5d7281fcc00123923ab130b94f7a843eee9af405918f924 \ + --hash=sha256:bd2a17b0193fbe4793c215d63ce1e01ae00a8183d81d7c04e77e1dfafc4b2b8a \ + --hash=sha256:c771b19dce2bfe06899247168c077d7ab4e273f6655d8174834f9a6034415096 \ + --hash=sha256:d941749bd8dc3f8be58fe37183143412a27bec3df8482d5abd6b4ec3f1ac2924 \ + --hash=sha256:dba6e32c87b4af29b5f475fb2f470f7ee3140bfc128644f17c6c59ddeb670680 \ + --hash=sha256:dd1e68c232fe01dd5312a8dbe52c50ecd2b5991d517d7f7446af4ba6334ba872 \ + --hash=sha256:e5614cf0960456d21d8a0f4902e3e5e3bcacc4e400bf22f196e5dd8aabb978b7 \ + --hash=sha256:e5c519a0d4ba1ab44a004fa144089738c59278233e2010b2cf4527dc667ff297 \ + --hash=sha256:e68dc4474f30cad11a965f0eb5d37720a032b4720afa0ec19dbcea2de73b5aae \ + --hash=sha256:e70d867c120d9849093b0ac24d861e378bc88af2552e743d83b9f642d2caa7c2 \ + --hash=sha256:e87cabac7969bdde309575edc2456357667a1b28262b2c1f12580ef48315b19d \ + --hash=sha256:eae27f9b16238e2aaee84c77b5923c6924d6dccb0bdd18435bf42acc8473ae1a \ + --hash=sha256:ec0e401e9a43d927d216d5169b03c61163fb52b665c5af2fed851357b15aef88 \ + --hash=sha256:ed30499340228d733ff69fcf4a66590ed7921f94eb5a2bf692258b1280b9dac7 \ + --hash=sha256:f10ef47460ce3c6fd400f05fe757b90df63486c9b84d1ecad42dcc5f80c8ac14 \ + --hash=sha256:f3d916606dcf5610d4367918245b3d9d8cd0d2ec0b7043d1bbb8c50fe9815c3a \ + --hash=sha256:f610384dee4b1ca705e8da66c5b5fe89a2de3d165c5282c3d1ddf40cb18924e4 \ + --hash=sha256:fb4df80868b3e397d5fbccc004c789d2668b622b51a9d2387b4c89c80d31e2c5 \ + --hash=sha256:fc01bc1079279ec342f0f1b6a107b3f5dc3169c33369cf96ada6e2e171f74e86 # via -r requirements.in -idna==3.2 \ - --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ - --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 # via requests -protobuf==4.24.3 \ - --hash=sha256:067f750169bc644da2e1ef18c785e85071b7c296f14ac53e0900e605da588719 \ - --hash=sha256:12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d \ - --hash=sha256:1b182c7181a2891e8f7f3a1b5242e4ec54d1f42582485a896e4de81aa17540c2 \ - --hash=sha256:20651f11b6adc70c0f29efbe8f4a94a74caf61b6200472a9aea6e19898f9fcf4 \ - --hash=sha256:2da777d34b4f4f7613cdf85c70eb9a90b1fbef9d36ae4a0ccfe014b0b07906f1 \ - --hash=sha256:3d42e9e4796a811478c783ef63dc85b5a104b44aaaca85d4864d5b886e4b05e3 \ - --hash=sha256:6e514e8af0045be2b56e56ae1bb14f43ce7ffa0f68b1c793670ccbe2c4fc7d2b \ - --hash=sha256:b0271a701e6782880d65a308ba42bc43874dabd1a0a0f41f72d2dac3b57f8e76 \ - --hash=sha256:ba53c2f04798a326774f0e53b9c759eaef4f6a568ea7072ec6629851c8435959 \ - --hash=sha256:e29d79c913f17a60cf17c626f1041e5288e9885c8579832580209de8b75f2a52 \ - --hash=sha256:f631bb982c5478e0c1c70eab383af74a84be66945ebf5dd6b06fc90079668d0b \ - --hash=sha256:f6ccbcf027761a2978c1406070c3788f6de4a4b2cc20800cc03d52df716ad675 \ - --hash=sha256:f6f8dc65625dadaad0c8545319c2e2f0424fede988368893ca3844261342c11a +protobuf==4.25.1 \ + --hash=sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd \ + --hash=sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb \ + --hash=sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0 \ + --hash=sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7 \ + --hash=sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b \ + --hash=sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2 \ + --hash=sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510 \ + --hash=sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6 \ + --hash=sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd \ + --hash=sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10 \ + --hash=sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7 # via # -r requirements.in # grpcio-tools @@ -133,7 +237,13 @@ requests==2.31.0 \ --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via -r requirements.in -urllib3==1.26.7 \ - --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ - --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e # via requests + +# The following packages are considered to be unsafe in a requirements file: +setuptools==68.2.2 \ + --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ + --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a + # via grpcio-tools diff --git a/examples/grpc-bridge/server/go.mod b/examples/grpc-bridge/server/go.mod index bd7672d03387..07334fa2cadd 100644 --- a/examples/grpc-bridge/server/go.mod +++ b/examples/grpc-bridge/server/go.mod @@ -1,12 +1,9 @@ -module github.com/envoyproxy/envoy +module github.com/envoyproxy/envoy/examples/grpc-bridge/server go 1.13 require ( - github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv v0.0.0-00010101000000-000000000000 - golang.org/x/net v0.8.0 - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect - google.golang.org/grpc v1.53.0 + github.com/golang/protobuf v1.5.3 + golang.org/x/net v0.19.0 + google.golang.org/grpc v1.60.0 ) - -replace github.com/envoyproxy/envoy/examples/grpc-bridge/server/kv => ./kv diff --git a/examples/grpc-bridge/server/go.sum b/examples/grpc-bridge/server/go.sum index 77970dfea09b..6df09c7c39fc 100644 --- a/examples/grpc-bridge/server/go.sum +++ b/examples/grpc-bridge/server/go.sum @@ -36,66 +36,110 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/aiplatform v1.50.0/go.mod h1:IRc2b8XAMTa9ZmfJV1BCCQbieWWvDnP1A8znyz5N7y4= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= +cloud.google.com/go/baremetalsolution v1.2.0/go.mod h1:68wi9AwPYkEWIUT4SvSGS9UJwKzNpshjHsH4lzk8iOw= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= +cloud.google.com/go/batch v1.4.1/go.mod h1:KdBmDD61K0ovcxoRHGrN6GmOBWeAOyCgKD0Mugx4Fkk= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -105,34 +149,59 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/billing v1.17.0/go.mod h1:Z9+vZXEq+HwH7bhJkyI4OQcR6TSbeMrjlpEjO2vzY64= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/binaryauthorization v1.7.0/go.mod h1:Zn+S6QqTMn6odcMU1zDZCJxPjU2tZPV1oDl45lWY154= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/channel v1.17.0/go.mod h1:RpbhJsGi/lXWAUM1eF4IbQGbsfVlg2o8Iiy2/YLfVT0= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.14.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/clouddms v1.7.0/go.mod h1:MW1dC6SOtI/tPNCciTsXtsGNEM0i0OccykPvv3hiYeM= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -146,6 +215,11 @@ cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARy cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -153,12 +227,22 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/container v1.26.0/go.mod h1:YJCmRet6+6jnYYRS000T6k0D0xUXQgBSaJ7VwI8FBj4= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/containeranalysis v0.11.0/go.mod h1:4n2e99ZwpGxpNcz+YsFT1dfOHPQFGcAC8FN2M2/ne/U= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= @@ -166,39 +250,67 @@ cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= +cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= +cloud.google.com/go/dataproc/v2 v2.2.0/go.mod h1:lZR7AQtwZPvmINx5J87DSOOpTfof9LVZju6/Qo4lmcY= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.14.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= @@ -206,57 +318,91 @@ cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dialogflow v1.43.0/go.mod h1:pDUJdi4elL0MFmt1REMvFkdsUTYSHq+rTCS8wg0S3+M= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.1/go.mod h1:LKs22aDHbJv7ufXuPypzRO7rG3ALLJxzdCXDPutw4Qc= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.12.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.13.0/go.mod h1:QojqqOh8IntInDUSTAh0c8ZsPYAr68Ma8c5DWOy8xb8= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= +cloud.google.com/go/gkebackup v1.3.1/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= @@ -265,97 +411,159 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/iap v1.9.0/go.mod h1:01OFxd1R+NFrg78S+hoPV5PxEzv22HXaNqUUlmNHFuY= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/language v1.11.0/go.mod h1:uDx+pFDdAKTY8ehpWbiXyQdz8tDSYLJbQcXsCkjYyvQ= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= +cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= +cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/monitoring v1.16.0/go.mod h1:Ptp15HgAyM1fNICAojDMoNc/wUmn67mLHQfyqbw+poY= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkconnectivity v1.13.0/go.mod h1:SAnGPes88pl7QRLUen2HmcBSE9AowVAcdug8c0RSBFk= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networkmanagement v1.9.0/go.mod h1:UTUaEU9YwbCAhhz3jEOHr+2/K/MrBk2XxOLS89LQzFw= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/notebooks v1.10.0/go.mod h1:SOPYMZnttHxqot0SGSFSkRrwE29eqnKPBJFqgWmiK2k= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/optimization v1.5.0/go.mod h1:evo1OvTxeBRBu6ydPlrIRizKY/LJKo/drDMMRKqGEUU= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= +cloud.google.com/go/policytroubleshooter v1.9.0/go.mod h1:+E2Lga7TycpeSTj2FsH4oXxTnrbHJGRlKhVZBLGgU64= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -363,8 +571,13 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= @@ -372,77 +585,111 @@ cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7d cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/recommender v1.11.0/go.mod h1:kPiRQhPyTJ9kyXPCG6u/dlPLbYfFlkwHNRwdzPVAoII= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/spanner v1.49.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -453,61 +700,94 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/translate v1.9.0/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/video v1.20.0/go.mod h1:U3G3FTnsvAGqglq9LxgqzOiBc/Nt8zis8S+850N2DUM= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +cloud.google.com/go/workflows v1.12.0/go.mod h1:PYhSk2b6DhZ508tj8HXKaBh+OFe+xdl0dHF/tJdzPQM= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -522,6 +802,8 @@ github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGW github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -547,11 +829,15 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -562,9 +848,16 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -584,9 +877,12 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -612,8 +908,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -630,7 +927,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -657,13 +953,18 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -674,6 +975,11 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -702,9 +1008,13 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= @@ -719,7 +1029,10 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -741,6 +1054,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -760,6 +1075,7 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -769,6 +1085,13 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -825,6 +1148,9 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -863,6 +1189,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -871,18 +1198,23 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= -golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -910,6 +1242,10 @@ golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -926,6 +1262,9 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -993,23 +1332,35 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d h1:/m5NbqQelATgoSPVC2Z23sR4kVNokFwDDyWh/3rGY+I= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1018,20 +1369,24 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1092,15 +1447,17 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= @@ -1165,6 +1522,14 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= +google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1172,6 +1537,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1257,8 +1623,6 @@ google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220708155623-50e5f4832e73 h1:sdZWfcGN37Dv0QWIhuasQGMzAQJOL2oqnvot4/kPgfQ= -google.golang.org/genproto v0.0.0-20220708155623-50e5f4832e73/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= @@ -1298,8 +1662,52 @@ google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= +google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920183334-c177e329c48b/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1331,15 +1739,22 @@ google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1354,10 +1769,12 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -1379,12 +1796,17 @@ lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= @@ -1394,19 +1816,31 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/examples/grpc-bridge/server/kv/empty.go b/examples/grpc-bridge/server/kv/empty.go new file mode 100644 index 000000000000..d0c265a0e2af --- /dev/null +++ b/examples/grpc-bridge/server/kv/empty.go @@ -0,0 +1,8 @@ +// make the kv module is not empty, make go mod tidy happy. +// a kv.pb.go file will be generated by protoc, while running the example. +// also, introduce the empty.go file to import the protobuf package, +// which will be imported from the generated kv.pb.go file. + +package kv + +import _ "github.com/golang/protobuf/proto" diff --git a/examples/grpc-bridge/server/kv/go.mod b/examples/grpc-bridge/server/kv/go.mod deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/kafka/Dockerfile-kafka b/examples/kafka/Dockerfile-kafka index 4bad0eb5baa2..444c2f8ec974 100644 --- a/examples/kafka/Dockerfile-kafka +++ b/examples/kafka/Dockerfile-kafka @@ -1 +1 @@ -FROM confluentinc/cp-kafka:latest@sha256:fbbb6fa11b258a88b83f54d4f0bddfcffbf2279f99d66a843486e3da7bdfbf41 +FROM confluentinc/cp-kafka:latest@sha256:51145a40d23336a11085ca695d02bdeee66fe01b582837c6d223384952226be9 diff --git a/examples/kafka/Dockerfile-zookeeper b/examples/kafka/Dockerfile-zookeeper index b67977df7422..a8e9ee756ae2 100644 --- a/examples/kafka/Dockerfile-zookeeper +++ b/examples/kafka/Dockerfile-zookeeper @@ -1 +1 @@ -FROM confluentinc/cp-zookeeper:latest@sha256:02f6c042bb9a7844382fc4cedc513a44585d8a5acae873fb9e510e3ca9dcabc6 +FROM confluentinc/cp-zookeeper:latest@sha256:000f1d11090f49fa8f67567e633bab4fea5dbd7d9119e7ee2ef259c509063593 diff --git a/examples/kafka/verify.sh b/examples/kafka/verify.sh index b234f0bf044b..efcf0f4afadd 100755 --- a/examples/kafka/verify.sh +++ b/examples/kafka/verify.sh @@ -4,6 +4,10 @@ export NAME=kafka export PORT_PROXY="${KAFKA_PORT_PROXY:-11100}" export PORT_ADMIN="${KAFKA_PORT_ADMIN:-11101}" +# Explicitly specified the service want to start, since the `kafka-client` is expected to +# not start. +UPARGS="proxy kafka-server zookeeper" + # shellcheck source=examples/verify-common.sh . "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" diff --git a/examples/load-reporting-service/go.mod b/examples/load-reporting-service/go.mod index 8dbb321ce1c7..505cd31104e2 100644 --- a/examples/load-reporting-service/go.mod +++ b/examples/load-reporting-service/go.mod @@ -3,7 +3,7 @@ module github.com/envoyproxy/envoy/examples/load-reporting-service go 1.13 require ( - github.com/envoyproxy/go-control-plane v0.11.1 + github.com/envoyproxy/go-control-plane v0.12.0 github.com/golang/protobuf v1.5.3 - google.golang.org/grpc v1.58.1 + google.golang.org/grpc v1.60.1 ) diff --git a/examples/load-reporting-service/go.sum b/examples/load-reporting-service/go.sum index e6f05609bfb6..25019aec0e7c 100644 --- a/examples/load-reporting-service/go.sum +++ b/examples/load-reporting-service/go.sum @@ -38,6 +38,9 @@ cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -55,12 +58,15 @@ cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6l cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/aiplatform v1.50.0/go.mod h1:IRc2b8XAMTa9ZmfJV1BCCQbieWWvDnP1A8znyz5N7y4= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= @@ -121,14 +127,19 @@ cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3 cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= +cloud.google.com/go/baremetalsolution v1.2.0/go.mod h1:68wi9AwPYkEWIUT4SvSGS9UJwKzNpshjHsH4lzk8iOw= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= +cloud.google.com/go/batch v1.4.1/go.mod h1:KdBmDD61K0ovcxoRHGrN6GmOBWeAOyCgKD0Mugx4Fkk= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -143,6 +154,8 @@ cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= @@ -150,12 +163,14 @@ cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOA cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/billing v1.17.0/go.mod h1:Z9+vZXEq+HwH7bhJkyI4OQcR6TSbeMrjlpEjO2vzY64= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/binaryauthorization v1.7.0/go.mod h1:Zn+S6QqTMn6odcMU1zDZCJxPjU2tZPV1oDl45lWY154= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= @@ -165,16 +180,20 @@ cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/channel v1.17.0/go.mod h1:RpbhJsGi/lXWAUM1eF4IbQGbsfVlg2o8Iiy2/YLfVT0= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/cloudbuild v1.14.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/clouddms v1.7.0/go.mod h1:MW1dC6SOtI/tPNCciTsXtsGNEM0i0OccykPvv3hiYeM= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= @@ -182,6 +201,7 @@ cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQky cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -200,6 +220,7 @@ cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IK cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= @@ -208,17 +229,21 @@ cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbT cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/container v1.26.0/go.mod h1:YJCmRet6+6jnYYRS000T6k0D0xUXQgBSaJ7VwI8FBj4= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/containeranalysis v0.11.0/go.mod h1:4n2e99ZwpGxpNcz+YsFT1dfOHPQFGcAC8FN2M2/ne/U= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= @@ -229,6 +254,8 @@ cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3 cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= @@ -252,9 +279,13 @@ cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0 cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataplex v1.9.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= +cloud.google.com/go/dataproc/v2 v2.2.0/go.mod h1:lZR7AQtwZPvmINx5J87DSOOpTfof9LVZju6/Qo4lmcY= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= @@ -265,6 +296,8 @@ cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.14.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= @@ -272,11 +305,13 @@ cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2 cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= @@ -286,6 +321,8 @@ cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHih cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dialogflow v1.43.0/go.mod h1:pDUJdi4elL0MFmt1REMvFkdsUTYSHq+rTCS8wg0S3+M= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= @@ -297,6 +334,8 @@ cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/documentai v1.22.1/go.mod h1:LKs22aDHbJv7ufXuPypzRO7rG3ALLJxzdCXDPutw4Qc= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= @@ -316,6 +355,7 @@ cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEu cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= @@ -323,6 +363,8 @@ cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466d cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.12.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/firestore v1.13.0/go.mod h1:QojqqOh8IntInDUSTAh0c8ZsPYAr68Ma8c5DWOy8xb8= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= @@ -340,6 +382,8 @@ cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= +cloud.google.com/go/gkebackup v1.3.1/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= @@ -353,6 +397,7 @@ cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dP cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= @@ -371,12 +416,14 @@ cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCta cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/iap v1.9.0/go.mod h1:01OFxd1R+NFrg78S+hoPV5PxEzv22HXaNqUUlmNHFuY= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= @@ -395,18 +442,22 @@ cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63 cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/language v1.11.0/go.mod h1:uDx+pFDdAKTY8ehpWbiXyQdz8tDSYLJbQcXsCkjYyvQ= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= @@ -420,6 +471,8 @@ cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHn cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= +cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= @@ -436,11 +489,13 @@ cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSox cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/monitoring v1.16.0/go.mod h1:Ptp15HgAyM1fNICAojDMoNc/wUmn67mLHQfyqbw+poY= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= @@ -448,10 +503,12 @@ cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5Mp cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkconnectivity v1.13.0/go.mod h1:SAnGPes88pl7QRLUen2HmcBSE9AowVAcdug8c0RSBFk= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networkmanagement v1.9.0/go.mod h1:UTUaEU9YwbCAhhz3jEOHr+2/K/MrBk2XxOLS89LQzFw= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= @@ -464,10 +521,12 @@ cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vu cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/notebooks v1.10.0/go.mod h1:SOPYMZnttHxqot0SGSFSkRrwE29eqnKPBJFqgWmiK2k= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/optimization v1.5.0/go.mod h1:evo1OvTxeBRBu6ydPlrIRizKY/LJKo/drDMMRKqGEUU= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= @@ -499,6 +558,8 @@ cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LK cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= +cloud.google.com/go/policytroubleshooter v1.9.0/go.mod h1:+E2Lga7TycpeSTj2FsH4oXxTnrbHJGRlKhVZBLGgU64= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= @@ -513,6 +574,7 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= @@ -536,6 +598,7 @@ cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclC cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/recommender v1.11.0/go.mod h1:kPiRQhPyTJ9kyXPCG6u/dlPLbYfFlkwHNRwdzPVAoII= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= @@ -562,6 +625,7 @@ cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldR cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= @@ -601,6 +665,7 @@ cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UV cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= @@ -617,6 +682,7 @@ cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5 cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/spanner v1.49.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= @@ -624,6 +690,7 @@ cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSy cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -666,6 +733,8 @@ cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXN cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/translate v1.9.0/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= @@ -673,6 +742,8 @@ cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1t cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/video v1.20.0/go.mod h1:U3G3FTnsvAGqglq9LxgqzOiBc/Nt8zis8S+850N2DUM= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= @@ -696,6 +767,7 @@ cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= @@ -716,6 +788,7 @@ cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vf cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +cloud.google.com/go/workflows v1.12.0/go.mod h1:PYhSk2b6DhZ508tj8HXKaBh+OFe+xdl0dHF/tJdzPQM= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -779,8 +852,9 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= -github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= @@ -809,6 +883,7 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGw github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -888,11 +963,13 @@ github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -907,11 +984,13 @@ github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -930,6 +1009,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -958,6 +1038,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -983,6 +1064,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1003,6 +1085,7 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1018,6 +1101,7 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1138,8 +1222,10 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1171,6 +1257,7 @@ golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1189,6 +1276,7 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1269,8 +1357,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1283,6 +1372,7 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1300,8 +1390,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1450,6 +1541,7 @@ google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZ google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1457,6 +1549,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1591,12 +1684,19 @@ google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -1604,6 +1704,11 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go. google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= @@ -1611,8 +1716,13 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920183334-c177e329c48b/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1655,9 +1765,12 @@ google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5v google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= -google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1676,8 +1789,9 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/examples/local_ratelimit/Dockerfile-nginx b/examples/local_ratelimit/Dockerfile-nginx index d39b894a827b..da576a2c01e6 100644 --- a/examples/local_ratelimit/Dockerfile-nginx +++ b/examples/local_ratelimit/Dockerfile-nginx @@ -1 +1 @@ -FROM nginx@sha256:6926dd802f40e5e7257fded83e0d8030039642e4e10c4a98a6478e9c6fe06153 +FROM nginx@sha256:10d1f5b58f74683ad34eb29287e07dab1e90f10af243f151bb50aa5dbb4d62ee diff --git a/examples/locality-load-balancing/verify.sh b/examples/locality-load-balancing/verify.sh index c6a2855df8ca..7f6727811257 100755 --- a/examples/locality-load-balancing/verify.sh +++ b/examples/locality-load-balancing/verify.sh @@ -79,7 +79,7 @@ make_healthy backend-local-1-1 make_healthy backend-local-2-1 run_log "Scale backend-local-1 to 5 replicas." -"${DOCKER_COMPOSE[@]}" -p ${NAME} up --scale backend-local-1=5 -d --build +"${DOCKER_COMPOSE[@]}" -p "${NAME}" up --scale backend-local-1=5 -d --build wait_for 5 check_health backend-local-1-2 healthy wait_for 5 check_health backend-local-1-3 healthy wait_for 5 check_health backend-local-1-4 healthy diff --git a/examples/lua-cluster-specifier/README.md b/examples/lua-cluster-specifier/README.md new file mode 100644 index 000000000000..0a1cd344b9b1 --- /dev/null +++ b/examples/lua-cluster-specifier/README.md @@ -0,0 +1,2 @@ +To learn about this sandbox and for instructions on how to run it please head over +to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/lua-cluster-specifier.html) diff --git a/examples/lua-cluster-specifier/docker-compose.yaml b/examples/lua-cluster-specifier/docker-compose.yaml new file mode 100644 index 000000000000..904b030f8417 --- /dev/null +++ b/examples/lua-cluster-specifier/docker-compose.yaml @@ -0,0 +1,14 @@ +services: + + proxy: + build: + context: . + dockerfile: ../shared/envoy/Dockerfile + ports: + - "${PORT_PROXY:-8000}:8000" + + web_service: + build: + context: ../shared/echo + ports: + - "${PORT_WEB:-8080}:8080" diff --git a/examples/lua-cluster-specifier/envoy.yaml b/examples/lua-cluster-specifier/envoy.yaml new file mode 100644 index 000000000000..058e28179395 --- /dev/null +++ b/examples/lua-cluster-specifier/envoy.yaml @@ -0,0 +1,58 @@ +static_resources: + listeners: + - name: main + address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + inline_cluster_specifier_plugin: + extension: + name: envoy.router.cluster_specifier_plugin.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.router.cluster_specifiers.lua.v3.LuaConfig + source_code: + inline_string: | + function envoy_on_route(route_handle) + local header_value = route_handle:headers():get("header_key") + if header_value == "fake" then + return "fake_service" + end + return "web_service" + end + default_cluster: web_service + + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + clusters: + - name: web_service + type: STRICT_DNS # static + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: web_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: httpbin.org + port_value: 80 diff --git a/examples/lua-cluster-specifier/verify.sh b/examples/lua-cluster-specifier/verify.sh new file mode 100755 index 000000000000..ec37d5b46d89 --- /dev/null +++ b/examples/lua-cluster-specifier/verify.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +export NAME=lua-cluster-specifier +export PORT_PROXY="${LUA_CLUSTER_PORT_PROXY:-12620}" +export PORT_WEB="${LUA_CLUSTER_PORT_WEB:-12621}" + +# shellcheck source=examples/verify-common.sh +. "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" + +run_log "Test Lua cluster specifier with normal cluster" +responds_with_header \ + "HTTP/1.1 200 OK" \ + "http://localhost:${PORT_PROXY}/" + +run_log "Test Lua cluster specifier with fake cluster" +responds_with_header \ + "HTTP/1.1 503 Service Unavailable" \ + "http://localhost:${PORT_PROXY}/" \ + -H 'header_key: fake' diff --git a/examples/mysql/Dockerfile-mysql b/examples/mysql/Dockerfile-mysql index 3c38055467fa..5d0714294e84 100644 --- a/examples/mysql/Dockerfile-mysql +++ b/examples/mysql/Dockerfile-mysql @@ -1 +1 @@ -FROM mysql:8.1.0@sha256:85ab57eb4a48ada2a341dcf7d96733ce2f370fffb8e8e216991b106e50fa6434 +FROM mysql:8.2.0@sha256:b359741e9fb794c4b6e39ea5d14047609c8ed1efcb03ababe61a239b48a165a9 diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index 49b4afcd8822..09edb08075ec 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ -FROM alpine:3.18@sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a as otelc_curl +FROM alpine:3.19@sha256:51b67269f354137895d43f3b3d810bfacd3945438e94dc5ac55fdac340352f48 as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:11cb391eff16542f6fef93be9536454dc5c38552c0285a349f81539e8b7caf85 +FROM otel/opentelemetry-collector:latest@sha256:d9df86048f9ea4e0a48e71c476ad140050ba2e80efe58abe4097f1a18a546272 COPY --from=otelc_curl / / diff --git a/examples/redis/Dockerfile-redis b/examples/redis/Dockerfile-redis index 575364232662..0d6ddffdacf5 100644 --- a/examples/redis/Dockerfile-redis +++ b/examples/redis/Dockerfile-redis @@ -1 +1 @@ -FROM redis@sha256:f92a0be0ba8c085e6a5e2d2bea386365443485bcd67ced5ca8ddcdacdd4656d2 +FROM redis@sha256:a7cee7c8178ff9b5297cb109e6240f5072cdaaafd775ce6b586c3c704b06458e diff --git a/examples/shared/build/Dockerfile b/examples/shared/build/Dockerfile index b6199e72dc13..cd8a748a4384 100644 --- a/examples/shared/build/Dockerfile +++ b/examples/shared/build/Dockerfile @@ -1,8 +1,7 @@ -FROM envoyproxy/envoy-build-ubuntu:56f235b141079013e64912d676fe7da981368402@sha256:d44499c6fd28a8a6a75dc61668b8a9e7bc3d99db11f9a61e8ea1d1f39c20a236 +FROM envoyproxy/envoy-build-ubuntu:fd9ec000fdd72d5c5e4e4ef16db4f9103058779e@sha256:1386a26f687826850ba488d66a6cd5337c5941b3b8793d08cfa6f9df12aa2fcf ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ - # apt-get -qq update \ apt-get -qq install --no-install-recommends -y gosu \ && groupadd -f envoygroup \ && useradd -g envoygroup -m -d /home/envoybuild envoybuild diff --git a/examples/shared/echo/Dockerfile b/examples/shared/echo/Dockerfile index 81f8c35019fd..16d356efc745 100644 --- a/examples/shared/echo/Dockerfile +++ b/examples/shared/echo/Dockerfile @@ -1 +1 @@ -FROM jmalloc/echo-server@sha256:57110914108448e6692cd28fc602332357f91951d74ca12217a347b1f7df599c +FROM jmalloc/echo-server@sha256:86f2c45aa7e7ebe1be30b21f8cfff25a7ed6e3b059751822d4b35bf244a688d5 diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index 72540485d314..29a66a0aef95 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -1,9 +1,9 @@ -FROM debian:bullseye-slim@sha256:3bc5e94a0e8329c102203c3f5f26fd67835f0c81633dd6949de0557867a87fac as os-base +FROM debian:bookworm-slim@sha256:f80c45482c8d147da87613cb6878a7238b8642bcc24fc11bad78c7bec726f340 as os-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache -FROM golang:1.21.1-bullseye@sha256:436969571fa091f02d34bf2b9bc8850af7de0527e5bc53c39eeda88bc01c38d3 as golang-base +FROM golang:1.21.5-bookworm@sha256:2d3b13c2a6368032e9697e64e7c923184e6e3be03cf01eadff27de124114e64e as golang-base FROM golang-base as golang-control-plane-builder @@ -19,7 +19,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ apt-get -qq update \ - && apt-get -qq install --no-install-recommends -y netcat + && apt-get -qq install --no-install-recommends -y netcat-traditional COPY --from=golang-control-plane-builder /go/go-control-plane/bin/example /usr/local/bin/example @@ -44,7 +44,6 @@ WORKDIR /build # Resolve and build Go dependencies as Docker cache COPY go.mod /build/go.mod COPY go.sum /build/go.sum -COPY kv/go.mod /build/kv/go.mod ENV GO111MODULE=on RUN go mod download COPY service.go /build/main.go diff --git a/examples/shared/jaeger/Dockerfile b/examples/shared/jaeger/Dockerfile index 64639486fdab..f9cf8f06478b 100644 --- a/examples/shared/jaeger/Dockerfile +++ b/examples/shared/jaeger/Dockerfile @@ -1,4 +1,4 @@ -FROM jaegertracing/all-in-one@sha256:8f090c17b7e5886d37395f8fc80726651d8adf6bfbcf5505b07adb0c6f1ffe0f +FROM jaegertracing/all-in-one@sha256:c7f7e9460de8eb06ab43732b41f6c9308f0a11c563459bd3717753232a3fafb3 HEALTHCHECK \ --interval=1s \ --timeout=1s \ diff --git a/examples/shared/node/Dockerfile b/examples/shared/node/Dockerfile index 2710e467b3b7..d8d6e371dab7 100644 --- a/examples/shared/node/Dockerfile +++ b/examples/shared/node/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.6-bullseye-slim@sha256:ee905d8492c443aebe41f4cc525ebabefef757df43556c444be67391cc031cba as node-base +FROM node:21.5-bookworm-slim@sha256:c88704924204ee6d4e9da02275a8fd27355502f7177e4175cd0d3a4375a9c9c8 as node-base FROM node-base as node-http-auth @@ -9,3 +9,27 @@ ENV NODE_APP_PATH "/app/${NODE_APP_NAME}" COPY "$NODE_APP" /app # Dont use exec form to interpolate correctly CMD node $NODE_APP_PATH + + +FROM node-base as yarn +ARG SERVICE_PORT=3000 +ENV DEBIAN_FRONTEND=noninteractive \ + SERVICE_PORT=$SERVICE_PORT +COPY --chmod=755 ./scripts/entrypoint.sh /entrypoint.sh +COPY --chmod=755 ./scripts/build.sh /usr/local/bin/build.sh +COPY --chmod=755 ./scripts/dev.sh /usr/local/bin/dev.sh +COPY ./routes.jq /usr/local/share/routes.jq +RUN apt-get update \ + && apt-get -qq install -y --no-install-recommends gosu jq netcat-traditional yq +ENTRYPOINT ["/entrypoint.sh"] +CMD ["dev.sh"] +HEALTHCHECK \ + --interval=2s \ + --timeout=1s \ + --start-period=1s \ + --retries=30 \ + CMD nc -zv localhost "$SERVICE_PORT" + + +FROM yarn as yarn-routed +COPY --chmod=755 ./scripts/build-routed.sh /usr/local/bin/build.sh diff --git a/examples/shared/node/routes.jq b/examples/shared/node/routes.jq new file mode 100644 index 000000000000..f5a61d71c558 --- /dev/null +++ b/examples/shared/node/routes.jq @@ -0,0 +1,49 @@ +# Generate direct_response routes for a passed in file list +# Expects an xDS config to be passed as the `base` argument to +# inject the routes into. + +def getMimeType: + {".js": "text/javascript", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", + ".css": "text/css", + ".html": "text/html", + ".txt": "text/plain", + ".xml": "application/xml", + ".json": "application/json", + ".pdf": "application/pdf", + ".zip": "application/zip", + ".tar": "application/x-tar", + ".gz": "application/gzip" + }[match("\\.[^.]*$").string // "application/octet-stream"] +; + +def pathToDirectResponse: + . as $path + | sub("^dist/"; "/") as $asset + | $asset + | sub("^/index.html$"; "/") as $path + | if $path == "/" then + {prefix: $path} + else {$path} end + | {match: ., + direct_response: { + status: 200, + body: {filename: "/var/www/html\($asset)"} + }, + response_headers_to_add: [ + {header: { + key: "Content-Type", + value: ($asset | getMimeType)}}]} +; + +split("\n") as $assets +| ($base | fromjson) as $base +| $assets +| map(select(. != "") | pathToDirectResponse) as $routes +| $base +| .resources[0].filter_chains[0].filters[0].typed_config.route_config.virtual_hosts[0].routes = $routes diff --git a/examples/shared/node/scripts/build-routed.sh b/examples/shared/node/scripts/build-routed.sh new file mode 100755 index 000000000000..254f4dcb84b5 --- /dev/null +++ b/examples/shared/node/scripts/build-routed.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +yarn +yarn build + +BASE=$(yq -c < ../xds/lds.tmpl.yml) +ASSETS=$(find dist -type f) +echo "$ASSETS" \ + | jq --arg base "$BASE" -Rs -f /usr/local/share/routes.jq \ + > ../xds/lds.update.yml +mv -f ../xds/lds.update.yml ../xds/lds.yml diff --git a/examples/shared/node/scripts/build.sh b/examples/shared/node/scripts/build.sh new file mode 100644 index 000000000000..4c3c1994bb0b --- /dev/null +++ b/examples/shared/node/scripts/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +yarn +yarn build diff --git a/examples/shared/node/scripts/dev.sh b/examples/shared/node/scripts/dev.sh new file mode 100755 index 000000000000..fdf461a027f5 --- /dev/null +++ b/examples/shared/node/scripts/dev.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +yarn +yarn dev --host 0.0.0.0 --port 3000 diff --git a/examples/shared/node/scripts/entrypoint.sh b/examples/shared/node/scripts/entrypoint.sh new file mode 100755 index 000000000000..bea8c478a328 --- /dev/null +++ b/examples/shared/node/scripts/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +# Add local user +# Either use the LOCAL_USER_ID if passed in at runtime or +# fallback + +DEFAULT_USER_NAME=node + +USER_UID=${LOCAL_UID:-1000} +USER_NAME=${LOCAL_USER_NAME:-worker} +USER_HOME=${LOCAL_USER_HOME:-"/home/${USER_NAME}"} + +echo "Starting (${*}) with user: $USER_UID $USER_NAME $USER_HOME" +usermod -l "$USER_NAME" -md "$USER_HOME" "$DEFAULT_USER_NAME" || : +chown "$USER_NAME" "$USER_HOME" +export HOME="${USER_HOME}" +export PATH=$PATH:"${HOME}/.yarn/bin/" +mkdir -p ./dist +chown -R "$USER_NAME" ./dist +exec /usr/sbin/gosu "${USER_NAME}" "$@" diff --git a/examples/shared/postgres/Dockerfile b/examples/shared/postgres/Dockerfile index 66e023ea11a9..37152b74cf8b 100644 --- a/examples/shared/postgres/Dockerfile +++ b/examples/shared/postgres/Dockerfile @@ -1,3 +1,3 @@ -FROM postgres:latest@sha256:bf0c7de8c8eadc8c86c631999b050e988a21c80530808f011bd864c899763e0f +FROM postgres:latest@sha256:ff37e66b0a03594086c3734d73e750f13480ca9bf64b53fafea18be4d5afb9ad COPY docker-healthcheck.sh /usr/local/bin/ HEALTHCHECK CMD ["docker-healthcheck.sh"] diff --git a/examples/shared/python/Dockerfile b/examples/shared/python/Dockerfile index 3a93baf38ff7..9f8b48a0aeaf 100644 --- a/examples/shared/python/Dockerfile +++ b/examples/shared/python/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.5-slim-bullseye@sha256:de917502e531b3f6e4a5acef017e9feef392cf3eb76826fd46d6810c70ae9b5e as python-base +FROM python:3.11.5-slim-bookworm@sha256:edaf703dce209d774af3ff768fc92b1e3b60261e7602126276f9ceb0e3a96874 as python-base RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache ARG PYTHON_REQUIREMENTS_FILE=aiohttp/requirements.txt @@ -15,20 +15,22 @@ CMD tail -f /dev/null FROM python-base as aiohttp-service -ENV DEBIAN_FRONTEND=noninteractive +ARG SERVICE_PORT=8080 +ENV DEBIAN_FRONTEND=noninteractive \ + SERVICE_PORT=$SERVICE_PORT ADD "$PYTHON_REQUIREMENTS_FILE" /tmp/requirements.txt RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ pip3 install --require-hashes -qr /tmp/requirements.txt \ && apt-get -qq update \ - && apt-get -qq install -y --no-install-recommends netcat \ + && apt-get -qq install -y --no-install-recommends netcat-traditional \ && mkdir /code HEALTHCHECK \ --interval=1s \ --timeout=1s \ --start-period=1s \ --retries=3 \ - CMD nc -zv localhost 8080 + CMD nc -zv localhost "$SERVICE_PORT" ENTRYPOINT ["python3", "/code/service.py"] diff --git a/examples/shared/python/aiohttp/requirements.txt b/examples/shared/python/aiohttp/requirements.txt index f7d632e86ae6..bf302ca82db1 100644 --- a/examples/shared/python/aiohttp/requirements.txt +++ b/examples/shared/python/aiohttp/requirements.txt @@ -4,259 +4,154 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -aiohttp==3.8.5 \ - --hash=sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67 \ - --hash=sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c \ - --hash=sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda \ - --hash=sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755 \ - --hash=sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d \ - --hash=sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5 \ - --hash=sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548 \ - --hash=sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690 \ - --hash=sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84 \ - --hash=sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4 \ - --hash=sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a \ - --hash=sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a \ - --hash=sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9 \ - --hash=sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef \ - --hash=sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b \ - --hash=sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a \ - --hash=sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d \ - --hash=sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945 \ - --hash=sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634 \ - --hash=sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7 \ - --hash=sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691 \ - --hash=sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802 \ - --hash=sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c \ - --hash=sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0 \ - --hash=sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8 \ - --hash=sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82 \ - --hash=sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a \ - --hash=sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975 \ - --hash=sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b \ - --hash=sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d \ - --hash=sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3 \ - --hash=sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7 \ - --hash=sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e \ - --hash=sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5 \ - --hash=sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649 \ - --hash=sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff \ - --hash=sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e \ - --hash=sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c \ - --hash=sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22 \ - --hash=sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df \ - --hash=sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e \ - --hash=sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780 \ - --hash=sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905 \ - --hash=sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51 \ - --hash=sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543 \ - --hash=sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6 \ - --hash=sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873 \ - --hash=sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f \ - --hash=sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35 \ - --hash=sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938 \ - --hash=sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b \ - --hash=sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d \ - --hash=sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8 \ - --hash=sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c \ - --hash=sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af \ - --hash=sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42 \ - --hash=sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3 \ - --hash=sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc \ - --hash=sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8 \ - --hash=sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410 \ - --hash=sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c \ - --hash=sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825 \ - --hash=sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9 \ - --hash=sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53 \ - --hash=sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a \ - --hash=sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc \ - --hash=sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8 \ - --hash=sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c \ - --hash=sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a \ - --hash=sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b \ - --hash=sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd \ - --hash=sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14 \ - --hash=sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2 \ - --hash=sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c \ - --hash=sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9 \ - --hash=sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692 \ - --hash=sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1 \ - --hash=sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa \ - --hash=sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a \ - --hash=sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de \ - --hash=sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91 \ - --hash=sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761 \ - --hash=sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd \ - --hash=sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced \ - --hash=sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28 \ - --hash=sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8 \ - --hash=sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824 +aiohttp==3.9.1 \ + --hash=sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f \ + --hash=sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c \ + --hash=sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af \ + --hash=sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4 \ + --hash=sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a \ + --hash=sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489 \ + --hash=sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213 \ + --hash=sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01 \ + --hash=sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5 \ + --hash=sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361 \ + --hash=sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26 \ + --hash=sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0 \ + --hash=sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4 \ + --hash=sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8 \ + --hash=sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1 \ + --hash=sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7 \ + --hash=sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6 \ + --hash=sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a \ + --hash=sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd \ + --hash=sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4 \ + --hash=sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499 \ + --hash=sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183 \ + --hash=sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544 \ + --hash=sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821 \ + --hash=sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501 \ + --hash=sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f \ + --hash=sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe \ + --hash=sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f \ + --hash=sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672 \ + --hash=sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5 \ + --hash=sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2 \ + --hash=sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57 \ + --hash=sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87 \ + --hash=sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0 \ + --hash=sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f \ + --hash=sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7 \ + --hash=sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed \ + --hash=sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70 \ + --hash=sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0 \ + --hash=sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f \ + --hash=sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d \ + --hash=sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f \ + --hash=sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d \ + --hash=sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431 \ + --hash=sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff \ + --hash=sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf \ + --hash=sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83 \ + --hash=sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690 \ + --hash=sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587 \ + --hash=sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e \ + --hash=sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb \ + --hash=sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3 \ + --hash=sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66 \ + --hash=sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014 \ + --hash=sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35 \ + --hash=sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f \ + --hash=sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0 \ + --hash=sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449 \ + --hash=sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23 \ + --hash=sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5 \ + --hash=sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd \ + --hash=sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4 \ + --hash=sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b \ + --hash=sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558 \ + --hash=sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd \ + --hash=sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766 \ + --hash=sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a \ + --hash=sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636 \ + --hash=sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d \ + --hash=sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590 \ + --hash=sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e \ + --hash=sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d \ + --hash=sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c \ + --hash=sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28 \ + --hash=sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065 \ + --hash=sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca # via -r requirements.in aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via aiohttp -async-timeout==4.0.2 \ - --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ - --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 # via aiohttp -attrs==22.2.0 \ - --hash=sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836 \ - --hash=sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99 - # via aiohttp -charset-normalizer==3.1.0 \ - --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ - --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ - --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ - --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ - --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ - --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ - --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ - --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ - --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ - --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ - --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ - --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ - --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ - --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ - --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ - --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ - --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ - --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ - --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ - --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ - --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ - --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ - --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ - --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ - --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ - --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ - --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ - --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ - --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ - --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ - --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ - --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ - --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ - --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ - --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ - --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ - --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ - --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ - --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ - --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ - --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ - --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ - --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ - --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ - --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ - --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ - --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ - --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ - --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ - --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ - --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ - --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ - --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ - --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ - --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ - --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ - --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ - --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ - --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ - --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ - --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ - --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ - --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ - --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ - --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ - --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ - --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ - --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ - --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ - --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ - --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ - --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ - --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ - --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ - --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab - # via aiohttp -frozenlist==1.3.3 \ - --hash=sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c \ - --hash=sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f \ - --hash=sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a \ - --hash=sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784 \ - --hash=sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27 \ - --hash=sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d \ - --hash=sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3 \ - --hash=sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678 \ - --hash=sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a \ - --hash=sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483 \ - --hash=sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8 \ - --hash=sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf \ - --hash=sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99 \ - --hash=sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c \ - --hash=sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48 \ - --hash=sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5 \ - --hash=sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56 \ - --hash=sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e \ - --hash=sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1 \ - --hash=sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401 \ - --hash=sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4 \ - --hash=sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e \ - --hash=sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649 \ - --hash=sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a \ - --hash=sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d \ - --hash=sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0 \ - --hash=sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6 \ - --hash=sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d \ - --hash=sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b \ - --hash=sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6 \ - --hash=sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf \ - --hash=sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef \ - --hash=sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7 \ - --hash=sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842 \ - --hash=sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba \ - --hash=sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420 \ - --hash=sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b \ - --hash=sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d \ - --hash=sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332 \ - --hash=sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936 \ - --hash=sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816 \ - --hash=sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91 \ - --hash=sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420 \ - --hash=sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448 \ - --hash=sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411 \ - --hash=sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4 \ - --hash=sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32 \ - --hash=sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b \ - --hash=sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0 \ - --hash=sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530 \ - --hash=sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669 \ - --hash=sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7 \ - --hash=sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1 \ - --hash=sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5 \ - --hash=sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce \ - --hash=sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4 \ - --hash=sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e \ - --hash=sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2 \ - --hash=sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d \ - --hash=sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9 \ - --hash=sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642 \ - --hash=sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0 \ - --hash=sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703 \ - --hash=sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb \ - --hash=sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1 \ - --hash=sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13 \ - --hash=sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab \ - --hash=sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38 \ - --hash=sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb \ - --hash=sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb \ - --hash=sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81 \ - --hash=sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8 \ - --hash=sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd \ - --hash=sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4 +frozenlist==1.4.0 \ + --hash=sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6 \ + --hash=sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01 \ + --hash=sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251 \ + --hash=sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9 \ + --hash=sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b \ + --hash=sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87 \ + --hash=sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf \ + --hash=sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f \ + --hash=sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0 \ + --hash=sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2 \ + --hash=sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b \ + --hash=sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc \ + --hash=sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c \ + --hash=sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467 \ + --hash=sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9 \ + --hash=sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1 \ + --hash=sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a \ + --hash=sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79 \ + --hash=sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167 \ + --hash=sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300 \ + --hash=sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf \ + --hash=sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea \ + --hash=sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2 \ + --hash=sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab \ + --hash=sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3 \ + --hash=sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb \ + --hash=sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087 \ + --hash=sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc \ + --hash=sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8 \ + --hash=sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62 \ + --hash=sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f \ + --hash=sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326 \ + --hash=sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c \ + --hash=sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431 \ + --hash=sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963 \ + --hash=sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7 \ + --hash=sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef \ + --hash=sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3 \ + --hash=sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956 \ + --hash=sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781 \ + --hash=sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472 \ + --hash=sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc \ + --hash=sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839 \ + --hash=sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672 \ + --hash=sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3 \ + --hash=sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503 \ + --hash=sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d \ + --hash=sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8 \ + --hash=sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b \ + --hash=sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc \ + --hash=sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f \ + --hash=sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559 \ + --hash=sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b \ + --hash=sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95 \ + --hash=sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb \ + --hash=sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963 \ + --hash=sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919 \ + --hash=sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f \ + --hash=sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3 \ + --hash=sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1 \ + --hash=sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e # via # aiohttp # aiosignal @@ -343,7 +238,9 @@ multidict==6.0.4 \ # aiohttp # yarl pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ @@ -351,7 +248,10 @@ pyyaml==6.0.1 \ --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ @@ -359,9 +259,12 @@ pyyaml==6.0.1 \ --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ @@ -376,7 +279,9 @@ pyyaml==6.0.1 \ --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ @@ -384,79 +289,79 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via -r requirements.in -yarl==1.8.2 \ - --hash=sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87 \ - --hash=sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89 \ - --hash=sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a \ - --hash=sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08 \ - --hash=sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996 \ - --hash=sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077 \ - --hash=sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901 \ - --hash=sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e \ - --hash=sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee \ - --hash=sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574 \ - --hash=sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165 \ - --hash=sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634 \ - --hash=sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229 \ - --hash=sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b \ - --hash=sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f \ - --hash=sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7 \ - --hash=sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf \ - --hash=sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89 \ - --hash=sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0 \ - --hash=sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1 \ - --hash=sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe \ - --hash=sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf \ - --hash=sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76 \ - --hash=sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951 \ - --hash=sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863 \ - --hash=sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06 \ - --hash=sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562 \ - --hash=sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6 \ - --hash=sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c \ - --hash=sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e \ - --hash=sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1 \ - --hash=sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3 \ - --hash=sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3 \ - --hash=sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778 \ - --hash=sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8 \ - --hash=sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2 \ - --hash=sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b \ - --hash=sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d \ - --hash=sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f \ - --hash=sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c \ - --hash=sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581 \ - --hash=sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918 \ - --hash=sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c \ - --hash=sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e \ - --hash=sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220 \ - --hash=sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37 \ - --hash=sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739 \ - --hash=sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77 \ - --hash=sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6 \ - --hash=sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42 \ - --hash=sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946 \ - --hash=sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5 \ - --hash=sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d \ - --hash=sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146 \ - --hash=sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a \ - --hash=sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83 \ - --hash=sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef \ - --hash=sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80 \ - --hash=sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588 \ - --hash=sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5 \ - --hash=sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2 \ - --hash=sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef \ - --hash=sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826 \ - --hash=sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05 \ - --hash=sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516 \ - --hash=sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0 \ - --hash=sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4 \ - --hash=sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2 \ - --hash=sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0 \ - --hash=sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd \ - --hash=sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8 \ - --hash=sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b \ - --hash=sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1 \ - --hash=sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c +yarl==1.9.2 \ + --hash=sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571 \ + --hash=sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3 \ + --hash=sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3 \ + --hash=sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c \ + --hash=sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7 \ + --hash=sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04 \ + --hash=sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191 \ + --hash=sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea \ + --hash=sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4 \ + --hash=sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4 \ + --hash=sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095 \ + --hash=sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e \ + --hash=sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74 \ + --hash=sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef \ + --hash=sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33 \ + --hash=sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde \ + --hash=sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45 \ + --hash=sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf \ + --hash=sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b \ + --hash=sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac \ + --hash=sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0 \ + --hash=sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528 \ + --hash=sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716 \ + --hash=sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb \ + --hash=sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18 \ + --hash=sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72 \ + --hash=sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6 \ + --hash=sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582 \ + --hash=sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5 \ + --hash=sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368 \ + --hash=sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc \ + --hash=sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9 \ + --hash=sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be \ + --hash=sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a \ + --hash=sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80 \ + --hash=sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8 \ + --hash=sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6 \ + --hash=sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417 \ + --hash=sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574 \ + --hash=sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59 \ + --hash=sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608 \ + --hash=sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82 \ + --hash=sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1 \ + --hash=sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3 \ + --hash=sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d \ + --hash=sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8 \ + --hash=sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc \ + --hash=sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac \ + --hash=sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8 \ + --hash=sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955 \ + --hash=sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0 \ + --hash=sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367 \ + --hash=sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb \ + --hash=sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a \ + --hash=sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623 \ + --hash=sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2 \ + --hash=sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6 \ + --hash=sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7 \ + --hash=sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4 \ + --hash=sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051 \ + --hash=sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938 \ + --hash=sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8 \ + --hash=sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9 \ + --hash=sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3 \ + --hash=sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5 \ + --hash=sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9 \ + --hash=sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333 \ + --hash=sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185 \ + --hash=sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3 \ + --hash=sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560 \ + --hash=sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b \ + --hash=sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7 \ + --hash=sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78 \ + --hash=sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7 # via aiohttp diff --git a/examples/shared/python/postgres/requirements.txt b/examples/shared/python/postgres/requirements.txt index 6e0f8988d7aa..272d6720b807 100644 --- a/examples/shared/python/postgres/requirements.txt +++ b/examples/shared/python/postgres/requirements.txt @@ -4,65 +4,74 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -psycopg2-binary==2.9.7 \ - --hash=sha256:00d8db270afb76f48a499f7bb8fa70297e66da67288471ca873db88382850bf4 \ - --hash=sha256:024eaeb2a08c9a65cd5f94b31ace1ee3bb3f978cd4d079406aef85169ba01f08 \ - --hash=sha256:094af2e77a1976efd4956a031028774b827029729725e136514aae3cdf49b87b \ - --hash=sha256:1011eeb0c51e5b9ea1016f0f45fa23aca63966a4c0afcf0340ccabe85a9f65bd \ - --hash=sha256:11abdbfc6f7f7dea4a524b5f4117369b0d757725798f1593796be6ece20266cb \ - --hash=sha256:122641b7fab18ef76b18860dd0c772290566b6fb30cc08e923ad73d17461dc63 \ - --hash=sha256:17cc17a70dfb295a240db7f65b6d8153c3d81efb145d76da1e4a096e9c5c0e63 \ - --hash=sha256:18f12632ab516c47c1ac4841a78fddea6508a8284c7cf0f292cb1a523f2e2379 \ - --hash=sha256:1b918f64a51ffe19cd2e230b3240ba481330ce1d4b7875ae67305bd1d37b041c \ - --hash=sha256:1c31c2606ac500dbd26381145684d87730a2fac9a62ebcfbaa2b119f8d6c19f4 \ - --hash=sha256:26484e913d472ecb6b45937ea55ce29c57c662066d222fb0fbdc1fab457f18c5 \ - --hash=sha256:2993ccb2b7e80844d534e55e0f12534c2871952f78e0da33c35e648bf002bbff \ - --hash=sha256:2b04da24cbde33292ad34a40db9832a80ad12de26486ffeda883413c9e1b1d5e \ - --hash=sha256:2dec5a75a3a5d42b120e88e6ed3e3b37b46459202bb8e36cd67591b6e5feebc1 \ - --hash=sha256:2df562bb2e4e00ee064779902d721223cfa9f8f58e7e52318c97d139cf7f012d \ - --hash=sha256:3fbb1184c7e9d28d67671992970718c05af5f77fc88e26fd7136613c4ece1f89 \ - --hash=sha256:42a62ef0e5abb55bf6ffb050eb2b0fcd767261fa3faf943a4267539168807522 \ - --hash=sha256:4ecc15666f16f97709106d87284c136cdc82647e1c3f8392a672616aed3c7151 \ - --hash=sha256:4eec5d36dbcfc076caab61a2114c12094c0b7027d57e9e4387b634e8ab36fd44 \ - --hash=sha256:4fe13712357d802080cfccbf8c6266a3121dc0e27e2144819029095ccf708372 \ - --hash=sha256:51d1b42d44f4ffb93188f9b39e6d1c82aa758fdb8d9de65e1ddfe7a7d250d7ad \ - --hash=sha256:59f7e9109a59dfa31efa022e94a244736ae401526682de504e87bd11ce870c22 \ - --hash=sha256:62cb6de84d7767164a87ca97e22e5e0a134856ebcb08f21b621c6125baf61f16 \ - --hash=sha256:642df77484b2dcaf87d4237792246d8068653f9e0f5c025e2c692fc56b0dda70 \ - --hash=sha256:6822c9c63308d650db201ba22fe6648bd6786ca6d14fdaf273b17e15608d0852 \ - --hash=sha256:692df8763b71d42eb8343f54091368f6f6c9cfc56dc391858cdb3c3ef1e3e584 \ - --hash=sha256:6d92e139ca388ccfe8c04aacc163756e55ba4c623c6ba13d5d1595ed97523e4b \ - --hash=sha256:7952807f95c8eba6a8ccb14e00bf170bb700cafcec3924d565235dffc7dc4ae8 \ - --hash=sha256:7db7b9b701974c96a88997d458b38ccb110eba8f805d4b4f74944aac48639b42 \ - --hash=sha256:81d5dd2dd9ab78d31a451e357315f201d976c131ca7d43870a0e8063b6b7a1ec \ - --hash=sha256:8a136c8aaf6615653450817a7abe0fc01e4ea720ae41dfb2823eccae4b9062a3 \ - --hash=sha256:8a7968fd20bd550431837656872c19575b687f3f6f98120046228e451e4064df \ - --hash=sha256:8c721ee464e45ecf609ff8c0a555018764974114f671815a0a7152aedb9f3343 \ - --hash=sha256:8f309b77a7c716e6ed9891b9b42953c3ff7d533dc548c1e33fddc73d2f5e21f9 \ - --hash=sha256:8f94cb12150d57ea433e3e02aabd072205648e86f1d5a0a692d60242f7809b15 \ - --hash=sha256:95a7a747bdc3b010bb6a980f053233e7610276d55f3ca506afff4ad7749ab58a \ - --hash=sha256:9b0c2b466b2f4d89ccc33784c4ebb1627989bd84a39b79092e560e937a11d4ac \ - --hash=sha256:9dcfd5d37e027ec393a303cc0a216be564b96c80ba532f3d1e0d2b5e5e4b1e6e \ - --hash=sha256:a5ee89587696d808c9a00876065d725d4ae606f5f7853b961cdbc348b0f7c9a1 \ - --hash=sha256:a6a8b575ac45af1eaccbbcdcf710ab984fd50af048fe130672377f78aaff6fc1 \ - --hash=sha256:ac83ab05e25354dad798401babaa6daa9577462136ba215694865394840e31f8 \ - --hash=sha256:ad26d4eeaa0d722b25814cce97335ecf1b707630258f14ac4d2ed3d1d8415265 \ - --hash=sha256:ad5ec10b53cbb57e9a2e77b67e4e4368df56b54d6b00cc86398578f1c635f329 \ - --hash=sha256:c82986635a16fb1fa15cd5436035c88bc65c3d5ced1cfaac7f357ee9e9deddd4 \ - --hash=sha256:ced63c054bdaf0298f62681d5dcae3afe60cbae332390bfb1acf0e23dcd25fc8 \ - --hash=sha256:d0b16e5bb0ab78583f0ed7ab16378a0f8a89a27256bb5560402749dbe8a164d7 \ - --hash=sha256:dbbc3c5d15ed76b0d9db7753c0db40899136ecfe97d50cbde918f630c5eb857a \ - --hash=sha256:ded8e15f7550db9e75c60b3d9fcbc7737fea258a0f10032cdb7edc26c2a671fd \ - --hash=sha256:e02bc4f2966475a7393bd0f098e1165d470d3fa816264054359ed4f10f6914ea \ - --hash=sha256:e5666632ba2b0d9757b38fc17337d84bdf932d38563c5234f5f8c54fd01349c9 \ - --hash=sha256:ea5f8ee87f1eddc818fc04649d952c526db4426d26bab16efbe5a0c52b27d6ab \ - --hash=sha256:eb1c0e682138f9067a58fc3c9a9bf1c83d8e08cfbee380d858e63196466d5c86 \ - --hash=sha256:eb3b8d55924a6058a26db69fb1d3e7e32695ff8b491835ba9f479537e14dcf9f \ - --hash=sha256:ee919b676da28f78f91b464fb3e12238bd7474483352a59c8a16c39dfc59f0c5 \ - --hash=sha256:f02f4a72cc3ab2565c6d9720f0343cb840fb2dc01a2e9ecb8bc58ccf95dc5c06 \ - --hash=sha256:f4f37bbc6588d402980ffbd1f3338c871368fb4b1cfa091debe13c68bb3852b3 \ - --hash=sha256:f8651cf1f144f9ee0fa7d1a1df61a9184ab72962531ca99f077bbdcba3947c58 \ - --hash=sha256:f955aa50d7d5220fcb6e38f69ea126eafecd812d96aeed5d5f3597f33fad43bb \ - --hash=sha256:fc10da7e7df3380426521e8c1ed975d22df678639da2ed0ec3244c3dc2ab54c8 \ - --hash=sha256:fdca0511458d26cf39b827a663d7d87db6f32b93efc22442a742035728603d5f +psycopg2-binary==2.9.9 \ + --hash=sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9 \ + --hash=sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77 \ + --hash=sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e \ + --hash=sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84 \ + --hash=sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3 \ + --hash=sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2 \ + --hash=sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67 \ + --hash=sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876 \ + --hash=sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152 \ + --hash=sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f \ + --hash=sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a \ + --hash=sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6 \ + --hash=sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503 \ + --hash=sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f \ + --hash=sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493 \ + --hash=sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996 \ + --hash=sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f \ + --hash=sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e \ + --hash=sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59 \ + --hash=sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94 \ + --hash=sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7 \ + --hash=sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682 \ + --hash=sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420 \ + --hash=sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae \ + --hash=sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291 \ + --hash=sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe \ + --hash=sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980 \ + --hash=sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692 \ + --hash=sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119 \ + --hash=sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716 \ + --hash=sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472 \ + --hash=sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b \ + --hash=sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2 \ + --hash=sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc \ + --hash=sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c \ + --hash=sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5 \ + --hash=sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984 \ + --hash=sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9 \ + --hash=sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf \ + --hash=sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0 \ + --hash=sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f \ + --hash=sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212 \ + --hash=sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb \ + --hash=sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be \ + --hash=sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90 \ + --hash=sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041 \ + --hash=sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7 \ + --hash=sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860 \ + --hash=sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245 \ + --hash=sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27 \ + --hash=sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417 \ + --hash=sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359 \ + --hash=sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202 \ + --hash=sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0 \ + --hash=sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7 \ + --hash=sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba \ + --hash=sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1 \ + --hash=sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd \ + --hash=sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07 \ + --hash=sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98 \ + --hash=sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55 \ + --hash=sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d \ + --hash=sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972 \ + --hash=sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f \ + --hash=sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e \ + --hash=sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26 \ + --hash=sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957 \ + --hash=sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53 \ + --hash=sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52 # via -r requirements.in diff --git a/examples/shared/websocket/Dockerfile b/examples/shared/websocket/Dockerfile index 9fd37ad2c579..86511d061002 100644 --- a/examples/shared/websocket/Dockerfile +++ b/examples/shared/websocket/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim@sha256:3bc5e94a0e8329c102203c3f5f26fd67835f0c81633dd6949de0557867a87fac as websocket-base +FROM debian:bookworm-slim@sha256:f80c45482c8d147da87613cb6878a7238b8642bcc24fc11bad78c7bec726f340 as websocket-base ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ diff --git a/examples/skywalking/Dockerfile-elasticsearch b/examples/skywalking/Dockerfile-elasticsearch index 892c716d6115..d80cf424178f 100644 --- a/examples/skywalking/Dockerfile-elasticsearch +++ b/examples/skywalking/Dockerfile-elasticsearch @@ -1 +1 @@ -FROM elasticsearch:8.9.2@sha256:38082e945768e8e98fc0d828c810baf0b63d58ddc9059db2a62e4c293e44e817 +FROM elasticsearch:8.11.1@sha256:e9b31098201dc40200d7c445f328845d77e9cbb4fe0b38219d989462a9a4280a diff --git a/examples/skywalking/Dockerfile-skywalking-oap b/examples/skywalking/Dockerfile-skywalking-oap index 838dec147019..74f1a6bbd3d4 100644 --- a/examples/skywalking/Dockerfile-skywalking-oap +++ b/examples/skywalking/Dockerfile-skywalking-oap @@ -1 +1 @@ -FROM apache/skywalking-oap-server:latest@sha256:bb29cafa89438686eb888f2bc3ba92d38e1bebb64957ff7c53e631bc550ac739 +FROM apache/skywalking-oap-server:latest@sha256:9af311a030f2a106dceecffddef72038e8a7335b92e2c2bd3a30be5942d663ea diff --git a/examples/skywalking/Dockerfile-skywalking-ui b/examples/skywalking/Dockerfile-skywalking-ui index 193eae7aa99c..d15faf95b70e 100644 --- a/examples/skywalking/Dockerfile-skywalking-ui +++ b/examples/skywalking/Dockerfile-skywalking-ui @@ -1 +1 @@ -FROM apache/skywalking-ui:latest@sha256:f99c764048546462195b323a0c6c89bf8208ebbca1a764d7ed0d2b596c303463 +FROM apache/skywalking-ui:latest@sha256:3c7f7225d0512fcb8ea96a0e8820e1752b99ee3a651b33fd4a69ead629de3b1d diff --git a/examples/skywalking/verify.sh b/examples/skywalking/verify.sh index 9eca5f37e7ae..b115b7ca2d82 100755 --- a/examples/skywalking/verify.sh +++ b/examples/skywalking/verify.sh @@ -1,8 +1,8 @@ #!/bin/bash -e export NAME=skywalking -export PORT_PROXY="${SKYWALKING_PORT_PROXY:-12600}" -export PORT_UI="${SKYWALKING_PORT_UI:-12601}" +export PORT_PROXY="${SKYWALKING_PORT_PROXY:-11910}" +export PORT_UI="${SKYWALKING_PORT_UI:-11911}" # NB: This allows ES to run in a low-resource environment, # dont do this in a production environment. diff --git a/examples/verify-common.sh b/examples/verify-common.sh index 540add9fe21d..c6c0692b944c 100644 --- a/examples/verify-common.sh +++ b/examples/verify-common.sh @@ -119,6 +119,10 @@ cleanup () { bring_down_example + if type -t finally &> /dev/null; then + finally + fi + if [[ "$code" -ne 0 ]]; then run_log Failed else @@ -142,14 +146,23 @@ _curl () { } } +move_if_exists () { + if [ -e "$1" ]; then + mv "$1" "$2" + else + echo "Warning: $1 does not exist. Skipping move operation." + fi +} + responds_with () { local expected response expected="$1" shift response=$(_curl "${@}") - grep -s "$expected" <<< "$response" || { + grep -Fs "$expected" <<< "$response" || { echo "ERROR: curl (${*})" >&2 - echo "EXPECTED: $expected" >&2 + echo "EXPECTED:" >&2 + echo "$expected" >&2 echo "RECEIVED:" >&2 echo "$response" >&2 return 1 @@ -178,7 +191,8 @@ responds_with_header () { response=$(_curl --head "${@}") grep -s "$expected" <<< "$response" || { echo "ERROR: curl (${*})" >&2 - echo "EXPECTED HEADER: $expected" >&2 + echo "EXPECTED HEADER:" >&2 + echo "$expected" >&2 echo "RECEIVED:" >&2 echo "$response" >&2 return 1 diff --git a/examples/websocket/verify.sh b/examples/websocket/verify.sh index dd76fef1fcd7..e241d19e8224 100755 --- a/examples/websocket/verify.sh +++ b/examples/websocket/verify.sh @@ -21,6 +21,8 @@ mkdir -p certs openssl req -batch -new -x509 -nodes -keyout certs/key.pem -out certs/cert.pem openssl pkcs12 -export -passout pass: -out certs/output.pkcs12 -inkey certs/key.pem -in certs/cert.pem +UPARGS="proxy-ws proxy-wss-wss proxy-wss-passthrough service-ws service-wss" + bring_up_example run_log "Interact with web socket ws -> ws" diff --git a/examples/zipkin/Dockerfile-zipkin b/examples/zipkin/Dockerfile-zipkin index f9c1f8485c77..ae7c849d0d59 100644 --- a/examples/zipkin/Dockerfile-zipkin +++ b/examples/zipkin/Dockerfile-zipkin @@ -1 +1 @@ -FROM openzipkin/zipkin:latest@sha256:5fd55e6a109233b36d419d7fd2449588d17a6e4da7ed7a3fd0d09c86f1c75a15 +FROM openzipkin/zipkin:latest@sha256:197a9692f6a9416488c34741190a1cbc0d52733ed0c24c0303de065215a0a588 diff --git a/go.mod b/go.mod index 61b62bbe8bfe..f9c5998977de 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/envoyproxy/envoy -go 1.18 +go 1.20 -require google.golang.org/protobuf v1.31.0 +require google.golang.org/protobuf v1.32.0 diff --git a/go.sum b/go.sum index 9ea5597b8346..b010a4f69c81 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,4 @@ -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/mobile/.bazelrc b/mobile/.bazelrc index 582cc555d838..26769b110898 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -117,6 +117,18 @@ build:mobile-remote-release-clang --remote_download_toplevel build:mobile-remote-release-clang --config=ci build:mobile-remote-release-clang --config=remote +build:mobile-remote-release-clang-android --config=mobile-remote-release-clang +build:mobile-remote-release-clang-android --fat_apk_cpu=x86_64 +build:mobile-remote-release-clang-android --linkopt=-fuse-ld=lld + +build:mobile-remote-release-clang-android-publish --config=mobile-remote-release-clang +build:mobile-remote-release-clang-android-publish --config=mobile-release-android +build:mobile-remote-release-clang-android-publish --fat_apk_cpu=x86,x86_64,armeabi-v7a,arm64-v8a +build:mobile-remote-release-clang-android-publish --linkopt=-fuse-ld=lld + +build:mobile-remote-release-clang-android-publish-xds --config=mobile-remote-release-clang-android-publish +build:mobile-remote-release-clang-android-publish-xds --define=envoy_mobile_xds=enabled + # Compile releases optimizing for size (eg -Os, etc). build:mobile-release-common --config=sizeopt @@ -126,6 +138,9 @@ build:mobile-release-common --copt=-fvisibility=hidden # Disable google_grpc in production by default build:mobile-release-common --define=google_grpc=disabled +# Disable envoy_mobile_xds in production by default +build:mobile-release-common --define=envoy_mobile_xds=disabled + # Enable automatic extension factory registration for release builds build:mobile-release-common --define=static_extension_registration=enabled @@ -183,6 +198,7 @@ build:mobile-remote-ci-linux-clang --config=mobile-remote-ci-linux build:mobile-remote-ci-linux-asan --config=clang-asan build:mobile-remote-ci-linux-asan --config=mobile-remote-ci-linux-clang build:mobile-remote-ci-linux-asan --config=remote-ci +test:mobile-remote-ci-linux-asan --test_env=ENVOY_IP_TEST_VERSIONS=v4only ############################################################################# # mobile-remote-ci-linux-tsan: These options are Linux-only using Clang and ThreadSanitizer @@ -190,6 +206,7 @@ build:mobile-remote-ci-linux-asan --config=remote-ci build:mobile-remote-ci-linux-tsan --config=clang-tsan build:mobile-remote-ci-linux-tsan --config=mobile-remote-ci-linux-clang build:mobile-remote-ci-linux-tsan --config=remote-ci +test:mobile-remote-ci-linux-tsan --test_env=ENVOY_IP_TEST_VERSIONS=v4only ############################################################################# # ci-linux-coverage: These options are Linux-only using Clang and LLVM coverage @@ -232,3 +249,78 @@ build:mobile-remote-ci-macos --config=remote build:mobile-remote-ci --config=mobile-remote-ci-linux-clang build:mobile-remote-ci --config=remote-ci + +build:mobile-remote-ci-android --config=mobile-remote-ci +test:mobile-remote-ci-android --build_tests_only +test:mobile-remote-ci-android --config=test-android +test:mobile-remote-ci-android --config=mobile-remote-ci +test:mobile-remote-ci-android --define=signal_trace=disabled + +build:mobile-remote-ci-cc --config=mobile-remote-ci +test:mobile-remote-ci-cc --action_env=LD_LIBRARY_PATH +test:mobile-remote-ci-cc --copt=-DUSE_API_LISTENER + +build:mobile-remote-ci-cc-no-yaml --config=mobile-remote-ci +build:mobile-remote-ci-cc-no-yaml --define=envoy_yaml=disabled +build:mobile-remote-ci-cc-no-yaml --define=envoy_full_protos=disabled +build:mobile-remote-ci-cc-no-yaml --test_env=ENVOY_IP_TEST_VERSIONS=v4only + +build:mobile-remote-ci-cc-no-exceptions --config=mobile-remote-ci +build:mobile-remote-ci-cc-no-exceptions --define=envoy_yaml=disabled +build:mobile-remote-ci-cc-no-exceptions --define envoy_exceptions=disabled +build:mobile-remote-ci-cc-no-exceptions --copt=-fno-unwind-tables +build:mobile-remote-ci-cc-no-exceptions --copt=-fno-exceptions +build:mobile-remote-ci-cc-no-exceptions --define=google_grpc=disabled +build:mobile-remote-ci-cc-no-exceptions --define=envoy_mobile_xds=disabled + + +build:mobile-remote-ci-cc-test --config=mobile-remote-ci +test:mobile-remote-ci-cc-test --test_output=all +test:mobile-remote-ci-cc-test --config=mobile-remote-ci +test:mobile-remote-ci-cc-test --define=signal_trace=disabled +test:mobile-remote-ci-cc-test --define=google_grpc=disabled +test:mobile-remote-ci-cc-test --define=envoy_mobile_xds=disabled +test:mobile-remote-ci-cc-test --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= +test:mobile-remote-ci-cc-test --define=envoy_yaml=disabled + +build:mobile-remote-ci-macos-kotlin --config=mobile-remote-ci-macos +build:mobile-remote-ci-macos-kotlin --fat_apk_cpu=x86_64 +build:mobile-remote-ci-macos-kotlin --define=signal_trace=disabled +build:mobile-remote-ci-macos-kotlin --define=envoy_mobile_request_compression=disabled +build:mobile-remote-ci-macos-kotlin --define=envoy_enable_http_datagrams=disabled +build:mobile-remote-ci-macos-kotlin --define=google_grpc=disabled +build:mobile-remote-ci-macos-kotlin --define=envoy_mobile_xds=disabled +build:mobile-remote-ci-macos-kotlin --define=envoy_yaml=disabled +build:mobile-remote-ci-macos-kotlin --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= + +build:mobile-remote-ci-macos-swift --config=mobile-remote-ci-macos +build:mobile-remote-ci-macos-swift --config=ios +build:mobile-remote-ci-macos-swift --config=mobile-remote-ci-macos +build:mobile-remote-ci-macos-swift --define=signal_trace=disabled +build:mobile-remote-ci-macos-swift --define=envoy_mobile_request_compression=disabled +build:mobile-remote-ci-macos-swift --define=envoy_mobile_stats_reporting=disabled +build:mobile-remote-ci-macos-swift --define=envoy_mobile_swift_cxx_interop=disabled +build:mobile-remote-ci-macos-swift --define=google_grpc=disabled +build:mobile-remote-ci-macos-swift --define=envoy_mobile_xds=disabled +build:mobile-remote-ci-macos-swift --@envoy//bazel:http3=False +build:mobile-remote-ci-macos-swift --@com_envoyproxy_protoc_gen_validate//bazel:template-flavor= + +build:mobile-remote-ci-core --config=mobile-remote-ci +test:mobile-remote-ci-core --build_tests_only +test:mobile-remote-ci-core --action_env=LD_LIBRARY_PATH +test:mobile-remote-ci-core --test_env=ENVOY_IP_TEST_VERSIONS=v4only +test:mobile-remote-ci-core --define=envoy_yaml=disabled + +build:mobile-remote-ci-macos-ios --config=mobile-remote-ci-macos +build:mobile-remote-ci-macos-ios --config=ios + +build:mobile-remote-ci-macos-ios-admin --config=mobile-remote-ci-macos-ios +build:mobile-remote-ci-macos-ios-admin --define=admin_functionality=enabled + + +build:mobile-remote-ci-macos-ios-swift --config=mobile-remote-ci-macos-ios +test:mobile-remote-ci-macos-ios-swift --experimental_ui_max_stdouterr_bytes=10485760 +test:mobile-remote-ci-macos-ios-swift --build_tests_only + +build:mobile-remote-ci-macos-ios-obj-c --config=mobile-remote-ci-macos-ios +test:mobile-remote-ci-macos-ios-obj-c --build_tests_only diff --git a/mobile/BUILD b/mobile/BUILD index e9f3f96752e7..bc5ee81a8a36 100644 --- a/mobile/BUILD +++ b/mobile/BUILD @@ -7,11 +7,16 @@ load( "xcode_schemes", "xcodeproj", ) +load("@envoy//tools/python:namespace.bzl", "envoy_py_namespace") load("@io_bazel_rules_kotlin//kotlin/internal:toolchains.bzl", "define_kt_toolchain") load("//bazel:framework_imports_extractor.bzl", "framework_imports_extractor") licenses(["notice"]) # Apache 2 +envoy_py_namespace() + +exports_files(["VERSION"]) + alias( name = "ios_xcframework", actual = "//library/swift:Envoy", @@ -60,6 +65,11 @@ alias( actual = "//library/kotlin/io/envoyproxy/envoymobile:envoy_aar_with_artifacts", ) +alias( + name = "android_xds_dist", + actual = "//library/kotlin/io/envoyproxy/envoymobile:envoy_xds_aar_with_artifacts", +) + define_kt_toolchain( name = "kotlin_toolchain", jvm_target = "1.8", diff --git a/mobile/Gemfile b/mobile/Gemfile deleted file mode 100644 index d1bf7c6fbeb0..000000000000 --- a/mobile/Gemfile +++ /dev/null @@ -1,3 +0,0 @@ -source "https://rubygems.org" - -gem "cocoapods" diff --git a/mobile/Gemfile.lock b/mobile/Gemfile.lock deleted file mode 100644 index 31a996d9878e..000000000000 --- a/mobile/Gemfile.lock +++ /dev/null @@ -1,99 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - CFPropertyList (3.0.5) - rexml - activesupport (6.1.7.3) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 1.6, < 2) - minitest (>= 5.1) - tzinfo (~> 2.0) - zeitwerk (~> 2.3) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) - algoliasearch (1.27.5) - httpclient (~> 2.8, >= 2.8.3) - json (>= 1.5.1) - atomos (0.1.3) - claide (1.1.0) - cocoapods (1.11.3) - addressable (~> 2.8) - claide (>= 1.0.2, < 2.0) - cocoapods-core (= 1.11.3) - cocoapods-deintegrate (>= 1.0.3, < 2.0) - cocoapods-downloader (>= 1.4.0, < 2.0) - cocoapods-plugins (>= 1.0.0, < 2.0) - cocoapods-search (>= 1.0.0, < 2.0) - cocoapods-trunk (>= 1.4.0, < 2.0) - cocoapods-try (>= 1.1.0, < 2.0) - colored2 (~> 3.1) - escape (~> 0.0.4) - fourflusher (>= 2.3.0, < 3.0) - gh_inspector (~> 1.0) - molinillo (~> 0.8.0) - nap (~> 1.0) - ruby-macho (>= 1.0, < 3.0) - xcodeproj (>= 1.21.0, < 2.0) - cocoapods-core (1.11.3) - activesupport (>= 5.0, < 7) - addressable (~> 2.8) - algoliasearch (~> 1.0) - concurrent-ruby (~> 1.1) - fuzzy_match (~> 2.0.4) - nap (~> 1.0) - netrc (~> 0.11) - public_suffix (~> 4.0) - typhoeus (~> 1.0) - cocoapods-deintegrate (1.0.5) - cocoapods-downloader (1.6.3) - cocoapods-plugins (1.0.0) - nap - cocoapods-search (1.0.1) - cocoapods-trunk (1.6.0) - nap (>= 0.8, < 2.0) - netrc (~> 0.11) - cocoapods-try (1.2.0) - colored2 (3.1.2) - concurrent-ruby (1.2.2) - escape (0.0.4) - ethon (0.15.0) - ffi (>= 1.15.0) - ffi (1.15.5) - fourflusher (2.3.1) - fuzzy_match (2.0.4) - gh_inspector (1.1.3) - httpclient (2.8.3) - i18n (1.12.0) - concurrent-ruby (~> 1.0) - json (2.6.1) - minitest (5.18.0) - molinillo (0.8.0) - nanaimo (0.3.0) - nap (1.1.0) - netrc (0.11.0) - public_suffix (4.0.7) - rexml (3.2.5) - ruby-macho (2.5.1) - typhoeus (1.4.0) - ethon (>= 0.9.0) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - xcodeproj (1.21.0) - CFPropertyList (>= 2.3.3, < 4.0) - atomos (~> 0.1.3) - claide (>= 1.0.2, < 2.0) - colored2 (~> 3.1) - nanaimo (~> 0.3.0) - rexml (~> 3.2.4) - zeitwerk (2.6.7) - -PLATFORMS - arm64-darwin-21 - x86_64-darwin-19 - x86_64-linux - -DEPENDENCIES - cocoapods - -BUNDLED WITH - 2.3.8 diff --git a/mobile/bazel/BUILD b/mobile/bazel/BUILD index aaf6ed2759b3..f5f28ce57895 100644 --- a/mobile/bazel/BUILD +++ b/mobile/bazel/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() config_setting( name = "envoy_mobile_swift_cxx_interop", diff --git a/mobile/bazel/android_artifacts.bzl b/mobile/bazel/android_artifacts.bzl index f039190800f4..69ccb56213cf 100644 --- a/mobile/bazel/android_artifacts.bzl +++ b/mobile/bazel/android_artifacts.bzl @@ -1,7 +1,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary") load("@envoy_mobile//bazel:dokka.bzl", "sources_javadocs") -load("@rules_java//java:defs.bzl", "java_binary") load("@google_bazel_common//tools/maven:pom_file.bzl", "pom_file") +load("@rules_java//java:defs.bzl", "java_binary") # This file is based on https://github.com/aj-michael/aar_with_jni which is # subject to the following copyright and license: @@ -27,7 +27,7 @@ load("@google_bazel_common//tools/maven:pom_file.bzl", "pom_file") # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -def android_artifacts(name, android_library, manifest, archive_name, native_deps = [], proguard_rules = "", visibility = []): +def android_artifacts(name, android_library, manifest, archive_name, native_deps = [], proguard_rules = "", visibility = [], substitutions = {}): """ NOTE: The bazel android_library's implicit aar output doesn't flatten its transitive dependencies. Additionally, when using the kotlin rules, the kt_android_library rule @@ -62,7 +62,7 @@ def android_artifacts(name, android_library, manifest, archive_name, native_deps # Generate other needed files for a maven publish _sources_name, _javadocs_name = _create_sources_javadocs(name, android_library) - _pom_name = _create_pom_xml(name, android_library, visibility) + _pom_name = _create_pom_xml(name, android_library, visibility, substitutions) native.genrule( name = name + "_with_artifacts", srcs = [ @@ -277,7 +277,7 @@ def _create_sources_javadocs(name, android_library): return _sources_name, _javadocs_name -def _create_pom_xml(name, android_library, visibility): +def _create_pom_xml(name, android_library, visibility, substitutions): """ Creates a pom xml associated with the android_library target. @@ -291,6 +291,7 @@ def _create_pom_xml(name, android_library, visibility): name = _pom_name, targets = [android_library], visibility = visibility, + substitutions = substitutions, template_file = "@envoy_mobile//bazel:pom_template.xml", ) diff --git a/mobile/bazel/envoy_mobile_dependencies.bzl b/mobile/bazel/envoy_mobile_dependencies.bzl index 031e9b3cfaaf..0fd01684171b 100644 --- a/mobile/bazel/envoy_mobile_dependencies.bzl +++ b/mobile/bazel/envoy_mobile_dependencies.bzl @@ -1,13 +1,13 @@ -load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies") -load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") -load("@rules_jvm_external//:defs.bzl", "maven_install") -load("@rules_detekt//detekt:dependencies.bzl", "rules_detekt_dependencies") +load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") +load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies") load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") -load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("@robolectric//bazel:robolectric.bzl", "robolectric_repositories") +load("@rules_detekt//detekt:dependencies.bzl", "rules_detekt_dependencies") load("@rules_java//java:repositories.bzl", "rules_java_dependencies") +load("@rules_jvm_external//:defs.bzl", "maven_install") +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") def _default_extra_swift_sources_impl(ctx): ctx.file("WORKSPACE", "") @@ -60,6 +60,8 @@ def kotlin_dependencies(extra_maven_dependencies = []): maven_install( artifacts = [ "com.google.code.findbugs:jsr305:3.0.2", + # Java Proto Lite + "com.google.protobuf:protobuf-javalite:3.24.4", # Kotlin "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.21", "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.21", diff --git a/mobile/bazel/envoy_mobile_repositories.bzl b/mobile/bazel/envoy_mobile_repositories.bzl index fdcb05785be4..1468a30fab2f 100644 --- a/mobile/bazel/envoy_mobile_repositories.bzl +++ b/mobile/bazel/envoy_mobile_repositories.bzl @@ -1,6 +1,4 @@ -load("@bazel_gazelle//:deps.bzl", "go_repository") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") def envoy_mobile_repositories(): http_archive( diff --git a/mobile/bazel/envoy_mobile_toolchains.bzl b/mobile/bazel/envoy_mobile_toolchains.bzl index 231b8c035056..0d5fa867e3a7 100644 --- a/mobile/bazel/envoy_mobile_toolchains.bzl +++ b/mobile/bazel/envoy_mobile_toolchains.bzl @@ -1,7 +1,7 @@ load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") load("@rules_detekt//detekt:toolchains.bzl", "rules_detekt_toolchains") -load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains") load("@rules_java//java:repositories.bzl", "rules_java_toolchains") +load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains") def envoy_mobile_toolchains(): rules_java_toolchains() diff --git a/mobile/bazel/kotlin_lib.bzl b/mobile/bazel/kotlin_lib.bzl index f31f8934c362..c0ee0f198343 100644 --- a/mobile/bazel/kotlin_lib.bzl +++ b/mobile/bazel/kotlin_lib.bzl @@ -1,5 +1,5 @@ -load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") # This is the magic function which helps get the name of the native library # from the native dependency. In general, the bazel cc_binary rules will diff --git a/mobile/bazel/kotlin_test.bzl b/mobile/bazel/kotlin_test.bzl index 9fcefa7dde03..2eea069b7275 100644 --- a/mobile/bazel/kotlin_test.bzl +++ b/mobile/bazel/kotlin_test.bzl @@ -36,24 +36,24 @@ def jvm_flags(lib_name): return [ "-Djava.library.path=library/common/jni:test/common/jni", "-Denvoy_jni_library_name={}".format(lib_name), - # TODO(RyanTheOptimist): Uncomment this when when - # https://github.com/envoyproxy/envoy/issues/28981 is fixed. - # "-Xcheck:jni", + "-Xcheck:jni", ] + select({ "@envoy//bazel:disable_google_grpc": ["-Denvoy_jni_google_grpc_disabled=true"], "//conditions:default": [], + }) + select({ + "@envoy//bazel:disable_envoy_mobile_xds": ["-Denvoy_jni_envoy_mobile_xds_disabled=true"], + "//conditions:default": [], }) # A basic macro to make it easier to declare and run kotlin tests which depend on a JNI lib # This will create the native .so binary (for linux) and a .jnilib (for macOS) look up -def envoy_mobile_jni_kt_test(name, srcs, native_deps = [], deps = [], repository = "", exec_properties = {}): - lib_name = native_lib_name(native_deps[0])[3:] +def envoy_mobile_jni_kt_test(name, srcs, native_lib_name, native_deps = [], deps = [], repository = "", exec_properties = {}): _internal_kt_test( name, srcs, deps, data = native_deps, - jvm_flags = jvm_flags(lib_name), + jvm_flags = jvm_flags(native_lib_name), repository = repository, exec_properties = exec_properties, ) @@ -78,8 +78,7 @@ def envoy_mobile_kt_test(name, srcs, deps = [], repository = "", exec_properties _internal_kt_test(name, srcs, deps, repository = repository, exec_properties = exec_properties) # A basic macro to run android based (robolectric) tests with native dependencies -def envoy_mobile_android_test(name, srcs, deps = [], native_deps = [], repository = "", exec_properties = {}): - lib_name = native_lib_name(native_deps[0])[3:] +def envoy_mobile_android_test(name, srcs, native_lib_name, deps = [], native_deps = [], repository = "", exec_properties = {}): android_library( name = name + "_test_lib", custom_package = "io.envoyproxy.envoymobile.test", @@ -116,6 +115,6 @@ def envoy_mobile_android_test(name, srcs, deps = [], native_deps = [], repositor manifest = repository + "//bazel:test_manifest.xml", custom_package = "io.envoyproxy.envoymobile.tests", test_class = "io.envoyproxy.envoymobile.bazel.EnvoyMobileTestSuite", - jvm_flags = jvm_flags(lib_name), + jvm_flags = jvm_flags(native_lib_name), exec_properties = exec_properties, ) diff --git a/mobile/bazel/pom_template.xml b/mobile/bazel/pom_template.xml index 9d8f17e80a7e..a480e794e1c0 100644 --- a/mobile/bazel/pom_template.xml +++ b/mobile/bazel/pom_template.xml @@ -5,11 +5,12 @@ 4.0.0 io.envoyproxy.envoymobile - envoy + {pom_artifact_id} {pom_version} aar {generated_bzl_deps} + {pom_extra_dependencies} Envoy Mobile diff --git a/mobile/ci/linux_ci_setup.sh b/mobile/ci/linux_ci_setup.sh new file mode 100755 index 000000000000..c74829272178 --- /dev/null +++ b/mobile/ci/linux_ci_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +# Set up necessary Android SDK and NDK. +ANDROID_HOME=$ANDROID_SDK_ROOT +SDKMANAGER="${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager" +$SDKMANAGER --uninstall "ndk-bundle" +echo "y" | $SDKMANAGER "ndk;21.4.7075529" +$SDKMANAGER --install "build-tools;30.0.3" +echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/21.4.7075529" >> "$GITHUB_ENV" diff --git a/mobile/ci/mac_ci_setup.sh b/mobile/ci/mac_ci_setup.sh index 41c5e3c26873..4a6016372016 100755 --- a/mobile/ci/mac_ci_setup.sh +++ b/mobile/ci/mac_ci_setup.sh @@ -34,7 +34,7 @@ function is_installed { function install { echo "Installing $1" - if ! retry brew install "$1"; then + if ! retry brew install --quiet "$1"; then echo "Failed to install $1" exit 1 fi @@ -67,5 +67,6 @@ if [[ "${1:-}" == "--android" ]]; then # Download and set up build-tools 30.0.3, 31.0.0 is missing dx.jar. $SDKMANAGER --install "build-tools;30.0.3" - echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/21.4.7075529" >> "$GITHUB_ENV" + ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/21.4.7075529" + export ANDROID_NDK_HOME fi diff --git a/mobile/ci/sonatype_nexus_upload.py b/mobile/ci/sonatype_nexus_upload.py index bd04883de05c..c62644ad3b03 100755 --- a/mobile/ci/sonatype_nexus_upload.py +++ b/mobile/ci/sonatype_nexus_upload.py @@ -21,7 +21,6 @@ _ARTIFACT_HOST_URL = "https://oss.sonatype.org/service/local/staging" _GROUP_ID = "io.envoyproxy.envoymobile" -_ARTIFACT_ID = "envoy" _LOCAL_INSTALL_PATH = os.path.expanduser( "~/.m2/repository/{directory}/envoy".format(directory=_GROUP_ID.replace(".", "/"))) @@ -52,7 +51,7 @@ def _resolve_name(file): return "", extension -def _install_locally(version, files): +def _install_locally(artifact_id, version, files): path = "{}/{}".format(_LOCAL_INSTALL_PATH, version) if os.path.exists(path): @@ -63,7 +62,7 @@ def _install_locally(version, files): for file in files: suffix, file_extension = _resolve_name(file) basename = "{name}-{version}{suffix}.{extension}".format( - name=_ARTIFACT_ID, version=version, suffix=suffix, extension=file_extension) + name=artifact_id, version=version, suffix=suffix, extension=file_extension) shutil.copyfile(file, os.path.join(path, basename)) print("{file_name}\n{sha}\n".format(file_name=file, sha=_sha256(file))) @@ -116,7 +115,7 @@ def _create_staging_repository(profile_id): raise e -def _upload_files(staging_id, version, files, ascs, sha256): +def _upload_files(staging_id, artifact_id, version, files, ascs, sha256): uploaded_file_count = 0 # aggregate all the files for uploading @@ -126,11 +125,11 @@ def _upload_files(staging_id, version, files, ascs, sha256): print("Uploading file {}".format(file)) suffix, file_extension = _resolve_name(file) basename = "{name}-{version}{suffix}.{extension}".format( - name=_ARTIFACT_ID, version=version, suffix=suffix, extension=file_extension) + name=artifact_id, version=version, suffix=suffix, extension=file_extension) artifact_url = os.path.join( _ARTIFACT_HOST_URL, "deployByRepositoryId/{}".format(staging_id), - _GROUP_ID.replace('.', "/"), _ARTIFACT_ID, version, basename) + _GROUP_ID.replace('.', "/"), artifact_id, version, basename) try: with open(file, "rb") as f: @@ -233,6 +232,12 @@ def _build_parser(): curl -u {usr}:{psswrd} -H "Accept: application/json" https://oss.sonatype.org//nexus/service/local/staging/profile_repositories """) + parser.add_argument( + "--artifact_id", + required=True, + help=""" + The artifact ID to be published. + """) parser.add_argument( "--version", default="LOCAL-SNAPSHOT", @@ -285,7 +290,7 @@ def _build_parser(): version = args.version if args.local: - _install_locally(version, args.files) + _install_locally(args.artifact_id, version, args.files) else: staging_id = "" @@ -301,7 +306,7 @@ def _build_parser(): print("Uploading files...") sha256_files = _create_sha256_files(args.files) uploaded_file_count = _upload_files( - staging_id, version, args.files, args.signed_files, sha256_files) + staging_id, args.artifact_id, version, args.files, args.signed_files, sha256_files) if uploaded_file_count > 0: print("Uploading files complete!") print("Closing staging repository...") diff --git a/mobile/ci/start_android_emulator.sh b/mobile/ci/start_android_emulator.sh index ea9f633e13ea..8b582c343304 100755 --- a/mobile/ci/start_android_emulator.sh +++ b/mobile/ci/start_android_emulator.sh @@ -2,11 +2,33 @@ set -e -echo "y" | "${ANDROID_HOME}/tools/bin/sdkmanager" --install 'system-images;android-29;google_apis;x86_64' --channel=3 -echo "no" | "${ANDROID_HOME}/tools/bin/avdmanager" create avd -n test_android_emulator -k 'system-images;android-29;google_apis;x86_64' --force -ls "${ANDROID_HOME}/tools/bin/" +check_emulator_status() { + while true; do + if grep -q "Running on a system with less than 6 logical cores. Setting number of virtual cores to 1" nohup.out; then + echo "==================================================================================" + echo "ERROR: Starting an emulator on this machine is likely to fail, please run /retest" + echo "==================================================================================" + exit 1 + elif grep -q "Boot completed" nohup.out; then + break + fi + sleep 1 + done +} + +echo "y" | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --install 'system-images;android-30;google_apis;x86_64' --channel=3 +echo "no" | "${ANDROID_HOME}/cmdline-tools/latest/bin/avdmanager" create avd -n test_android_emulator -k 'system-images;android-30;google_apis;x86_64' --device pixel_4 --force +"${ANDROID_HOME}"/emulator/emulator -accel-check +# This is only available on macOS. +if [[ -n $(which system_profiler) ]]; then + system_profiler SPHardwareDataType +fi -nohup "${ANDROID_HOME}/emulator/emulator" -partition-size 1024 -avd test_android_emulator -no-snapshot > /dev/null 2>&1 & { +# shellcheck disable=SC2094 +nohup "${ANDROID_HOME}/emulator/emulator" -no-window -accel on -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -avd test_android_emulator > nohup.out 2>&1 | tail -f nohup.out & { + if [[ "$(uname -s)" == "Darwin" ]]; then + check_emulator_status + fi # shellcheck disable=SC2016 "${ANDROID_HOME}/platform-tools/adb" wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\''\r'\'') ]]; do sleep 1; done; input keyevent 82' } diff --git a/mobile/ci/test_size_regression.sh b/mobile/ci/test_size_regression.sh index d5b03618a8a2..c80afb73c10a 100755 --- a/mobile/ci/test_size_regression.sh +++ b/mobile/ci/test_size_regression.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/bin/bash -e + +set -o pipefail # Checks the absolute size and the relative size increase of a file. diff --git a/mobile/docs/BUILD b/mobile/docs/BUILD index fcddec507579..53b43f10847a 100644 --- a/mobile/docs/BUILD +++ b/mobile/docs/BUILD @@ -1,15 +1,22 @@ load("@base_pip3//:requirements.bzl", "requirement") -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy//tools/base:envoy_python.bzl", "envoy_entry_point") +load("@envoy//tools/python:namespace.bzl", "envoy_py_namespace") load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() + +envoy_py_namespace() envoy_entry_point( name = "sphinx", + init_data = [ + "//:py-init", + ":py-init", + ], pkg = "sphinx", script = "sphinx-build", deps = [ @@ -58,10 +65,42 @@ pkg_tar( ) genrule( - name = "docs", - outs = ["docs.tar.gz"], + name = "html_release", + outs = ["html_release.tar.gz"], + cmd = """ + . $(location @envoy//bazel:volatile_env) \ + && VERSION_NUMBER="$$(cat $(location //:VERSION))" \ + && export ENVOY_BLOB_SHA=$${BUILD_SHA:-$${ENVOY_BUILD_SCM_REVISION:-$${{BUILD_SCM_REVISION}}} \ + && export ENVOY_DOCS_VERSION_STRING="$${VERSION_NUMBER}"-"$${ENVOY_BLOB_SHA:0:6}" \ + && export ENVOY_DOCS_RELEASE_LEVEL=pre-release \ + && mkdir -p build \ + && tar xf $(location :rst) -C build \ + && $(location :sphinx) \ + -W \ + --keep-going \ + -b html \ + build \ + output \ + && tar czf $@ -C output . + """, + stamp = 1, + tools = [ + ":rst", + ":sphinx", + "//:VERSION", + "@envoy//bazel:volatile_env", + ], +) + +genrule( + name = "html", + outs = ["html.tar.gz"], cmd = """ mkdir -p build \ + && VERSION_NUMBER="$$(cat $(location //:VERSION))" \ + && export ENVOY_BLOB_SHA="$${BUILD_SHA:-UNKNOWN}" \ + && export ENVOY_DOCS_VERSION_STRING="$${VERSION_NUMBER}"-"$${ENVOY_BLOB_SHA:0:6}" \ + && export ENVOY_DOCS_RELEASE_LEVEL=pre-release \ && tar xf $(location :rst) -C build \ && $(location :sphinx) \ -W \ @@ -74,5 +113,11 @@ genrule( tools = [ ":rst", ":sphinx", + "//:VERSION", ], ) + +alias( + name = "docs", + actual = ":html_release", +) diff --git a/mobile/docs/build.sh b/mobile/docs/build.sh index a45286c397e0..c53756660690 100755 --- a/mobile/docs/build.sh +++ b/mobile/docs/build.sh @@ -19,16 +19,9 @@ then # Check the version_history.rst contains current release version. grep --fixed-strings "$VERSION_NUMBER" docs/root/intro/version_history.rst \ || (echo "Git tag not found in version_history.rst" && exit 1) - - # Now that we now there is a match, we can use the tag. - export ENVOY_DOCS_VERSION_STRING="tag-$GITHUB_REF_NAME" - export ENVOY_DOCS_RELEASE_LEVEL=tagged - export ENVOY_BLOB_SHA="$GITHUB_REF_NAME" + DOCS_TARGET=//docs else - BUILD_SHA=$(git rev-parse HEAD) - export ENVOY_DOCS_VERSION_STRING="${VERSION_NUMBER}"-"${BUILD_SHA:0:6}" - export ENVOY_DOCS_RELEASE_LEVEL=pre-release - export ENVOY_BLOB_SHA="$BUILD_SHA" + DOCS_TARGET=//docs:html fi [[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs @@ -37,10 +30,7 @@ rm -rf "${DOCS_OUTPUT_DIR}" mkdir -p "${DOCS_OUTPUT_DIR}" DOCS_OUTPUT_DIR="$(realpath "$DOCS_OUTPUT_DIR")" -./bazelw build \ - --action_env=ENVOY_BLOB_SHA \ - --action_env=ENVOY_DOCS_RELEASE_LEVEL \ - --action_env=ENVOY_DOCS_VERSION_STRING \ - //docs - -tar xf bazel-bin/docs/docs.tar.gz -C "$DOCS_OUTPUT_DIR" . +./bazelw run \ + "--@envoy//tools/tarball:target=$DOCS_TARGET" \ + @envoy//tools/tarball:unpack \ + "$DOCS_OUTPUT_DIR" diff --git a/mobile/docs/conf.py b/mobile/docs/conf.py index d1b09cf5a7d6..9dbeba35fc13 100644 --- a/mobile/docs/conf.py +++ b/mobile/docs/conf.py @@ -48,11 +48,11 @@ def setup(app): app.add_directive('substitution-code-block', SubstitutionCodeBlock) -if not os.environ.get('ENVOY_DOCS_RELEASE_LEVEL'): +if not (release_level := os.environ.get('ENVOY_DOCS_RELEASE_LEVEL')): raise Exception("ENVOY_DOCS_RELEASE_LEVEL env var must be defined") -release_level = os.environ['ENVOY_DOCS_RELEASE_LEVEL'] -blob_sha = os.environ['ENVOY_BLOB_SHA'] +if not (blob_sha := os.environ.get("ENVOY_BLOB_SHA")): + raise Exception("ENVOY_BLOB_SHA env var must be defined") # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -103,13 +103,12 @@ def setup(app): # |version| and |release|, also used in various other places throughout the # built documents. -if not os.environ.get('ENVOY_DOCS_VERSION_STRING'): +# The short X.Y version. +if not (version := os.environ.get("ENVOY_DOCS_VERSION_STRING")): raise Exception("ENVOY_DOCS_VERSION_STRING env var must be defined") -# The short X.Y version. -version = os.environ['ENVOY_DOCS_VERSION_STRING'] # The full version, including alpha/beta/rc tags. -release = os.environ['ENVOY_DOCS_VERSION_STRING'] +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/mobile/docs/publish.sh b/mobile/docs/publish.sh index dc821274f934..03b40a059e77 100755 --- a/mobile/docs/publish.sh +++ b/mobile/docs/publish.sh @@ -1,44 +1,41 @@ -#!/bin/bash +#!/bin/bash -e # This is run on every commit that GitHub Actions picks up. It assumes that docs have already been -# built via docs/build.sh. The push behavior differs depending on the nature of the commit: -# * Tag commit (e.g. v1.6.0): pushes docs to versioned location. -# * Main commit: pushes docs to latest. Note that envoy-mobile.github.io uses `master` rather than +# built via docs/build.sh. The commit behavior differs depending on the nature of the commit: +# * Tag commit (e.g. v1.6.0): commits docs to versioned location. +# * Main commit: commits docs to latest. Note that envoy-mobile.github.io uses `master` rather than # `main` because using `main` as the default branch currently results in 404s. # * Otherwise: noop. -set -e +set -o pipefail DOCS_DIR=generated/docs -CHECKOUT_DIR=../envoy-mobile-docs BUILD_SHA="$(git rev-parse HEAD)" -if [ "$GITHUB_REF_TYPE" == "tag" ] -then - PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy-mobile/"$GITHUB_REF_NAME" -elif [ "$GITHUB_REF_NAME" == "main" ] -then - PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy-mobile/latest +if [[ -z "$MOBILE_DOCS_CHECKOUT_DIR" ]]; then + echo "MOBILE_DOCS_CHECKOUT_DIR is not set, exiting" >&2 + exit 1 +fi + +if [[ "$GITHUB_REF_TYPE" == "tag" ]]; then + PUBLISH_DIR="$MOBILE_DOCS_CHECKOUT_DIR"/docs/envoy-mobile/"$GITHUB_REF_NAME" else - echo "Ignoring docs push" - exit 0 + PUBLISH_DIR="$MOBILE_DOCS_CHECKOUT_DIR"/docs/envoy-mobile/latest fi -echo 'cloning' -git clone git@github.com:envoy-mobile/envoy-mobile.github.io "$CHECKOUT_DIR" +echo "Publishing docs in ${PUBLISH_DIR}" -git -C "$CHECKOUT_DIR" fetch -git -C "$CHECKOUT_DIR" checkout -B master origin/master +git -C "$MOBILE_DOCS_CHECKOUT_DIR" checkout -B master origin/master rm -fr "$PUBLISH_DIR" mkdir -p "$PUBLISH_DIR" cp -r "$DOCS_DIR"/* "$PUBLISH_DIR" -cd "$CHECKOUT_DIR" -git config user.name "envoy-mobile-docs(ci)" -git config user.email envoy-mobile-docs@users.noreply.github.com +git -C "${MOBILE_DOCS_CHECKOUT_DIR}" config user.name "envoy-mobile-docs(ci)" +git -C "${MOBILE_DOCS_CHECKOUT_DIR}" config user.email envoy-mobile-docs@users.noreply.github.com echo 'add' -git add . -echo 'commit' -git commit -m "docs envoy-mobile@$BUILD_SHA" -echo 'push' -git push origin master +git -C "${MOBILE_DOCS_CHECKOUT_DIR}" add . + +if [[ "$(git -C "${MOBILE_DOCS_CHECKOUT_DIR}" status --porcelain)" ]]; then + echo 'commit' + git -C "${MOBILE_DOCS_CHECKOUT_DIR}" commit -m "docs envoy-mobile@$BUILD_SHA" +fi diff --git a/mobile/docs/root/api/http.rst b/mobile/docs/root/api/http.rst index f09b946d7296..c8dce9ee4673 100644 --- a/mobile/docs/root/api/http.rst +++ b/mobile/docs/root/api/http.rst @@ -78,7 +78,6 @@ then passing it to a previously created :ref:`StreamClient instance `_ - `Retry semantics `_ ------------------------- -``CompressionAlgorithm`` ------------------------- - -The ``CompressionAlgorithm`` type allows for compressing request bodies using either gzip or Brotli. -This can be set by calling ``enableRequestCompression(...)`` on the ``RequestHeadersBuilder``. - -See the compression filter documentation for details about how and when this compression is applied: - -- `gzip compressor proto `_ -- `brotli compressor proto `_ - ---------- ``Stream`` ---------- diff --git a/mobile/docs/root/api/starting_envoy.rst b/mobile/docs/root/api/starting_envoy.rst index 2bff205e92fb..cb65abea481c 100644 --- a/mobile/docs/root/api/starting_envoy.rst +++ b/mobile/docs/root/api/starting_envoy.rst @@ -123,44 +123,6 @@ Specify the log level to be used when running the underlying Envoy engine. // Swift builder.addLogLevel(.warn) -~~~~~~~~~~~~~~~~~~~~~~ -``addGrpcStatsDomain`` -~~~~~~~~~~~~~~~~~~~~~~ - -Specify a domain which implements the -:tree:`stats endpoint <83908423d46a37574e9a35627df1f3dd9634e5ec/library/common/config_template.cc#L139-L145>` -in order to take advantage of the -`stats emitted by Envoy `_ -(and subsequently Envoy Mobile). - -Note that only stats specified in the configuration's -:tree:`inclusion list <83908423d46a37574e9a35627df1f3dd9634e5ec/library/common/config_template.cc#L146-L167>` -will be emitted. - -Passing ``nil``/``null`` disables stats emission, and this is the default value. - -**Example**:: - - // Kotlin - builder.addGrpcStatsDomain("envoy-mobile.envoyproxy.io") - - // Swift - builder.addGrpcStatsDomain("envoy-mobile.envoyproxy.io") - -~~~~~~~~~~~~~~~~~~~~~~~~ -``addStatsFlushSeconds`` -~~~~~~~~~~~~~~~~~~~~~~~~ - -Specify the rate at which Envoy Mobile should flush its queued stats. - -**Example**:: - - // Kotlin - builder.addStatsFlushSeconds(5L) - - // Swift - builder.addStatsFlushSeconds(5) - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``addStreamIdleTimeoutSeconds`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -200,7 +162,6 @@ for further information. ~~~~~~~~~~~~~~~~~ Specify the version of the app using Envoy Mobile. -This information is sent as metadata when flushing stats. **Example**:: @@ -215,7 +176,6 @@ This information is sent as metadata when flushing stats. ~~~~~~~~~~~~ Specify the version of the app using Envoy Mobile. -This information is sent as metadata when flushing stats. **Example**:: @@ -363,6 +323,55 @@ Specify whether to enable transparent response Brotli decompression. Defaults to Default values from the `brotli decompressor proto `_ are used. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``enableHttp3`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Specify whether to enable HTTP/3. Defaults to true. Only available when the Envoy Mobile build has HTTP/3 included. +When HTTP/3 is enabled, the client will first talk HTTP/2 with the servers and upon receiving alt-svc in the response, +following traffic will be sent via HTTP/3. + +**Example**:: + + // Kotlin + builder.enableHttp3(true) + + // Swift + builder.enableHttp3(true) + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``addQuicHint`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add a host port pair that's known to support QUIC. Only available when HTTP/3 is enabled. +It can be called multiple times to append a list of QUIC hints. +This allows HTTP/3 to be used for the first request to the hosts and avoid the HTTP/2 -> HTTP/3 switch as mentioned in enableHttp3. + +**Example**:: + + // Kotlin + builder.addQuicHint("www.example.com", 443) + + // Swift + builder.addQuicHint("www.example.com", 443) + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``addQuicCanonicalSuffix`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add a canonical suffix that's known to speak QUIC. +This feature works as a extension to QUIC hints in such way that: +if `.abc.com` is added to canonical suffix, and `foo.abc.com` is added to QUIC hint, then all requests to +`*.abc.com` will be considered QUIC ready. + +**Example**:: + + // Kotlin + builder.addQuicCanonicalSuffix(".example.com") + + // Swift + builder.addQuicCanonicalSuffix(".example.com") + ~~~~~~~~~~~~~~~~~~~~~~~ ``enableSocketTagging`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -585,7 +594,6 @@ This may be done by initializing a builder with the contents of the YAML file yo val streamClient = AndroidEngineBuilder(baseContext, Yaml(yamlFileString)) .addLogLevel(LogLevel.WARN) - .addStatsFlushSeconds(60) ... .build() .streamClient() @@ -594,7 +602,6 @@ This may be done by initializing a builder with the contents of the YAML file yo let streamClient = try EngineBuilder(yaml: yamlFileString) .addLogLevel(.warn) - .addStatsFlushSeconds(60) ... .build() .streamClient() diff --git a/mobile/docs/root/development/performance/binary_size.rst b/mobile/docs/root/development/performance/binary_size.rst index 12ad77d9cc5f..b97af38b1537 100644 --- a/mobile/docs/root/development/performance/binary_size.rst +++ b/mobile/docs/root/development/performance/binary_size.rst @@ -67,6 +67,7 @@ binary size: 6. ``--define=signal_trace=disabled``: more info in the `envoy docs `_. Due to the project's ``.bazelrc``. 7. ``--define=tcmalloc=disabled``: more info in the `envoy docs `_. Due to the project's ``.bazelrc``. 8. ``--define=hot_restart=disabled``: more info in the `envoy docs `_. Due to the project's ``.bazelrc``. +9. ``--define=envoy_mobile_xds=disabled``: more info in the `envoy docs `_. Due to the project's ``.bazelrc``. After compiling, the binary can be stripped of all symbols by using ``strip``:: diff --git a/mobile/envoy_build_config/BUILD b/mobile/envoy_build_config/BUILD index d83d85bedc2d..847d38256bc4 100644 --- a/mobile/envoy_build_config/BUILD +++ b/mobile/envoy_build_config/BUILD @@ -1,8 +1,15 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", "envoy_select_enable_http3", "envoy_select_envoy_mobile_listener", "envoy_select_envoy_mobile_request_compression", "envoy_select_envoy_mobile_stats_reporting") +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_mobile_package", + "envoy_select_enable_http3", + "envoy_select_envoy_mobile_listener", + "envoy_select_envoy_mobile_xds", +) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "extension_registry", @@ -19,6 +26,7 @@ envoy_cc_library( "@envoy//source/common/upstream:default_local_address_selector_factory", "@envoy//source/common/watchdog:abort_action_config", "@envoy//source/extensions/clusters/dynamic_forward_proxy:cluster", + "@envoy//source/extensions/compression/brotli/compressor:config", "@envoy//source/extensions/compression/brotli/decompressor:config", "@envoy//source/extensions/compression/gzip/decompressor:config", "@envoy//source/extensions/filters/http/alternate_protocols_cache:config", @@ -28,6 +36,7 @@ envoy_cc_library( "@envoy//source/extensions/filters/http/router:config", "@envoy//source/extensions/filters/network/http_connection_manager:config", "@envoy//source/extensions/http/header_formatters/preserve_case:config", + "@envoy//source/extensions/load_balancing_policies/cluster_provided:config", "@envoy//source/extensions/network/dns_resolver/getaddrinfo:config", "@envoy//source/extensions/path/match/uri_template:config", "@envoy//source/extensions/path/rewrite/uri_template:config", @@ -50,21 +59,6 @@ envoy_cc_library( "@envoy//source/common/quic:quic_transport_socket_factory_lib", ], "@envoy", - ) + envoy_select_envoy_mobile_stats_reporting( - [ - "@envoy//source/extensions/stat_sinks/metrics_service:config", - "@envoy//source/extensions/stat_sinks/statsd:config", - "@envoy//source/extensions/clusters/logical_dns:logical_dns_cluster_lib", - ], - "@envoy", - ) + envoy_select_envoy_mobile_request_compression( - [ - "@envoy//source/extensions/compression/brotli/compressor:config", - "@envoy//source/extensions/compression/gzip/compressor:config", - "@envoy//source/extensions/filters/http/compressor:config", - "@envoy//source/extensions/filters/http/composite:config", - ], - "@envoy", ) + envoy_select_envoy_mobile_listener( [ "@envoy//source/extensions/udp_packet_writer/default:config", @@ -72,6 +66,13 @@ envoy_cc_library( "@envoy//source/extensions/listener_managers/listener_manager:connection_handler_lib", ], "@envoy", + ) + envoy_select_envoy_mobile_xds( + [ + "@envoy//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", + "@envoy//source/extensions/config_subscription/grpc:grpc_mux_lib", + "@envoy//source/extensions/config_subscription/grpc:grpc_subscription_lib", + ], + "@envoy", ), ) @@ -116,6 +117,7 @@ envoy_cc_library( deps = [ "@envoy//source/extensions/clusters/static:static_cluster_lib", "@envoy//source/extensions/filters/http/buffer:config", + "@envoy//source/extensions/load_balancing_policies/round_robin:config", "@envoy_mobile//test/common/http/filters/assertion:config", "@envoy_mobile//test/common/http/filters/assertion:filter_cc_proto_descriptor", "@envoy_mobile//test/common/http/filters/route_cache_reset:config", diff --git a/mobile/envoy_build_config/extension_registry.cc b/mobile/envoy_build_config/extension_registry.cc index af1922af939b..fd6178ba778c 100644 --- a/mobile/envoy_build_config/extension_registry.cc +++ b/mobile/envoy_build_config/extension_registry.cc @@ -1,21 +1,15 @@ #include "extension_registry.h" -#ifdef ENVOY_MOBILE_REQUEST_COMPRESSION -#include "source/extensions/compression/brotli/compressor/config.h" -#include "source/extensions/compression/gzip/compressor/config.h" -#include "source/extensions/filters/http/composite/action.h" -#include "source/extensions/filters/http/composite/config.h" -#include "source/extensions/filters/http/compressor/config.h" -#endif - #include "source/common/http/match_delegate/config.h" #include "source/common/http/matching/inputs.h" #include "source/common/network/default_client_connection_factory.h" +#include "source/common/network/resolver_impl.h" #include "source/common/network/socket_interface_impl.h" #include "source/common/router/upstream_codec_filter.h" #include "source/common/upstream/default_local_address_selector_factory.h" #include "source/common/watchdog/abort_action_config.h" #include "source/extensions/clusters/dynamic_forward_proxy/cluster.h" +#include "source/extensions/compression/brotli/compressor/config.h" #include "source/extensions/compression/brotli/decompressor/config.h" #include "source/extensions/compression/gzip/decompressor/config.h" #include "source/extensions/early_data/default_early_data_policy.h" @@ -28,6 +22,7 @@ #include "source/extensions/http/header_formatters/preserve_case/config.h" #include "source/extensions/http/header_validators/envoy_default/config.h" #include "source/extensions/http/original_ip_detection/xff/config.h" +#include "source/extensions/load_balancing_policies/cluster_provided/config.h" #include "source/extensions/network/dns_resolver/getaddrinfo/getaddrinfo.h" #include "source/extensions/path/match/uri_template/config.h" #include "source/extensions/path/rewrite/uri_template/config.h" @@ -51,13 +46,7 @@ #include "source/extensions/quic/proof_source/envoy_quic_proof_source_factory_impl.h" #include "source/extensions/udp_packet_writer/default/config.h" #endif -#include "source/common/quic/quic_transport_socket_factory.h" -#endif - -#ifdef ENVOY_MOBILE_STATS_REPORTING -#include "source/extensions/clusters/logical_dns/logical_dns_cluster.h" -#include "source/extensions/stat_sinks/metrics_service/config.h" -#include "source/extensions/stat_sinks/statsd/config.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" #endif #include "extension_registry_platform_additions.h" @@ -70,6 +59,14 @@ #include "library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h" #include "library/common/extensions/retry/options/network_configuration/config.h" +#ifdef ENVOY_MOBILE_XDS +#include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" +#include "source/extensions/transport_sockets/tls/cert_validator/default_validator.h" +#endif + namespace Envoy { void ExtensionRegistry::registerFactories() { @@ -167,6 +164,8 @@ void ExtensionRegistry::registerFactories() { // This could be compiled out for iOS. Network::forceRegisterGetAddrInfoDnsResolverFactory(); + Network::Address::forceRegisterIpResolver(); + // This is Envoy's lightweight listener manager which lets E-M avoid the 1M // hit of compiling in downstream code. Server::forceRegisterApiListenerManagerFactoryImpl(); @@ -178,12 +177,8 @@ void ExtensionRegistry::registerFactories() { // This is required for the default upstream local address selector. Upstream::forceRegisterDefaultUpstreamLocalAddressSelectorFactory(); -#ifdef ENVOY_MOBILE_STATS_REPORTING - Network::Address::forceRegisterIpResolver(); - Upstream::forceRegisterLogicalDnsClusterFactory(); - Extensions::StatSinks::MetricsService::forceRegisterMetricsServiceSinkFactory(); - Extensions::StatSinks::Statsd::forceRegisterStatsdSinkFactory(); -#endif + // This is required for load balancers of upstream clusters `base` and `base_clear`. + Envoy::Extensions::LoadBalancingPolices::ClusterProvided::forceRegisterFactory(); #ifdef ENVOY_MOBILE_ENABLE_LISTENER // These are downstream factories required if Envoy Mobile is compiled with @@ -202,6 +197,7 @@ void ExtensionRegistry::registerFactories() { Network::forceRegisterUdpDefaultWriterFactoryFactory(); Server::forceRegisterConnectionHandlerFactoryImpl(); Quic::forceRegisterQuicHttpServerConnectionFactoryImpl(); + Quic::forceRegisterEnvoyQuicCryptoServerStreamFactoryImpl(); Quic::forceRegisterQuicServerTransportSocketConfigFactory(); Quic::forceRegisterEnvoyQuicProofSourceFactoryImpl(); Quic::forceRegisterEnvoyDeterministicConnectionIdGeneratorConfigFactory(); @@ -209,13 +205,19 @@ void ExtensionRegistry::registerFactories() { Quic::forceRegisterQuicClientTransportSocketConfigFactory(); #endif -#ifdef ENVOY_MOBILE_REQUEST_COMPRESSION - // These are factories required for request decompression. + // TODO(alyssawilk) figure out why these are needed. Extensions::Compression::Brotli::Compressor::forceRegisterBrotliCompressorLibraryFactory(); - Extensions::Compression::Gzip::Compressor::forceRegisterGzipCompressorLibraryFactory(); - Extensions::HttpFilters::Composite::forceRegisterCompositeFilterFactory(); - Extensions::HttpFilters::Composite::forceRegisterExecuteFilterActionFactory(); - Extensions::HttpFilters::Compressor::forceRegisterCompressorFilterFactory(); + +#ifdef ENVOY_MOBILE_XDS + // These extensions are required for xDS over gRPC using ADS, which is what Envoy Mobile + // supports for xDS. + Config::forceRegisterAdsConfigSubscriptionFactory(); + Config::forceRegisterGrpcConfigSubscriptionFactory(); + Config::forceRegisterAggregatedGrpcCollectionConfigSubscriptionFactory(); + Config::forceRegisterAdsCollectionConfigSubscriptionFactory(); + Config::forceRegisterGrpcMuxFactory(); + Config::forceRegisterNewGrpcMuxFactory(); + Extensions::TransportSockets::Tls::forceRegisterDefaultCertValidatorFactory(); #endif } diff --git a/mobile/envoy_build_config/extensions_build_config.bzl b/mobile/envoy_build_config/extensions_build_config.bzl index faa6efaf3fc0..a1135c44c941 100644 --- a/mobile/envoy_build_config/extensions_build_config.bzl +++ b/mobile/envoy_build_config/extensions_build_config.bzl @@ -4,7 +4,6 @@ EXTENSION_CONFIG_VISIBILITY = ["//visibility:public"] EXTENSION_PACKAGE_VISIBILITY = ["//visibility:public"] EXTENSIONS = { "envoy.clusters.dynamic_forward_proxy": "//source/extensions/clusters/dynamic_forward_proxy:cluster", - "envoy.clusters.logical_dns": "//source/extensions/clusters/logical_dns:logical_dns_cluster_lib", "envoy.clusters.static": "//source/extensions/clusters/static:static_cluster_lib", "envoy.filters.connection_pools.http.generic": "//source/extensions/upstreams/http/generic:config", "envoy.filters.http.alternate_protocols_cache": "//source/extensions/filters/http/alternate_protocols_cache:config", @@ -23,7 +22,6 @@ EXTENSIONS = { "envoy.network.dns_resolver.apple": "//source/extensions/network/dns_resolver/apple:config", "envoy.network.dns_resolver.getaddrinfo": "//source/extensions/network/dns_resolver/getaddrinfo:config", "envoy.retry.options.network_configuration": "@envoy_mobile//library/common/extensions/retry/options/network_configuration:config", - "envoy.stat_sinks.metrics_service": "//source/extensions/stat_sinks/metrics_service:config", "envoy.transport_sockets.http_11_proxy": "//source/extensions/transport_sockets/http_11_proxy:upstream_config", "envoy.transport_sockets.raw_buffer": "//source/extensions/transport_sockets/raw_buffer:config", "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", @@ -31,6 +29,8 @@ EXTENSIONS = { "envoy_mobile.cert_validator.platform_bridge_cert_validator": "@envoy_mobile//library/common/extensions/cert_validator/platform_bridge:config", "envoy.listener_manager_impl.api": "@envoy_mobile//library/common/extensions/listener_managers/api_listener_manager:api_listener_manager_lib", "envoy.connection_handler.default": "//source/extensions/listener_managers/listener_manager:connection_handler_lib", + "envoy.load_balancing_policies.round_robin": "//source/extensions/load_balancing_policies/round_robin:config", + "envoy.load_balancing_policies.cluster_provided": "//source/extensions/load_balancing_policies/cluster_provided:config", } WINDOWS_EXTENSIONS = {} LEGACY_ALWAYSLINK = 1 diff --git a/mobile/envoy_build_config/test_extensions.cc b/mobile/envoy_build_config/test_extensions.cc index 5f3663c6cd56..9fcd8213026c 100644 --- a/mobile/envoy_build_config/test_extensions.cc +++ b/mobile/envoy_build_config/test_extensions.cc @@ -1,5 +1,6 @@ #include "source/extensions/clusters/static/static_cluster.h" #include "source/extensions/filters/http/buffer/config.h" +#include "source/extensions/load_balancing_policies/round_robin/config.h" #include "test/common/http/filters/assertion/config.h" #include "test/common/http/filters/route_cache_reset/config.h" @@ -36,6 +37,7 @@ void register_test_extensions() { Envoy::Extensions::HttpFilters::TestLogger::forceRegisterFactory(); Envoy::Extensions::HttpFilters::TestRemoteResponse:: forceRegisterTestRemoteResponseFilterFactory(); + Envoy::Extensions::LoadBalancingPolices::RoundRobin::forceRegisterFactory(); Envoy::HttpFilters::TestRead::forceRegisterTestReadFilterFactory(); Envoy::Upstream::forceRegisterStaticClusterFactory(); diff --git a/mobile/examples/BUILD b/mobile/examples/BUILD index 2eac13b1569c..127d2469bc55 100644 --- a/mobile/examples/BUILD +++ b/mobile/examples/BUILD @@ -3,9 +3,12 @@ load( "@com_github_buildbuddy_io_rules_xcodeproj//xcodeproj:defs.bzl", "xcode_provisioning_profile", ) +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + # Change to your Apple Developer Team ID as shown in # https://developer.apple.com/account/#!/membership TEAM_ID = "X4ST43AL9W" diff --git a/mobile/examples/cc/fetch_client/BUILD b/mobile/examples/cc/fetch_client/BUILD index 15b0e1892882..70f4c4900a6f 100644 --- a/mobile/examples/cc/fetch_client/BUILD +++ b/mobile/examples/cc/fetch_client/BUILD @@ -1,12 +1,12 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "fetch_client_lib", @@ -16,6 +16,7 @@ envoy_cc_library( hdrs = [ "fetch_client.h", ], + external_deps = ["abseil_synchronization"], repository = "@envoy", deps = [ "//library/cc:engine_builder_lib", diff --git a/mobile/examples/cc/fetch_client/fetch_client.cc b/mobile/examples/cc/fetch_client/fetch_client.cc index 430c3252c9cf..90b163f0cdbc 100644 --- a/mobile/examples/cc/fetch_client/fetch_client.cc +++ b/mobile/examples/cc/fetch_client/fetch_client.cc @@ -37,7 +37,10 @@ void Fetch::fetch(const std::vector& urls) { } dispatcher_->exit(); envoy_thread->join(); - engine_->terminate(); + { + absl::MutexLock lock(&engine_mutex_); + engine_->terminate(); + } } void Fetch::sendRequest(const absl::string_view url_string) { @@ -49,8 +52,11 @@ void Fetch::sendRequest(const absl::string_view url_string) { std::cout << "Fetching url: " << url.toString() << "\n"; absl::Notification request_finished; - Platform::StreamPrototypeSharedPtr stream_prototype = - engine_->streamClient()->newStreamPrototype(); + Platform::StreamPrototypeSharedPtr stream_prototype; + { + absl::MutexLock lock(&engine_mutex_); + stream_prototype = engine_->streamClient()->newStreamPrototype(); + } stream_prototype->setOnHeaders( [](Platform::ResponseHeadersSharedPtr headers, bool, envoy_stream_intel intel) { std::cerr << "Received headers on connection: " << intel.connection_id << "\n"; @@ -103,7 +109,12 @@ void Fetch::runEngine(absl::Notification& engine_running) { Platform::EngineBuilder engine_builder; engine_builder.addLogLevel(Envoy::Platform::LogLevel::debug); engine_builder.setOnEngineRunning([&engine_running]() { engine_running.Notify(); }); - engine_ = engine_builder.build(); + + { + absl::MutexLock lock(&engine_mutex_); + engine_ = engine_builder.build(); + } + dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } diff --git a/mobile/examples/cc/fetch_client/fetch_client.h b/mobile/examples/cc/fetch_client/fetch_client.h index 453b2d844d1a..7e28ace30b2c 100644 --- a/mobile/examples/cc/fetch_client/fetch_client.h +++ b/mobile/examples/cc/fetch_client/fetch_client.h @@ -15,6 +15,8 @@ #include "source/exe/platform_impl.h" #include "source/exe/process_wide.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" #include "absl/synchronization/notification.h" #include "library/cc/engine_builder.h" #include "library/cc/stream.h" @@ -46,7 +48,9 @@ class Fetch { Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; - Platform::EngineSharedPtr engine_; + + absl::Mutex engine_mutex_; + Platform::EngineSharedPtr engine_ ABSL_GUARDED_BY(engine_mutex_); }; } // namespace Envoy diff --git a/mobile/examples/java/hello_world/BUILD b/mobile/examples/java/hello_world/BUILD index fe3739a27e6c..04c8163d40db 100644 --- a/mobile/examples/java/hello_world/BUILD +++ b/mobile/examples/java/hello_world/BUILD @@ -1,9 +1,12 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_binary( name = "hello_envoy", custom_package = "io.envoyproxy.envoymobile.helloenvoy", @@ -33,5 +36,6 @@ kt_android_library( artifact("androidx.recyclerview:recyclerview"), artifact("androidx.annotation:annotation"), artifact("com.google.code.findbugs:jsr305"), + artifact("com.google.protobuf:protobuf-javalite"), ], ) diff --git a/mobile/examples/kotlin/hello_world/AsyncDemoFilter.kt b/mobile/examples/kotlin/hello_world/AsyncDemoFilter.kt index a45ec5f7af1d..1436a6b6a8e7 100644 --- a/mobile/examples/kotlin/hello_world/AsyncDemoFilter.kt +++ b/mobile/examples/kotlin/hello_world/AsyncDemoFilter.kt @@ -30,9 +30,7 @@ class AsyncDemoFilter : AsyncResponseFilter { ): FilterHeadersStatus { // If this is the end of the stream, asynchronously resume response processing via callback. if (endStream) { - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } } return FilterHeadersStatus.StopIteration() } @@ -44,9 +42,7 @@ class AsyncDemoFilter : AsyncResponseFilter { ): FilterDataStatus { // If this is the end of the stream, asynchronously resume response processing via callback. if (endStream) { - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } } return FilterDataStatus.StopIterationAndBuffer() } @@ -56,9 +52,7 @@ class AsyncDemoFilter : AsyncResponseFilter { streamIntel: StreamIntel ): FilterTrailersStatus { // Trailers imply end of stream, so asynchronously resume response processing via callbacka - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } return FilterTrailersStatus.StopIteration() } @@ -73,23 +67,14 @@ class AsyncDemoFilter : AsyncResponseFilter { endStream: Boolean, streamIntel: StreamIntel ): FilterResumeStatus { - val builder = headers!!.toResponseHeadersBuilder() - .add("async-filter-demo", "1") + val builder = headers!!.toResponseHeadersBuilder().add("async-filter-demo", "1") return FilterResumeStatus.ResumeIteration(builder.build(), data, trailers) } @Suppress("EmptyFunctionBlock") - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onCancel(finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/examples/kotlin/hello_world/BUILD b/mobile/examples/kotlin/hello_world/BUILD index 07fbe4fbb765..72eec5686f1d 100644 --- a/mobile/examples/kotlin/hello_world/BUILD +++ b/mobile/examples/kotlin/hello_world/BUILD @@ -1,10 +1,13 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") load("@rules_detekt//detekt:defs.bzl", "detekt") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_binary( name = "hello_envoy_kt", custom_package = "io.envoyproxy.envoymobile.helloenvoykotlin", @@ -34,6 +37,7 @@ kt_android_library( artifact("androidx.recyclerview:recyclerview"), artifact("androidx.annotation:annotation"), artifact("com.google.code.findbugs:jsr305"), + artifact("com.google.protobuf:protobuf-javalite"), ], ) diff --git a/mobile/examples/kotlin/hello_world/BufferDemoFilter.kt b/mobile/examples/kotlin/hello_world/BufferDemoFilter.kt index f158d1b1e65a..f37c330e357e 100644 --- a/mobile/examples/kotlin/hello_world/BufferDemoFilter.kt +++ b/mobile/examples/kotlin/hello_world/BufferDemoFilter.kt @@ -13,8 +13,7 @@ import java.nio.ByteBuffer /** * Example of a more complex HTTP filter that pauses processing on the response filter chain, - * buffers until the response is complete, then resumes filter iteration while setting a new - * header. + * buffers until the response is complete, then resumes filter iteration while setting a new header. */ class BufferDemoFilter : ResponseFilter { private lateinit var headers: ResponseHeaders @@ -39,8 +38,7 @@ class BufferDemoFilter : ResponseFilter { // If this is the end of the stream, resume processing of the (now fully-buffered) response. if (endStream) { - val builder = headers.toResponseHeadersBuilder() - .add("buffer-filter-demo", "1") + val builder = headers.toResponseHeadersBuilder().add("buffer-filter-demo", "1") return FilterDataStatus.ResumeIteration(builder.build(), body) } return FilterDataStatus.StopIterationAndBuffer() @@ -51,23 +49,14 @@ class BufferDemoFilter : ResponseFilter { streamIntel: StreamIntel ): FilterTrailersStatus { // Trailers imply end of stream; resume processing of the (now fully-buffered) response. - val builder = headers.toResponseHeadersBuilder() - .add("buffer-filter-demo", "1") + val builder = headers.toResponseHeadersBuilder().add("buffer-filter-demo", "1") return FilterTrailersStatus.ResumeIteration(builder.build(), this.body, trailers) } @Suppress("EmptyFunctionBlock") - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onCancel(finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/examples/kotlin/hello_world/DemoFilter.kt b/mobile/examples/kotlin/hello_world/DemoFilter.kt index 4f8244499e5e..d7ebf70c2193 100644 --- a/mobile/examples/kotlin/hello_world/DemoFilter.kt +++ b/mobile/examples/kotlin/hello_world/DemoFilter.kt @@ -12,9 +12,7 @@ import io.envoyproxy.envoymobile.ResponseTrailers import io.envoyproxy.envoymobile.StreamIntel import java.nio.ByteBuffer -/** - * A filter implemented as a simple example of Envoy response filter. - */ +/** A filter implemented as a simple example of Envoy response filter. */ class DemoFilter : ResponseFilter { override fun onResponseHeaders( headers: ResponseHeaders, @@ -44,10 +42,7 @@ class DemoFilter : ResponseFilter { return FilterTrailersStatus.Continue(trailers) } - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) { Log.d("DemoFilter", "On error!") } @@ -55,7 +50,5 @@ class DemoFilter : ResponseFilter { Log.d("DemoFilter", "On cancel!") } - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/examples/kotlin/hello_world/MainActivity.kt b/mobile/examples/kotlin/hello_world/MainActivity.kt index ee7199fd67c6..c117ddb87304 100644 --- a/mobile/examples/kotlin/hello_world/MainActivity.kt +++ b/mobile/examples/kotlin/hello_world/MainActivity.kt @@ -25,17 +25,16 @@ private const val REQUEST_HANDLER_THREAD_NAME = "hello_envoy_kt" private const val REQUEST_AUTHORITY = "api.lyft.com" private const val REQUEST_PATH = "/ping" private const val REQUEST_SCHEME = "https" -private val FILTERED_HEADERS = setOf( - "server", - "filter-demo", - "buffer-filter-demo", - "async-filter-demo", - "x-envoy-upstream-service-time" -) +private val FILTERED_HEADERS = + setOf( + "server", + "filter-demo", + "buffer-filter-demo", + "async-filter-demo", + "x-envoy-upstream-service-time" + ) -/** - * The main activity of the app. - */ +/** The main activity of the app. */ class MainActivity : Activity() { private val thread = HandlerThread(REQUEST_HANDLER_THREAD_NAME) private lateinit var recyclerView: RecyclerView @@ -47,33 +46,34 @@ class MainActivity : Activity() { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - engine = AndroidEngineBuilder(application) - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .addPlatformFilter(::DemoFilter) - .addPlatformFilter(::BufferDemoFilter) - .addPlatformFilter(::AsyncDemoFilter) - .addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}") - .addStringAccessor("demo-accessor", { "PlatformString" }) - .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") } - .setEventTracker({ - for (entry in it.entries) { - Log.d("MainActivity", "Event emitted: ${entry.key}, ${entry.value}") - } - }) - .setLogger { - Log.d("MainActivity", it) - } - .build() + engine = + AndroidEngineBuilder(application) + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .addPlatformFilter(::DemoFilter) + .addPlatformFilter(::BufferDemoFilter) + .addPlatformFilter(::AsyncDemoFilter) + .addNativeFilter( + "envoy.filters.http.buffer", + "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}" + ) + .addStringAccessor("demo-accessor", { "PlatformString" }) + .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") } + .setEventTracker({ + for (entry in it.entries) { + Log.d("MainActivity", "Event emitted: ${entry.key}, ${entry.value}") + } + }) + .setLogger { Log.d("MainActivity", it) } + .build() recyclerView = findViewById(R.id.recycler_view) as RecyclerView recyclerView.layoutManager = LinearLayoutManager(this) viewAdapter = ResponseRecyclerViewAdapter() recyclerView.adapter = viewAdapter - val dividerItemDecoration = DividerItemDecoration( - recyclerView.context, DividerItemDecoration.VERTICAL - ) + val dividerItemDecoration = + DividerItemDecoration(recyclerView.context, DividerItemDecoration.VERTICAL) recyclerView.addItemDecoration(dividerItemDecoration) thread.start() val handler = Handler(thread.looper) @@ -106,10 +106,9 @@ class MainActivity : Activity() { // Note: this request will use an h2 stream for the upstream request due to https scheme. // The Java example uses http so http/1.1. This is done on purpose to test both paths in // end-to-end tests in CI. - val requestHeaders = RequestHeadersBuilder( - RequestMethod.GET, REQUEST_SCHEME, REQUEST_AUTHORITY, REQUEST_PATH - ) - .build() + val requestHeaders = + RequestHeadersBuilder(RequestMethod.GET, REQUEST_SCHEME, REQUEST_AUTHORITY, REQUEST_PATH) + .build() engine .streamClient() .newStreamPrototype() diff --git a/mobile/examples/kotlin/shared/BUILD b/mobile/examples/kotlin/shared/BUILD index 778bf958ef0a..4639ee467606 100644 --- a/mobile/examples/kotlin/shared/BUILD +++ b/mobile/examples/kotlin/shared/BUILD @@ -1,8 +1,11 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + kt_android_library( name = "hello_envoy_shared_lib", srcs = [ diff --git a/mobile/examples/kotlin/shared/Response.kt b/mobile/examples/kotlin/shared/Response.kt index 8d205c2177ab..5aa9953af3ca 100644 --- a/mobile/examples/kotlin/shared/Response.kt +++ b/mobile/examples/kotlin/shared/Response.kt @@ -2,10 +2,11 @@ package io.envoyproxy.envoymobile.shared // Response is a class to handle HTTP responses. sealed class Response { - fun fold(success: (Success) -> Unit, failure: (Failure) -> Unit) = when (this) { - is Success -> success(this) - is Failure -> failure(this) - } + fun fold(success: (Success) -> Unit, failure: (Failure) -> Unit) = + when (this) { + is Success -> success(this) + is Failure -> failure(this) + } } data class Success(val title: String, val header: String) : Response() diff --git a/mobile/examples/kotlin/shared/ResponseViewHolder.kt b/mobile/examples/kotlin/shared/ResponseViewHolder.kt index 325f843b6bfd..87b6f06aa396 100644 --- a/mobile/examples/kotlin/shared/ResponseViewHolder.kt +++ b/mobile/examples/kotlin/shared/ResponseViewHolder.kt @@ -5,27 +5,26 @@ import android.widget.TextView import androidx.recyclerview.widget.RecyclerView class ResponseViewHolder(itemView: View) : RecyclerView.ViewHolder(itemView) { - private val countTextView: TextView = itemView - .findViewById(R.id.response_text_view_count) as TextView - private val responseTextView: TextView = itemView - .findViewById(R.id.response_text_view) as TextView - private val headerTextView: TextView = itemView - .findViewById(R.id.header_text_view) as TextView + private val countTextView: TextView = + itemView.findViewById(R.id.response_text_view_count) as TextView + private val responseTextView: TextView = + itemView.findViewById(R.id.response_text_view) as TextView + private val headerTextView: TextView = itemView.findViewById(R.id.header_text_view) as TextView fun setResult(count: Int, response: Response) { countTextView.text = count.toString() response.fold( { success -> - responseTextView.text = responseTextView.resources - .getString(R.string.title_string, success.title) - headerTextView.text = headerTextView.resources - .getString(R.string.header_string, success.header) + responseTextView.text = + responseTextView.resources.getString(R.string.title_string, success.title) + headerTextView.text = + headerTextView.resources.getString(R.string.header_string, success.header) headerTextView.visibility = View.VISIBLE itemView.setBackgroundResource(R.color.success_color) }, { failure -> - responseTextView.text = responseTextView.resources - .getString(R.string.title_string, failure.message) + responseTextView.text = + responseTextView.resources.getString(R.string.title_string, failure.message) headerTextView.visibility = View.GONE itemView.setBackgroundResource(R.color.failed_color) } diff --git a/mobile/examples/objective-c/hello_world/BUILD b/mobile/examples/objective-c/hello_world/BUILD index 958727140ab5..8bdeda34786a 100644 --- a/mobile/examples/objective-c/hello_world/BUILD +++ b/mobile/examples/objective-c/hello_world/BUILD @@ -1,8 +1,11 @@ -load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") +load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + objc_library( name = "appmain", srcs = glob([ diff --git a/mobile/examples/swift/async_await/BUILD b/mobile/examples/swift/async_await/BUILD index 3e38045ef5d4..8dc5323dce21 100644 --- a/mobile/examples/swift/async_await/BUILD +++ b/mobile/examples/swift/async_await/BUILD @@ -1,8 +1,11 @@ load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application") load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + swift_library( name = "appmain", srcs = glob(["*.swift"]), diff --git a/mobile/examples/swift/hello_world/BUILD b/mobile/examples/swift/hello_world/BUILD index 3ac85595124a..0eccd319f4e3 100644 --- a/mobile/examples/swift/hello_world/BUILD +++ b/mobile/examples/swift/hello_world/BUILD @@ -1,9 +1,12 @@ -load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application") load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") +load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + swift_library( name = "appmain", srcs = glob(["*.swift"]), diff --git a/mobile/experimental/swift/BUILD b/mobile/experimental/swift/BUILD index 09251ada9f2e..a0a2e2482a10 100644 --- a/mobile/experimental/swift/BUILD +++ b/mobile/experimental/swift/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_swift_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_swift_test( name = "quic_stream_test", srcs = [ diff --git a/mobile/library/BUILD b/mobile/library/BUILD index e581fc891e27..292cbb9b5357 100644 --- a/mobile/library/BUILD +++ b/mobile/library/BUILD @@ -1,7 +1,14 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") + licenses(["notice"]) # Apache 2 +envoy_mobile_package() + filegroup( name = "proguard_rules", - srcs = ["proguard.txt"], + srcs = [ + "java_proto_proguard.txt", + "proguard.txt", + ], visibility = ["//visibility:public"], ) diff --git a/mobile/library/cc/BUILD b/mobile/library/cc/BUILD index 193e6b40857b..4b04dcb82b89 100644 --- a/mobile/library/cc/BUILD +++ b/mobile/library/cc/BUILD @@ -1,8 +1,12 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", "envoy_select_envoy_mobile_request_compression") +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_mobile_package", +) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "engine_builder_lib", @@ -22,7 +26,9 @@ envoy_cc_library( deps = [ ":envoy_engine_cc_lib_no_stamp", "@envoy//source/common/common:assert_lib", + "@envoy//source/common/protobuf", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/compression/brotli/decompressor/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/compression/gzip/decompressor/v3:pkg_cc_proto", @@ -41,15 +47,7 @@ envoy_cc_library( "@envoy_mobile//library/common/extensions/filters/http/network_configuration:filter_cc_proto", "@envoy_mobile//library/common/extensions/filters/http/socket_tag:filter_cc_proto", "@envoy_mobile//library/common/types:matcher_data_lib", - ] + envoy_select_envoy_mobile_request_compression( - [ - "@envoy_api//envoy/extensions/compression/brotli/compressor/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/compression/gzip/compressor/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/http/composite/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/http/compressor/v3:pkg_cc_proto", - ], - "@envoy", - ), + ], ) envoy_cc_library( diff --git a/mobile/library/cc/engine.cc b/mobile/library/cc/engine.cc index 1b44741fccdd..bfd5a973307c 100644 --- a/mobile/library/cc/engine.cc +++ b/mobile/library/cc/engine.cc @@ -38,7 +38,8 @@ std::string Engine::dumpStats() { envoy_status_t Engine::terminate() { if (terminated_) { - throw std::runtime_error("attempting to double terminate Engine"); + IS_ENVOY_BUG("attempted to double terminate engine"); + return ENVOY_FAILURE; } envoy_status_t ret = terminate_engine(engine_, /* release */ false); terminated_ = true; diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index c8cfc5b99cd3..5db4f9731f84 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -1,7 +1,5 @@ #include "engine_builder.h" -#include - #include "envoy/config/metrics/v3/metrics_service.pb.h" #include "envoy/extensions/compression/brotli/decompressor/v3/brotli.pb.h" #include "envoy/extensions/compression/gzip/decompressor/v3/gzip.pb.h" @@ -9,21 +7,18 @@ #include "envoy/extensions/filters/http/decompressor/v3/decompressor.pb.h" #include "envoy/extensions/filters/http/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" #include "envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.pb.h" + +#if defined(__APPLE__) #include "envoy/extensions/network/dns_resolver/apple/v3/apple_dns_resolver.pb.h" +#endif + #include "envoy/extensions/network/dns_resolver/getaddrinfo/v3/getaddrinfo_dns_resolver.pb.h" #include "envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.pb.h" #include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" #include "envoy/extensions/transport_sockets/raw_buffer/v3/raw_buffer.pb.h" -#ifdef ENVOY_MOBILE_REQUEST_COMPRESSION -#include "envoy/extensions/compression/brotli/compressor/v3/brotli.pb.h" -#include "envoy/extensions/compression/gzip/compressor/v3/gzip.pb.h" -#include "envoy/extensions/filters/http/composite/v3/composite.pb.h" -#include "envoy/extensions/filters/http/compressor/v3/compressor.pb.h" -#include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h" -#endif - #include "source/common/http/matching/inputs.h" +#include "envoy/config/core/v3/base.pb.h" #include "source/extensions/clusters/dynamic_forward_proxy/cluster.h" #include "absl/strings/str_join.h" @@ -40,21 +35,15 @@ namespace Envoy { namespace Platform { -#ifdef ENVOY_GOOGLE_GRPC -XdsBuilder::XdsBuilder(std::string xds_server_address, const int xds_server_port) +#ifdef ENVOY_MOBILE_XDS +XdsBuilder::XdsBuilder(std::string xds_server_address, const uint32_t xds_server_port) : xds_server_address_(std::move(xds_server_address)), xds_server_port_(xds_server_port) {} -XdsBuilder& XdsBuilder::setAuthenticationToken(std::string token_header, std::string token) { - authentication_token_header_ = std::move(token_header); - authentication_token_ = std::move(token); - return *this; -} - -XdsBuilder& XdsBuilder::setJwtAuthenticationToken(std::string token, - const int token_lifetime_in_seconds) { - jwt_token_ = std::move(token); - jwt_token_lifetime_in_seconds_ = - token_lifetime_in_seconds > 0 ? token_lifetime_in_seconds : DefaultJwtTokenLifetimeSeconds; +XdsBuilder& XdsBuilder::addInitialStreamHeader(std::string header, std::string value) { + envoy::config::core::v3::HeaderValue header_value; + header_value.set_key(std::move(header)); + header_value.set_value(std::move(value)); + xds_initial_grpc_metadata_.emplace_back(std::move(header_value)); return *this; } @@ -63,11 +52,6 @@ XdsBuilder& XdsBuilder::setSslRootCerts(std::string root_certs) { return *this; } -XdsBuilder& XdsBuilder::setSni(std::string sni) { - sni_ = std::move(sni); - return *this; -} - XdsBuilder& XdsBuilder::addRuntimeDiscoveryService(std::string resource_name, const int timeout_in_seconds) { rtds_resource_name_ = std::move(resource_name); @@ -83,42 +67,24 @@ XdsBuilder& XdsBuilder::addClusterDiscoveryService(std::string cds_resources_loc return *this; } -void XdsBuilder::build(envoy::config::bootstrap::v3::Bootstrap* bootstrap) const { - auto* ads_config = bootstrap->mutable_dynamic_resources()->mutable_ads_config(); +void XdsBuilder::build(envoy::config::bootstrap::v3::Bootstrap& bootstrap) const { + auto* ads_config = bootstrap.mutable_dynamic_resources()->mutable_ads_config(); ads_config->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); ads_config->set_set_node_on_first_message_only(true); ads_config->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); + auto& grpc_service = *ads_config->add_grpc_services(); - grpc_service.mutable_google_grpc()->set_target_uri( - fmt::format(R"({}:{})", xds_server_address_, xds_server_port_)); - grpc_service.mutable_google_grpc()->set_stat_prefix("ads"); - if (!ssl_root_certs_.empty()) { - grpc_service.mutable_google_grpc() - ->mutable_channel_credentials() - ->mutable_ssl_credentials() - ->mutable_root_certs() - ->set_inline_string(ssl_root_certs_); - } + grpc_service.mutable_envoy_grpc()->set_cluster_name("base"); + grpc_service.mutable_envoy_grpc()->set_authority( + absl::StrCat(xds_server_address_, ":", xds_server_port_)); - if (!authentication_token_header_.empty() && !authentication_token_.empty()) { - auto* auth_token_metadata = grpc_service.add_initial_metadata(); - auth_token_metadata->set_key(authentication_token_header_); - auth_token_metadata->set_value(authentication_token_); - } else if (!jwt_token_.empty()) { - auto& jwt = *grpc_service.mutable_google_grpc() - ->add_call_credentials() - ->mutable_service_account_jwt_access(); - jwt.set_json_key(jwt_token_); - jwt.set_token_lifetime_seconds(jwt_token_lifetime_in_seconds_); - } - if (!sni_.empty()) { - auto& channel_args = - *grpc_service.mutable_google_grpc()->mutable_channel_args()->mutable_args(); - channel_args["grpc.default_authority"].set_string_value(sni_); + if (!xds_initial_grpc_metadata_.empty()) { + grpc_service.mutable_initial_metadata()->Assign(xds_initial_grpc_metadata_.begin(), + xds_initial_grpc_metadata_.end()); } if (!rtds_resource_name_.empty()) { - auto* layered_runtime = bootstrap->mutable_layered_runtime(); + auto* layered_runtime = bootstrap.mutable_layered_runtime(); auto* layer = layered_runtime->add_layers(); layer->set_name("rtds_layer"); auto* rtds_layer = layer->mutable_rtds_layer(); @@ -130,11 +96,11 @@ void XdsBuilder::build(envoy::config::bootstrap::v3::Bootstrap* bootstrap) const } if (enable_cds_) { - auto* cds_config = bootstrap->mutable_dynamic_resources()->mutable_cds_config(); + auto* cds_config = bootstrap.mutable_dynamic_resources()->mutable_cds_config(); if (cds_resources_locator_.empty()) { cds_config->mutable_ads(); } else { - bootstrap->mutable_dynamic_resources()->set_cds_resources_locator(cds_resources_locator_); + bootstrap.mutable_dynamic_resources()->set_cds_resources_locator(cds_resources_locator_); cds_config->mutable_api_config_source()->set_api_type( envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC); cds_config->mutable_api_config_source()->set_transport_api_version( @@ -142,12 +108,14 @@ void XdsBuilder::build(envoy::config::bootstrap::v3::Bootstrap* bootstrap) const } cds_config->mutable_initial_fetch_timeout()->set_seconds(cds_timeout_in_seconds_); cds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - bootstrap->add_node_context_params("cluster"); + bootstrap.add_node_context_params("cluster"); // Stat prefixes that we use in tests. auto* list = - bootstrap->mutable_stats_config()->mutable_stats_matcher()->mutable_inclusion_list(); + bootstrap.mutable_stats_config()->mutable_stats_matcher()->mutable_inclusion_list(); list->add_patterns()->set_exact("cluster_manager.active_clusters"); list->add_patterns()->set_exact("cluster_manager.cluster_added"); + list->add_patterns()->set_exact("cluster_manager.cluster_updated"); + list->add_patterns()->set_exact("cluster_manager.cluster_removed"); // Allow SDS related stats. list->add_patterns()->mutable_safe_regex()->set_regex("sds\\..*"); list->add_patterns()->mutable_safe_regex()->set_regex(".*\\.ssl_context_update_by_sds"); @@ -167,27 +135,10 @@ EngineBuilder& EngineBuilder::addLogLevel(LogLevel log_level) { } EngineBuilder& EngineBuilder::setOnEngineRunning(std::function closure) { - callbacks_->on_engine_running = closure; + callbacks_->on_engine_running = std::move(closure); return *this; } -#ifdef ENVOY_MOBILE_STATS_REPORTING -EngineBuilder& EngineBuilder::addStatsSinks(std::vector stats_sinks) { - stats_sinks_ = std::move(stats_sinks); - return *this; -} - -EngineBuilder& EngineBuilder::addGrpcStatsDomain(std::string stats_domain) { - stats_domain_ = std::move(stats_domain); - return *this; -} - -EngineBuilder& EngineBuilder::addStatsFlushSeconds(int stats_flush_seconds) { - stats_flush_seconds_ = stats_flush_seconds; - return *this; -} -#endif - EngineBuilder& EngineBuilder::addConnectTimeoutSeconds(int connect_timeout_seconds) { connect_timeout_seconds_ = connect_timeout_seconds; return *this; @@ -215,7 +166,12 @@ EngineBuilder& EngineBuilder::addDnsQueryTimeoutSeconds(int dns_query_timeout_se } EngineBuilder& EngineBuilder::addDnsPreresolveHostnames(const std::vector& hostnames) { - dns_preresolve_hostnames_ = std::move(hostnames); + // Add a default port of 443 for all hosts. We'll eventually change this API so it takes a single + // {host, pair} and it can be called multiple times. + dns_preresolve_hostnames_.clear(); + for (const std::string& hostname : hostnames) { + dns_preresolve_hostnames_.push_back({hostname /* host */, 443 /* port */}); + } return *this; } @@ -309,6 +265,11 @@ EngineBuilder& EngineBuilder::addQuicHint(std::string host, int port) { return *this; } +EngineBuilder& EngineBuilder::addQuicCanonicalSuffix(std::string suffix) { + quic_suffixes_.emplace_back(std::move(suffix)); + return *this; +} + #endif EngineBuilder& EngineBuilder::setForceAlwaysUsev6(bool value) { @@ -338,13 +299,22 @@ EngineBuilder& EngineBuilder::setNodeId(std::string node_id) { EngineBuilder& EngineBuilder::setNodeLocality(std::string region, std::string zone, std::string sub_zone) { - node_locality_ = {region, zone, sub_zone}; + node_locality_ = {std::move(region), std::move(zone), std::move(sub_zone)}; return *this; } -#ifdef ENVOY_GOOGLE_GRPC +EngineBuilder& EngineBuilder::setNodeMetadata(ProtobufWkt::Struct node_metadata) { + node_metadata_ = std::move(node_metadata); + return *this; +} + +#ifdef ENVOY_MOBILE_XDS EngineBuilder& EngineBuilder::setXds(XdsBuilder xds_builder) { xds_builder_ = std::move(xds_builder); + // Add the XdsBuilder's xDS server hostname and port to the list of DNS addresses to preresolve in + // the `base` DFP cluster. + dns_preresolve_hostnames_.push_back( + {xds_builder_->xds_server_address_ /* host */, xds_builder_->xds_server_port_ /* port */}); return *this; } #endif @@ -372,7 +342,7 @@ EngineBuilder& EngineBuilder::addNativeFilter(std::string name, std::string type return *this; } -EngineBuilder& EngineBuilder::addPlatformFilter(std::string name) { +EngineBuilder& EngineBuilder::addPlatformFilter(const std::string& name) { addNativeFilter( "envoy.filters.http.platform_bridge", absl::StrCat( @@ -384,7 +354,7 @@ EngineBuilder& EngineBuilder::addPlatformFilter(std::string name) { } EngineBuilder& EngineBuilder::setRuntimeGuard(std::string guard, bool value) { - runtime_guards_.push_back({guard, value}); + runtime_guards_.emplace_back(std::move(guard), value); return *this; } @@ -458,6 +428,9 @@ std::unique_ptr EngineBuilder::generate entry->set_hostname(host); entry->set_port(port); } + for (const auto& suffix : quic_suffixes_) { + cache_config.mutable_alternate_protocols_cache_options()->add_canonical_suffixes(suffix); + } auto* cache_filter = hcm->add_http_filters(); cache_filter->set_name("alternate_protocols_cache"); cache_filter->mutable_typed_config()->PackFrom(cache_config); @@ -501,63 +474,6 @@ std::unique_ptr EngineBuilder::generate brotli_filter->set_name("envoy.filters.http.decompressor"); brotli_filter->mutable_typed_config()->PackFrom(decompressor_config); } -#ifdef ENVOY_MOBILE_REQUEST_COMPRESSION - auto* compressor_filter = hcm->add_http_filters(); - compressor_filter->set_name("envoy.filters.http.compressor"); - envoy::extensions::common::matching::v3::ExtensionWithMatcher extension_config; - extension_config.mutable_extension_config()->set_name("composite"); - envoy::extensions::filters::http::composite::v3::Composite composite_config; - extension_config.mutable_extension_config()->mutable_typed_config()->PackFrom(composite_config); - auto* matcher_tree = extension_config.mutable_xds_matcher()->mutable_matcher_tree(); - auto* matcher_input = matcher_tree->mutable_input(); - matcher_input->set_name("request-headers"); - envoy::type::matcher::v3::HttpRequestHeaderMatchInput request_header_match_input; - request_header_match_input.set_header_name("x-envoy-mobile-compression"); - matcher_input->mutable_typed_config()->PackFrom(request_header_match_input); - auto* exact_match_map = matcher_tree->mutable_exact_match_map()->mutable_map(); - ::xds::type::matcher::v3::Matcher_OnMatch on_gzip_match; - auto* on_gzip_match_action = on_gzip_match.mutable_action(); - on_gzip_match_action->set_name("composite-action"); - envoy::extensions::filters::http::composite::v3::ExecuteFilterAction execute_gzip_filter_action; - envoy::extensions::compression::gzip::compressor::v3::Gzip gzip_config; - gzip_config.mutable_window_bits()->set_value(15); - envoy::extensions::filters::http::compressor::v3::Compressor gzip_compressor_config; - gzip_compressor_config.mutable_compressor_library()->set_name("gzip"); - gzip_compressor_config.mutable_compressor_library()->mutable_typed_config()->PackFrom( - gzip_config); - auto* gzip_common_request = - gzip_compressor_config.mutable_request_direction_config()->mutable_common_config(); - gzip_common_request->mutable_enabled()->mutable_default_value()->set_value(true); - auto* gzip_common_response = - gzip_compressor_config.mutable_response_direction_config()->mutable_common_config(); - gzip_common_response->mutable_enabled()->mutable_default_value()->set_value(false); - execute_gzip_filter_action.mutable_typed_config()->set_name("envoy.filters.http.compressor"); - execute_gzip_filter_action.mutable_typed_config()->mutable_typed_config()->PackFrom( - gzip_compressor_config); - on_gzip_match_action->mutable_typed_config()->PackFrom(execute_gzip_filter_action); - (*exact_match_map)["gzip"] = on_gzip_match; - ::xds::type::matcher::v3::Matcher_OnMatch on_brotli_match; - auto* on_brotli_match_action = on_brotli_match.mutable_action(); - on_brotli_match_action->set_name("composite-action"); - envoy::extensions::filters::http::composite::v3::ExecuteFilterAction execute_brotli_filter_action; - envoy::extensions::compression::brotli::compressor::v3::Brotli brotli_config; - envoy::extensions::filters::http::compressor::v3::Compressor brotli_compressor_config; - brotli_compressor_config.mutable_compressor_library()->set_name("text_optimized"); - brotli_compressor_config.mutable_compressor_library()->mutable_typed_config()->PackFrom( - brotli_config); - auto* brotli_common_request = - brotli_compressor_config.mutable_request_direction_config()->mutable_common_config(); - brotli_common_request->mutable_enabled()->mutable_default_value()->set_value(true); - auto* brotli_common_response = - brotli_compressor_config.mutable_response_direction_config()->mutable_common_config(); - brotli_common_response->mutable_enabled()->mutable_default_value()->set_value(false); - execute_brotli_filter_action.mutable_typed_config()->set_name("envoy.filters.http.compressor"); - execute_brotli_filter_action.mutable_typed_config()->mutable_typed_config()->PackFrom( - brotli_compressor_config); - on_brotli_match_action->mutable_typed_config()->PackFrom(execute_brotli_filter_action); - (*exact_match_map)["brotli"] = on_brotli_match; - compressor_filter->mutable_typed_config()->PackFrom(extension_config); -#endif if (socket_tagging_filter_) { envoymobile::extensions::filters::http::socket_tag::SocketTag tag_config; auto* tag_filter = hcm->add_http_filters(); @@ -617,10 +533,10 @@ std::unique_ptr EngineBuilder::generate dns_cache_config->mutable_typed_dns_resolver_config()->mutable_typed_config()->PackFrom( resolver_config); - for (auto& hostname : dns_preresolve_hostnames_) { + for (const auto& [host, port] : dns_preresolve_hostnames_) { envoy::config::core::v3::SocketAddress* address = dns_cache_config->add_preresolve_hostnames(); - address->set_address(hostname); - address->set_port_value(443); + address->set_address(host); + address->set_port_value(port); } auto* dfp_filter = hcm->add_http_filters(); @@ -656,23 +572,34 @@ std::unique_ptr EngineBuilder::generate validation->set_trust_chain_verification(envoy::extensions::transport_sockets::tls::v3:: CertificateValidationContext::ACCEPT_UNTRUSTED); } + if (platform_certificates_validation_on_) { envoy_mobile::extensions::cert_validator::platform_bridge::PlatformBridgeCertValidator validator; validation->mutable_custom_validator_config()->set_name( "envoy_mobile.cert_validator.platform_bridge_cert_validator"); validation->mutable_custom_validator_config()->mutable_typed_config()->PackFrom(validator); - } else { - const char* inline_certs = "" + std::string certs; +#ifdef ENVOY_MOBILE_XDS + if (xds_builder_ && !xds_builder_->ssl_root_certs_.empty()) { + certs = xds_builder_->ssl_root_certs_; + } +#endif + + if (certs.empty()) { + // The xDS builder doesn't supply root certs, so we'll use the certs packed with Envoy Mobile, + // if the build config allows it. + const char* inline_certs = "" #ifndef EXCLUDE_CERTIFICATES #include "library/common/config/certificates.inc" #endif - ""; - // The certificates in certificates.inc are prefixed with 2 spaces per - // line to be ingressed into YAML. - std::string certs = inline_certs; - absl::StrReplaceAll({{"\n ", "\n"}}, &certs); + ""; + certs = inline_certs; + // The certificates in certificates.inc are prefixed with 2 spaces per + // line to be ingressed into YAML. + absl::StrReplaceAll({{"\n ", "\n"}}, &certs); + } validation->mutable_trusted_ca()->set_inline_string(certs); } envoy::extensions::transport_sockets::http_11_proxy::v3::Http11ProxyUpstreamTransport @@ -686,27 +613,6 @@ std::unique_ptr EngineBuilder::generate envoy::extensions::upstreams::http::v3::HttpProtocolOptions h2_protocol_options; h2_protocol_options.mutable_explicit_http_config()->mutable_http2_protocol_options(); - if (!stats_domain_.empty()) { - // Stats cluster - auto* stats_cluster = static_resources->add_clusters(); - stats_cluster->set_name("stats"); - stats_cluster->set_type(envoy::config::cluster::v3::Cluster::LOGICAL_DNS); - stats_cluster->mutable_connect_timeout()->set_seconds(connect_timeout_seconds_); - stats_cluster->mutable_dns_refresh_rate()->set_seconds(dns_refresh_seconds_); - stats_cluster->mutable_transport_socket()->CopyFrom(base_tls_socket); - stats_cluster->mutable_load_assignment()->set_cluster_name("stats"); - auto* address = stats_cluster->mutable_load_assignment() - ->add_endpoints() - ->add_lb_endpoints() - ->mutable_endpoint() - ->mutable_address(); - address->mutable_socket_address()->set_address(stats_domain_); - address->mutable_socket_address()->set_port_value(443); - (*stats_cluster->mutable_typed_extension_protocol_options()) - ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] - .PackFrom(h2_protocol_options); - stats_cluster->mutable_wait_for_warm_on_init(); - } // Base cluster config (DFP cluster config) auto* base_cluster = static_resources->add_clusters(); @@ -826,6 +732,11 @@ std::unique_ptr EngineBuilder::generate entry->set_hostname(host); entry->set_port(port); } + for (const auto& suffix : quic_suffixes_) { + alpn_options.mutable_auto_config() + ->mutable_alternate_protocols_cache_options() + ->add_canonical_suffixes(suffix); + } base_cluster->mutable_transport_socket()->mutable_typed_config()->PackFrom(h3_proxy_socket); (*base_cluster->mutable_typed_extension_protocol_options()) @@ -834,8 +745,6 @@ std::unique_ptr EngineBuilder::generate } // Set up stats. - bootstrap->mutable_stats_flush_interval()->set_seconds(stats_flush_seconds_); - bootstrap->mutable_stats_sinks(); auto* list = bootstrap->mutable_stats_config()->mutable_stats_matcher()->mutable_inclusion_list(); list->add_patterns()->set_prefix("cluster.base.upstream_rq_"); list->add_patterns()->set_prefix("cluster.stats.upstream_rq_"); @@ -868,6 +777,9 @@ std::unique_ptr EngineBuilder::generate node->mutable_locality()->set_zone(node_locality_->zone); node->mutable_locality()->set_sub_zone(node_locality_->sub_zone); } + if (node_metadata_.has_value()) { + *node->mutable_metadata() = *node_metadata_; + } ProtobufWkt::Struct& metadata = *node->mutable_metadata(); (*metadata.mutable_fields())["app_id"].set_string_value(app_id_); (*metadata.mutable_fields())["app_version"].set_string_value(app_version_); @@ -895,30 +807,10 @@ std::unique_ptr EngineBuilder::generate bootstrap->mutable_typed_dns_resolver_config()->CopyFrom( *dns_cache_config->mutable_typed_dns_resolver_config()); - for (const std::string& sink_yaml : stats_sinks_) { -#ifdef ENVOY_ENABLE_YAML - auto* sink = bootstrap->add_stats_sinks(); - MessageUtil::loadFromYaml(sink_yaml, *sink, ProtobufMessage::getStrictValidationVisitor()); -#else - UNREFERENCED_PARAMETER(sink_yaml); - IS_ENVOY_BUG("stats sinks can not be added when YAML is compiled out."); -#endif - } - if (!stats_domain_.empty()) { - envoy::config::metrics::v3::MetricsServiceConfig metrics_config; - metrics_config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("stats"); - metrics_config.mutable_report_counters_as_deltas()->set_value(true); - metrics_config.set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); - metrics_config.set_emit_tags_as_labels(true); - auto* sink = bootstrap->add_stats_sinks(); - sink->set_name("envoy.metrics_service"); - sink->mutable_typed_config()->PackFrom(metrics_config); - } - bootstrap->mutable_dynamic_resources(); -#ifdef ENVOY_GOOGLE_GRPC +#ifdef ENVOY_MOBILE_XDS if (xds_builder_) { - xds_builder_->build(bootstrap.get()); + xds_builder_->build(*bootstrap); } #endif @@ -943,27 +835,31 @@ EngineSharedPtr EngineBuilder::build() { for (const auto& [name, store] : key_value_stores_) { // TODO(goaway): This leaks, but it's tied to the life of the engine. - auto* api = new envoy_kv_store(); - *api = store->asEnvoyKeyValueStore(); - register_platform_api(name.c_str(), api); + if (!Api::External::retrieveApi(name, true)) { + auto* api = new envoy_kv_store(); + *api = store->asEnvoyKeyValueStore(); + register_platform_api(name.c_str(), api); + } } for (const auto& [name, accessor] : string_accessors_) { // TODO(RyanTheOptimist): This leaks, but it's tied to the life of the engine. - auto* api = new envoy_string_accessor(); - *api = StringAccessor::asEnvoyStringAccessor(accessor); - register_platform_api(name.c_str(), api); + if (!Api::External::retrieveApi(name, true)) { + auto* api = new envoy_string_accessor(); + *api = StringAccessor::asEnvoyStringAccessor(accessor); + register_platform_api(name.c_str(), api); + } } Engine* engine = new Engine(envoy_engine); if (auto cast_engine = reinterpret_cast(envoy_engine)) { - auto options = std::make_unique(); + auto options = std::make_unique(); std::unique_ptr bootstrap = generateBootstrap(); if (bootstrap) { options->setConfigProto(std::move(bootstrap)); } - options->setLogLevel(options->parseAndValidateLogLevel(logLevelToString(log_level_).c_str())); + ENVOY_BUG(options->setLogLevel(logLevelToString(log_level_)).ok(), "invalid log level"); options->setConcurrency(1); cast_engine->run(std::move(options)); } diff --git a/mobile/library/cc/engine_builder.h b/mobile/library/cc/engine_builder.h index 70d3a3dc526d..8e4b8b42aaa6 100644 --- a/mobile/library/cc/engine_builder.h +++ b/mobile/library/cc/engine_builder.h @@ -6,6 +6,9 @@ #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/core/v3/base.pb.h" + +#include "source/common/protobuf/protobuf.h" #include "absl/container/flat_hash_map.h" #include "absl/types/optional.h" @@ -20,7 +23,6 @@ namespace Envoy { namespace Platform { -constexpr int DefaultJwtTokenLifetimeSeconds = 60 * 60 * 24 * 90; // 90 days constexpr int DefaultXdsTimeout = 5; // Forward declaration so it can be referenced by XdsBuilder. @@ -34,7 +36,7 @@ struct NodeLocality { std::string sub_zone; }; -#ifdef ENVOY_GOOGLE_GRPC +#ifdef ENVOY_MOBILE_XDS // A class for building the xDS configuration for the Envoy Mobile engine. // xDS is a protocol for dynamic configuration of Envoy instances, more information can be found in: // https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol. @@ -47,42 +49,24 @@ class XdsBuilder final { // (https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#aggregated-xds-ads). // `xds_server_port`: the port on which the xDS management server listens for ADS discovery // requests. - XdsBuilder(std::string xds_server_address, const int xds_server_port); + XdsBuilder(std::string xds_server_address, const uint32_t xds_server_port); - // Sets the authentication token in the gRPC headers used to authenticate to the xDS management + // Adds a header to the initial HTTP metadata headers sent on the gRPC stream. + // + // A common use for the initial metadata headers is for authentication to the xDS management // server. // // For example, if using API keys to authenticate to Traffic Director on GCP (see // https://cloud.google.com/docs/authentication/api-keys for details), invoke: - // builder.setAuthenticationToken("x-goog-api-key", api_key_token) - // - // If this method is called, then don't call setJwtAuthenticationToken. - // - // `token_header`: the header name for which the the `token` will be set as a value. - // `token`: the authentication token. - XdsBuilder& setAuthenticationToken(std::string token_header, std::string token); - - // Sets JWT as the authentication method to the xDS management server, using the given token. - // - // If setAuthenticationToken is called, then invocations of this method will be ignored. - // - // `token`: the JWT token used to authenticate the client to the xDS management server. - // `token_lifetime_in_seconds`: the lifetime of the JWT token, in seconds. If none - // (or 0) is specified, then DefaultJwtTokenLifetimeSeconds is used. - // TODO(abeyad): Deprecate and remove this. - XdsBuilder& - setJwtAuthenticationToken(std::string token, - int token_lifetime_in_seconds = DefaultJwtTokenLifetimeSeconds); + // builder.addInitialStreamHeader("x-goog-api-key", api_key_token) + // .addInitialStreamHeader("X-Android-Package", app_package_name) + // .addInitialStreamHeader("X-Android-Cert", sha1_key_fingerprint); + XdsBuilder& addInitialStreamHeader(std::string header, std::string value); // Sets the PEM-encoded server root certificates used to negotiate the TLS handshake for the gRPC // connection. If no root certs are specified, the operating system defaults are used. XdsBuilder& setSslRootCerts(std::string root_certs); - // Sets the SNI (https://datatracker.ietf.org/doc/html/rfc6066#section-3) on the TLS handshake - // and the authority HTTP header. If not set, the SNI is set by default to the xDS server address - // and the authority HTTP header is not set. - XdsBuilder& setSni(std::string sni); - // Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration, // to retrieve dynamic runtime configuration via the xDS management server. // @@ -115,20 +99,16 @@ class XdsBuilder final { // This method takes in a modifiable Bootstrap proto pointer because returning a new Bootstrap // proto would rely on proto's MergeFrom behavior, which can lead to unexpected results in the // Bootstrap config. - void build(envoy::config::bootstrap::v3::Bootstrap* bootstrap) const; + void build(envoy::config::bootstrap::v3::Bootstrap& bootstrap) const; private: // Required so that EngineBuilder can call the XdsBuilder's protected build() method. friend class EngineBuilder; std::string xds_server_address_; - int xds_server_port_; - std::string authentication_token_header_; - std::string authentication_token_; - std::string jwt_token_; - int jwt_token_lifetime_in_seconds_ = DefaultJwtTokenLifetimeSeconds; + uint32_t xds_server_port_; + std::vector xds_initial_grpc_metadata_; std::string ssl_root_certs_; - std::string sni_; std::string rtds_resource_name_; int rtds_timeout_in_seconds_ = DefaultXdsTimeout; bool enable_cds_ = false; @@ -173,6 +153,7 @@ class EngineBuilder { EngineBuilder& setHttp3ConnectionOptions(std::string options); EngineBuilder& setHttp3ClientConnectionOptions(std::string options); EngineBuilder& addQuicHint(std::string host, int port); + EngineBuilder& addQuicCanonicalSuffix(std::string suffix); #endif EngineBuilder& enableInterfaceBinding(bool interface_binding_on); EngineBuilder& enableDrainPostDnsRefresh(bool drain_post_dns_refresh_on); @@ -182,7 +163,9 @@ class EngineBuilder { EngineBuilder& setNodeId(std::string node_id); // Sets the node.locality field in the Bootstrap configuration. EngineBuilder& setNodeLocality(std::string region, std::string zone, std::string sub_zone); -#ifdef ENVOY_GOOGLE_GRPC + // Sets the node.metadata field in the Bootstrap configuration. + EngineBuilder& setNodeMetadata(ProtobufWkt::Struct node_metadata); +#ifdef ENVOY_MOBILE_XDS // Sets the xDS configuration for the Envoy Mobile engine. // // `xds_builder`: the XdsBuilder instance used to specify the xDS configuration options. @@ -190,15 +173,15 @@ class EngineBuilder { #endif EngineBuilder& enableDnsCache(bool dns_cache_on, int save_interval_seconds = 1); EngineBuilder& setForceAlwaysUsev6(bool value); + // Adds the hostnames that should be pre-resolved by DNS prior to the first request issued for + // that host. When invoked, any previous preresolve hostname entries get cleared and only the ones + // provided in the hostnames argument get set. + // TODO(abeyad): change this method and the other language APIs to take a {host,port} pair. + // E.g. addDnsPreresolveHost(std::string host, uint32_t port); EngineBuilder& addDnsPreresolveHostnames(const std::vector& hostnames); EngineBuilder& addNativeFilter(std::string name, std::string typed_config); -#ifdef ENVOY_MOBILE_STATS_REPORTING - EngineBuilder& addStatsSinks(std::vector stat_sinks); - EngineBuilder& addGrpcStatsDomain(std::string stats_domain); - EngineBuilder& addStatsFlushSeconds(int stats_flush_seconds); -#endif - EngineBuilder& addPlatformFilter(std::string name); + EngineBuilder& addPlatformFilter(const std::string& name); EngineBuilder& setRuntimeGuard(std::string guard, bool value); @@ -223,7 +206,6 @@ class EngineBuilder { LogLevel log_level_ = LogLevel::info; EngineCallbacksSharedPtr callbacks_; - std::string stats_domain_; int connect_timeout_seconds_ = 30; int dns_refresh_seconds_ = 60; int dns_failure_refresh_seconds_base_ = 2; @@ -232,7 +214,6 @@ class EngineBuilder { bool use_system_resolver_ = true; int h2_connection_keepalive_idle_interval_milliseconds_ = 100000000; int h2_connection_keepalive_timeout_seconds_ = 10; - int stats_flush_seconds_ = 60; std::string app_version_ = "unspecified"; std::string app_id_ = "unspecified"; std::string device_os_ = "unspecified"; @@ -244,6 +225,7 @@ class EngineBuilder { bool platform_certificates_validation_on_ = false; std::string node_id_; absl::optional node_locality_ = absl::nullopt; + absl::optional node_metadata_ = absl::nullopt; bool dns_cache_on_ = false; int dns_cache_save_interval_seconds_ = 1; @@ -256,18 +238,18 @@ class EngineBuilder { std::string http3_connection_options_ = ""; std::string http3_client_connection_options_ = ""; std::vector> quic_hints_; + std::vector quic_suffixes_; bool always_use_v6_ = false; int dns_min_refresh_seconds_ = 60; int max_connections_per_host_ = 7; - std::vector stats_sinks_; std::vector native_filter_chain_; - std::vector dns_preresolve_hostnames_; + std::vector> dns_preresolve_hostnames_; std::vector> runtime_guards_; absl::flat_hash_map string_accessors_; -#ifdef ENVOY_GOOGLE_GRPC +#ifdef ENVOY_MOBILE_XDS absl::optional xds_builder_ = absl::nullopt; #endif }; diff --git a/mobile/library/cc/engine_callbacks.cc b/mobile/library/cc/engine_callbacks.cc index d76bfa616b94..12995f293950 100644 --- a/mobile/library/cc/engine_callbacks.cc +++ b/mobile/library/cc/engine_callbacks.cc @@ -7,7 +7,9 @@ namespace { void c_on_engine_running(void* context) { auto engine_callbacks = *static_cast(context); - engine_callbacks->on_engine_running(); + std::function on_engine_running_cb = std::move(engine_callbacks->on_engine_running); + engine_callbacks->on_engine_running = {}; + on_engine_running_cb(); } void c_on_exit(void* context) { diff --git a/mobile/library/cc/log_level.cc b/mobile/library/cc/log_level.cc index 066771d90eec..5fe40abe5144 100644 --- a/mobile/library/cc/log_level.cc +++ b/mobile/library/cc/log_level.cc @@ -2,6 +2,9 @@ #include +#include "source/common/common/assert.h" +#include "source/common/common/macros.h" + namespace Envoy { namespace Platform { @@ -18,7 +21,8 @@ std::string logLevelToString(LogLevel method) { } } - throw std::out_of_range("unknown log level type"); + IS_ENVOY_BUG("unknown log level, defaulting to off"); + return LOG_LEVEL_LOOKUP[ARRAY_SIZE(LOG_LEVEL_LOOKUP) - 1].second; } LogLevel logLevelFromString(const std::string& str) { @@ -28,7 +32,8 @@ LogLevel logLevelFromString(const std::string& str) { } } - throw std::out_of_range("unknown log level type"); + IS_ENVOY_BUG("unknown log level, defaulting to off"); + return LOG_LEVEL_LOOKUP[ARRAY_SIZE(LOG_LEVEL_LOOKUP) - 1].first; } } // namespace Platform diff --git a/mobile/library/cc/request_headers.cc b/mobile/library/cc/request_headers.cc index 4be6cbdf1178..2563b5769230 100644 --- a/mobile/library/cc/request_headers.cc +++ b/mobile/library/cc/request_headers.cc @@ -14,11 +14,7 @@ const std::string& RequestHeaders::authority() const { return (*this)[":authorit const std::string& RequestHeaders::path() const { return (*this)[":path"][0]; } absl::optional RequestHeaders::retryPolicy() const { - try { - return absl::optional(RetryPolicy::fromRawHeaderMap(allHeaders())); - } catch (const std::exception&) { - return absl::optional(); - } + return absl::optional(RetryPolicy::fromRawHeaderMap(allHeaders())); } RequestHeadersBuilder RequestHeaders::toRequestHeadersBuilder() const { diff --git a/mobile/library/cc/request_headers_builder.cc b/mobile/library/cc/request_headers_builder.cc index e0755e2848ee..ca1193acfdc1 100644 --- a/mobile/library/cc/request_headers_builder.cc +++ b/mobile/library/cc/request_headers_builder.cc @@ -36,21 +36,6 @@ RequestHeadersBuilder& RequestHeadersBuilder::addRetryPolicy(const RetryPolicy& return *this; } -RequestHeadersBuilder& -RequestHeadersBuilder::enableRequestCompression(CompressionAlgorithm algorithm) { - std::string value; - switch (algorithm) { - case CompressionAlgorithm::Gzip: - value = "gzip"; - break; - case CompressionAlgorithm::Brotli: - value = "brotli"; - break; - } - internalSet("x-envoy-mobile-compression", std::vector{value}); - return *this; -} - RequestHeaders RequestHeadersBuilder::build() const { return RequestHeaders(allHeaders()); } } // namespace Platform diff --git a/mobile/library/cc/request_headers_builder.h b/mobile/library/cc/request_headers_builder.h index 1e9a71466582..4daa65196e36 100644 --- a/mobile/library/cc/request_headers_builder.h +++ b/mobile/library/cc/request_headers_builder.h @@ -10,12 +10,6 @@ namespace Envoy { namespace Platform { -// Available algorithms to compress requests. -enum CompressionAlgorithm { - Gzip, - Brotli, -}; - class RequestHeaders; struct RetryPolicy; @@ -26,7 +20,6 @@ class RequestHeadersBuilder : public HeadersBuilder { RequestHeadersBuilder(RequestMethod request_method, absl::string_view url); RequestHeadersBuilder& addRetryPolicy(const RetryPolicy& retry_policy); - RequestHeadersBuilder& enableRequestCompression(CompressionAlgorithm algorithm); RequestHeaders build() const; diff --git a/mobile/library/cc/request_method.cc b/mobile/library/cc/request_method.cc index b2e3e37023ee..5fe3913be8cc 100644 --- a/mobile/library/cc/request_method.cc +++ b/mobile/library/cc/request_method.cc @@ -2,6 +2,8 @@ #include +#include "source/common/common/assert.h" + #include "absl/strings/string_view.h" namespace Envoy { @@ -25,7 +27,8 @@ absl::string_view requestMethodToString(RequestMethod method) { } } - throw std::out_of_range("unknown request method type"); + IS_ENVOY_BUG("unknown method"); + return ""; } RequestMethod requestMethodFromString(absl::string_view str) { @@ -35,7 +38,8 @@ RequestMethod requestMethodFromString(absl::string_view str) { } } - throw std::out_of_range("unknown request method type"); + IS_ENVOY_BUG("unknown method"); + return REQUEST_METHOD_LOOKUP[0].first; } } // namespace Platform diff --git a/mobile/library/cc/response_headers.cc b/mobile/library/cc/response_headers.cc index 995825b35b2f..1dcf44cee074 100644 --- a/mobile/library/cc/response_headers.cc +++ b/mobile/library/cc/response_headers.cc @@ -5,7 +5,7 @@ namespace Platform { int ResponseHeaders::httpStatus() const { if (!contains(":status")) { - throw std::logic_error("ResponseHeaders does not contain :status"); + return 0; } return stoi((*this)[":status"][0]); } diff --git a/mobile/library/cc/retry_policy.cc b/mobile/library/cc/retry_policy.cc index f08b0fdffbac..37f457a5ee39 100644 --- a/mobile/library/cc/retry_policy.cc +++ b/mobile/library/cc/retry_policy.cc @@ -3,34 +3,6 @@ namespace Envoy { namespace Platform { -static const std::pair RETRY_RULE_LOOKUP[]{ - {RetryRule::Status5xx, "5xx"}, - {RetryRule::GatewayError, "gateway-error"}, - {RetryRule::ConnectFailure, "connect-failure"}, - {RetryRule::RefusedStream, "refused-stream"}, - {RetryRule::Retriable4xx, "retriable-4xx"}, - {RetryRule::RetriableHeaders, "retriable-headers"}, - {RetryRule::Reset, "reset"}, -}; - -std::string retryRuleToString(RetryRule retry_rule) { - for (const auto& pair : RETRY_RULE_LOOKUP) { - if (pair.first == retry_rule) { - return pair.second; - } - } - throw std::invalid_argument("invalid retry rule"); -} - -RetryRule retryRuleFromString(const std::string& str) { - for (const auto& pair : RETRY_RULE_LOOKUP) { - if (pair.second == str) { - return pair.first; - } - } - throw std::invalid_argument("invalid retry rule"); -} - RawHeaderMap RetryPolicy::asRawHeaderMap() const { RawHeaderMap outbound_headers{ {"x-envoy-max-retries", {std::to_string(max_retry_count)}}, @@ -45,7 +17,7 @@ RawHeaderMap RetryPolicy::asRawHeaderMap() const { std::vector retry_on_copy; retry_on_copy.reserve(retry_on.size()); for (const auto& retry_rule : retry_on) { - retry_on_copy.push_back(retryRuleToString(retry_rule)); + retry_on_copy.push_back(retry_rule); } if (!retry_status_codes.empty()) { @@ -89,7 +61,7 @@ RetryPolicy RetryPolicy::fromRawHeaderMap(const RawHeaderMap& headers) { has_retriable_status_codes = true; continue; } - retry_policy.retry_on.push_back(retryRuleFromString(retry_rule_str)); + retry_policy.retry_on.push_back(retry_rule_str); } } diff --git a/mobile/library/cc/retry_policy.h b/mobile/library/cc/retry_policy.h index cd9142331597..b6f76ad064e3 100644 --- a/mobile/library/cc/retry_policy.h +++ b/mobile/library/cc/retry_policy.h @@ -12,22 +12,9 @@ namespace Platform { class RequestHeaders; -enum RetryRule { - Status5xx, - GatewayError, - ConnectFailure, - RefusedStream, - Retriable4xx, - RetriableHeaders, - Reset, -}; - -std::string retryRuleToString(RetryRule retry_rule); -RetryRule retryRuleFromString(const std::string& str); - struct RetryPolicy { int max_retry_count; - std::vector retry_on; + std::vector retry_on; std::vector retry_status_codes; absl::optional per_try_timeout_ms; absl::optional total_upstream_timeout_ms; diff --git a/mobile/library/cc/stream_prototype.cc b/mobile/library/cc/stream_prototype.cc index 7c03da7e9d64..8a86ee100c3d 100644 --- a/mobile/library/cc/stream_prototype.cc +++ b/mobile/library/cc/stream_prototype.cc @@ -9,10 +9,10 @@ StreamPrototype::StreamPrototype(EngineSharedPtr engine) : engine_(engine) { callbacks_ = std::make_shared(); } -StreamSharedPtr StreamPrototype::start(bool explicit_flow_control, uint64_t min_delivery_size) { +StreamSharedPtr StreamPrototype::start(bool explicit_flow_control) { auto envoy_stream = init_stream(engine_->engine_); start_stream(engine_->engine_, envoy_stream, callbacks_->asEnvoyHttpCallbacks(), - explicit_flow_control, min_delivery_size); + explicit_flow_control); return std::make_shared(engine_->engine_, envoy_stream); } diff --git a/mobile/library/cc/stream_prototype.h b/mobile/library/cc/stream_prototype.h index 80185c2aaff7..3278a7a2f0de 100644 --- a/mobile/library/cc/stream_prototype.h +++ b/mobile/library/cc/stream_prototype.h @@ -20,7 +20,7 @@ class StreamPrototype { public: StreamPrototype(EngineSharedPtr engine); - StreamSharedPtr start(bool explicit_flow_control = false, uint64_t min_delivery_size = 0); + StreamSharedPtr start(bool explicit_flow_control = false); StreamPrototype& setOnHeaders(OnHeadersCallback closure); StreamPrototype& setOnData(OnDataCallback closure); diff --git a/mobile/library/common/BUILD b/mobile/library/common/BUILD index d46174daba41..9969ada7b673 100644 --- a/mobile/library/common/BUILD +++ b/mobile/library/common/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", "envoy_select_signal_trace") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package", "envoy_select_signal_trace") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "envoy_main_interface_lib", @@ -18,11 +18,9 @@ envoy_cc_library( srcs = [ "engine.cc", "engine.h", - "engine_handle.cc", "main_interface.cc", ], hdrs = [ - "engine_handle.h", "main_interface.h", ], repository = "@envoy", @@ -39,6 +37,7 @@ envoy_cc_library( "//library/common/types:c_types_lib", "@envoy//envoy/server:lifecycle_notifier_interface", "@envoy//envoy/stats:stats_interface", + "@envoy//source/common/runtime:runtime_lib", "@envoy_build_config//:extension_registry", ], ) @@ -60,6 +59,7 @@ envoy_cc_library( "@envoy//source/common/common:random_generator_lib", "@envoy//source/common/runtime:runtime_lib", "@envoy//source/exe:stripped_main_base_lib", + "@envoy//source/server:null_overload_manager_lib", ] + envoy_select_signal_trace( ["@envoy//source/common/signal:sigaction_lib"], "@envoy", diff --git a/mobile/library/common/api/BUILD b/mobile/library/common/api/BUILD index babc5d83a910..47f6169e9f96 100644 --- a/mobile/library/common/api/BUILD +++ b/mobile/library/common/api/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "external_api_lib", diff --git a/mobile/library/common/api/external.cc b/mobile/library/common/api/external.cc index 34c705d75f80..6b61712e34e0 100644 --- a/mobile/library/common/api/external.cc +++ b/mobile/library/common/api/external.cc @@ -24,9 +24,11 @@ void registerApi(std::string name, void* api) { // before any reads occur. // TODO(alyssawilk, abeyad): gracefully handle the case where an Api by the given name is not // registered. -void* retrieveApi(std::string name) { +void* retrieveApi(std::string name, bool allow_absent) { void* api = registry_[name]; - RELEASE_ASSERT(api != nullptr, fmt::format("{} not registered", name)); + if (!allow_absent) { + RELEASE_ASSERT(api != nullptr, fmt::format("{} not registered", name)); + } return api; } diff --git a/mobile/library/common/api/external.h b/mobile/library/common/api/external.h index 3617179f2538..1097ff49cf2a 100644 --- a/mobile/library/common/api/external.h +++ b/mobile/library/common/api/external.h @@ -14,7 +14,7 @@ void registerApi(std::string name, void* api); /** * Retrieve an external runtime API for usage (e.g. in extensions). */ -void* retrieveApi(std::string name); +void* retrieveApi(std::string name, bool allow_absent = false); } // namespace External } // namespace Api diff --git a/mobile/library/common/bridge/BUILD b/mobile/library/common/bridge/BUILD index a9836b16346c..41cdaf96b2df 100644 --- a/mobile/library/common/bridge/BUILD +++ b/mobile/library/common/bridge/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "utility_lib", diff --git a/mobile/library/common/buffer/BUILD b/mobile/library/common/buffer/BUILD index 2df0625dad07..d85687bb8036 100644 --- a/mobile/library/common/buffer/BUILD +++ b/mobile/library/common/buffer/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "bridge_fragment_lib", diff --git a/mobile/library/common/common/BUILD b/mobile/library/common/common/BUILD index 49e1e0342fb8..a0cc2959038d 100644 --- a/mobile/library/common/common/BUILD +++ b/mobile/library/common/common/BUILD @@ -1,9 +1,9 @@ load("@bazel_skylib//lib:selects.bzl", "selects") -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() selects.config_setting_group( name = "use_android_system_helper", diff --git a/mobile/library/common/common/default_system_helper_android.cc b/mobile/library/common/common/default_system_helper_android.cc index 0bc2062217ea..fca30e45e38f 100644 --- a/mobile/library/common/common/default_system_helper_android.cc +++ b/mobile/library/common/common/default_system_helper_android.cc @@ -5,15 +5,15 @@ namespace Envoy { bool DefaultSystemHelper::isCleartextPermitted(absl::string_view hostname) { - return is_cleartext_permitted(hostname); + return JNI::isCleartextPermitted(hostname); } envoy_cert_validation_result DefaultSystemHelper::validateCertificateChain(const std::vector& certs, absl::string_view hostname) { - return verify_x509_cert_chain(certs, hostname); + return JNI::verifyX509CertChain(certs, hostname); } -void DefaultSystemHelper::cleanupAfterCertificateValidation() { jvm_detach_thread(); } +void DefaultSystemHelper::cleanupAfterCertificateValidation() { JNI::jvmDetachThread(); } } // namespace Envoy diff --git a/mobile/library/common/config/BUILD b/mobile/library/common/config/BUILD index f088148ada3c..c4902b6253c8 100644 --- a/mobile/library/common/config/BUILD +++ b/mobile/library/common/config/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "certificates_lib", diff --git a/mobile/library/common/data/BUILD b/mobile/library/common/data/BUILD index 7381b3e565d9..9f9a8b0e6c04 100644 --- a/mobile/library/common/data/BUILD +++ b/mobile/library/common/data/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "utility_lib", diff --git a/mobile/library/common/engine.cc b/mobile/library/common/engine.cc index f73cda268298..b7e6e2e4a423 100644 --- a/mobile/library/common/engine.cc +++ b/mobile/library/common/engine.cc @@ -2,6 +2,7 @@ #include "source/common/api/os_sys_calls_impl.h" #include "source/common/common/lock_guard.h" +#include "source/common/runtime/runtime_features.h" #include "library/common/bridge/utility.h" #include "library/common/data/utility.h" @@ -19,6 +20,11 @@ Engine::Engine(envoy_engine_callbacks callbacks, envoy_logger logger, // registry may lead to crashes at Engine shutdown. To be figured out as part of // https://github.com/envoyproxy/envoy-mobile/issues/332 Envoy::Api::External::registerApi(std::string(envoy_event_tracker_api_name), &event_tracker_); + // Envoy Mobile always requires dfp_mixed_scheme for the TLS and cleartext DFP clusters. + // While dfp_mixed_scheme defaults to true, some environments force it to false (e.g. within + // Google), so we force it back to true in Envoy Mobile. + // TODO(abeyad): Remove once this is no longer needed. + Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.dfp_mixed_scheme", true); } envoy_status_t Engine::run(const std::string config, const std::string log_level) { @@ -26,21 +32,21 @@ envoy_status_t Engine::run(const std::string config, const std::string log_level // std::thread, main_thread_ is the same object after this call, but its state is replaced with // that of the temporary. The temporary object's state becomes the default state, which does // nothing. - auto options = std::make_unique(); + auto options = std::make_unique(); options->setConfigYaml(config); if (!log_level.empty()) { - options->setLogLevel(options->parseAndValidateLogLevel(log_level.c_str())); + ENVOY_BUG(options->setLogLevel(log_level.c_str()).ok(), "invalid log level"); } options->setConcurrency(1); return run(std::move(options)); } -envoy_status_t Engine::run(std::unique_ptr&& options) { +envoy_status_t Engine::run(std::unique_ptr&& options) { main_thread_ = std::thread(&Engine::main, this, std::move(options)); return ENVOY_SUCCESS; } -envoy_status_t Engine::main(std::unique_ptr&& options) { +envoy_status_t Engine::main(std::unique_ptr&& options) { // Using unique_ptr ensures main_common's lifespan is strictly scoped to this function. std::unique_ptr main_common; { @@ -90,8 +96,10 @@ envoy_status_t Engine::main(std::unique_ptr&& options) { Envoy::Server::ServerLifecycleNotifier::Stage::PostInit, [this]() -> void { ASSERT(Thread::MainThread::isMainOrTestThread()); - connectivity_manager_ = - Network::ConnectivityManagerFactory{server_->serverFactoryContext()}.get(); + Envoy::Server::GenericFactoryContextImpl generic_context( + server_->serverFactoryContext(), + server_->serverFactoryContext().messageValidationVisitor()); + connectivity_manager_ = Network::ConnectivityManagerFactory{generic_context}.get(); auto v4_interfaces = connectivity_manager_->enumerateV4Interfaces(); auto v6_interfaces = connectivity_manager_->enumerateV6Interfaces(); logInterfaces("netconf_get_v4_interfaces", v4_interfaces); @@ -236,19 +244,13 @@ void handlerStats(Stats::Store& stats, Buffer::Instance& response) { } Envoy::Buffer::OwnedImpl Engine::dumpStats() { - ASSERT(dispatcher_->isThreadSafe(), "flushStats must be called from the dispatcher's context"); + ASSERT(dispatcher_->isThreadSafe(), "dumpStats must be called from the dispatcher's context"); Envoy::Buffer::OwnedImpl instance; handlerStats(server_->stats(), instance); return instance; } -void Engine::flushStats() { - ASSERT(dispatcher_->isThreadSafe(), "flushStats must be called from the dispatcher's context"); - - server_->flushStats(); -} - Upstream::ClusterManager& Engine::getClusterManager() { ASSERT(dispatcher_->isThreadSafe(), "getClusterManager must be called from the dispatcher's context"); diff --git a/mobile/library/common/engine.h b/mobile/library/common/engine.h index 7a61b8366bf7..3a130ad5d89d 100644 --- a/mobile/library/common/engine.h +++ b/mobile/library/common/engine.h @@ -36,7 +36,7 @@ class Engine : public Logger::Loggable { * @param log_level, the log level. */ envoy_status_t run(std::string config, std::string log_level); - envoy_status_t run(std::unique_ptr&& options); + envoy_status_t run(std::unique_ptr&& options); /** * Immediately terminate the engine, if running. @@ -76,13 +76,6 @@ class Engine : public Logger::Loggable { */ Buffer::OwnedImpl dumpStats(); - /** - * Flush the stats sinks outside of a flushing interval. - * Note: stat flushing is done asynchronously, this function will never block. - * This is a noop if called before the underlying EnvoyEngine has started. - */ - void flushStats(); - /** * Get cluster manager from the Engine. */ @@ -94,7 +87,7 @@ class Engine : public Logger::Loggable { Stats::Store& getStatsStore(); private: - envoy_status_t main(std::unique_ptr&& options); + envoy_status_t main(std::unique_ptr&& options); static void logInterfaces(absl::string_view event, std::vector& interfaces); diff --git a/mobile/library/common/engine_common.cc b/mobile/library/common/engine_common.cc index c49e2141df8b..a7f5fb3bcfec 100644 --- a/mobile/library/common/engine_common.cc +++ b/mobile/library/common/engine_common.cc @@ -2,6 +2,7 @@ #include "source/common/common/random_generator.h" #include "source/common/runtime/runtime_impl.h" +#include "source/server/null_overload_manager.h" #if !defined(ENVOY_ENABLE_FULL_PROTOS) @@ -54,17 +55,45 @@ void registerMobileProtoDescriptors() { namespace Envoy { -EngineCommon::EngineCommon(std::unique_ptr&& options) +class ServerLite : public Server::InstanceBase { +public: + using Server::InstanceBase::InstanceBase; + void maybeCreateHeapShrinker() override {} + std::unique_ptr createOverloadManager() override { + return std::make_unique(threadLocal(), true); + } + std::unique_ptr maybeCreateGuardDog(absl::string_view) override { + return nullptr; + } +}; + +EngineCommon::EngineCommon(std::unique_ptr&& options) : options_(std::move(options)) { #if !defined(ENVOY_ENABLE_FULL_PROTOS) registerMobileProtoDescriptors(); #endif + StrippedMainBase::CreateInstanceFunction create_instance = + [](Init::Manager& init_manager, const Server::Options& options, + Event::TimeSystem& time_system, ListenerHooks& hooks, Server::HotRestart& restarter, + Stats::StoreRoot& store, Thread::BasicLockable& access_log_lock, + Server::ComponentFactory& component_factory, Random::RandomGeneratorPtr&& random_generator, + ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, + Filesystem::Instance& file_system, std::unique_ptr process_context, + Buffer::WatermarkFactorySharedPtr watermark_factory) { + auto local_address = Network::Utility::getLocalAddress(options.localAddressIpVersion()); + auto server = std::make_unique( + init_manager, options, time_system, hooks, restarter, store, access_log_lock, + std::move(random_generator), tls, thread_factory, file_system, + std::move(process_context), watermark_factory); + server->initialize(local_address, component_factory); + return server; + }; base_ = std::make_unique( *options_, real_time_system_, default_listener_hooks_, prod_component_factory_, - std::make_unique(), std::make_unique(), nullptr); - + std::make_unique(), std::make_unique(), nullptr, + create_instance); // Disabling signal handling in the options makes it so that the server's event dispatcher _does // not_ listen for termination signals such as SIGTERM, SIGINT, etc // (https://github.com/envoyproxy/envoy/blob/048f4231310fbbead0cbe03d43ffb4307fff0517/source/server/server.cc#L519). diff --git a/mobile/library/common/engine_common.h b/mobile/library/common/engine_common.h index 13803984bdfe..f96722c63c69 100644 --- a/mobile/library/common/engine_common.h +++ b/mobile/library/common/engine_common.h @@ -7,7 +7,7 @@ #include "source/exe/platform_impl.h" #include "source/exe/stripped_main_base.h" #include "source/server/listener_hooks.h" -#include "source/server/options_impl.h" +#include "source/server/options_impl_base.h" #ifdef ENVOY_HANDLE_SIGNALS #include "source/common/signal/signal_action.h" @@ -22,7 +22,7 @@ namespace Envoy { */ class EngineCommon { public: - EngineCommon(std::unique_ptr&& options); + EngineCommon(std::unique_ptr&& options); bool run() { base_->runServer(); return true; @@ -41,7 +41,7 @@ class EngineCommon { Envoy::SignalAction handle_sigs_; Envoy::TerminateHandler log_on_terminate_; #endif - std::unique_ptr options_; + std::unique_ptr options_; Event::RealTimeSystem real_time_system_; // NO_CHECK_FORMAT(real_time) DefaultListenerHooks default_listener_hooks_; ProdComponentFactory prod_component_factory_; diff --git a/mobile/library/common/engine_handle.cc b/mobile/library/common/engine_handle.cc deleted file mode 100644 index 51287f6daebe..000000000000 --- a/mobile/library/common/engine_handle.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include "library/common/engine_handle.h" - -namespace Envoy { - -envoy_status_t EngineHandle::runOnEngineDispatcher(envoy_engine_t handle, - std::function func) { - if (auto engine = reinterpret_cast(handle)) { - return engine->dispatcher().post([engine, func]() { func(*engine); }); - } - return ENVOY_FAILURE; -} - -envoy_engine_t EngineHandle::initEngine(envoy_engine_callbacks callbacks, envoy_logger logger, - envoy_event_tracker event_tracker) { - auto engine = new Envoy::Engine(callbacks, logger, event_tracker); - return reinterpret_cast(engine); -} - -envoy_status_t EngineHandle::runEngine(envoy_engine_t handle, const char* config, - const char* log_level) { - if (auto engine = reinterpret_cast(handle)) { - engine->run(config, log_level); - return ENVOY_SUCCESS; - } - return ENVOY_FAILURE; -} - -envoy_status_t EngineHandle::terminateEngine(envoy_engine_t handle, bool release) { - auto engine = reinterpret_cast(handle); - envoy_status_t ret = engine->terminate(); - if (release) { - // TODO(jpsim): Always delete engine to avoid leaking it - delete engine; - } - return ret; -} - -} // namespace Envoy diff --git a/mobile/library/common/engine_handle.h b/mobile/library/common/engine_handle.h deleted file mode 100644 index 3b91029e0a3c..000000000000 --- a/mobile/library/common/engine_handle.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "library/common/engine.h" -#include "library/common/main_interface.h" -#include "library/common/types/c_types.h" - -namespace Envoy { - -/** - * Wrapper class around the singleton engine handle. This allows us to use C++ access modifiers to - * control what functionality dependent code is able to access. Furthermore, this allows C++ access - * to scheduling work on the dispatcher beyond what is available through the public C API. - */ -class EngineHandle { -public: - /** - * Execute function on the dispatcher of the provided engine, if this engine is currently running. - * @param envoy_engine_t, handle to the engine which will be used to execute the function. - * @param func, function that will be executed if engine exists. - * @return bool, true if the function was scheduled on the dispatcher. - */ - static envoy_status_t runOnEngineDispatcher(envoy_engine_t engine, - std::function func); - -private: - static envoy_engine_t initEngine(envoy_engine_callbacks callbacks, envoy_logger logger, - envoy_event_tracker event_tracker); - static envoy_status_t runEngine(envoy_engine_t, const char* config, const char* log_level); - static envoy_status_t terminateEngine(envoy_engine_t handle, bool release); - - // Allow a specific list of functions to access the internal setup/teardown functionality. - friend envoy_engine_t(::init_engine)(envoy_engine_callbacks callbacks, envoy_logger logger, - envoy_event_tracker event_tracker); - friend envoy_status_t(::run_engine)(envoy_engine_t, const char* config, const char* log_level); - friend envoy_status_t(::terminate_engine)(envoy_engine_t engine, bool release); -}; - -} // namespace Envoy diff --git a/mobile/library/common/event/BUILD b/mobile/library/common/event/BUILD index c19e4a003d59..811e559e3b67 100644 --- a/mobile/library/common/event/BUILD +++ b/mobile/library/common/event/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "provisional_dispatcher_lib", diff --git a/mobile/library/common/extensions/filters/http/platform_bridge/config.cc b/mobile/library/common/extensions/filters/http/platform_bridge/config.cc index 3dffd3b4dc98..5e5ad0f4b628 100644 --- a/mobile/library/common/extensions/filters/http/platform_bridge/config.cc +++ b/mobile/library/common/extensions/filters/http/platform_bridge/config.cc @@ -14,8 +14,8 @@ Http::FilterFactoryCb PlatformBridgeFilterFactory::createFilterFactoryFromProtoT PlatformBridgeFilterConfigSharedPtr filter_config = std::make_shared(context, proto_config); return [filter_config, &context](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter( - std::make_shared(filter_config, context.mainThreadDispatcher())); + callbacks.addStreamFilter(std::make_shared( + filter_config, context.serverFactoryContext().mainThreadDispatcher())); }; } diff --git a/mobile/library/common/extensions/key_value/platform/config.cc b/mobile/library/common/extensions/key_value/platform/config.cc index aa9c7b26e683..8f4eb7f21e46 100644 --- a/mobile/library/common/extensions/key_value/platform/config.cc +++ b/mobile/library/common/extensions/key_value/platform/config.cc @@ -27,7 +27,9 @@ class PlatformInterfaceImpl : public PlatformInterface, std::string read(const std::string& key) const override { envoy_data bridged_key = Data::Utility::copyToBridgeData(key); envoy_data bridged_value = bridged_store_.read(bridged_key, bridged_store_.context); - return Data::Utility::copyToString(bridged_value); + std::string result = Data::Utility::copyToString(bridged_value); + release_envoy_data(bridged_value); + return result; } void save(const std::string& key, const std::string& contents) override { diff --git a/mobile/library/common/extensions/listener_managers/api_listener_manager/BUILD b/mobile/library/common/extensions/listener_managers/api_listener_manager/BUILD index 1edcd81a93d2..09f58511b280 100644 --- a/mobile/library/common/extensions/listener_managers/api_listener_manager/BUILD +++ b/mobile/library/common/extensions/listener_managers/api_listener_manager/BUILD @@ -22,5 +22,6 @@ envoy_cc_extension( "@envoy//envoy/server:listener_manager_interface", "@envoy//source/server:api_listener_lib", "@envoy//source/server:listener_manager_factory_lib", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], ) diff --git a/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h b/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h index 0f152df9e502..493beb8dec44 100644 --- a/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h +++ b/mobile/library/common/extensions/listener_managers/api_listener_manager/api_listener_manager.h @@ -29,7 +29,7 @@ class ApiListenerManagerImpl : public ListenerManager, Logger::Loggable callback) override { callback(); } + void startWorkers(OptRef, std::function callback) override { callback(); } void stopListeners(StopListenersType, const Network::ExtraShutdownListenerOptions&) override {} void stopWorkers() override {} void beginListenerUpdate() override {} diff --git a/mobile/library/common/http/BUILD b/mobile/library/common/http/BUILD index 22c4886712a4..d62e943d4dd7 100644 --- a/mobile/library/common/http/BUILD +++ b/mobile/library/common/http/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "client_lib", @@ -19,7 +19,6 @@ envoy_cc_library( "//library/common/extensions/filters/http/local_error:local_error_filter_lib", "//library/common/extensions/filters/http/network_configuration:network_configuration_filter_lib", "//library/common/http:header_utility_lib", - "//library/common/jni:android_jni_utility_lib", "//library/common/network:connectivity_manager_lib", "//library/common/network:synthetic_address_lib", "//library/common/stream_info:extra_stream_info_lib", diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index 37188d1521c2..2ca3a7901342 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -1,6 +1,5 @@ #include "library/common/http/client.h" -#include "source/common/buffer/buffer_impl.h" #include "source/common/common/dump_state_utils.h" #include "source/common/common/scope_tracker.h" #include "source/common/http/codes.h" @@ -14,8 +13,6 @@ #include "library/common/data/utility.h" #include "library/common/http/header_utility.h" #include "library/common/http/headers.h" -#include "library/common/jni/android_jni_utility.h" -#include "library/common/network/connectivity_manager.h" #include "library/common/stream_info/extra_stream_info.h" namespace Envoy { @@ -39,8 +36,7 @@ Client::DirectStreamCallbacks::DirectStreamCallbacks(DirectStream& direct_stream envoy_http_callbacks bridge_callbacks, Client& http_client) : direct_stream_(direct_stream), bridge_callbacks_(bridge_callbacks), http_client_(http_client), - explicit_flow_control_(direct_stream_.explicit_flow_control_), - min_delivery_size_(direct_stream_.min_delivery_size_) {} + explicit_flow_control_(direct_stream_.explicit_flow_control_) {} void Client::DirectStreamCallbacks::encodeHeaders(const ResponseHeaderMap& headers, bool end_stream) { @@ -116,7 +112,7 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ // The response_data_ is systematically assigned here because resumeData can // incur an asynchronous callback to sendDataToBridge. - if ((explicit_flow_control_ || min_delivery_size_ != 0) && !response_data_) { + if (explicit_flow_control_ && !response_data_) { response_data_ = std::make_unique( [this]() -> void { onBufferedDataDrained(); }, [this]() -> void { onHasBufferedData(); }, []() -> void {}); @@ -126,10 +122,8 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ response_data_->setWatermarks(1000000); } - // Try to send data if - // 1) in default flow control mode - // 2) if resumeData has been called in explicit flow control mode. - // sendDataToBridge will enforce delivery size limits. + // Send data if in default flow control mode, or if resumeData has been called in explicit + // flow control mode. if (bytes_to_send_ > 0 || !explicit_flow_control_) { ASSERT(!hasBufferedData()); sendDataToBridge(data, end_stream); @@ -137,9 +131,10 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ // If not all the bytes have been sent up, buffer any remaining data in response_data. if (data.length() != 0) { - ENVOY_LOG(debug, "[S{}] buffering {} bytes. {} total bytes buffered.", - direct_stream_.stream_handle_, data.length(), - data.length() + response_data_->length()); + ASSERT(explicit_flow_control_); + ENVOY_LOG( + debug, "[S{}] buffering {} bytes due to explicit flow control. {} total bytes buffered.", + direct_stream_.stream_handle_, data.length(), data.length() + response_data_->length()); response_data_->move(data); } } @@ -147,15 +142,6 @@ void Client::DirectStreamCallbacks::encodeData(Buffer::Instance& data, bool end_ void Client::DirectStreamCallbacks::sendDataToBridge(Buffer::Instance& data, bool end_stream) { ASSERT(!explicit_flow_control_ || bytes_to_send_ > 0); - if (min_delivery_size_ > 0 && (data.length() < min_delivery_size_) && !end_stream) { - ENVOY_LOG( - debug, - "[S{}] defering sending {} bytes due to delivery size limits (limit={} end stream={})", - direct_stream_.stream_handle_, data.length(), min_delivery_size_, end_stream); - - return; // Not enough data to justify sending up to the bridge. - } - // Cap by bytes_to_send_ if and only if applying explicit flow control. uint32_t bytes_to_send = calculateBytesToSend(data, bytes_to_send_); // Update the number of bytes consumed by this non terminal callback. @@ -475,11 +461,10 @@ void Client::DirectStream::dumpState(std::ostream&, int indent_level) const { } void Client::startStream(envoy_stream_t new_stream_handle, envoy_http_callbacks bridge_callbacks, - bool explicit_flow_control, uint64_t min_delivery_size) { + bool explicit_flow_control) { ASSERT(dispatcher_.isThreadSafe()); Client::DirectStreamSharedPtr direct_stream{new DirectStream(new_stream_handle, *this)}; direct_stream->explicit_flow_control_ = explicit_flow_control; - direct_stream->min_delivery_size_ = min_delivery_size; direct_stream->callbacks_ = std::make_unique(*direct_stream, bridge_callbacks, *this); @@ -514,7 +499,7 @@ void Client::sendHeaders(envoy_stream_t stream, envoy_headers headers, bool end_ ScopeTrackerScopeState scope(direct_stream.get(), scopeTracker()); RequestHeaderMapPtr internal_headers = Utility::toRequestHeaders(headers); - // This is largely a check for the android platform: is_cleartext_permitted + // This is largely a check for the android platform: isCleartextPermitted // is a no-op for other platforms. if (internal_headers->getSchemeValue() != "https" && !SystemHelper::getInstance().isCleartextPermitted(internal_headers->getHostValue())) { diff --git a/mobile/library/common/http/client.h b/mobile/library/common/http/client.h index d8945cfbb531..33e62f48a440 100644 --- a/mobile/library/common/http/client.h +++ b/mobile/library/common/http/client.h @@ -67,11 +67,9 @@ class Client : public Logger::Loggable { * @param stream, the stream to start. * @param bridge_callbacks, wrapper for callbacks for events on this stream. * @param explicit_flow_control, whether the stream will require explicit flow control. - * @param min_delivery_size, if greater than zero, indicates the smallest number of bytes that - * will be delivered up via the on_data callbacks without end stream. */ void startStream(envoy_stream_t stream, envoy_http_callbacks bridge_callbacks, - bool explicit_flow_control, uint64_t min_delivery_size); + bool explicit_flow_control); /** * Send headers over an open HTTP stream. This method can be invoked once and needs to be called @@ -206,15 +204,12 @@ class Client : public Logger::Loggable { absl::optional error_; bool success_{}; - // Buffered response data when in explicit flow control or buffering due to min delivery size. + // Buffered response data when in explicit flow control mode. Buffer::InstancePtr response_data_; ResponseTrailerMapPtr response_trailers_; // True if the bridge should operate in explicit flow control mode, and only send // data when it is requested by the caller. bool explicit_flow_control_{}; - // If greater than zero, indicates the minimum size of data that should be - // delivered up via on_data without end stream. - uint64_t min_delivery_size_{}; // Set true when the response headers have been forwarded to the bridge. bool response_headers_forwarded_{}; // Called in closeStream() to communicate that the end of the stream has @@ -341,10 +336,6 @@ class Client : public Logger::Loggable { // back, avoids excessive buffering of response bodies if the response body is // read faster than the mobile caller can process it. bool explicit_flow_control_ = false; - // If this is non-zero, Envoy will buffer at the C++ layer until either - // min_delivery_size_ bytes are received or end_stream is received. If this is - // zero, it will deliver data as it arrivies, modulo explicit flow control rules. - uint64_t min_delivery_size_{}; // Latest intel data retrieved from the StreamInfo. envoy_stream_intel stream_intel_{-1, -1, 0, 0}; envoy_final_stream_intel envoy_final_stream_intel_{-1, -1, -1, -1, -1, -1, -1, -1, diff --git a/mobile/library/common/jni/BUILD b/mobile/library/common/jni/BUILD index b28b52f2c0ee..6159a9d5517d 100644 --- a/mobile/library/common/jni/BUILD +++ b/mobile/library/common/jni/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines", "envoy_mobile_package") load("//bazel:android_debug_info.bzl", "android_debug_info") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() # TODO(jpsim): Migrate these to use `envoy_cc_library` @@ -35,6 +35,7 @@ cc_library( "jni_utility.h", ], deps = [ + ":jni_helper_lib", ":jni_support_lib", "//library/common/jni/import:jni_import_lib", "//library/common/jni/types:jni_env_lib", @@ -43,18 +44,30 @@ cc_library( "//library/common/types:managed_types_lib", "//library/common/types:matcher_data_lib", "@envoy//source/common/common:assert_lib", + "@envoy//source/common/protobuf", + ], +) + +cc_library( + name = "jni_helper_lib", + srcs = [ + "jni_helper.cc", + ], + hdrs = [ + "jni_helper.h", + ], + deps = [ + "//library/common/jni/import:jni_import_lib", + "@envoy//source/common/common:assert_lib", ], ) # Implementations of the various "native" Java methods for classes # in library/java/io/envoyproxy/envoymobile. -# TODO(RyanTheOptimist): Is there a better name for this? I'm not sure what -# "interface" refers to in this file/lib. Maybe we should have a convention -# where the name of the java classes are part of the file name? cc_library( - name = "jni_interface_lib", + name = "jni_impl_lib", srcs = [ - "jni_interface.cc", + "jni_impl.cc", ], defines = envoy_mobile_defines("@envoy"), deps = [ @@ -68,6 +81,7 @@ cc_library( "//library/common/jni/types:jni_exception_lib", "//library/common/jni/types:jni_javavm_lib", "//library/common/types:managed_types_lib", + "@envoy//source/common/protobuf", ], # We need this to ensure that we link this into the .so even though there are no code references. alwayslink = True, @@ -97,13 +111,13 @@ cc_library( # Implementations of the "native" Java methods for AndroidJniLibrary.java cc_library( - name = "android_jni_interface_lib", + name = "android_jni_impl_lib", srcs = [ - "android_jni_interface.cc", + "android_jni_impl.cc", ], deps = [ ":android_network_utility_lib", - ":jni_interface_lib", + ":jni_impl_lib", ":jni_support_lib", ":jni_utility_lib", "//library/common:envoy_main_interface_lib", @@ -121,9 +135,9 @@ cc_library( "//conditions:default": [], }), deps = [ - ":android_jni_interface_lib", + ":android_jni_impl_lib", ":android_network_utility_lib", - ":jni_interface_lib", + ":jni_impl_lib", ":jni_support_lib", ":jni_utility_lib", ], diff --git a/mobile/library/common/jni/android_jni_interface.cc b/mobile/library/common/jni/android_jni_impl.cc similarity index 70% rename from mobile/library/common/jni/android_jni_interface.cc rename to mobile/library/common/jni/android_jni_impl.cc index 834e41f83a2a..46dc546f99be 100644 --- a/mobile/library/common/jni/android_jni_interface.cc +++ b/mobile/library/common/jni/android_jni_impl.cc @@ -1,8 +1,5 @@ -#include "library/common/jni/android_network_utility.h" #include "library/common/jni/import/jni_import.h" -#include "library/common/jni/jni_support.h" #include "library/common/jni/jni_utility.h" -#include "library/common/main_interface.h" // NOLINT(namespace-envoy) @@ -12,6 +9,6 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_AndroidJniLibrary_initialize(JNIEnv* env, jclass, // class jobject class_loader) { - set_class_loader(env->NewGlobalRef(class_loader)); + Envoy::JNI::setClassLoader(env->NewGlobalRef(class_loader)); return ENVOY_SUCCESS; } diff --git a/mobile/library/common/jni/android_jni_utility.cc b/mobile/library/common/jni/android_jni_utility.cc index 3744cdbaa4be..c059b108950a 100644 --- a/mobile/library/common/jni/android_jni_utility.cc +++ b/mobile/library/common/jni/android_jni_utility.cc @@ -1,8 +1,5 @@ #include "library/common/jni/android_jni_utility.h" -#include -#include - #include "source/common/common/assert.h" #if defined(__ANDROID_API__) @@ -12,20 +9,20 @@ #include "library/common/jni/jni_utility.h" #endif -// NOLINT(namespace-envoy) +namespace Envoy { +namespace JNI { -bool is_cleartext_permitted(absl::string_view hostname) { +bool isCleartextPermitted(absl::string_view hostname) { #if defined(__ANDROID_API__) envoy_data host = Envoy::Data::Utility::copyToBridgeData(hostname); - JNIEnv* env = get_env(); - jstring java_host = native_data_to_string(env, host); - jclass jcls_AndroidNetworkLibrary = - find_class("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); - jmethodID jmid_isCleartextTrafficPermitted = env->GetStaticMethodID( - jcls_AndroidNetworkLibrary, "isCleartextTrafficPermitted", "(Ljava/lang/String;)Z"); - jboolean result = env->CallStaticBooleanMethod(jcls_AndroidNetworkLibrary, - jmid_isCleartextTrafficPermitted, java_host); - env->DeleteLocalRef(java_host); + JniHelper jni_helper(getEnv()); + LocalRefUniquePtr java_host = envoyDataToJavaString(jni_helper, host); + LocalRefUniquePtr jcls_AndroidNetworkLibrary = + findClass("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); + jmethodID jmid_isCleartextTrafficPermitted = jni_helper.getStaticMethodId( + jcls_AndroidNetworkLibrary.get(), "isCleartextTrafficPermitted", "(Ljava/lang/String;)Z"); + jboolean result = jni_helper.callStaticBooleanMethod( + jcls_AndroidNetworkLibrary.get(), jmid_isCleartextTrafficPermitted, java_host.get()); release_envoy_data(host); return result == JNI_TRUE; #else @@ -34,17 +31,20 @@ bool is_cleartext_permitted(absl::string_view hostname) { #endif } -void tag_socket(int ifd, int uid, int tag) { +void tagSocket(int ifd, int uid, int tag) { #if defined(__ANDROID_API__) - JNIEnv* env = get_env(); - jclass jcls_AndroidNetworkLibrary = - find_class("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); + JniHelper jni_helper(getEnv()); + LocalRefUniquePtr jcls_AndroidNetworkLibrary = + findClass("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); jmethodID jmid_tagSocket = - env->GetStaticMethodID(jcls_AndroidNetworkLibrary, "tagSocket", "(III)V"); - env->CallStaticVoidMethod(jcls_AndroidNetworkLibrary, jmid_tagSocket, ifd, uid, tag); + jni_helper.getStaticMethodId(jcls_AndroidNetworkLibrary.get(), "tagSocket", "(III)V"); + jni_helper.callStaticVoidMethod(jcls_AndroidNetworkLibrary.get(), jmid_tagSocket, ifd, uid, tag); #else UNREFERENCED_PARAMETER(ifd); UNREFERENCED_PARAMETER(uid); UNREFERENCED_PARAMETER(tag); #endif } + +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/android_jni_utility.h b/mobile/library/common/jni/android_jni_utility.h index 271ba790c37d..67c3d125c15b 100644 --- a/mobile/library/common/jni/android_jni_utility.h +++ b/mobile/library/common/jni/android_jni_utility.h @@ -2,16 +2,19 @@ #include "absl/strings/string_view.h" -// NOLINT(namespace-envoy) +namespace Envoy { +namespace JNI { -/* For android, calls up through JNI to see if cleartext is permitted for this - * host. +/** + * For android, calls up through JNI to see if cleartext is permitted for this host. * For other platforms simply returns true. */ -bool is_cleartext_permitted(absl::string_view hostname); +bool isCleartextPermitted(absl::string_view hostname); -/* For android, calls up through JNI to apply - * host. - * For other platforms simply returns true. +/** + * For android, calls up through JNI to apply host. For other platforms simply returns true. */ -void tag_socket(int ifd, int uid, int tag); +void tagSocket(int ifd, int uid, int tag); + +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/android_network_utility.cc b/mobile/library/common/jni/android_network_utility.cc index 976a8e9c6967..270a3b10828e 100644 --- a/mobile/library/common/jni/android_network_utility.cc +++ b/mobile/library/common/jni/android_network_utility.cc @@ -7,111 +7,107 @@ #include "library/common/jni/types/java_virtual_machine.h" #include "openssl/ssl.h" -// NOLINT(namespace-envoy) +namespace Envoy { +namespace JNI { +namespace { // Helper functions call into AndroidNetworkLibrary, but they are not platform dependent // because AndroidNetworkLibray can be called in non-Android platform with mock interfaces. -bool jvm_cert_is_issued_by_known_root(JNIEnv* env, jobject result) { - jclass jcls_AndroidCertVerifyResult = - find_class("io.envoyproxy.envoymobile.utilities.AndroidCertVerifyResult"); +bool jvmCertIsIssuedByKnownRoot(JniHelper& jni_helper, jobject result) { + LocalRefUniquePtr jcls_AndroidCertVerifyResult = + findClass("io.envoyproxy.envoymobile.utilities.AndroidCertVerifyResult"); jmethodID jmid_isIssuedByKnownRoot = - env->GetMethodID(jcls_AndroidCertVerifyResult, "isIssuedByKnownRoot", "()Z"); - Envoy::JNI::Exception::checkAndClear("jvm_cert_is_issued_by_known_root:GetMethodID"); + jni_helper.getMethodId(jcls_AndroidCertVerifyResult.get(), "isIssuedByKnownRoot", "()Z"); ASSERT(jmid_isIssuedByKnownRoot); - bool is_issued_by_known_root = env->CallBooleanMethod(result, jmid_isIssuedByKnownRoot); - Envoy::JNI::Exception::checkAndClear("jvm_cert_is_issued_by_known_root:CallBooleanMethod"); - env->DeleteLocalRef(jcls_AndroidCertVerifyResult); + bool is_issued_by_known_root = jni_helper.callBooleanMethod(result, jmid_isIssuedByKnownRoot); return is_issued_by_known_root; } -envoy_cert_verify_status_t jvm_cert_get_status(JNIEnv* env, jobject j_result) { - jclass jcls_AndroidCertVerifyResult = - find_class("io.envoyproxy.envoymobile.utilities.AndroidCertVerifyResult"); - jmethodID jmid_getStatus = env->GetMethodID(jcls_AndroidCertVerifyResult, "getStatus", "()I"); - Envoy::JNI::Exception::checkAndClear("jvm_cert_get_status:GetMethodID"); +envoy_cert_verify_status_t jvmCertGetStatus(JniHelper& jni_helper, jobject j_result) { + LocalRefUniquePtr jcls_AndroidCertVerifyResult = + findClass("io.envoyproxy.envoymobile.utilities.AndroidCertVerifyResult"); + jmethodID jmid_getStatus = + jni_helper.getMethodId(jcls_AndroidCertVerifyResult.get(), "getStatus", "()I"); ASSERT(jmid_getStatus); - envoy_cert_verify_status_t result = CERT_VERIFY_STATUS_FAILED; - result = static_cast(env->CallIntMethod(j_result, jmid_getStatus)); - Envoy::JNI::Exception::checkAndClear("jvm_cert_get_status:CallIntMethod"); - - env->DeleteLocalRef(jcls_AndroidCertVerifyResult); + envoy_cert_verify_status_t result = + static_cast(jni_helper.callIntMethod(j_result, jmid_getStatus)); return result; } -jobjectArray jvm_cert_get_certificate_chain_encoded(JNIEnv* env, jobject result) { - jclass jcls_AndroidCertVerifyResult = - find_class("io.envoyproxy.envoymobile.utilities.AndroidCertVerifyResult"); - jmethodID jmid_getCertificateChainEncoded = - env->GetMethodID(jcls_AndroidCertVerifyResult, "getCertificateChainEncoded", "()[[B"); - Envoy::JNI::Exception::checkAndClear("jvm_cert_get_certificate_chain_encoded:GetMethodID"); - jobjectArray certificate_chain = - static_cast(env->CallObjectMethod(result, jmid_getCertificateChainEncoded)); - Envoy::JNI::Exception::checkAndClear("jvm_cert_get_certificate_chain_encoded:CallObjectMethod"); - env->DeleteLocalRef(jcls_AndroidCertVerifyResult); +LocalRefUniquePtr jvmCertGetCertificateChainEncoded(JniHelper& jni_helper, + jobject result) { + LocalRefUniquePtr jcls_AndroidCertVerifyResult = + findClass("io.envoyproxy.envoymobile.utilities.AndroidCertVerifyResult"); + jmethodID jmid_getCertificateChainEncoded = jni_helper.getMethodId( + jcls_AndroidCertVerifyResult.get(), "getCertificateChainEncoded", "()[[B"); + LocalRefUniquePtr certificate_chain = + jni_helper.callObjectMethod(result, jmid_getCertificateChainEncoded); return certificate_chain; } -static void ExtractCertVerifyResult(JNIEnv* env, jobject result, envoy_cert_verify_status_t* status, +static void extractCertVerifyResult(JniHelper& jni_helper, jobject result, + envoy_cert_verify_status_t* status, bool* is_issued_by_known_root, std::vector* verified_chain) { - *status = jvm_cert_get_status(env, result); + *status = jvmCertGetStatus(jni_helper, result); if (*status == CERT_VERIFY_STATUS_OK) { - *is_issued_by_known_root = jvm_cert_is_issued_by_known_root(env, result); - jobjectArray chain_byte_array = jvm_cert_get_certificate_chain_encoded(env, result); + *is_issued_by_known_root = jvmCertIsIssuedByKnownRoot(jni_helper, result); + LocalRefUniquePtr chain_byte_array = + jvmCertGetCertificateChainEncoded(jni_helper, result); if (chain_byte_array != nullptr) { - JavaArrayOfByteArrayToStringVector(env, chain_byte_array, verified_chain); + javaArrayOfByteArrayToStringVector(jni_helper, chain_byte_array.get(), verified_chain); } } } // `auth_type` and `host` are expected to be UTF-8 encoded. -jobject call_jvm_verify_x509_cert_chain(JNIEnv* env, const std::vector& cert_chain, - std::string auth_type, absl::string_view hostname) { - jni_log("[Envoy]", "jvm_verify_x509_cert_chain"); - jclass jcls_AndroidNetworkLibrary = - find_class("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); - jmethodID jmid_verifyServerCertificates = env->GetStaticMethodID( - jcls_AndroidNetworkLibrary, "verifyServerCertificates", - "([[B[B[B)Lio/envoyproxy/envoymobile/utilities/AndroidCertVerifyResult;"); - Envoy::JNI::Exception::checkAndClear("call_jvm_verify_x509_cert_chain:GetStaticMethodID"); - jobjectArray chain_byte_array = ToJavaArrayOfByteArray(env, cert_chain); - jbyteArray auth_string = ToJavaByteArray(env, auth_type); - jbyteArray host_string = - ToJavaByteArray(env, reinterpret_cast(hostname.data()), hostname.length()); - jobject result = - env->CallStaticObjectMethod(jcls_AndroidNetworkLibrary, jmid_verifyServerCertificates, - chain_byte_array, auth_string, host_string); - Envoy::JNI::Exception::checkAndClear("call_jvm_verify_x509_cert_chain:CallStaticObjectMethod"); - env->DeleteLocalRef(chain_byte_array); - env->DeleteLocalRef(auth_string); - env->DeleteLocalRef(host_string); - env->DeleteLocalRef(jcls_AndroidNetworkLibrary); - return result; -} - -// `auth_type` and `host` are expected to be UTF-8 encoded. -static void jvm_verify_x509_cert_chain(const std::vector& cert_chain, - std::string auth_type, absl::string_view hostname, - envoy_cert_verify_status_t* status, - bool* is_issued_by_known_root, - std::vector* verified_chain) { - JNIEnv* env = get_env(); - jobject result = call_jvm_verify_x509_cert_chain(env, cert_chain, auth_type, hostname); - if (Envoy::JNI::Exception::checkAndClear()) { +static void jvmVerifyX509CertChain(const std::vector& cert_chain, + std::string auth_type, absl::string_view hostname, + envoy_cert_verify_status_t* status, + bool* is_issued_by_known_root, + std::vector* verified_chain) { + JniHelper jni_helper(getEnv()); + LocalRefUniquePtr result = + callJvmVerifyX509CertChain(jni_helper, cert_chain, auth_type, hostname); + if (Exception::checkAndClear()) { *status = CERT_VERIFY_STATUS_NOT_YET_VALID; } else { - ExtractCertVerifyResult(get_env(), result, status, is_issued_by_known_root, verified_chain); - if (Envoy::JNI::Exception::checkAndClear()) { + extractCertVerifyResult(jni_helper, result.get(), status, is_issued_by_known_root, + verified_chain); + if (Exception::checkAndClear()) { *status = CERT_VERIFY_STATUS_FAILED; } } - env->DeleteLocalRef(result); } -envoy_cert_validation_result verify_x509_cert_chain(const std::vector& certs, - absl::string_view hostname) { - jni_log("[Envoy]", "verify_x509_cert_chain"); +} // namespace + +// `auth_type` and `host` are expected to be UTF-8 encoded. +LocalRefUniquePtr callJvmVerifyX509CertChain(Envoy::JNI::JniHelper& jni_helper, + const std::vector& cert_chain, + std::string auth_type, + absl::string_view hostname) { + jni_log("[Envoy]", "jvmVerifyX509CertChain"); + LocalRefUniquePtr jcls_AndroidNetworkLibrary = + findClass("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); + jmethodID jmid_verifyServerCertificates = jni_helper.getStaticMethodId( + jcls_AndroidNetworkLibrary.get(), "verifyServerCertificates", + "([[B[B[B)Lio/envoyproxy/envoymobile/utilities/AndroidCertVerifyResult;"); + LocalRefUniquePtr chain_byte_array = + vectorStringToJavaArrayOfByteArray(jni_helper, cert_chain); + LocalRefUniquePtr auth_string = stringToJavaByteArray(jni_helper, auth_type); + LocalRefUniquePtr host_string = byteArrayToJavaByteArray( + jni_helper, reinterpret_cast(hostname.data()), hostname.length()); + LocalRefUniquePtr result = jni_helper.callStaticObjectMethod( + jcls_AndroidNetworkLibrary.get(), jmid_verifyServerCertificates, chain_byte_array.get(), + auth_string.get(), host_string.get()); + return result; +} + +envoy_cert_validation_result verifyX509CertChain(const std::vector& certs, + absl::string_view hostname) { + jni_log("[Envoy]", "verifyX509CertChain"); envoy_cert_verify_status_t result; bool is_issued_by_known_root; @@ -123,8 +119,8 @@ envoy_cert_validation_result verify_x509_cert_chain(const std::vector& cert_chain, - std::string auth_type, absl::string_view hostname); +LocalRefUniquePtr callJvmVerifyX509CertChain(JniHelper& jni_helper, + const std::vector& cert_chain, + std::string auth_type, + absl::string_view hostname); -envoy_cert_validation_result verify_x509_cert_chain(const std::vector& certs, - absl::string_view hostname); +envoy_cert_validation_result verifyX509CertChain(const std::vector& certs, + absl::string_view hostname); -void jvm_detach_thread(); +void jvmDetachThread(); + +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/import/BUILD b/mobile/library/common/jni/import/BUILD index 16fb814475ba..d872c484e23a 100644 --- a/mobile/library/common/jni/import/BUILD +++ b/mobile/library/common/jni/import/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() cc_library( name = "jni_import_lib", diff --git a/mobile/library/common/jni/jni_helper.cc b/mobile/library/common/jni/jni_helper.cc new file mode 100644 index 000000000000..e22871d0c99e --- /dev/null +++ b/mobile/library/common/jni/jni_helper.cc @@ -0,0 +1,205 @@ +#include "library/common/jni/jni_helper.h" + +#include "source/common/common/assert.h" + +namespace Envoy { +namespace JNI { + +JNIEnv* JniHelper::getEnv() { return env_; } + +jmethodID JniHelper::getMethodId(jclass clazz, const char* name, const char* signature) { + jmethodID method_id = env_->GetMethodID(clazz, name, signature); + rethrowException(); + return method_id; +} + +jmethodID JniHelper::getStaticMethodId(jclass clazz, const char* name, const char* signature) { + jmethodID method_id = env_->GetStaticMethodID(clazz, name, signature); + rethrowException(); + return method_id; +} + +LocalRefUniquePtr JniHelper::findClass(const char* class_name) { + LocalRefUniquePtr result(env_->FindClass(class_name), LocalRefDeleter(env_)); + rethrowException(); + return result; +} + +LocalRefUniquePtr JniHelper::getObjectClass(jobject object) { + return {env_->GetObjectClass(object), LocalRefDeleter(env_)}; +} + +void JniHelper::throwNew(const char* java_class_name, const char* message) { + LocalRefUniquePtr java_class = findClass(java_class_name); + if (java_class != nullptr) { + jint error = env_->ThrowNew(java_class.get(), message); + RELEASE_ASSERT(error == JNI_OK, fmt::format("Failed calling ThrowNew.")); + } +} + +LocalRefUniquePtr JniHelper::exceptionOccurred() { + return {env_->ExceptionOccurred(), LocalRefDeleter(env_)}; +} + +GlobalRefUniquePtr JniHelper::newGlobalRef(jobject object) { + GlobalRefUniquePtr result(env_->NewGlobalRef(object), GlobalRefDeleter(env_)); + return result; +} + +LocalRefUniquePtr JniHelper::newObject(jclass clazz, jmethodID method_id, ...) { + va_list args; + va_start(args, method_id); + LocalRefUniquePtr result(env_->NewObjectV(clazz, method_id, args), + LocalRefDeleter(env_)); + rethrowException(); + va_end(args); + return result; +} + +LocalRefUniquePtr JniHelper::newStringUtf(const char* str) { + LocalRefUniquePtr result(env_->NewStringUTF(str), LocalRefDeleter(env_)); + rethrowException(); + return result; +} + +StringUtfUniquePtr JniHelper::getStringUtfChars(jstring str, jboolean* is_copy) { + StringUtfUniquePtr result(env_->GetStringUTFChars(str, is_copy), StringUtfDeleter(env_, str)); + rethrowException(); + return result; +} + +jsize JniHelper::getArrayLength(jarray array) { return env_->GetArrayLength(array); } + +#define DEFINE_NEW_ARRAY(JAVA_TYPE, JNI_TYPE) \ + LocalRefUniquePtr JniHelper::new##JAVA_TYPE##Array(jsize length) { \ + LocalRefUniquePtr result(env_->New##JAVA_TYPE##Array(length), \ + LocalRefDeleter(env_)); \ + rethrowException(); \ + return result; \ + } + +DEFINE_NEW_ARRAY(Byte, jbyteArray) +DEFINE_NEW_ARRAY(Char, jcharArray) +DEFINE_NEW_ARRAY(Short, jshortArray) +DEFINE_NEW_ARRAY(Int, jintArray) +DEFINE_NEW_ARRAY(Long, jlongArray) +DEFINE_NEW_ARRAY(Float, jfloatArray) +DEFINE_NEW_ARRAY(Double, jdoubleArray) +DEFINE_NEW_ARRAY(Boolean, jbooleanArray) + +LocalRefUniquePtr JniHelper::newObjectArray(jsize length, jclass element_class, + jobject initial_element) { + LocalRefUniquePtr result( + env_->NewObjectArray(length, element_class, initial_element), LocalRefDeleter(env_)); + + return result; +} + +#define DEFINE_GET_ARRAY_ELEMENTS(JAVA_TYPE, JNI_ARRAY_TYPE, JNI_ELEMENT_TYPE) \ + ArrayElementsUniquePtr \ + JniHelper::get##JAVA_TYPE##ArrayElements(JNI_ARRAY_TYPE array, jboolean* is_copy) { \ + ArrayElementsUniquePtr result( \ + env_->Get##JAVA_TYPE##ArrayElements(array, is_copy), \ + ArrayElementsDeleter(env_, array)); \ + rethrowException(); \ + return result; \ + } + +DEFINE_GET_ARRAY_ELEMENTS(Byte, jbyteArray, jbyte) +DEFINE_GET_ARRAY_ELEMENTS(Char, jcharArray, jchar) +DEFINE_GET_ARRAY_ELEMENTS(Short, jshortArray, jshort) +DEFINE_GET_ARRAY_ELEMENTS(Int, jintArray, jint) +DEFINE_GET_ARRAY_ELEMENTS(Long, jlongArray, jlong) +DEFINE_GET_ARRAY_ELEMENTS(Float, jfloatArray, jfloat) +DEFINE_GET_ARRAY_ELEMENTS(Double, jdoubleArray, jdouble) +DEFINE_GET_ARRAY_ELEMENTS(Boolean, jbooleanArray, jboolean) + +void JniHelper::setObjectArrayElement(jobjectArray array, jsize index, jobject value) { + env_->SetObjectArrayElement(array, index, value); + rethrowException(); +} + +#define DEFINE_SET_ARRAY_REGION(JAVA_TYPE, JNI_ARRAY_TYPE, JNI_ELEMENT_TYPE) \ + void JniHelper::set##JAVA_TYPE##ArrayRegion(JNI_ARRAY_TYPE array, jsize start, jsize length, \ + const JNI_ELEMENT_TYPE* buffer) { \ + env_->Set##JAVA_TYPE##ArrayRegion(array, start, length, buffer); \ + rethrowException(); \ + } + +DEFINE_SET_ARRAY_REGION(Byte, jbyteArray, jbyte) +DEFINE_SET_ARRAY_REGION(Char, jcharArray, jchar) +DEFINE_SET_ARRAY_REGION(Short, jshortArray, jshort) +DEFINE_SET_ARRAY_REGION(Int, jintArray, jint) +DEFINE_SET_ARRAY_REGION(Long, jlongArray, jlong) +DEFINE_SET_ARRAY_REGION(Float, jfloatArray, jfloat) +DEFINE_SET_ARRAY_REGION(Double, jdoubleArray, jdouble) +DEFINE_SET_ARRAY_REGION(Boolean, jbooleanArray, jboolean) + +#define DEFINE_CALL_METHOD(JAVA_TYPE, JNI_TYPE) \ + JNI_TYPE JniHelper::call##JAVA_TYPE##Method(jobject object, jmethodID method_id, ...) { \ + va_list args; \ + va_start(args, method_id); \ + JNI_TYPE result = env_->Call##JAVA_TYPE##MethodV(object, method_id, args); \ + va_end(args); \ + rethrowException(); \ + return result; \ + } + +DEFINE_CALL_METHOD(Byte, jbyte) +DEFINE_CALL_METHOD(Char, jchar) +DEFINE_CALL_METHOD(Short, jshort) +DEFINE_CALL_METHOD(Int, jint) +DEFINE_CALL_METHOD(Long, jlong) +DEFINE_CALL_METHOD(Float, jfloat) +DEFINE_CALL_METHOD(Double, jdouble) +DEFINE_CALL_METHOD(Boolean, jboolean) + +void JniHelper::callVoidMethod(jobject object, jmethodID method_id, ...) { + va_list args; + va_start(args, method_id); + env_->CallVoidMethodV(object, method_id, args); + va_end(args); + rethrowException(); +} + +#define DEFINE_CALL_STATIC_METHOD(JAVA_TYPE, JNI_TYPE) \ + JNI_TYPE JniHelper::callStatic##JAVA_TYPE##Method(jclass clazz, jmethodID method_id, ...) { \ + va_list args; \ + va_start(args, method_id); \ + JNI_TYPE result = env_->CallStatic##JAVA_TYPE##MethodV(clazz, method_id, args); \ + va_end(args); \ + rethrowException(); \ + return result; \ + } + +DEFINE_CALL_STATIC_METHOD(Byte, jbyte) +DEFINE_CALL_STATIC_METHOD(Char, jchar) +DEFINE_CALL_STATIC_METHOD(Short, jshort) +DEFINE_CALL_STATIC_METHOD(Int, jint) +DEFINE_CALL_STATIC_METHOD(Long, jlong) +DEFINE_CALL_STATIC_METHOD(Float, jfloat) +DEFINE_CALL_STATIC_METHOD(Double, jdouble) +DEFINE_CALL_STATIC_METHOD(Boolean, jboolean) + +void JniHelper::callStaticVoidMethod(jclass clazz, jmethodID method_id, ...) { + va_list args; + va_start(args, method_id); + env_->CallStaticVoidMethodV(clazz, method_id, args); + va_end(args); + rethrowException(); +} + +jlong JniHelper::getDirectBufferCapacity(jobject buffer) { + return env_->GetDirectBufferCapacity(buffer); +} + +void JniHelper::rethrowException() { + if (env_->ExceptionCheck()) { + auto throwable = exceptionOccurred(); + env_->ExceptionClear(); + env_->Throw(throwable.release()); + } +} + +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/jni_helper.h b/mobile/library/common/jni/jni_helper.h new file mode 100644 index 000000000000..7c6c6bde2ba6 --- /dev/null +++ b/mobile/library/common/jni/jni_helper.h @@ -0,0 +1,394 @@ +#pragma once + +#include + +#include "library/common/jni/import/jni_import.h" + +namespace Envoy { +namespace JNI { + +/** A custom deleter to delete JNI global ref. */ +class GlobalRefDeleter { +public: + explicit GlobalRefDeleter(JNIEnv* env) : env_(env) {} + + void operator()(jobject object) const { + if (object != nullptr) { + env_->DeleteGlobalRef(object); + } + } + +private: + JNIEnv* const env_; +}; + +/** A unique pointer for JNI global ref. */ +template +using GlobalRefUniquePtr = std::unique_ptr::type, GlobalRefDeleter>; + +/** A custom deleter to delete JNI local ref. */ +class LocalRefDeleter { +public: + explicit LocalRefDeleter(JNIEnv* env) : env_(env) {} + + // This is to allow move semantics in `LocalRefUniquePtr`. + LocalRefDeleter& operator=(const LocalRefDeleter&) { return *this; } + + void operator()(jobject object) const { + if (object != nullptr) { + env_->DeleteLocalRef(object); + } + } + +private: + JNIEnv* const env_; +}; + +/** A unique pointer for JNI local ref. */ +template +using LocalRefUniquePtr = std::unique_ptr::type, LocalRefDeleter>; + +/** A custom deleter for UTF strings. */ +class StringUtfDeleter { +public: + StringUtfDeleter(JNIEnv* env, jstring j_str) : env_(env), j_str_(j_str) {} + + void operator()(const char* c_str) const { + if (c_str != nullptr) { + env_->ReleaseStringUTFChars(j_str_, c_str); + } + } + +private: + JNIEnv* const env_; + jstring j_str_; +}; + +/** A unique pointer for JNI UTF string. */ +using StringUtfUniquePtr = std::unique_ptr; + +/** A custom deleter to delete JNI array elements. */ +template class ArrayElementsDeleter { +public: + ArrayElementsDeleter(JNIEnv* env, ArrayType array) : env_(env), array_(array) {} + + void operator()(ElementType* elements) const { + if (elements == nullptr) { + return; + } + if constexpr (std::is_same_v) { + env_->ReleaseByteArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseCharArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseShortArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseIntArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseLongArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseFloatArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseDoubleArrayElements(array_, elements, 0); + } else if constexpr (std::is_same_v) { + env_->ReleaseBooleanArrayElements(array_, elements, 0); + } + } + +private: + JNIEnv* const env_; + ArrayType array_; +}; + +/** A unique pointer for JNI array elements. */ +template +using ArrayElementsUniquePtr = std::unique_ptr< + typename std::remove_pointer::type, + ArrayElementsDeleter::type>>; + +/** A custom deleter for JNI primitive array critical. */ +class PrimitiveArrayCriticalDeleter { +public: + PrimitiveArrayCriticalDeleter(JNIEnv* env, jarray array) : env_(env), array_(array) {} + + void operator()(void* c_array) const { + if (c_array != nullptr) { + env_->ReleasePrimitiveArrayCritical(array_, c_array, 0); + } + } + +private: + JNIEnv* const env_; + jarray array_; +}; + +/** A unique pointer for JNI primitive array critical. */ +template +using PrimitiveArrayCriticalUniquePtr = + std::unique_ptr::type, PrimitiveArrayCriticalDeleter>; + +/** + * A thin wrapper around JNI API with automatic memory management. + * + * NOTE: Do not put any other helper functions that are not part of the JNI API here. + */ +class JniHelper { +public: + explicit JniHelper(JNIEnv* env) : env_(env) {} + + /** Gets the underlying `JNIEnv`. */ + JNIEnv* getEnv(); + + /** + * Gets the object method with the given signature. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getmethodid + */ + jmethodID getMethodId(jclass clazz, const char* name, const char* signature); + + /** + * Gets the static method with the given signature. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getstaticmethodid + */ + jmethodID getStaticMethodId(jclass clazz, const char* name, const char* signature); + + /** + * Finds the given `class_name` using Java classloader. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#findclass + */ + [[nodiscard]] LocalRefUniquePtr findClass(const char* class_name); + + /** + * Returns the class of a given `object`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getobjectclass + */ + [[nodiscard]] LocalRefUniquePtr getObjectClass(jobject object); + + /** + * Throws Java exception with the specified class name and error message. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#thrownew + */ + void throwNew(const char* java_class_name, const char* message); + + /** + * Determines if an exception is being thrown. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#exceptionoccurred + */ + [[nodiscard]] LocalRefUniquePtr exceptionOccurred(); + + /** + * Creates a new global reference to the object referred to by the `object` argument. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#newglobalref + */ + [[nodiscard]] GlobalRefUniquePtr newGlobalRef(jobject object); + + /** + * Creates a new instance of a given `clazz` from the given `method_id`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#newobject-newobjecta-newobjectv + */ + [[nodiscard]] LocalRefUniquePtr newObject(jclass clazz, jmethodID method_id, ...); + + /** + * Creates a new Java string from the given `str`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#newstringutf + */ + [[nodiscard]] LocalRefUniquePtr newStringUtf(const char* str); + + /** Gets the pointer to an array of bytes representing `str`. */ + [[nodiscard]] StringUtfUniquePtr getStringUtfChars(jstring str, jboolean* is_copy); + + /** + * Gets the size of the array. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getarraylength + */ + jsize getArrayLength(jarray array); + +/** A macro to create `NewArray`. helper function. */ +#define DECLARE_NEW_ARRAY(JAVA_TYPE, JNI_TYPE) \ + [[nodiscard]] LocalRefUniquePtr new##JAVA_TYPE##Array(jsize length); + + /** + * Helper functions for `NewArray`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#newprimitivetypearray-routines + */ + DECLARE_NEW_ARRAY(Byte, jbyteArray) + DECLARE_NEW_ARRAY(Char, jcharArray) + DECLARE_NEW_ARRAY(Short, jshortArray) + DECLARE_NEW_ARRAY(Int, jintArray) + DECLARE_NEW_ARRAY(Long, jlongArray) + DECLARE_NEW_ARRAY(Float, jfloatArray) + DECLARE_NEW_ARRAY(Double, jdoubleArray) + DECLARE_NEW_ARRAY(Boolean, jbooleanArray) + [[nodiscard]] LocalRefUniquePtr newObjectArray(jsize length, jclass element_class, + jobject initial_element = nullptr); + +/** A macro to create `GetArrayElement` function. */ +#define DECLARE_GET_ARRAY_ELEMENTS(JAVA_TYPE, JNI_ARRAY_TYPE, JNI_ELEMENT_TYPE) \ + [[nodiscard]] ArrayElementsUniquePtr \ + get##JAVA_TYPE##ArrayElements(JNI_ARRAY_TYPE array, jboolean* is_copy); + + /** + * Helper functions for `GetArrayElements`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getprimitivetypearrayelements-routines + */ + DECLARE_GET_ARRAY_ELEMENTS(Byte, jbyteArray, jbyte) + DECLARE_GET_ARRAY_ELEMENTS(Char, jcharArray, jchar) + DECLARE_GET_ARRAY_ELEMENTS(Short, jshortArray, jshort) + DECLARE_GET_ARRAY_ELEMENTS(Int, jintArray, jint) + DECLARE_GET_ARRAY_ELEMENTS(Long, jlongArray, jlong) + DECLARE_GET_ARRAY_ELEMENTS(Float, jfloatArray, jfloat) + DECLARE_GET_ARRAY_ELEMENTS(Double, jdoubleArray, jdouble) + DECLARE_GET_ARRAY_ELEMENTS(Boolean, jbooleanArray, jboolean) + + /** + * Gets an element of a given `array` with the specified `index`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getobjectarrayelement + */ + template + [[nodiscard]] LocalRefUniquePtr getObjectArrayElement(jobjectArray array, jsize index) { + LocalRefUniquePtr result(static_cast(env_->GetObjectArrayElement(array, index)), + LocalRefDeleter(env_)); + rethrowException(); + return result; + } + + /** + * Sets an element of a given `array` with the specified `index. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#setobjectarrayelement + */ + void setObjectArrayElement(jobjectArray array, jsize index, jobject value); + + /** + * Returns the pointer into the primitive array. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getprimitivearraycritical-releaseprimitivearraycritical + */ + template + [[nodiscard]] PrimitiveArrayCriticalUniquePtr getPrimitiveArrayCritical(jarray array, + jboolean* is_copy) { + PrimitiveArrayCriticalUniquePtr result( + static_cast(env_->GetPrimitiveArrayCritical(array, is_copy)), + PrimitiveArrayCriticalDeleter(env_, array)); + return result; + } + + /** + * Sets a region of an `array` from a `buffer` with the specified `start` index and `length`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#setprimitivetypearrayregion-routines + */ +#define DECLARE_SET_ARRAY_REGION(JAVA_TYPE, JNI_ARRAY_TYPE, JNI_ELEMENT_TYPE) \ + void set##JAVA_TYPE##ArrayRegion(JNI_ARRAY_TYPE array, jsize start, jsize length, \ + const JNI_ELEMENT_TYPE* buffer); + + DECLARE_SET_ARRAY_REGION(Byte, jbyteArray, jbyte) + DECLARE_SET_ARRAY_REGION(Char, jcharArray, jchar) + DECLARE_SET_ARRAY_REGION(Short, jshortArray, jshort) + DECLARE_SET_ARRAY_REGION(Int, jintArray, jint) + DECLARE_SET_ARRAY_REGION(Long, jlongArray, jlong) + DECLARE_SET_ARRAY_REGION(Float, jfloatArray, jfloat) + DECLARE_SET_ARRAY_REGION(Double, jdoubleArray, jdouble) + DECLARE_SET_ARRAY_REGION(Boolean, jbooleanArray, jboolean) + +/** A macro to create `CallMethod` helper function. */ +#define DECLARE_CALL_METHOD(JAVA_TYPE, JNI_TYPE) \ + JNI_TYPE call##JAVA_TYPE##Method(jobject object, jmethodID method_id, ...); + + /** + * Helper functions for `CallMethod`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#calltypemethod-routines-calltypemethoda-routines-calltypemethodv-routines + */ + DECLARE_CALL_METHOD(Byte, jbyte) + DECLARE_CALL_METHOD(Char, jchar) + DECLARE_CALL_METHOD(Short, jshort) + DECLARE_CALL_METHOD(Int, jint) + DECLARE_CALL_METHOD(Long, jlong) + DECLARE_CALL_METHOD(Float, jfloat) + DECLARE_CALL_METHOD(Double, jdouble) + DECLARE_CALL_METHOD(Boolean, jboolean) + + void callVoidMethod(jobject object, jmethodID method_id, ...); + + template + [[nodiscard]] LocalRefUniquePtr callObjectMethod(jobject object, jmethodID method_id, ...) { + va_list args; + va_start(args, method_id); + LocalRefUniquePtr result(static_cast(env_->CallObjectMethodV(object, method_id, args)), + LocalRefDeleter(env_)); + va_end(args); + rethrowException(); + return result; + } + +/** A macro to create `CallStaticMethod` helper function. */ +#define DECLARE_CALL_STATIC_METHOD(JAVA_TYPE, JNI_TYPE) \ + JNI_TYPE callStatic##JAVA_TYPE##Method(jclass clazz, jmethodID method_id, ...); + + /** + * Helper functions for `CallStaticMethod`. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#callstatictypemethod-routines-callstatictypemethoda-routines-callstatictypemethodv-routines + */ + DECLARE_CALL_STATIC_METHOD(Byte, jbyte) + DECLARE_CALL_STATIC_METHOD(Char, jchar) + DECLARE_CALL_STATIC_METHOD(Short, jshort) + DECLARE_CALL_STATIC_METHOD(Int, jint) + DECLARE_CALL_STATIC_METHOD(Long, jlong) + DECLARE_CALL_STATIC_METHOD(Float, jfloat) + DECLARE_CALL_STATIC_METHOD(Double, jdouble) + DECLARE_CALL_STATIC_METHOD(Boolean, jboolean) + + void callStaticVoidMethod(jclass clazz, jmethodID method_id, ...); + + template + [[nodiscard]] LocalRefUniquePtr callStaticObjectMethod(jclass clazz, jmethodID method_id, + ...) { + va_list args; + va_start(args, method_id); + LocalRefUniquePtr result( + static_cast(env_->CallStaticObjectMethodV(clazz, method_id, args)), + LocalRefDeleter(env_)); + va_end(args); + rethrowException(); + return result; + } + + /** + * Returns the capacity of the memory region referenced by the given `java.nio.Buffer` object. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getdirectbuffercapacity + */ + jlong getDirectBufferCapacity(jobject buffer); + + /** + * Gets the address of memory associated with the given NIO direct buffer. + * + * https://docs.oracle.com/en/java/javase/17/docs/specs/jni/functions.html#getdirectbufferaddress + */ + template T getDirectBufferAddress(jobject buffer) { + return static_cast(env_->GetDirectBufferAddress(buffer)); + } + +private: + /** Rethrows the Java exception occurred. */ + void rethrowException(); + + JNIEnv* const env_; +}; + +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_impl.cc similarity index 58% rename from mobile/library/common/jni/jni_interface.cc rename to mobile/library/common/jni/jni_impl.cc index 4a7ec5a50a69..fa7f00659aee 100644 --- a/mobile/library/common/jni/jni_interface.cc +++ b/mobile/library/common/jni/jni_impl.cc @@ -2,6 +2,8 @@ #include #include +#include "source/common/protobuf/protobuf.h" + #include "library/cc/engine_builder.h" #include "library/common/api/c_types.h" #include "library/common/data/utility.h" @@ -38,17 +40,18 @@ static void jvm_on_engine_running(void* context) { } jni_log("[Envoy]", "jvm_on_engine_running"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); - jclass jcls_JvmonEngineRunningContext = env->GetObjectClass(j_context); - jmethodID jmid_onEngineRunning = env->GetMethodID( - jcls_JvmonEngineRunningContext, "invokeOnEngineRunning", "()Ljava/lang/Object;"); - env->CallObjectMethod(j_context, jmid_onEngineRunning); + Envoy::JNI::LocalRefUniquePtr jcls_JvmonEngineRunningContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_onEngineRunning = jni_helper.getMethodId( + jcls_JvmonEngineRunningContext.get(), "invokeOnEngineRunning", "()Ljava/lang/Object;"); + Envoy::JNI::LocalRefUniquePtr unused = + jni_helper.callObjectMethod(j_context, jmid_onEngineRunning); - env->DeleteLocalRef(jcls_JvmonEngineRunningContext); // TODO(goaway): This isn't re-used by other engine callbacks, so it's safe to delete here. // This will need to be updated for https://github.com/envoyproxy/envoy-mobile/issues/332 - env->DeleteGlobalRef(j_context); + jni_helper.getEnv()->DeleteGlobalRef(j_context); } static void jvm_on_log(envoy_data data, const void* context) { @@ -56,17 +59,17 @@ static void jvm_on_log(envoy_data data, const void* context) { return; } - JNIEnv* env = get_env(); - jstring str = native_data_to_string(env, data); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); + Envoy::JNI::LocalRefUniquePtr str = Envoy::JNI::envoyDataToJavaString(jni_helper, data); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmLoggerContext = env->GetObjectClass(j_context); - jmethodID jmid_onLog = env->GetMethodID(jcls_JvmLoggerContext, "log", "(Ljava/lang/String;)V"); - env->CallVoidMethod(j_context, jmid_onLog, str); + Envoy::JNI::LocalRefUniquePtr jcls_JvmLoggerContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_onLog = + jni_helper.getMethodId(jcls_JvmLoggerContext.get(), "log", "(Ljava/lang/String;)V"); + jni_helper.callVoidMethod(j_context, jmid_onLog, str.get()); release_envoy_data(data); - env->DeleteLocalRef(str); - env->DeleteLocalRef(jcls_JvmLoggerContext); } static void jvm_on_exit(void*) { @@ -84,17 +87,18 @@ static void jvm_on_track(envoy_map events, const void* context) { return; } - JNIEnv* env = get_env(); - jobject events_hashmap = native_map_to_map(env, events); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); + Envoy::JNI::LocalRefUniquePtr events_hashmap = + Envoy::JNI::envoyMapToJavaMap(jni_helper, events); jobject j_context = static_cast(const_cast(context)); - jclass jcls_EnvoyEventTracker = env->GetObjectClass(j_context); - jmethodID jmid_onTrack = env->GetMethodID(jcls_EnvoyEventTracker, "track", "(Ljava/util/Map;)V"); - env->CallVoidMethod(j_context, jmid_onTrack, events_hashmap); + Envoy::JNI::LocalRefUniquePtr jcls_EnvoyEventTracker = + jni_helper.getObjectClass(j_context); + jmethodID jmid_onTrack = + jni_helper.getMethodId(jcls_EnvoyEventTracker.get(), "track", "(Ljava/util/Map;)V"); + jni_helper.callVoidMethod(j_context, jmid_onTrack, events_hashmap.get()); release_envoy_map(events); - env->DeleteLocalRef(events_hashmap); - env->DeleteLocalRef(jcls_EnvoyEventTracker); } extern "C" JNIEXPORT jint JNICALL @@ -114,7 +118,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr const jobject retained_logger_context = env->NewGlobalRef(envoy_logger_context); envoy_logger logger = {nullptr, nullptr, nullptr}; if (envoy_logger_context != nullptr) { - logger = envoy_logger{jvm_on_log, jni_delete_const_global_ref, retained_logger_context}; + logger = envoy_logger{jvm_on_log, Envoy::JNI::jniDeleteConstGlobalRef, retained_logger_context}; } envoy_event_tracker event_tracker = {nullptr, nullptr}; @@ -143,9 +147,9 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra if (!bootstrap) { result = run_engine(engine, string_config, string_log_level); } else { - auto options = std::make_unique(); + auto options = std::make_unique(); options->setConfigProto(std::move(bootstrap)); - options->setLogLevel(options->parseAndValidateLogLevel(string_log_level)); + ENVOY_BUG(options->setLogLevel(string_log_level).ok(), "invalid log level"); options->setConcurrency(1); result = reinterpret_cast(engine)->run(std::move(options)); } @@ -164,20 +168,14 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra JNIEnv* env, jclass, // class jlong engine, jstring elements, jobjectArray tags, jint count) { - const char* native_elements = env->GetStringUTFChars(elements, nullptr); - jint result = record_counter_inc(engine, native_elements, to_native_tags(env, tags), count); - env->ReleaseStringUTFChars(elements, native_elements); + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr native_elements = jni_helper.getStringUtfChars(elements, nullptr); + jint result = record_counter_inc( + engine, native_elements.get(), + Envoy::JNI::javaArrayOfObjectArrayToEnvoyStatsTags(jni_helper, tags), count); return result; } -extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_JniLibrary_flushStats(JNIEnv* env, - jclass, // class - jlong engine) { - jni_log("[Envoy]", "flushStats"); - flush_stats(engine); -} - extern "C" JNIEXPORT jstring JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_dumpStats(JNIEnv* env, jclass, // class @@ -189,19 +187,22 @@ Java_io_envoyproxy_envoymobile_engine_JniLibrary_dumpStats(JNIEnv* env, return env->NewStringUTF(""); } - jstring str = native_data_to_string(env, data); + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::LocalRefUniquePtr str = Envoy::JNI::envoyDataToJavaString(jni_helper, data); release_envoy_data(data); - return str; + return str.release(); } // JvmCallbackContext static void passHeaders(const char* method, const Envoy::Types::ManagedEnvoyHeaders& headers, jobject j_context) { - JNIEnv* env = get_env(); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); - jmethodID jmid_passHeader = env->GetMethodID(jcls_JvmCallbackContext, method, "([B[BZ)V"); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_passHeader = + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), method, "([B[BZ)V"); jboolean start_headers = JNI_TRUE; for (envoy_map_size_t i = 0; i < headers.get().length; i++) { @@ -211,21 +212,20 @@ static void passHeaders(const char* method, const Envoy::Types::ManagedEnvoyHead // requires a null-terminated *modified* UTF-8 string. // Create platform byte array for header key - jbyteArray j_key = native_data_to_array(env, headers.get().entries[i].key); + Envoy::JNI::LocalRefUniquePtr j_key = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, headers.get().entries[i].key); // Create platform byte array for header value - jbyteArray j_value = native_data_to_array(env, headers.get().entries[i].value); + Envoy::JNI::LocalRefUniquePtr j_value = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, headers.get().entries[i].value); // Pass this header pair to the platform - env->CallVoidMethod(j_context, jmid_passHeader, j_key, j_value, start_headers); - env->DeleteLocalRef(j_key); - env->DeleteLocalRef(j_value); + jni_helper.callVoidMethod(j_context, jmid_passHeader, j_key.get(), j_value.get(), + start_headers); // We don't release local refs currently because we've pushed a large enough frame, but we could // consider this and/or periodically popping the frame. start_headers = JNI_FALSE; } - - env->DeleteLocalRef(jcls_JvmCallbackContext); } // Platform callback implementation @@ -235,50 +235,52 @@ static void passHeaders(const char* method, const Envoy::Types::ManagedEnvoyHead static void* jvm_on_headers(const char* method, const Envoy::Types::ManagedEnvoyHeaders& headers, bool end_stream, envoy_stream_intel stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_headers"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); passHeaders("passHeader", headers, j_context); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); jmethodID jmid_onHeaders = - env->GetMethodID(jcls_JvmCallbackContext, method, "(JZ[J)Ljava/lang/Object;"); + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), method, "(JZ[J)Ljava/lang/Object;"); - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); // Note: be careful of JVM types. Before we casted to jlong we were getting integer problems. // TODO: make this cast safer. - jobject result = env->CallObjectMethod(j_context, jmid_onHeaders, (jlong)headers.get().length, - end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel); + Envoy::JNI::LocalRefUniquePtr result = + jni_helper.callObjectMethod(j_context, jmid_onHeaders, (jlong)headers.get().length, + end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel.get()); // TODO(Augustyniak): Pass the name of the filter in here so that we can instrument the origin of // the JNI exception better. bool exception_cleared = Envoy::JNI::Exception::checkAndClear(method); - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(jcls_JvmCallbackContext); - if (!exception_cleared) { - return result; + return result.release(); } // Create a "no operation" result: // 1. Tell the filter chain to continue the iteration. // 2. Return headers received on as method's input as part of the method's output. - jclass jcls_object_array = env->FindClass("java/lang/Object"); - jobjectArray noopResult = env->NewObjectArray(2, jcls_object_array, NULL); - - jclass jcls_int = env->FindClass("java/lang/Integer"); - jmethodID jmid_intInit = env->GetMethodID(jcls_int, "", "(I)V"); - jobject j_status = env->NewObject(jcls_int, jmid_intInit, 0); + Envoy::JNI::LocalRefUniquePtr jcls_object_array = + jni_helper.findClass("java/lang/Object"); + Envoy::JNI::LocalRefUniquePtr noopResult = + jni_helper.newObjectArray(2, jcls_object_array.get(), NULL); + + Envoy::JNI::LocalRefUniquePtr jcls_int = jni_helper.findClass("java/lang/Integer"); + jmethodID jmid_intInit = jni_helper.getMethodId(jcls_int.get(), "", "(I)V"); + Envoy::JNI::LocalRefUniquePtr j_status = + jni_helper.newObject(jcls_int.get(), jmid_intInit, 0); // Set status to "0" (FilterHeadersStatus::Continue). Signal that the intent // is to continue the iteration of the filter chain. - env->SetObjectArrayElement(noopResult, 0, j_status); + jni_helper.setObjectArrayElement(noopResult.get(), 0, j_status.get()); // Since the "on headers" call threw an exception set input headers as output headers. - env->SetObjectArrayElement(noopResult, 1, ToJavaArrayOfObjectArray(env, headers)); - - env->DeleteLocalRef(jcls_object_array); - env->DeleteLocalRef(jcls_int); + Envoy::JNI::LocalRefUniquePtr j_headers = + Envoy::JNI::envoyHeadersToJavaArrayOfObjectArray(jni_helper, headers); + jni_helper.setObjectArrayElement(noopResult.get(), 1, j_headers.get()); - return noopResult; + return noopResult.release(); } static void* jvm_on_response_headers(envoy_headers headers, bool end_stream, @@ -290,26 +292,26 @@ static void* jvm_on_response_headers(envoy_headers headers, bool end_stream, static envoy_filter_headers_status jvm_http_filter_on_request_headers(envoy_headers input_headers, bool end_stream, envoy_stream_intel stream_intel, const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); const auto headers = Envoy::Types::ManagedEnvoyHeaders(input_headers); jobjectArray result = static_cast(jvm_on_headers( "onRequestHeaders", headers, end_stream, stream_intel, const_cast(context))); - if (result == NULL || env->GetArrayLength(result) < 2) { - env->DeleteLocalRef(result); + if (result == NULL || jni_helper.getArrayLength(result) < 2) { + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_headers_status){/*status*/ kEnvoyFilterHeadersStatusStopIteration, /*headers*/ {}}; } - jobject status = env->GetObjectArrayElement(result, 0); - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 1)); + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result, 0); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result, 1); - int unboxed_status = unbox_integer(env, status); - envoy_headers native_headers = to_native_headers(env, j_headers); + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_headers native_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, j_headers.get()); - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_headers); + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_headers_status){/*status*/ unboxed_status, /*headers*/ native_headers}; @@ -318,26 +320,26 @@ jvm_http_filter_on_request_headers(envoy_headers input_headers, bool end_stream, static envoy_filter_headers_status jvm_http_filter_on_response_headers(envoy_headers input_headers, bool end_stream, envoy_stream_intel stream_intel, const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); const auto headers = Envoy::Types::ManagedEnvoyHeaders(input_headers); jobjectArray result = static_cast(jvm_on_headers( "onResponseHeaders", headers, end_stream, stream_intel, const_cast(context))); - if (result == NULL || env->GetArrayLength(result) < 2) { - env->DeleteLocalRef(result); + if (result == NULL || jni_helper.getArrayLength(result) < 2) { + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_headers_status){/*status*/ kEnvoyFilterHeadersStatusStopIteration, /*headers*/ {}}; } - jobject status = env->GetObjectArrayElement(result, 0); - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 1)); + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result, 0); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result, 1); - int unboxed_status = unbox_integer(env, status); - envoy_headers native_headers = to_native_headers(env, j_headers); + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_headers native_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, j_headers.get()); - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_headers); + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_headers_status){/*status*/ unboxed_status, /*headers*/ native_headers}; @@ -346,21 +348,23 @@ jvm_http_filter_on_response_headers(envoy_headers input_headers, bool end_stream static void* jvm_on_data(const char* method, envoy_data data, bool end_stream, envoy_stream_intel stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_data"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); jmethodID jmid_onData = - env->GetMethodID(jcls_JvmCallbackContext, method, "([BZ[J)Ljava/lang/Object;"); + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), method, "([BZ[J)Ljava/lang/Object;"); - jbyteArray j_data = native_data_to_array(env, data); - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); - jobject result = env->CallObjectMethod(j_context, jmid_onData, j_data, - end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel); + Envoy::JNI::LocalRefUniquePtr j_data = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, data); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); + jobject result = jni_helper + .callObjectMethod(j_context, jmid_onData, j_data.get(), + end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel.get()) + .release(); - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(j_data); - env->DeleteLocalRef(jcls_JvmCallbackContext); release_envoy_data(data); return result; @@ -374,34 +378,34 @@ static void* jvm_on_response_data(envoy_data data, bool end_stream, envoy_stream static envoy_filter_data_status jvm_http_filter_on_request_data(envoy_data data, bool end_stream, envoy_stream_intel stream_intel, const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobjectArray result = static_cast( jvm_on_data("onRequestData", data, end_stream, stream_intel, const_cast(context))); - if (result == NULL || env->GetArrayLength(result) < 2) { - env->DeleteLocalRef(result); + if (result == NULL || jni_helper.getArrayLength(result) < 2) { + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_data_status){/*status*/ kEnvoyFilterHeadersStatusStopIteration, /*data*/ {}, /*pending_headers*/ {}}; } - jobject status = env->GetObjectArrayElement(result, 0); - jobject j_data = static_cast(env->GetObjectArrayElement(result, 1)); + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result, 0); + Envoy::JNI::LocalRefUniquePtr j_data = + jni_helper.getObjectArrayElement(result, 1); - int unboxed_status = unbox_integer(env, status); - envoy_data native_data = buffer_to_native_data(env, j_data); + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_data native_data = Envoy::JNI::javaByteBufferToEnvoyData(jni_helper, j_data.get()); envoy_headers* pending_headers = nullptr; // Avoid out-of-bounds access to array when checking for optional pending entities. if (unboxed_status == kEnvoyFilterDataStatusResumeIteration) { - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 2)); - pending_headers = to_native_headers_ptr(env, j_headers); - env->DeleteLocalRef(j_headers); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result, 2); + pending_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeadersPtr(jni_helper, j_headers.get()); } - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_data); + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_data_status){/*status*/ unboxed_status, /*data*/ native_data, @@ -411,34 +415,34 @@ static envoy_filter_data_status jvm_http_filter_on_request_data(envoy_data data, static envoy_filter_data_status jvm_http_filter_on_response_data(envoy_data data, bool end_stream, envoy_stream_intel stream_intel, const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobjectArray result = static_cast( jvm_on_data("onResponseData", data, end_stream, stream_intel, const_cast(context))); - if (result == NULL || env->GetArrayLength(result) < 2) { - env->DeleteLocalRef(result); + if (result == NULL || jni_helper.getArrayLength(result) < 2) { + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_data_status){/*status*/ kEnvoyFilterHeadersStatusStopIteration, /*data*/ {}, /*pending_headers*/ {}}; } - jobject status = env->GetObjectArrayElement(result, 0); - jobject j_data = static_cast(env->GetObjectArrayElement(result, 1)); + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result, 0); + Envoy::JNI::LocalRefUniquePtr j_data = + jni_helper.getObjectArrayElement(result, 1); - int unboxed_status = unbox_integer(env, status); - envoy_data native_data = buffer_to_native_data(env, j_data); + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_data native_data = Envoy::JNI::javaByteBufferToEnvoyData(jni_helper, j_data.get()); envoy_headers* pending_headers = nullptr; // Avoid out-of-bounds access to array when checking for optional pending entities. if (unboxed_status == kEnvoyFilterDataStatusResumeIteration) { - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 2)); - pending_headers = to_native_headers_ptr(env, j_headers); - env->DeleteLocalRef(j_headers); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result, 2); + pending_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeadersPtr(jni_helper, j_headers.get()); } - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_data); + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_data_status){/*status*/ unboxed_status, /*data*/ native_data, @@ -456,23 +460,23 @@ static void* jvm_on_trailers(const char* method, envoy_headers trailers, envoy_stream_intel stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_trailers"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); passHeaders("passHeader", trailers, j_context); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); jmethodID jmid_onTrailers = - env->GetMethodID(jcls_JvmCallbackContext, method, "(J[J)Ljava/lang/Object;"); + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), method, "(J[J)Ljava/lang/Object;"); - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); // Note: be careful of JVM types. Before we casted to jlong we were getting integer problems. // TODO: make this cast safer. - // TODO(Augustyniak): check for pending exceptions after returning from JNI call. - jobject result = - env->CallObjectMethod(j_context, jmid_onTrailers, (jlong)trailers.length, j_stream_intel); - - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(jcls_JvmCallbackContext); + jobject result = jni_helper + .callObjectMethod(j_context, jmid_onTrailers, (jlong)trailers.length, + j_stream_intel.get()) + .release(); return result; } @@ -485,40 +489,40 @@ static void* jvm_on_response_trailers(envoy_headers trailers, envoy_stream_intel static envoy_filter_trailers_status jvm_http_filter_on_request_trailers(envoy_headers trailers, envoy_stream_intel stream_intel, const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobjectArray result = static_cast( jvm_on_trailers("onRequestTrailers", trailers, stream_intel, const_cast(context))); - if (result == NULL || env->GetArrayLength(result) < 2) { - env->DeleteLocalRef(result); + if (result == NULL || jni_helper.getArrayLength(result) < 2) { + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_trailers_status){/*status*/ kEnvoyFilterHeadersStatusStopIteration, /*trailers*/ {}, /*pending_headers*/ {}, /*pending_data*/ {}}; } - jobject status = env->GetObjectArrayElement(result, 0); - jobjectArray j_trailers = static_cast(env->GetObjectArrayElement(result, 1)); + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result, 0); + Envoy::JNI::LocalRefUniquePtr j_trailers = + jni_helper.getObjectArrayElement(result, 1); - int unboxed_status = unbox_integer(env, status); - envoy_headers native_trailers = to_native_headers(env, j_trailers); + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_headers native_trailers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, j_trailers.get()); envoy_headers* pending_headers = nullptr; envoy_data* pending_data = nullptr; // Avoid out-of-bounds access to array when checking for optional pending entities. if (unboxed_status == kEnvoyFilterTrailersStatusResumeIteration) { - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 2)); - pending_headers = to_native_headers_ptr(env, j_headers); - env->DeleteLocalRef(j_headers); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result, 2); + pending_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeadersPtr(jni_helper, j_headers.get()); - jobject j_data = static_cast(env->GetObjectArrayElement(result, 3)); - pending_data = buffer_to_native_data_ptr(env, j_data); - env->DeleteLocalRef(j_data); + Envoy::JNI::LocalRefUniquePtr j_data = jni_helper.getObjectArrayElement(result, 3); + pending_data = Envoy::JNI::javaByteBufferToEnvoyDataPtr(jni_helper, j_data.get()); } - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_trailers); + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_trailers_status){/*status*/ unboxed_status, /*trailers*/ native_trailers, @@ -529,40 +533,40 @@ jvm_http_filter_on_request_trailers(envoy_headers trailers, envoy_stream_intel s static envoy_filter_trailers_status jvm_http_filter_on_response_trailers(envoy_headers trailers, envoy_stream_intel stream_intel, const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobjectArray result = static_cast( jvm_on_trailers("onResponseTrailers", trailers, stream_intel, const_cast(context))); - if (result == NULL || env->GetArrayLength(result) < 2) { - env->DeleteLocalRef(result); + if (result == NULL || jni_helper.getArrayLength(result) < 2) { + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_trailers_status){/*status*/ kEnvoyFilterHeadersStatusStopIteration, /*trailers*/ {}, /*pending_headers*/ {}, /*pending_data*/ {}}; } - jobject status = env->GetObjectArrayElement(result, 0); - jobjectArray j_trailers = static_cast(env->GetObjectArrayElement(result, 1)); + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result, 0); + Envoy::JNI::LocalRefUniquePtr j_trailers = + jni_helper.getObjectArrayElement(result, 1); - int unboxed_status = unbox_integer(env, status); - envoy_headers native_trailers = to_native_headers(env, j_trailers); + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_headers native_trailers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, j_trailers.get()); envoy_headers* pending_headers = nullptr; envoy_data* pending_data = nullptr; // Avoid out-of-bounds access to array when checking for optional pending entities. if (unboxed_status == kEnvoyFilterTrailersStatusResumeIteration) { - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 2)); - pending_headers = to_native_headers_ptr(env, j_headers); - env->DeleteLocalRef(j_headers); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result, 2); + pending_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeadersPtr(jni_helper, j_headers.get()); - jobject j_data = static_cast(env->GetObjectArrayElement(result, 3)); - pending_data = buffer_to_native_data_ptr(env, j_data); - env->DeleteLocalRef(j_data); + Envoy::JNI::LocalRefUniquePtr j_data = jni_helper.getObjectArrayElement(result, 3); + pending_data = Envoy::JNI::javaByteBufferToEnvoyDataPtr(jni_helper, j_data.get()); } - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_trailers); + jni_helper.getEnv()->DeleteLocalRef(result); return (envoy_filter_trailers_status){/*status*/ unboxed_status, /*trailers*/ native_trailers, @@ -575,9 +579,10 @@ static void jvm_http_filter_set_request_callbacks(envoy_http_filter_callbacks ca jni_log("[Envoy]", "jvm_http_filter_set_request_callbacks"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); envoy_http_filter_callbacks* on_heap_callbacks = static_cast(safe_malloc(sizeof(envoy_http_filter_callbacks))); @@ -585,10 +590,8 @@ static void jvm_http_filter_set_request_callbacks(envoy_http_filter_callbacks ca jlong callback_handle = reinterpret_cast(on_heap_callbacks); jmethodID jmid_setRequestFilterCallbacks = - env->GetMethodID(jcls_JvmCallbackContext, "setRequestFilterCallbacks", "(J)V"); - env->CallVoidMethod(j_context, jmid_setRequestFilterCallbacks, callback_handle); - - env->DeleteLocalRef(jcls_JvmCallbackContext); + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), "setRequestFilterCallbacks", "(J)V"); + jni_helper.callVoidMethod(j_context, jmid_setRequestFilterCallbacks, callback_handle); } static void jvm_http_filter_set_response_callbacks(envoy_http_filter_callbacks callbacks, @@ -596,9 +599,10 @@ static void jvm_http_filter_set_response_callbacks(envoy_http_filter_callbacks c jni_log("[Envoy]", "jvm_http_filter_set_response_callbacks"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); envoy_http_filter_callbacks* on_heap_callbacks = static_cast(safe_malloc(sizeof(envoy_http_filter_callbacks))); @@ -606,10 +610,8 @@ static void jvm_http_filter_set_response_callbacks(envoy_http_filter_callbacks c jlong callback_handle = reinterpret_cast(on_heap_callbacks); jmethodID jmid_setResponseFilterCallbacks = - env->GetMethodID(jcls_JvmCallbackContext, "setResponseFilterCallbacks", "(J)V"); - env->CallVoidMethod(j_context, jmid_setResponseFilterCallbacks, callback_handle); - - env->DeleteLocalRef(jcls_JvmCallbackContext); + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), "setResponseFilterCallbacks", "(J)V"); + jni_helper.callVoidMethod(j_context, jmid_setResponseFilterCallbacks, callback_handle); } static envoy_filter_resume_status @@ -618,54 +620,49 @@ jvm_http_filter_on_resume(const char* method, envoy_headers* headers, envoy_data const void* context) { jni_log("[Envoy]", "jvm_on_resume"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); jlong headers_length = -1; if (headers) { headers_length = (jlong)headers->length; passHeaders("passHeader", *headers, j_context); } - jbyteArray j_in_data = nullptr; + Envoy::JNI::LocalRefUniquePtr j_in_data = Envoy::JNI::LocalRefUniquePtr( + nullptr, Envoy::JNI::LocalRefDeleter(jni_helper.getEnv())); if (data) { - j_in_data = native_data_to_array(env, *data); + j_in_data = Envoy::JNI::envoyDataToJavaByteArray(jni_helper, *data); } jlong trailers_length = -1; if (trailers) { trailers_length = (jlong)trailers->length; passHeaders("passTrailer", *trailers, j_context); } - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); - jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmCallbackContext = + jni_helper.getObjectClass(j_context); jmethodID jmid_onResume = - env->GetMethodID(jcls_JvmCallbackContext, method, "(J[BJZ[J)Ljava/lang/Object;"); + jni_helper.getMethodId(jcls_JvmCallbackContext.get(), method, "(J[BJZ[J)Ljava/lang/Object;"); // Note: be careful of JVM types. Before we casted to jlong we were getting integer problems. // TODO: make this cast safer. - jobjectArray result = static_cast( - env->CallObjectMethod(j_context, jmid_onResume, headers_length, j_in_data, trailers_length, - end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel)); - - env->DeleteLocalRef(jcls_JvmCallbackContext); - env->DeleteLocalRef(j_stream_intel); - if (j_in_data != nullptr) { - env->DeleteLocalRef(j_in_data); - } - - jobject status = env->GetObjectArrayElement(result, 0); - jobjectArray j_headers = static_cast(env->GetObjectArrayElement(result, 1)); - jobject j_data = static_cast(env->GetObjectArrayElement(result, 2)); - jobjectArray j_trailers = static_cast(env->GetObjectArrayElement(result, 3)); - - int unboxed_status = unbox_integer(env, status); - envoy_headers* pending_headers = to_native_headers_ptr(env, j_headers); - envoy_data* pending_data = buffer_to_native_data_ptr(env, j_data); - envoy_headers* pending_trailers = to_native_headers_ptr(env, j_trailers); - - env->DeleteLocalRef(result); - env->DeleteLocalRef(status); - env->DeleteLocalRef(j_headers); - env->DeleteLocalRef(j_data); - env->DeleteLocalRef(j_trailers); + Envoy::JNI::LocalRefUniquePtr result = jni_helper.callObjectMethod( + j_context, jmid_onResume, headers_length, j_in_data.get(), trailers_length, + end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel.get()); + + Envoy::JNI::LocalRefUniquePtr status = jni_helper.getObjectArrayElement(result.get(), 0); + Envoy::JNI::LocalRefUniquePtr j_headers = + jni_helper.getObjectArrayElement(result.get(), 1); + Envoy::JNI::LocalRefUniquePtr j_data = jni_helper.getObjectArrayElement(result.get(), 2); + Envoy::JNI::LocalRefUniquePtr j_trailers = + jni_helper.getObjectArrayElement(result.get(), 3); + + int unboxed_status = Envoy::JNI::javaIntegerTotInt(jni_helper, status.get()); + envoy_headers* pending_headers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeadersPtr(jni_helper, j_headers.get()); + envoy_data* pending_data = Envoy::JNI::javaByteBufferToEnvoyDataPtr(jni_helper, j_data.get()); + envoy_headers* pending_trailers = + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeadersPtr(jni_helper, j_trailers.get()); return (envoy_filter_resume_status){/*status*/ unboxed_status, /*pending_headers*/ pending_headers, @@ -693,49 +690,50 @@ static void* call_jvm_on_complete(envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_complete"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); - jclass jcls_JvmObserverContext = env->GetObjectClass(j_context); - jmethodID jmid_onComplete = - env->GetMethodID(jcls_JvmObserverContext, "onComplete", "([J[J)Ljava/lang/Object;"); - - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); - jlongArray j_final_stream_intel = native_final_stream_intel_to_array(env, final_stream_intel); - jobject result = - env->CallObjectMethod(j_context, jmid_onComplete, j_stream_intel, j_final_stream_intel); + Envoy::JNI::LocalRefUniquePtr jcls_JvmObserverContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_onComplete = jni_helper.getMethodId(jcls_JvmObserverContext.get(), "onComplete", + "([J[J)Ljava/lang/Object;"); - Envoy::JNI::Exception::checkAndClear(); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_final_stream_intel = + Envoy::JNI::envoyFinalStreamIntelToJavaLongArray(jni_helper, final_stream_intel); + jobject result = jni_helper + .callObjectMethod(j_context, jmid_onComplete, j_stream_intel.get(), + j_final_stream_intel.get()) + .release(); - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(j_final_stream_intel); - env->DeleteLocalRef(jcls_JvmObserverContext); return result; } static void* call_jvm_on_error(envoy_error error, envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_error"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); - jclass jcls_JvmObserverContext = env->GetObjectClass(j_context); - jmethodID jmid_onError = - env->GetMethodID(jcls_JvmObserverContext, "onError", "(I[BI[J[J)Ljava/lang/Object;"); + Envoy::JNI::LocalRefUniquePtr jcls_JvmObserverContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_onError = jni_helper.getMethodId(jcls_JvmObserverContext.get(), "onError", + "(I[BI[J[J)Ljava/lang/Object;"); - jbyteArray j_error_message = native_data_to_array(env, error.message); - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); - jlongArray j_final_stream_intel = native_final_stream_intel_to_array(env, final_stream_intel); + Envoy::JNI::LocalRefUniquePtr j_error_message = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, error.message); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_final_stream_intel = + Envoy::JNI::envoyFinalStreamIntelToJavaLongArray(jni_helper, final_stream_intel); - jobject result = env->CallObjectMethod(j_context, jmid_onError, error.error_code, j_error_message, - error.attempt_count, j_stream_intel, j_final_stream_intel); - - Envoy::JNI::Exception::checkAndClear(); + jobject result = + jni_helper + .callObjectMethod(j_context, jmid_onError, error.error_code, j_error_message.get(), + error.attempt_count, j_stream_intel.get(), j_final_stream_intel.get()) + .release(); - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(j_final_stream_intel); - env->DeleteLocalRef(j_error_message); - env->DeleteLocalRef(jcls_JvmObserverContext); release_envoy_error(error); return result; } @@ -743,7 +741,7 @@ static void* call_jvm_on_error(envoy_error error, envoy_stream_intel stream_inte static void* jvm_on_error(envoy_error error, envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel, void* context) { void* result = call_jvm_on_error(error, stream_intel, final_stream_intel, context); - jni_delete_global_ref(context); + Envoy::JNI::jniDeleteGlobalRef(context); return result; } @@ -751,38 +749,38 @@ static void* call_jvm_on_cancel(envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_cancel"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); - jclass jcls_JvmObserverContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmObserverContext = + jni_helper.getObjectClass(j_context); jmethodID jmid_onCancel = - env->GetMethodID(jcls_JvmObserverContext, "onCancel", "([J[J)Ljava/lang/Object;"); + jni_helper.getMethodId(jcls_JvmObserverContext.get(), "onCancel", "([J[J)Ljava/lang/Object;"); - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); - jlongArray j_final_stream_intel = native_final_stream_intel_to_array(env, final_stream_intel); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_final_stream_intel = + Envoy::JNI::envoyFinalStreamIntelToJavaLongArray(jni_helper, final_stream_intel); - jobject result = - env->CallObjectMethod(j_context, jmid_onCancel, j_stream_intel, j_final_stream_intel); - - Envoy::JNI::Exception::checkAndClear(); + jobject result = jni_helper + .callObjectMethod(j_context, jmid_onCancel, j_stream_intel.get(), + j_final_stream_intel.get()) + .release(); - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(j_final_stream_intel); - env->DeleteLocalRef(jcls_JvmObserverContext); return result; } static void* jvm_on_complete(envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel, void* context) { void* result = call_jvm_on_complete(stream_intel, final_stream_intel, context); - jni_delete_global_ref(context); + Envoy::JNI::jniDeleteGlobalRef(context); return result; } static void* jvm_on_cancel(envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel, void* context) { void* result = call_jvm_on_cancel(stream_intel, final_stream_intel, context); - jni_delete_global_ref(context); + Envoy::JNI::jniDeleteGlobalRef(context); return result; } @@ -801,72 +799,74 @@ static void jvm_http_filter_on_cancel(envoy_stream_intel stream_intel, static void* jvm_on_send_window_available(envoy_stream_intel stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_send_window_available"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(context); - jclass jcls_JvmObserverContext = env->GetObjectClass(j_context); - jmethodID jmid_onSendWindowAvailable = - env->GetMethodID(jcls_JvmObserverContext, "onSendWindowAvailable", "([J)Ljava/lang/Object;"); + Envoy::JNI::LocalRefUniquePtr jcls_JvmObserverContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_onSendWindowAvailable = jni_helper.getMethodId( + jcls_JvmObserverContext.get(), "onSendWindowAvailable", "([J)Ljava/lang/Object;"); - jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); + Envoy::JNI::LocalRefUniquePtr j_stream_intel = + Envoy::JNI::envoyStreamIntelToJavaLongArray(jni_helper, stream_intel); - jobject result = env->CallObjectMethod(j_context, jmid_onSendWindowAvailable, j_stream_intel); + jobject result = + jni_helper.callObjectMethod(j_context, jmid_onSendWindowAvailable, j_stream_intel.get()) + .release(); - env->DeleteLocalRef(j_stream_intel); - env->DeleteLocalRef(jcls_JvmObserverContext); return result; } // JvmKeyValueStoreContext static envoy_data jvm_kv_store_read(envoy_data key, const void* context) { jni_log("[Envoy]", "jvm_kv_store_read"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmKeyValueStoreContext = env->GetObjectClass(j_context); - jmethodID jmid_read = env->GetMethodID(jcls_JvmKeyValueStoreContext, "read", "([B)[B"); - jbyteArray j_key = native_data_to_array(env, key); - jbyteArray j_value = (jbyteArray)env->CallObjectMethod(j_context, jmid_read, j_key); - envoy_data native_data = array_to_native_data(env, j_value); - - env->DeleteLocalRef(j_value); - env->DeleteLocalRef(j_key); - env->DeleteLocalRef(jcls_JvmKeyValueStoreContext); + Envoy::JNI::LocalRefUniquePtr jcls_JvmKeyValueStoreContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_read = + jni_helper.getMethodId(jcls_JvmKeyValueStoreContext.get(), "read", "([B)[B"); + Envoy::JNI::LocalRefUniquePtr j_key = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, key); + Envoy::JNI::LocalRefUniquePtr j_value = + jni_helper.callObjectMethod(j_context, jmid_read, j_key.get()); + envoy_data native_data = Envoy::JNI::javaByteArrayToEnvoyData(jni_helper, j_value.get()); return native_data; } static void jvm_kv_store_remove(envoy_data key, const void* context) { jni_log("[Envoy]", "jvm_kv_store_remove"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmKeyValueStoreContext = env->GetObjectClass(j_context); - jmethodID jmid_remove = env->GetMethodID(jcls_JvmKeyValueStoreContext, "remove", "([B)V"); - jbyteArray j_key = native_data_to_array(env, key); - env->CallVoidMethod(j_context, jmid_remove, j_key); - - env->DeleteLocalRef(j_key); - env->DeleteLocalRef(jcls_JvmKeyValueStoreContext); + Envoy::JNI::LocalRefUniquePtr jcls_JvmKeyValueStoreContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_remove = + jni_helper.getMethodId(jcls_JvmKeyValueStoreContext.get(), "remove", "([B)V"); + Envoy::JNI::LocalRefUniquePtr j_key = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, key); + jni_helper.callVoidMethod(j_context, jmid_remove, j_key.get()); } static void jvm_kv_store_save(envoy_data key, envoy_data value, const void* context) { jni_log("[Envoy]", "jvm_kv_store_save"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmKeyValueStoreContext = env->GetObjectClass(j_context); - jmethodID jmid_save = env->GetMethodID(jcls_JvmKeyValueStoreContext, "save", "([B[B)V"); - jbyteArray j_key = native_data_to_array(env, key); - jbyteArray j_value = native_data_to_array(env, value); - env->CallVoidMethod(j_context, jmid_save, j_key, j_value); - - env->DeleteLocalRef(j_value); - env->DeleteLocalRef(j_key); - env->DeleteLocalRef(jcls_JvmKeyValueStoreContext); + Envoy::JNI::LocalRefUniquePtr jcls_JvmKeyValueStoreContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_save = + jni_helper.getMethodId(jcls_JvmKeyValueStoreContext.get(), "save", "([B[B)V"); + Envoy::JNI::LocalRefUniquePtr j_key = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, key); + Envoy::JNI::LocalRefUniquePtr j_value = + Envoy::JNI::envoyDataToJavaByteArray(jni_helper, value); + jni_helper.callVoidMethod(j_context, jmid_save, j_key.get(), j_value.get()); } // JvmFilterFactoryContext @@ -874,40 +874,39 @@ static void jvm_kv_store_save(envoy_data key, envoy_data value, const void* cont static const void* jvm_http_filter_init(const void* context) { jni_log("[Envoy]", "jvm_filter_init"); - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); envoy_http_filter* c_filter = static_cast(const_cast(context)); jobject j_context = static_cast(const_cast(c_filter->static_context)); jni_log_fmt("[Envoy]", "j_context: %p", j_context); - jclass jcls_JvmFilterFactoryContext = env->GetObjectClass(j_context); - jmethodID jmid_create = env->GetMethodID(jcls_JvmFilterFactoryContext, "create", - "()Lio/envoyproxy/envoymobile/engine/JvmFilterContext;"); + Envoy::JNI::LocalRefUniquePtr jcls_JvmFilterFactoryContext = + jni_helper.getObjectClass(j_context); + jmethodID jmid_create = + jni_helper.getMethodId(jcls_JvmFilterFactoryContext.get(), "create", + "()Lio/envoyproxy/envoymobile/engine/JvmFilterContext;"); - jobject j_filter = env->CallObjectMethod(j_context, jmid_create); - jni_log_fmt("[Envoy]", "j_filter: %p", j_filter); - jobject retained_filter = env->NewGlobalRef(j_filter); + Envoy::JNI::LocalRefUniquePtr j_filter = + jni_helper.callObjectMethod(j_context, jmid_create); + jni_log_fmt("[Envoy]", "j_filter: %p", j_filter.get()); + Envoy::JNI::GlobalRefUniquePtr retained_filter = jni_helper.newGlobalRef(j_filter.get()); - env->DeleteLocalRef(jcls_JvmFilterFactoryContext); - env->DeleteLocalRef(j_filter); - - return retained_filter; + return retained_filter.release(); } // EnvoyStringAccessor static envoy_data jvm_get_string(const void* context) { - JNIEnv* env = get_env(); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); jobject j_context = static_cast(const_cast(context)); - jclass jcls_JvmStringAccessorContext = env->GetObjectClass(j_context); + Envoy::JNI::LocalRefUniquePtr jcls_JvmStringAccessorContext = + jni_helper.getObjectClass(j_context); jmethodID jmid_getString = - env->GetMethodID(jcls_JvmStringAccessorContext, "getEnvoyString", "()[B"); - jbyteArray j_data = (jbyteArray)env->CallObjectMethod(j_context, jmid_getString); - envoy_data native_data = array_to_native_data(env, j_data); - - env->DeleteLocalRef(jcls_JvmStringAccessorContext); - env->DeleteLocalRef(j_data); + jni_helper.getMethodId(jcls_JvmStringAccessorContext.get(), "getEnvoyString", "()[B"); + Envoy::JNI::LocalRefUniquePtr j_data = + jni_helper.callObjectMethod(j_context, jmid_getString); + envoy_data native_data = Envoy::JNI::javaByteArrayToEnvoyData(jni_helper, j_data.get()); return native_data; } @@ -922,7 +921,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_startStream( JNIEnv* env, jclass, jlong engine_handle, jlong stream_handle, jobject j_context, - jboolean explicit_flow_control, jlong min_delivery_size) { + jboolean explicit_flow_control) { // TODO: To be truly safe we may need stronger guarantees of operation ordering on this ref. jobject retained_context = env->NewGlobalRef(j_context); @@ -937,7 +936,7 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra retained_context}; envoy_status_t result = start_stream(static_cast(engine_handle), static_cast(stream_handle), native_callbacks, - explicit_flow_control, min_delivery_size); + explicit_flow_control); if (result != ENVOY_SUCCESS) { env->DeleteGlobalRef(retained_context); // No callbacks are fired and we need to release } @@ -994,7 +993,7 @@ Java_io_envoyproxy_envoymobile_engine_JniLibrary_registerFilterFactory(JNIEnv* e api->on_resume_response = jvm_http_filter_on_resume_response; api->on_cancel = jvm_http_filter_on_cancel; api->on_error = jvm_http_filter_on_error; - api->release_filter = jni_delete_const_global_ref; + api->release_filter = Envoy::JNI::jniDeleteConstGlobalRef; api->static_context = retained_context; api->instance_context = nullptr; @@ -1051,12 +1050,13 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_sendData( JNIEnv* env, jclass, jlong engine_handle, jlong stream_handle, jobject data, jint length, jboolean end_stream) { + Envoy::JNI::JniHelper jni_helper(env); if (end_stream) { jni_log("[Envoy]", "jvm_send_data_end_stream"); } return send_data(static_cast(engine_handle), static_cast(stream_handle), - buffer_to_native_data(env, data, length), end_stream); + Envoy::JNI::javaByteBufferToEnvoyData(jni_helper, data, length), end_stream); } // The Java counterpart guarantees to invoke this method with a non-null jbyteArray where the @@ -1069,28 +1069,31 @@ Java_io_envoyproxy_envoymobile_engine_JniLibrary_sendDataByteArray(JNIEnv* env, jlong stream_handle, jbyteArray data, jint length, jboolean end_stream) { + Envoy::JNI::JniHelper jni_helper(env); if (end_stream) { jni_log("[Envoy]", "jvm_send_data_end_stream"); } return send_data(static_cast(engine_handle), static_cast(stream_handle), - array_to_native_data(env, data, length), end_stream); + Envoy::JNI::javaByteArrayToEnvoyData(jni_helper, data, length), end_stream); } extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_sendHeaders( JNIEnv* env, jclass, jlong engine_handle, jlong stream_handle, jobjectArray headers, jboolean end_stream) { - return send_headers(static_cast(engine_handle), - static_cast(stream_handle), to_native_headers(env, headers), - end_stream); + Envoy::JNI::JniHelper jni_helper(env); + return send_headers( + static_cast(engine_handle), static_cast(stream_handle), + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, headers), end_stream); } extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_sendTrailers( JNIEnv* env, jclass, jlong engine_handle, jlong stream_handle, jobjectArray trailers) { + Envoy::JNI::JniHelper jni_helper(env); jni_log("[Envoy]", "jvm_send_trailers"); return send_trailers(static_cast(engine_handle), static_cast(stream_handle), - to_native_headers(env, trailers)); + Envoy::JNI::javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, trailers)); } extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_resetStream( @@ -1122,55 +1125,45 @@ Java_io_envoyproxy_envoymobile_engine_JniLibrary_registerStringAccessor(JNIEnv* // Takes a jstring from Java, converts it to a C++ string, calls the supplied // setter on it. -void setString(JNIEnv* env, jstring java_string, EngineBuilder* builder, +void setString(Envoy::JNI::JniHelper& jni_helper, jstring java_string, EngineBuilder* builder, EngineBuilder& (EngineBuilder::*setter)(std::string)) { if (!java_string) { return; } - const char* native_java_string = env->GetStringUTFChars(java_string, nullptr); - std::string java_string_str(native_java_string); + Envoy::JNI::StringUtfUniquePtr native_java_string = + jni_helper.getStringUtfChars(java_string, nullptr); + std::string java_string_str(native_java_string.get()); if (!java_string_str.empty()) { (builder->*setter)(java_string_str); - env->ReleaseStringUTFChars(java_string, native_java_string); - } -} - -// Convert jstring to std::string -std::string getCppString(JNIEnv* env, jstring java_string) { - if (!java_string) { - return ""; } - const char* native_java_string = env->GetStringUTFChars(java_string, nullptr); - std::string cpp_string(native_java_string); - env->ReleaseStringUTFChars(java_string, native_java_string); - return cpp_string; } // Converts a java byte array to a C++ string. -std::string javaByteArrayToString(JNIEnv* env, jbyteArray j_data) { - size_t data_length = static_cast(env->GetArrayLength(j_data)); - char* critical_data = static_cast(env->GetPrimitiveArrayCritical(j_data, 0)); - std::string ret(critical_data, data_length); - env->ReleasePrimitiveArrayCritical(j_data, critical_data, 0); +std::string javaByteArrayToString(Envoy::JNI::JniHelper& jni_helper, jbyteArray j_data) { + size_t data_length = static_cast(jni_helper.getArrayLength(j_data)); + Envoy::JNI::PrimitiveArrayCriticalUniquePtr critical_data = + jni_helper.getPrimitiveArrayCritical(j_data, nullptr); + std::string ret(critical_data.get(), data_length); return ret; } -// Converts a java object array to C++ vector of of strings. -std::vector javaObjectArrayToStringVector(JNIEnv* env, jobjectArray entries) { +// Converts a java object array to C++ vector of strings. +std::vector javaObjectArrayToStringVector(Envoy::JNI::JniHelper& jni_helper, + jobjectArray entries) { std::vector ret; // Note that headers is a flattened array of key/value pairs. // Therefore, the length of the native header array is n envoy_data or n/2 envoy_map_entry. - envoy_map_size_t length = env->GetArrayLength(entries); + envoy_map_size_t length = jni_helper.getArrayLength(entries); if (length == 0) { return ret; } for (envoy_map_size_t i = 0; i < length; ++i) { // Copy native byte array for header key - jbyteArray j_str = static_cast(env->GetObjectArrayElement(entries, i)); - std::string str = javaByteArrayToString(env, j_str); - ret.push_back(javaByteArrayToString(env, j_str)); - env->DeleteLocalRef(j_str); + Envoy::JNI::LocalRefUniquePtr j_str = + jni_helper.getObjectArrayElement(entries, i); + std::string str = javaByteArrayToString(jni_helper, j_str.get()); + ret.push_back(javaByteArrayToString(jni_helper, j_str.get())); } return ret; @@ -1178,48 +1171,46 @@ std::vector javaObjectArrayToStringVector(JNIEnv* env, jobjectArray // Converts a java object array to C++ vector of pairs of strings. std::vector> -javaObjectArrayToStringPairVector(JNIEnv* env, jobjectArray entries) { +javaObjectArrayToStringPairVector(Envoy::JNI::JniHelper& jni_helper, jobjectArray entries) { std::vector> ret; // Note that headers is a flattened array of key/value pairs. // Therefore, the length of the native header array is n envoy_data or n/2 envoy_map_entry. - envoy_map_size_t length = env->GetArrayLength(entries); + envoy_map_size_t length = jni_helper.getArrayLength(entries); if (length == 0) { return ret; } for (envoy_map_size_t i = 0; i < length; i += 2) { // Copy native byte array for header key - jbyteArray j_key = static_cast(env->GetObjectArrayElement(entries, i)); - jbyteArray j_value = static_cast(env->GetObjectArrayElement(entries, i + 1)); - std::string first = javaByteArrayToString(env, j_key); - std::string second = javaByteArrayToString(env, j_value); + Envoy::JNI::LocalRefUniquePtr j_key = + jni_helper.getObjectArrayElement(entries, i); + Envoy::JNI::LocalRefUniquePtr j_value = + jni_helper.getObjectArrayElement(entries, i + 1); + std::string first = javaByteArrayToString(jni_helper, j_key.get()); + std::string second = javaByteArrayToString(jni_helper, j_value.get()); ret.push_back(std::make_pair(first, second)); - - env->DeleteLocalRef(j_key); - env->DeleteLocalRef(j_value); } return ret; } -void configureBuilder(JNIEnv* env, jstring grpc_stats_domain, jlong connect_timeout_seconds, +void configureBuilder(Envoy::JNI::JniHelper& jni_helper, jlong connect_timeout_seconds, jlong dns_refresh_seconds, jlong dns_failure_refresh_seconds_base, jlong dns_failure_refresh_seconds_max, jlong dns_query_timeout_seconds, jlong dns_min_refresh_seconds, jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache, jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh, jboolean enable_http3, jstring http3_connection_options, jstring http3_client_connection_options, - jobjectArray quic_hints, jboolean enable_gzip_decompression, - jboolean enable_brotli_decompression, jboolean enable_socket_tagging, - jboolean enable_interface_binding, + jobjectArray quic_hints, jobjectArray quic_canonical_suffixes, + jboolean enable_gzip_decompression, jboolean enable_brotli_decompression, + jboolean enable_socket_tagging, jboolean enable_interface_binding, jlong h2_connection_keepalive_idle_interval_milliseconds, jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, - jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, - jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, - jboolean trust_chain_verification, jobjectArray filter_chain, - jobjectArray stat_sinks, jboolean enable_platform_certificates_validation, + jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, + jstring app_version, jstring app_id, jboolean trust_chain_verification, + jobjectArray filter_chain, jboolean enable_platform_certificates_validation, jobjectArray runtime_guards, jstring node_id, jstring node_region, - jstring node_zone, jstring node_sub_zone, + jstring node_zone, jstring node_sub_zone, jbyteArray serialized_node_metadata, Envoy::Platform::EngineBuilder& builder) { builder.addConnectTimeoutSeconds((connect_timeout_seconds)); builder.addDnsRefreshSeconds((dns_refresh_seconds)); @@ -1233,8 +1224,8 @@ void configureBuilder(JNIEnv* env, jstring grpc_stats_domain, jlong connect_time (h2_connection_keepalive_idle_interval_milliseconds)); builder.addH2ConnectionKeepaliveTimeoutSeconds((h2_connection_keepalive_timeout_seconds)); - setString(env, app_version, &builder, &EngineBuilder::setAppVersion); - setString(env, app_id, &builder, &EngineBuilder::setAppId); + setString(jni_helper, app_version, &builder, &EngineBuilder::setAppVersion); + setString(jni_helper, app_id, &builder, &EngineBuilder::setAppId); builder.setDeviceOs("Android"); builder.setStreamIdleTimeoutSeconds((stream_idle_timeout_seconds)); @@ -1244,12 +1235,20 @@ void configureBuilder(JNIEnv* env, jstring grpc_stats_domain, jlong connect_time builder.enableSocketTagging(enable_socket_tagging == JNI_TRUE); #ifdef ENVOY_ENABLE_QUIC builder.enableHttp3(enable_http3 == JNI_TRUE); - builder.setHttp3ConnectionOptions(getCppString(env, http3_connection_options)); - builder.setHttp3ClientConnectionOptions(getCppString(env, http3_client_connection_options)); - auto hints = javaObjectArrayToStringPairVector(env, quic_hints); - for (std::pair& entry : hints) { + builder.setHttp3ConnectionOptions( + Envoy::JNI::javaStringToString(jni_helper, http3_connection_options)); + builder.setHttp3ClientConnectionOptions( + Envoy::JNI::javaStringToString(jni_helper, http3_client_connection_options)); + auto hints = javaObjectArrayToStringPairVector(jni_helper, quic_hints); + for (const std::pair& entry : hints) { builder.addQuicHint(entry.first, stoi(entry.second)); } + std::vector suffixes = + javaObjectArrayToStringVector(jni_helper, quic_canonical_suffixes); + for (const std::string& suffix : suffixes) { + builder.addQuicCanonicalSuffix(suffix); + } + #endif builder.enableInterfaceBinding(enable_interface_binding == JNI_TRUE); builder.enableDrainPostDnsRefresh(enable_drain_post_dns_refresh == JNI_TRUE); @@ -1257,106 +1256,101 @@ void configureBuilder(JNIEnv* env, jstring grpc_stats_domain, jlong connect_time builder.enablePlatformCertificatesValidation(enable_platform_certificates_validation == JNI_TRUE); builder.setForceAlwaysUsev6(true); - auto guards = javaObjectArrayToStringPairVector(env, runtime_guards); + auto guards = javaObjectArrayToStringPairVector(jni_helper, runtime_guards); for (std::pair& entry : guards) { builder.setRuntimeGuard(entry.first, entry.second == "true"); } - auto filters = javaObjectArrayToStringPairVector(env, filter_chain); + auto filters = javaObjectArrayToStringPairVector(jni_helper, filter_chain); for (std::pair& filter : filters) { builder.addNativeFilter(filter.first, filter.second); } - std::vector sinks = javaObjectArrayToStringVector(env, stat_sinks); - -#ifdef ENVOY_MOBILE_STATS_REPORTING - builder.addStatsSinks(std::move(sinks)); - builder.addStatsFlushSeconds((stats_flush_seconds)); - setString(env, grpc_stats_domain, &builder, &EngineBuilder::addGrpcStatsDomain); -#endif - - std::vector hostnames = javaObjectArrayToStringVector(env, dns_preresolve_hostnames); + std::vector hostnames = + javaObjectArrayToStringVector(jni_helper, dns_preresolve_hostnames); builder.addDnsPreresolveHostnames(hostnames); - std::string native_node_id = getCppString(env, node_id); + std::string native_node_id = Envoy::JNI::javaStringToString(jni_helper, node_id); if (!native_node_id.empty()) { builder.setNodeId(native_node_id); } - std::string native_node_region = getCppString(env, node_region); + std::string native_node_region = Envoy::JNI::javaStringToString(jni_helper, node_region); if (!native_node_region.empty()) { - builder.setNodeLocality(native_node_region, getCppString(env, node_zone), - getCppString(env, node_sub_zone)); + builder.setNodeLocality(native_node_region, + Envoy::JNI::javaStringToString(jni_helper, node_zone), + Envoy::JNI::javaStringToString(jni_helper, node_sub_zone)); } + Envoy::ProtobufWkt::Struct node_metadata; + Envoy::JNI::javaByteArrayToProto(jni_helper, serialized_node_metadata, &node_metadata); + builder.setNodeMetadata(node_metadata); } extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_createBootstrap( - JNIEnv* env, jclass, jstring grpc_stats_domain, jlong connect_timeout_seconds, - jlong dns_refresh_seconds, jlong dns_failure_refresh_seconds_base, - jlong dns_failure_refresh_seconds_max, jlong dns_query_timeout_seconds, - jlong dns_min_refresh_seconds, jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache, + JNIEnv* env, jclass, jlong connect_timeout_seconds, jlong dns_refresh_seconds, + jlong dns_failure_refresh_seconds_base, jlong dns_failure_refresh_seconds_max, + jlong dns_query_timeout_seconds, jlong dns_min_refresh_seconds, + jobjectArray dns_preresolve_hostnames, jboolean enable_dns_cache, jlong dns_cache_save_interval_seconds, jboolean enable_drain_post_dns_refresh, jboolean enable_http3, jstring http3_connection_options, jstring http3_client_connection_options, jobjectArray quic_hints, - jboolean enable_gzip_decompression, jboolean enable_brotli_decompression, - jboolean enable_socket_tagging, jboolean enable_interface_binding, - jlong h2_connection_keepalive_idle_interval_milliseconds, + jobjectArray quic_canonical_suffixes, jboolean enable_gzip_decompression, + jboolean enable_brotli_decompression, jboolean enable_socket_tagging, + jboolean enable_interface_binding, jlong h2_connection_keepalive_idle_interval_milliseconds, jlong h2_connection_keepalive_timeout_seconds, jlong max_connections_per_host, - jlong stats_flush_seconds, jlong stream_idle_timeout_seconds, - jlong per_try_idle_timeout_seconds, jstring app_version, jstring app_id, - jboolean trust_chain_verification, jobjectArray filter_chain, jobjectArray stat_sinks, + jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version, + jstring app_id, jboolean trust_chain_verification, jobjectArray filter_chain, jboolean enable_platform_certificates_validation, jobjectArray runtime_guards, jstring rtds_resource_name, jlong rtds_timeout_seconds, jstring xds_address, jlong xds_port, - jstring xds_auth_header, jstring xds_auth_token, jstring xds_jwt_token, - jlong xds_jwt_token_lifetime, jstring xds_root_certs, jstring xds_sni, jstring node_id, - jstring node_region, jstring node_zone, jstring node_sub_zone, jstring cds_resources_locator, - jlong cds_timeout_seconds, jboolean enable_cds) { + jobjectArray xds_grpc_initial_metadata, jstring xds_root_certs, jstring node_id, + jstring node_region, jstring node_zone, jstring node_sub_zone, + jbyteArray serialized_node_metadata, jstring cds_resources_locator, jlong cds_timeout_seconds, + jboolean enable_cds) { + Envoy::JNI::JniHelper jni_helper(env); Envoy::Platform::EngineBuilder builder; - configureBuilder( - env, grpc_stats_domain, connect_timeout_seconds, dns_refresh_seconds, - dns_failure_refresh_seconds_base, dns_failure_refresh_seconds_max, dns_query_timeout_seconds, - dns_min_refresh_seconds, dns_preresolve_hostnames, enable_dns_cache, - dns_cache_save_interval_seconds, enable_drain_post_dns_refresh, enable_http3, - http3_connection_options, http3_client_connection_options, quic_hints, - enable_gzip_decompression, enable_brotli_decompression, enable_socket_tagging, - enable_interface_binding, h2_connection_keepalive_idle_interval_milliseconds, - h2_connection_keepalive_timeout_seconds, max_connections_per_host, stats_flush_seconds, - stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id, - trust_chain_verification, filter_chain, stat_sinks, enable_platform_certificates_validation, - runtime_guards, node_id, node_region, node_zone, node_sub_zone, builder); - -#ifdef ENVOY_GOOGLE_GRPC - std::string native_xds_address = getCppString(env, xds_address); + configureBuilder(jni_helper, connect_timeout_seconds, dns_refresh_seconds, + dns_failure_refresh_seconds_base, dns_failure_refresh_seconds_max, + dns_query_timeout_seconds, dns_min_refresh_seconds, dns_preresolve_hostnames, + enable_dns_cache, dns_cache_save_interval_seconds, enable_drain_post_dns_refresh, + enable_http3, http3_connection_options, http3_client_connection_options, + quic_hints, quic_canonical_suffixes, enable_gzip_decompression, + enable_brotli_decompression, enable_socket_tagging, enable_interface_binding, + h2_connection_keepalive_idle_interval_milliseconds, + h2_connection_keepalive_timeout_seconds, max_connections_per_host, + stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id, + trust_chain_verification, filter_chain, enable_platform_certificates_validation, + runtime_guards, node_id, node_region, node_zone, node_sub_zone, + serialized_node_metadata, builder); + + std::string native_xds_address = Envoy::JNI::javaStringToString(jni_helper, xds_address); if (!native_xds_address.empty()) { +#ifdef ENVOY_MOBILE_XDS Envoy::Platform::XdsBuilder xds_builder(std::move(native_xds_address), xds_port); - std::string native_xds_auth_header = getCppString(env, xds_auth_header); - if (!native_xds_auth_header.empty()) { - xds_builder.setAuthenticationToken(std::move(native_xds_auth_header), - getCppString(env, xds_auth_token)); - } - std::string native_jwt_token = getCppString(env, xds_jwt_token); - if (!native_jwt_token.empty()) { - xds_builder.setJwtAuthenticationToken(std::move(native_jwt_token), xds_jwt_token_lifetime); + auto initial_metadata = + javaObjectArrayToStringPairVector(jni_helper, xds_grpc_initial_metadata); + for (const std::pair& entry : initial_metadata) { + xds_builder.addInitialStreamHeader(entry.first, entry.second); } - std::string native_root_certs = getCppString(env, xds_root_certs); + std::string native_root_certs = Envoy::JNI::javaStringToString(jni_helper, xds_root_certs); if (!native_root_certs.empty()) { xds_builder.setSslRootCerts(std::move(native_root_certs)); } - std::string native_sni = getCppString(env, xds_sni); - if (!native_sni.empty()) { - xds_builder.setSni(std::move(native_sni)); - } - std::string native_rtds_resource_name = getCppString(env, rtds_resource_name); + std::string native_rtds_resource_name = + Envoy::JNI::javaStringToString(jni_helper, rtds_resource_name); if (!native_rtds_resource_name.empty()) { xds_builder.addRuntimeDiscoveryService(std::move(native_rtds_resource_name), rtds_timeout_seconds); } if (enable_cds == JNI_TRUE) { - xds_builder.addClusterDiscoveryService(getCppString(env, cds_resources_locator), - cds_timeout_seconds); + xds_builder.addClusterDiscoveryService( + Envoy::JNI::javaStringToString(jni_helper, cds_resources_locator), cds_timeout_seconds); } builder.setXds(std::move(xds_builder)); - } +#else + jni_helper.throwNew("java/lang/UnsupportedOperationException", + "This library does not support xDS. Please use " + "io.envoyproxy.envoymobile:envoy-xds instead."); #endif + } return reinterpret_cast(builder.generateBootstrap().release()); } @@ -1396,49 +1390,50 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra static void jvm_add_test_root_certificate(const uint8_t* cert, size_t len) { jni_log("[Envoy]", "jvm_add_test_root_certificate"); - JNIEnv* env = get_env(); - jclass jcls_AndroidNetworkLibrary = - find_class("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); - jmethodID jmid_addTestRootCertificate = - env->GetStaticMethodID(jcls_AndroidNetworkLibrary, "addTestRootCertificate", "([B)V"); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); + Envoy::JNI::LocalRefUniquePtr jcls_AndroidNetworkLibrary = + Envoy::JNI::findClass("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); + jmethodID jmid_addTestRootCertificate = jni_helper.getStaticMethodId( + jcls_AndroidNetworkLibrary.get(), "addTestRootCertificate", "([B)V"); - jbyteArray cert_array = ToJavaByteArray(env, cert, len); - env->CallStaticVoidMethod(jcls_AndroidNetworkLibrary, jmid_addTestRootCertificate, cert_array); - env->DeleteLocalRef(cert_array); - env->DeleteLocalRef(jcls_AndroidNetworkLibrary); + Envoy::JNI::LocalRefUniquePtr cert_array = + Envoy::JNI::byteArrayToJavaByteArray(jni_helper, cert, len); + jni_helper.callStaticVoidMethod(jcls_AndroidNetworkLibrary.get(), jmid_addTestRootCertificate, + cert_array.get()); } static void jvm_clear_test_root_certificate() { jni_log("[Envoy]", "jvm_clear_test_root_certificate"); - JNIEnv* env = get_env(); - jclass jcls_AndroidNetworkLibrary = - find_class("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); - jmethodID jmid_clearTestRootCertificates = - env->GetStaticMethodID(jcls_AndroidNetworkLibrary, "clearTestRootCertificates", "()V"); + Envoy::JNI::JniHelper jni_helper(Envoy::JNI::getEnv()); + Envoy::JNI::LocalRefUniquePtr jcls_AndroidNetworkLibrary = + Envoy::JNI::findClass("io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary"); + jmethodID jmid_clearTestRootCertificates = jni_helper.getStaticMethodId( + jcls_AndroidNetworkLibrary.get(), "clearTestRootCertificates", "()V"); - env->CallStaticVoidMethod(jcls_AndroidNetworkLibrary, jmid_clearTestRootCertificates); - env->DeleteLocalRef(jcls_AndroidNetworkLibrary); + jni_helper.callStaticVoidMethod(jcls_AndroidNetworkLibrary.get(), jmid_clearTestRootCertificates); } extern "C" JNIEXPORT jobject JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_callCertificateVerificationFromNative( JNIEnv* env, jclass, jobjectArray certChain, jbyteArray jauthType, jbyteArray jhost) { + Envoy::JNI::JniHelper jni_helper(env); std::vector cert_chain; std::string auth_type; std::string host; - JavaArrayOfByteArrayToStringVector(env, certChain, &cert_chain); - JavaArrayOfByteToString(env, jauthType, &auth_type); - JavaArrayOfByteToString(env, jhost, &host); + Envoy::JNI::javaArrayOfByteArrayToStringVector(jni_helper, certChain, &cert_chain); + Envoy::JNI::javaByteArrayToString(jni_helper, jauthType, &auth_type); + Envoy::JNI::javaByteArrayToString(jni_helper, jhost, &host); - return call_jvm_verify_x509_cert_chain(env, cert_chain, auth_type, host); + return callJvmVerifyX509CertChain(jni_helper, cert_chain, auth_type, host).release(); } extern "C" JNIEXPORT void JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibrary_callAddTestRootCertificateFromNative( JNIEnv* env, jclass, jbyteArray jcert) { + Envoy::JNI::JniHelper jni_helper(env); std::vector cert; - JavaArrayOfByteToBytesVector(env, jcert, &cert); + Envoy::JNI::javaByteArrayToByteVector(jni_helper, jcert, &cert); jvm_add_test_root_certificate(cert.data(), cert.size()); } diff --git a/mobile/library/common/jni/jni_utility.cc b/mobile/library/common/jni/jni_utility.cc index c6a74a2c92f4..3105aa0d5be2 100644 --- a/mobile/library/common/jni/jni_utility.cc +++ b/mobile/library/common/jni/jni_utility.cc @@ -1,7 +1,7 @@ #include "library/common/jni/jni_utility.h" -#include -#include +#include +#include #include "source/common/common/assert.h" @@ -9,194 +9,177 @@ #include "library/common/jni/types/env.h" #include "library/common/jni/types/exception.h" -// NOLINT(namespace-envoy) +namespace Envoy { +namespace JNI { static jobject static_class_loader = nullptr; -void set_class_loader(jobject class_loader) { static_class_loader = class_loader; } +void setClassLoader(jobject class_loader) { static_class_loader = class_loader; } -jobject get_class_loader() { +jobject getClassLoader() { RELEASE_ASSERT(static_class_loader, - "find_class() is used before calling AndroidJniLibrary.load()"); + "findClass() is used before calling AndroidJniLibrary.load()"); return static_class_loader; } -jclass find_class(const char* class_name) { - JNIEnv* env = get_env(); - jclass class_loader = env->FindClass("java/lang/ClassLoader"); - Envoy::JNI::Exception::checkAndClear("find_class:FindClass"); - jmethodID find_class_method = - env->GetMethodID(class_loader, "loadClass", "(Ljava/lang/String;)Ljava/lang/Class;"); - Envoy::JNI::Exception::checkAndClear("find_class:GetMethodID"); - jstring str_class_name = env->NewStringUTF(class_name); - Envoy::JNI::Exception::checkAndClear("find_class:NewStringUTF"); - jclass clazz = - (jclass)(env->CallObjectMethod(get_class_loader(), find_class_method, str_class_name)); - Envoy::JNI::Exception::checkAndClear("find_class:CallObjectMethod"); - env->DeleteLocalRef(str_class_name); +LocalRefUniquePtr findClass(const char* class_name) { + JniHelper jni_helper(getEnv()); + LocalRefUniquePtr class_loader = jni_helper.findClass("java/lang/ClassLoader"); + jmethodID find_class_method = jni_helper.getMethodId(class_loader.get(), "loadClass", + "(Ljava/lang/String;)Ljava/lang/Class;"); + LocalRefUniquePtr str_class_name = jni_helper.newStringUtf(class_name); + LocalRefUniquePtr clazz = jni_helper.callObjectMethod( + getClassLoader(), find_class_method, str_class_name.get()); return clazz; } -JNIEnv* get_env() { return Envoy::JNI::Env::get(); } +JNIEnv* getEnv() { return Envoy::JNI::Env::get(); } -void jni_delete_global_ref(void* context) { - JNIEnv* env = get_env(); +void jniDeleteGlobalRef(void* context) { + JNIEnv* env = getEnv(); jobject ref = static_cast(context); env->DeleteGlobalRef(ref); } -void jni_delete_const_global_ref(const void* context) { - jni_delete_global_ref(const_cast(context)); +void jniDeleteConstGlobalRef(const void* context) { + jniDeleteGlobalRef(const_cast(context)); } -int unbox_integer(JNIEnv* env, jobject boxedInteger) { - jclass jcls_Integer = env->FindClass("java/lang/Integer"); - jmethodID jmid_intValue = env->GetMethodID(jcls_Integer, "intValue", "()I"); - env->DeleteLocalRef(jcls_Integer); - return env->CallIntMethod(boxedInteger, jmid_intValue); +int javaIntegerTotInt(JniHelper& jni_helper, jobject boxed_integer) { + LocalRefUniquePtr jcls_Integer = jni_helper.findClass("java/lang/Integer"); + jmethodID jmid_intValue = jni_helper.getMethodId(jcls_Integer.get(), "intValue", "()I"); + return jni_helper.callIntMethod(boxed_integer, jmid_intValue); } -envoy_data array_to_native_data(JNIEnv* env, jbyteArray j_data) { - size_t data_length = static_cast(env->GetArrayLength(j_data)); - return array_to_native_data(env, j_data, data_length); +envoy_data javaByteArrayToEnvoyData(JniHelper& jni_helper, jbyteArray j_data) { + size_t data_length = static_cast(jni_helper.getArrayLength(j_data)); + return javaByteArrayToEnvoyData(jni_helper, j_data, data_length); } -envoy_data array_to_native_data(JNIEnv* env, jbyteArray j_data, size_t data_length) { +envoy_data javaByteArrayToEnvoyData(JniHelper& jni_helper, jbyteArray j_data, size_t data_length) { uint8_t* native_bytes = static_cast(safe_malloc(data_length)); - void* critical_data = env->GetPrimitiveArrayCritical(j_data, 0); - memcpy(native_bytes, critical_data, data_length); // NOLINT(safe-memcpy) - env->ReleasePrimitiveArrayCritical(j_data, critical_data, 0); + Envoy::JNI::PrimitiveArrayCriticalUniquePtr critical_data = + jni_helper.getPrimitiveArrayCritical(j_data, nullptr); + memcpy(native_bytes, critical_data.get(), data_length); // NOLINT(safe-memcpy) return {data_length, native_bytes, free, native_bytes}; } -jstring native_data_to_string(JNIEnv* env, envoy_data data) { +LocalRefUniquePtr envoyDataToJavaString(JniHelper& jni_helper, envoy_data data) { // Ensure we get a null-terminated string, the data coming in via envoy_data might not be. std::string str(reinterpret_cast(data.bytes), data.length); - jstring jstrBuf = env->NewStringUTF(str.c_str()); - return jstrBuf; + return jni_helper.newStringUtf(str.c_str()); } -jbyteArray native_data_to_array(JNIEnv* env, envoy_data data) { - jbyteArray j_data = env->NewByteArray(data.length); - void* critical_data = env->GetPrimitiveArrayCritical(j_data, nullptr); +LocalRefUniquePtr envoyDataToJavaByteArray(JniHelper& jni_helper, envoy_data data) { + LocalRefUniquePtr j_data = jni_helper.newByteArray(data.length); + PrimitiveArrayCriticalUniquePtr critical_data = + jni_helper.getPrimitiveArrayCritical(j_data.get(), nullptr); RELEASE_ASSERT(critical_data != nullptr, "unable to allocate memory in jni_utility"); - memcpy(critical_data, data.bytes, data.length); // NOLINT(safe-memcpy) - // Here '0' (for which there is no named constant) indicates we want to commit the changes back - // to the JVM and free the c array, where applicable. - // TODO: potential perf improvement. Check if copied via isCopy, and optimize memory handling. - env->ReleasePrimitiveArrayCritical(j_data, critical_data, 0); + memcpy(critical_data.get(), data.bytes, data.length); // NOLINT(safe-memcpy) return j_data; } -jlongArray native_stream_intel_to_array(JNIEnv* env, envoy_stream_intel stream_intel) { - jlongArray j_array = env->NewLongArray(4); - jlong* critical_array = static_cast(env->GetPrimitiveArrayCritical(j_array, nullptr)); +LocalRefUniquePtr envoyStreamIntelToJavaLongArray(JniHelper& jni_helper, + envoy_stream_intel stream_intel) { + LocalRefUniquePtr j_array = jni_helper.newLongArray(4); + PrimitiveArrayCriticalUniquePtr critical_array = + jni_helper.getPrimitiveArrayCritical(j_array.get(), nullptr); RELEASE_ASSERT(critical_array != nullptr, "unable to allocate memory in jni_utility"); - critical_array[0] = static_cast(stream_intel.stream_id); - critical_array[1] = static_cast(stream_intel.connection_id); - critical_array[2] = static_cast(stream_intel.attempt_count); - critical_array[3] = static_cast(stream_intel.consumed_bytes_from_response); - // Here '0' (for which there is no named constant) indicates we want to commit the changes back - // to the JVM and free the c array, where applicable. - env->ReleasePrimitiveArrayCritical(j_array, critical_array, 0); + critical_array.get()[0] = static_cast(stream_intel.stream_id); + critical_array.get()[1] = static_cast(stream_intel.connection_id); + critical_array.get()[2] = static_cast(stream_intel.attempt_count); + critical_array.get()[3] = static_cast(stream_intel.consumed_bytes_from_response); return j_array; } -jlongArray native_final_stream_intel_to_array(JNIEnv* env, - envoy_final_stream_intel final_stream_intel) { - jlongArray j_array = env->NewLongArray(16); - jlong* critical_array = static_cast(env->GetPrimitiveArrayCritical(j_array, nullptr)); +LocalRefUniquePtr +envoyFinalStreamIntelToJavaLongArray(JniHelper& jni_helper, + envoy_final_stream_intel final_stream_intel) { + LocalRefUniquePtr j_array = jni_helper.newLongArray(16); + PrimitiveArrayCriticalUniquePtr critical_array = + jni_helper.getPrimitiveArrayCritical(j_array.get(), nullptr); RELEASE_ASSERT(critical_array != nullptr, "unable to allocate memory in jni_utility"); - critical_array[0] = static_cast(final_stream_intel.stream_start_ms); - critical_array[1] = static_cast(final_stream_intel.dns_start_ms); - critical_array[2] = static_cast(final_stream_intel.dns_end_ms); - critical_array[3] = static_cast(final_stream_intel.connect_start_ms); - critical_array[4] = static_cast(final_stream_intel.connect_end_ms); - critical_array[5] = static_cast(final_stream_intel.ssl_start_ms); - critical_array[6] = static_cast(final_stream_intel.ssl_end_ms); - critical_array[7] = static_cast(final_stream_intel.sending_start_ms); - critical_array[8] = static_cast(final_stream_intel.sending_end_ms); - critical_array[9] = static_cast(final_stream_intel.response_start_ms); - critical_array[10] = static_cast(final_stream_intel.stream_end_ms); - critical_array[11] = static_cast(final_stream_intel.socket_reused); - critical_array[12] = static_cast(final_stream_intel.sent_byte_count); - critical_array[13] = static_cast(final_stream_intel.received_byte_count); - critical_array[14] = static_cast(final_stream_intel.response_flags); - critical_array[15] = static_cast(final_stream_intel.upstream_protocol); - - // Here '0' (for which there is no named constant) indicates we want to commit the changes back - // to the JVM and free the c array, where applicable. - env->ReleasePrimitiveArrayCritical(j_array, critical_array, 0); + critical_array.get()[0] = static_cast(final_stream_intel.stream_start_ms); + critical_array.get()[1] = static_cast(final_stream_intel.dns_start_ms); + critical_array.get()[2] = static_cast(final_stream_intel.dns_end_ms); + critical_array.get()[3] = static_cast(final_stream_intel.connect_start_ms); + critical_array.get()[4] = static_cast(final_stream_intel.connect_end_ms); + critical_array.get()[5] = static_cast(final_stream_intel.ssl_start_ms); + critical_array.get()[6] = static_cast(final_stream_intel.ssl_end_ms); + critical_array.get()[7] = static_cast(final_stream_intel.sending_start_ms); + critical_array.get()[8] = static_cast(final_stream_intel.sending_end_ms); + critical_array.get()[9] = static_cast(final_stream_intel.response_start_ms); + critical_array.get()[10] = static_cast(final_stream_intel.stream_end_ms); + critical_array.get()[11] = static_cast(final_stream_intel.socket_reused); + critical_array.get()[12] = static_cast(final_stream_intel.sent_byte_count); + critical_array.get()[13] = static_cast(final_stream_intel.received_byte_count); + critical_array.get()[14] = static_cast(final_stream_intel.response_flags); + critical_array.get()[15] = static_cast(final_stream_intel.upstream_protocol); return j_array; } -jobject native_map_to_map(JNIEnv* env, envoy_map map) { - jclass jcls_hashMap = env->FindClass("java/util/HashMap"); - jmethodID jmid_hashMapInit = env->GetMethodID(jcls_hashMap, "", "(I)V"); - jobject j_hashMap = env->NewObject(jcls_hashMap, jmid_hashMapInit, map.length); - jmethodID jmid_hashMapPut = env->GetMethodID( - jcls_hashMap, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); +LocalRefUniquePtr envoyMapToJavaMap(JniHelper& jni_helper, envoy_map map) { + LocalRefUniquePtr jcls_hashMap = jni_helper.findClass("java/util/HashMap"); + jmethodID jmid_hashMapInit = jni_helper.getMethodId(jcls_hashMap.get(), "", "(I)V"); + LocalRefUniquePtr j_hashMap = + jni_helper.newObject(jcls_hashMap.get(), jmid_hashMapInit, map.length); + jmethodID jmid_hashMapPut = jni_helper.getMethodId( + jcls_hashMap.get(), "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); for (envoy_map_size_t i = 0; i < map.length; i++) { - auto key = native_data_to_string(env, map.entries[i].key); - auto value = native_data_to_string(env, map.entries[i].value); - env->CallObjectMethod(j_hashMap, jmid_hashMapPut, key, value); - env->DeleteLocalRef(key); - env->DeleteLocalRef(value); + LocalRefUniquePtr key = envoyDataToJavaString(jni_helper, map.entries[i].key); + LocalRefUniquePtr value = envoyDataToJavaString(jni_helper, map.entries[i].value); + LocalRefUniquePtr ignored = + jni_helper.callObjectMethod(j_hashMap.get(), jmid_hashMapPut, key.get(), value.get()); } - env->DeleteLocalRef(jcls_hashMap); return j_hashMap; } -envoy_data buffer_to_native_data(JNIEnv* env, jobject j_data) { +envoy_data javaByteBufferToEnvoyData(JniHelper& jni_helper, jobject j_data) { // Returns -1 if the buffer is not a direct buffer. - jlong data_length = env->GetDirectBufferCapacity(j_data); + jlong data_length = jni_helper.getDirectBufferCapacity(j_data); if (data_length < 0) { - jclass jcls_ByteBuffer = env->FindClass("java/nio/ByteBuffer"); + LocalRefUniquePtr jcls_ByteBuffer = jni_helper.findClass("java/nio/ByteBuffer"); // We skip checking hasArray() because only direct ByteBuffers or array-backed ByteBuffers // are supported. We will crash here if this is an invalid buffer, but guards may be // implemented in the JVM layer. - jmethodID jmid_array = env->GetMethodID(jcls_ByteBuffer, "array", "()[B"); - jbyteArray array = static_cast(env->CallObjectMethod(j_data, jmid_array)); - env->DeleteLocalRef(jcls_ByteBuffer); - - envoy_data native_data = array_to_native_data(env, array); - env->DeleteLocalRef(array); + jmethodID jmid_array = jni_helper.getMethodId(jcls_ByteBuffer.get(), "array", "()[B"); + LocalRefUniquePtr array = + jni_helper.callObjectMethod(j_data, jmid_array); + envoy_data native_data = javaByteArrayToEnvoyData(jni_helper, array.get()); return native_data; } - return buffer_to_native_data(env, j_data, static_cast(data_length)); + return javaByteBufferToEnvoyData(jni_helper, j_data, static_cast(data_length)); } -envoy_data buffer_to_native_data(JNIEnv* env, jobject j_data, size_t data_length) { +envoy_data javaByteBufferToEnvoyData(JniHelper& jni_helper, jobject j_data, size_t data_length) { // Returns nullptr if the buffer is not a direct buffer. - uint8_t* direct_address = static_cast(env->GetDirectBufferAddress(j_data)); + uint8_t* direct_address = jni_helper.getDirectBufferAddress(j_data); if (direct_address == nullptr) { - jclass jcls_ByteBuffer = env->FindClass("java/nio/ByteBuffer"); + LocalRefUniquePtr jcls_ByteBuffer = jni_helper.findClass("java/nio/ByteBuffer"); // We skip checking hasArray() because only direct ByteBuffers or array-backed ByteBuffers // are supported. We will crash here if this is an invalid buffer, but guards may be // implemented in the JVM layer. - jmethodID jmid_array = env->GetMethodID(jcls_ByteBuffer, "array", "()[B"); - jbyteArray array = static_cast(env->CallObjectMethod(j_data, jmid_array)); - env->DeleteLocalRef(jcls_ByteBuffer); - - envoy_data native_data = array_to_native_data(env, array, data_length); - env->DeleteLocalRef(array); + jmethodID jmid_array = jni_helper.getMethodId(jcls_ByteBuffer.get(), "array", "()[B"); + LocalRefUniquePtr array = + jni_helper.callObjectMethod(j_data, jmid_array); + envoy_data native_data = javaByteArrayToEnvoyData(jni_helper, array.get(), data_length); return native_data; } envoy_data native_data; native_data.bytes = direct_address; native_data.length = data_length; - native_data.release = jni_delete_global_ref; - native_data.context = env->NewGlobalRef(j_data); + native_data.release = jniDeleteGlobalRef; + native_data.context = jni_helper.newGlobalRef(j_data).release(); return native_data; } -envoy_data* buffer_to_native_data_ptr(JNIEnv* env, jobject j_data) { +envoy_data* javaByteBufferToEnvoyDataPtr(JniHelper& jni_helper, jobject j_data) { // Note: This check works for LocalRefs and GlobalRefs, but will not work for WeakGlobalRefs. // Such usage would generally be inappropriate anyways; like C++ weak_ptrs, one should // acquire a new strong reference before attempting to interact with an object held by @@ -207,15 +190,16 @@ envoy_data* buffer_to_native_data_ptr(JNIEnv* env, jobject j_data) { } envoy_data* native_data = static_cast(safe_malloc(sizeof(envoy_map_entry))); - *native_data = buffer_to_native_data(env, j_data); + *native_data = javaByteBufferToEnvoyData(jni_helper, j_data); return native_data; } -envoy_headers to_native_headers(JNIEnv* env, jobjectArray headers) { - return to_native_map(env, headers); +envoy_headers javaArrayOfObjectArrayToEnvoyHeaders(JniHelper& jni_helper, jobjectArray headers) { + return javaArrayOfObjectArrayToEnvoyMap(jni_helper, headers); } -envoy_headers* to_native_headers_ptr(JNIEnv* env, jobjectArray headers) { +envoy_headers* javaArrayOfObjectArrayToEnvoyHeadersPtr(JniHelper& jni_helper, + jobjectArray headers) { // Note: This check works for LocalRefs and GlobalRefs, but will not work for WeakGlobalRefs. // Such usage would generally be inappropriate anyways; like C++ weak_ptrs, one should // acquire a new strong reference before attempting to interact with an object held by @@ -226,16 +210,18 @@ envoy_headers* to_native_headers_ptr(JNIEnv* env, jobjectArray headers) { } envoy_headers* native_headers = static_cast(safe_malloc(sizeof(envoy_map_entry))); - *native_headers = to_native_headers(env, headers); + *native_headers = javaArrayOfObjectArrayToEnvoyHeaders(jni_helper, headers); return native_headers; } -envoy_stats_tags to_native_tags(JNIEnv* env, jobjectArray tags) { return to_native_map(env, tags); } +envoy_stats_tags javaArrayOfObjectArrayToEnvoyStatsTags(JniHelper& jni_helper, jobjectArray tags) { + return javaArrayOfObjectArrayToEnvoyMap(jni_helper, tags); +} -envoy_map to_native_map(JNIEnv* env, jobjectArray entries) { +envoy_map javaArrayOfObjectArrayToEnvoyMap(JniHelper& jni_helper, jobjectArray entries) { // Note that headers is a flattened array of key/value pairs. // Therefore, the length of the native header array is n envoy_data or n/2 envoy_map_entry. - envoy_map_size_t length = env->GetArrayLength(entries); + envoy_map_size_t length = jni_helper.getArrayLength(entries); if (length == 0) { return {0, nullptr}; } @@ -245,99 +231,103 @@ envoy_map to_native_map(JNIEnv* env, jobjectArray entries) { for (envoy_map_size_t i = 0; i < length; i += 2) { // Copy native byte array for header key - jbyteArray j_key = static_cast(env->GetObjectArrayElement(entries, i)); - envoy_data entry_key = array_to_native_data(env, j_key); + LocalRefUniquePtr j_key = jni_helper.getObjectArrayElement(entries, i); + envoy_data entry_key = javaByteArrayToEnvoyData(jni_helper, j_key.get()); // Copy native byte array for header value - jbyteArray j_value = static_cast(env->GetObjectArrayElement(entries, i + 1)); - envoy_data entry_value = array_to_native_data(env, j_value); + LocalRefUniquePtr j_value = + jni_helper.getObjectArrayElement(entries, i + 1); + envoy_data entry_value = javaByteArrayToEnvoyData(jni_helper, j_value.get()); entry_array[i / 2] = {entry_key, entry_value}; - env->DeleteLocalRef(j_key); - env->DeleteLocalRef(j_value); } envoy_map native_map = {length / 2, entry_array}; return native_map; } -jobjectArray ToJavaArrayOfObjectArray(JNIEnv* env, const Envoy::Types::ManagedEnvoyHeaders& map) { - jclass jcls_byte_array = env->FindClass("java/lang/Object"); - jobjectArray javaArray = env->NewObjectArray(2 * map.get().length, jcls_byte_array, nullptr); +LocalRefUniquePtr +envoyHeadersToJavaArrayOfObjectArray(JniHelper& jni_helper, + const Envoy::Types::ManagedEnvoyHeaders& map) { + LocalRefUniquePtr jcls_byte_array = jni_helper.findClass("java/lang/Object"); + LocalRefUniquePtr javaArray = + jni_helper.newObjectArray(2 * map.get().length, jcls_byte_array.get(), nullptr); for (envoy_map_size_t i = 0; i < map.get().length; i++) { - jbyteArray key = native_data_to_array(env, map.get().entries[i].key); - jbyteArray value = native_data_to_array(env, map.get().entries[i].value); + LocalRefUniquePtr key = + envoyDataToJavaByteArray(jni_helper, map.get().entries[i].key); + LocalRefUniquePtr value = + envoyDataToJavaByteArray(jni_helper, map.get().entries[i].value); - env->SetObjectArrayElement(javaArray, 2 * i, key); - env->SetObjectArrayElement(javaArray, 2 * i + 1, value); + jni_helper.setObjectArrayElement(javaArray.get(), 2 * i, key.get()); + jni_helper.setObjectArrayElement(javaArray.get(), 2 * i + 1, value.get()); } return javaArray; } -jobjectArray ToJavaArrayOfByteArray(JNIEnv* env, const std::vector& v) { - jclass jcls_byte_array = env->FindClass("[B"); - jobjectArray joa = env->NewObjectArray(v.size(), jcls_byte_array, nullptr); +LocalRefUniquePtr +vectorStringToJavaArrayOfByteArray(JniHelper& jni_helper, const std::vector& v) { + LocalRefUniquePtr jcls_byte_array = jni_helper.findClass("[B"); + LocalRefUniquePtr joa = + jni_helper.newObjectArray(v.size(), jcls_byte_array.get(), nullptr); for (size_t i = 0; i < v.size(); ++i) { - jbyteArray byte_array = - ToJavaByteArray(env, reinterpret_cast(v[i].data()), v[i].length()); - env->SetObjectArrayElement(joa, i, byte_array); + LocalRefUniquePtr byte_array = byteArrayToJavaByteArray( + jni_helper, reinterpret_cast(v[i].data()), v[i].length()); + jni_helper.setObjectArrayElement(joa.get(), i, byte_array.get()); } return joa; } -jbyteArray ToJavaByteArray(JNIEnv* env, const uint8_t* bytes, size_t len) { - jbyteArray byte_array = env->NewByteArray(len); +LocalRefUniquePtr byteArrayToJavaByteArray(JniHelper& jni_helper, const uint8_t* bytes, + size_t len) { + LocalRefUniquePtr byte_array = jni_helper.newByteArray(len); const jbyte* jbytes = reinterpret_cast(bytes); - env->SetByteArrayRegion(byte_array, /*start=*/0, len, jbytes); + jni_helper.setByteArrayRegion(byte_array.get(), /*start=*/0, len, jbytes); return byte_array; } -jbyteArray ToJavaByteArray(JNIEnv* env, const std::string& str) { +LocalRefUniquePtr stringToJavaByteArray(JniHelper& jni_helper, const std::string& str) { const uint8_t* str_bytes = reinterpret_cast(str.data()); - return ToJavaByteArray(env, str_bytes, str.size()); + return byteArrayToJavaByteArray(jni_helper, str_bytes, str.size()); } -void JavaArrayOfByteArrayToStringVector(JNIEnv* env, jobjectArray array, +void javaArrayOfByteArrayToStringVector(JniHelper& jni_helper, jobjectArray array, std::vector* out) { ASSERT(out); ASSERT(array); - size_t len = env->GetArrayLength(array); + size_t len = jni_helper.getArrayLength(array); out->resize(len); for (size_t i = 0; i < len; ++i) { - jbyteArray bytes_array = static_cast(env->GetObjectArrayElement(array, i)); - jsize bytes_len = env->GetArrayLength(bytes_array); + LocalRefUniquePtr bytes_array = + jni_helper.getObjectArrayElement(array, i); + jsize bytes_len = jni_helper.getArrayLength(bytes_array.get()); // It doesn't matter if the array returned by GetByteArrayElements is a copy // or not, as the data will be simply be copied into C++ owned memory below. - jbyte* bytes = env->GetByteArrayElements(bytes_array, /*isCopy=*/nullptr); - (*out)[i].assign(reinterpret_cast(bytes), bytes_len); - // There is nothing to write back, it is always safe to JNI_ABORT. - env->ReleaseByteArrayElements(bytes_array, bytes, JNI_ABORT); - // Explicitly delete to keep the local ref count low. - env->DeleteLocalRef(bytes_array); + ArrayElementsUniquePtr bytes = + jni_helper.getByteArrayElements(bytes_array.get(), /* is_copy= */ nullptr); + (*out)[i].assign(reinterpret_cast(bytes.get()), bytes_len); } } -void JavaArrayOfByteToString(JNIEnv* env, jbyteArray jbytes, std::string* out) { +void javaByteArrayToString(JniHelper& jni_helper, jbyteArray jbytes, std::string* out) { std::vector bytes; - JavaArrayOfByteToBytesVector(env, jbytes, &bytes); + javaByteArrayToByteVector(jni_helper, jbytes, &bytes); *out = std::string(bytes.begin(), bytes.end()); } -void JavaArrayOfByteToBytesVector(JNIEnv* env, jbyteArray array, std::vector* out) { - const size_t len = env->GetArrayLength(array); +void javaByteArrayToByteVector(JniHelper& jni_helper, jbyteArray array, std::vector* out) { + const size_t len = jni_helper.getArrayLength(array); out->resize(len); // It doesn't matter if the array returned by GetByteArrayElements is a copy // or not, as the data will be simply be copied into C++ owned memory below. - jbyte* jbytes = env->GetByteArrayElements(array, /*isCopy=*/nullptr); - uint8_t* bytes = reinterpret_cast(jbytes); + ArrayElementsUniquePtr jbytes = + jni_helper.getByteArrayElements(array, /* is_copy= */ nullptr); + uint8_t* bytes = reinterpret_cast(jbytes.get()); std::copy(bytes, bytes + len, out->begin()); - // There is nothing to write back, it is always safe to JNI_ABORT. - env->ReleaseByteArrayElements(array, jbytes, JNI_ABORT); } MatcherData::Type StringToType(std::string type_as_string) { @@ -356,31 +346,32 @@ MatcherData::Type StringToType(std::string type_as_string) { return MatcherData::EXACT; } -std::vector javaObjectArrayToMatcherData(JNIEnv* env, jobjectArray array, - std::string& cluster_name_out) { - const size_t len = env->GetArrayLength(array); - std::vector ret; - if (len == 0) { - return ret; - } - ASSERT((len - 1) % 3 == 0); - if ((len - 1) % 3 != 0) { - return ret; - } +void javaByteArrayToProto(JniHelper& jni_helper, jbyteArray source, + Envoy::Protobuf::MessageLite* dest) { + ArrayElementsUniquePtr bytes = + jni_helper.getByteArrayElements(source, /* is_copy= */ nullptr); + jsize size = jni_helper.getArrayLength(source); + bool success = dest->ParseFromArray(bytes.get(), size); + RELEASE_ASSERT(success, "Failed to parse protobuf message."); +} - JavaArrayOfByteToString(env, static_cast(env->GetObjectArrayElement(array, 0)), - &cluster_name_out); - for (int i = 1; i < len; i += 3) { - std::string name; - std::string type_as_string; - std::string value; - JavaArrayOfByteToString(env, static_cast(env->GetObjectArrayElement(array, i)), - &name); - JavaArrayOfByteToString(env, static_cast(env->GetObjectArrayElement(array, i + 1)), - &type_as_string); - JavaArrayOfByteToString(env, static_cast(env->GetObjectArrayElement(array, i + 2)), - &value); - ret.emplace_back(MatcherData(name, StringToType(type_as_string), value)); +LocalRefUniquePtr protoToJavaByteArray(JniHelper& jni_helper, + const Envoy::Protobuf::MessageLite& source) { + size_t size = source.ByteSizeLong(); + LocalRefUniquePtr byte_array = jni_helper.newByteArray(size); + auto bytes = jni_helper.getByteArrayElements(byte_array.get(), nullptr); + source.SerializeToArray(bytes.get(), size); + return byte_array; +} + +std::string javaStringToString(JniHelper& jni_helper, jstring java_string) { + if (!java_string) { + return ""; } - return ret; + StringUtfUniquePtr native_java_string = jni_helper.getStringUtfChars(java_string, nullptr); + std::string cpp_string(native_java_string.get()); + return cpp_string; } + +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/jni_utility.h b/mobile/library/common/jni/jni_utility.h index df1ff904b069..e7db6f4289a1 100644 --- a/mobile/library/common/jni/jni_utility.h +++ b/mobile/library/common/jni/jni_utility.h @@ -3,30 +3,34 @@ #include #include +#include "source/common/protobuf/protobuf.h" + #include "library/common/jni/import/jni_import.h" +#include "library/common/jni/jni_helper.h" #include "library/common/types/c_types.h" #include "library/common/types/managed_envoy_headers.h" #include "library/common/types/matcher_data.h" -// NOLINT(namespace-envoy) +namespace Envoy { +namespace JNI { // TODO(Augustyniak): Replace the usages of this global method with Envoy::JNI::Env::get() -JNIEnv* get_env(); +JNIEnv* getEnv(); -void set_class_loader(jobject class_loader); +void setClassLoader(jobject class_loader); /** * Finds a class with a given name using a class loader provided with the use - * of `set_class_loader` function. The class loader is supposed to come from + * of `setClassLoader` function. The class loader is supposed to come from * application's context and should be associated with project's code - Java classes * defined by the project. For finding classes of Java built in-types use * `env->FindClass(...)` method instead as it is lighter to use. * * Read more about why you cannot use `env->FindClass(...)` to look for Java classes - * defined by the project and a pattern used by the implementation of `find_class` helper + * defined by the project and a pattern used by the implementation of `findClass` helper * method at https://developer.android.com/training/articles/perf-jni#native-libraries. * - * The method works on Android targets only as the `set_class_loader` method is not + * The method works on Android targets only as the `setClassLoader` method is not * called by JVM-only targets. * * @param class_name, the name of the class to find (i.e. @@ -35,86 +39,97 @@ void set_class_loader(jobject class_loader); * @return jclass, the class with a provided `class_name` or NULL if * it couldn't be found. */ -jclass find_class(const char* class_name); +LocalRefUniquePtr findClass(const char* class_name); -void jni_delete_global_ref(void* context); +void jniDeleteGlobalRef(void* context); -void jni_delete_const_global_ref(const void* context); +void jniDeleteConstGlobalRef(const void* context); -/** - * Clears any pending exceptions that may have been rides in result to a call into Java code. - * - * @param env, the JNI env pointer. - * - * @return Whether any pending JNI exception was cleared. - */ -bool clear_pending_exceptions(JNIEnv* env); +/** Converts `java.lang.Integer` to C++ `int`. */ +int javaIntegerTotInt(JniHelper& jni_helper, jobject boxed_integer); -int unbox_integer(JNIEnv* env, jobject boxedInteger); +/** Converts from Java byte array to `envoy_data`. */ +envoy_data javaByteArrayToEnvoyData(JniHelper& jni_helper, jbyteArray j_data); -envoy_data array_to_native_data(JNIEnv* env, jbyteArray j_data); +/** Converts from Java byte array with the specified length to `envoy_data`. */ +envoy_data javaByteArrayToEnvoyData(JniHelper& jni_helper, jbyteArray j_data, size_t data_length); -envoy_data array_to_native_data(JNIEnv* env, jbyteArray j_data, size_t data_length); +/** Converts from `envoy_data` to Java byte array. */ +LocalRefUniquePtr envoyDataToJavaByteArray(JniHelper& jni_helper, envoy_data data); -/** - * Utility function that copies envoy_data to jbyteArray. - * - * @param env, the JNI env pointer. - * @param envoy_data, the source to copy from. - * - * @return jbyteArray, copied data. It is up to the function caller to clean up memory. - */ -jbyteArray native_data_to_array(JNIEnv* env, envoy_data data); +/** Converts from `envoy_stream_intel` to Java long array. */ +LocalRefUniquePtr envoyStreamIntelToJavaLongArray(JniHelper& jni_helper, + envoy_stream_intel stream_intel); -jlongArray native_stream_intel_to_array(JNIEnv* env, envoy_stream_intel stream_intel); +/** Converts from `envoy_final_stream_intel` to Java long array. */ +LocalRefUniquePtr +envoyFinalStreamIntelToJavaLongArray(JniHelper& jni_helper, + envoy_final_stream_intel final_stream_intel); -jlongArray native_final_stream_intel_to_array(JNIEnv* env, - envoy_final_stream_intel final_stream_intel); +/** Converts from Java `Map` to `envoy_map`. */ +LocalRefUniquePtr envoyMapToJavaMap(JniHelper& jni_helper, envoy_map map); -/** - * Utility function that copies envoy_map to a java HashMap jobject. - * - * @param env, the JNI env pointer. - * @param envoy_map, the source to copy from. - * - * @return jobject, copied data. It is up to the function caller to clean up memory. - */ -jobject native_map_to_map(JNIEnv* env, envoy_map map); +/** Converts from `envoy_data` to Java `String`. */ +LocalRefUniquePtr envoyDataToJavaString(JniHelper& jni_helper, envoy_data data); -jstring native_data_to_string(JNIEnv* env, envoy_data data); +/** Converts from Java `ByteBuffer` to `envoy_data`. */ +envoy_data javaByteBufferToEnvoyData(JniHelper& jni_helper, jobject j_data); -envoy_data buffer_to_native_data(JNIEnv* env, jobject j_data); +/** Converts from Java `ByteBuffer` to `envoy_data` with the given length. */ +envoy_data javaByteBufferToEnvoyData(JniHelper& jni_helper, jobject j_data, size_t data_length); -envoy_data buffer_to_native_data(JNIEnv* env, jobject j_data, size_t data_length); +/** Returns the pointer of conversion from Java `ByteBuffer` to `envoy_data`. */ +envoy_data* javaByteBufferToEnvoyDataPtr(JniHelper& jni_helper, jobject j_data); -envoy_data* buffer_to_native_data_ptr(JNIEnv* env, jobject j_data); +/** Converts from Java array of object array[2] (key-value pairs) to `envoy_headers`. */ +envoy_headers javaArrayOfObjectArrayToEnvoyHeaders(JniHelper& jni_helper, jobjectArray headers); -envoy_headers to_native_headers(JNIEnv* env, jobjectArray headers); +/** Returns the pointer of conversion from Java array of object array[2] (key-value pairs) to + * `envoy_headers`. */ +envoy_headers* javaArrayOfObjectArrayToEnvoyHeadersPtr(JniHelper& jni_helper, jobjectArray headers); -envoy_headers* to_native_headers_ptr(JNIEnv* env, jobjectArray headers); +/** Converts from Java array of object array[2] (key-value pairs) to `envoy_stats_tags`. */ +envoy_stats_tags javaArrayOfObjectArrayToEnvoyStatsTags(JniHelper& jni_helper, jobjectArray tags); -envoy_stats_tags to_native_tags(JNIEnv* env, jobjectArray tags); +/** Converts from Java array of object array[2] (key-value pairs) to `envoy_map`. */ +envoy_map javaArrayOfObjectArrayToEnvoyMap(JniHelper& jni_helper, jobjectArray entries); -envoy_map to_native_map(JNIEnv* env, jobjectArray entries); +/** Converts from `ManagedEnvoyHeaders` to Java array of object array[2] (key-value pairs). */ +LocalRefUniquePtr +envoyHeadersToJavaArrayOfObjectArray(JniHelper& jni_helper, + const Envoy::Types::ManagedEnvoyHeaders& map); -/** - * Utilities to translate C++ std library constructs to their Java counterpart. - * The underlying data is always copied to disentangle C++ and Java objects lifetime. - */ -jobjectArray ToJavaArrayOfByteArray(JNIEnv* env, const std::vector& v); +/** Converts from C++ vector of strings to Java array of byte array. */ +LocalRefUniquePtr +vectorStringToJavaArrayOfByteArray(JniHelper& jni_helper, const std::vector& v); -jbyteArray ToJavaByteArray(JNIEnv* env, const uint8_t* bytes, size_t len); +/** Converts from C++ byte array to Java byte array. */ +LocalRefUniquePtr byteArrayToJavaByteArray(JniHelper& jni_helper, const uint8_t* bytes, + size_t len); -jbyteArray ToJavaByteArray(JNIEnv* env, const std::string& str); +/** Converts from C++ string to Java byte array. */ +LocalRefUniquePtr stringToJavaByteArray(JniHelper& jni_helper, const std::string& str); -jobjectArray ToJavaArrayOfObjectArray(JNIEnv* env, const Envoy::Types::ManagedEnvoyHeaders& map); - -void JavaArrayOfByteArrayToStringVector(JNIEnv* env, jobjectArray array, +/** Converts from Java array of byte array to C++ vector of strings. */ +void javaArrayOfByteArrayToStringVector(JniHelper& jni_helper, jobjectArray array, std::vector* out); -void JavaArrayOfByteToBytesVector(JNIEnv* env, jbyteArray array, std::vector* out); +/** Converts from Java byte array to C++ vector of bytes. */ +void javaByteArrayToByteVector(JniHelper& jni_helper, jbyteArray array, std::vector* out); + +/** Converts from Java byte array to C++ string. */ +void javaByteArrayToString(JniHelper& jni_helper, jbyteArray jbytes, std::string* out); + +/** Parses the proto from Java byte array and stores the output into `dest` proto. */ +void javaByteArrayToProto(JniHelper& jni_helper, jbyteArray source, + Envoy::Protobuf::MessageLite* dest); + +/** Converts from Proto to Java byte array. */ +LocalRefUniquePtr protoToJavaByteArray(JniHelper& jni_helper, + const Envoy::Protobuf::MessageLite& source); -void JavaArrayOfByteToString(JNIEnv* env, jbyteArray jbytes, std::string* out); +/** Converts from Java `String` to C++ string. */ +std::string javaStringToString(JniHelper& jni_helper, jstring java_string); -std::vector javaObjectArrayToMatcherData(JNIEnv* env, jobjectArray array, - std::string& cluster_out); +} // namespace JNI +} // namespace Envoy diff --git a/mobile/library/common/jni/types/BUILD b/mobile/library/common/jni/types/BUILD index c1ada9a74967..a4ab38c46fc3 100644 --- a/mobile/library/common/jni/types/BUILD +++ b/mobile/library/common/jni/types/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() cc_library( name = "jni_exception_lib", diff --git a/mobile/library/common/main_interface.cc b/mobile/library/common/main_interface.cc index 8b280d272ac1..45745743a0a9 100644 --- a/mobile/library/common/main_interface.cc +++ b/mobile/library/common/main_interface.cc @@ -7,48 +7,52 @@ #include "library/common/api/external.h" #include "library/common/data/utility.h" #include "library/common/engine.h" -#include "library/common/engine_handle.h" #include "library/common/extensions/filters/http/platform_bridge/c_types.h" #include "library/common/http/client.h" #include "library/common/network/connectivity_manager.h" // NOLINT(namespace-envoy) +namespace { +envoy_status_t runOnEngineDispatcher(envoy_engine_t handle, + std::function func) { + if (auto engine = reinterpret_cast(handle)) { + return engine->dispatcher().post([engine, func]() { func(*engine); }); + } + return ENVOY_FAILURE; +} +} // namespace + static std::atomic current_stream_handle_{0}; envoy_stream_t init_stream(envoy_engine_t) { return current_stream_handle_++; } envoy_status_t start_stream(envoy_engine_t engine, envoy_stream_t stream, - envoy_http_callbacks callbacks, bool explicit_flow_control, - uint64_t min_delivery_size) { - return Envoy::EngineHandle::runOnEngineDispatcher( - engine, [stream, callbacks, explicit_flow_control, min_delivery_size](auto& engine) -> void { - engine.httpClient().startStream(stream, callbacks, explicit_flow_control, - min_delivery_size); + envoy_http_callbacks callbacks, bool explicit_flow_control) { + return runOnEngineDispatcher( + engine, [stream, callbacks, explicit_flow_control](auto& engine) -> void { + engine.httpClient().startStream(stream, callbacks, explicit_flow_control); }); } envoy_status_t send_headers(envoy_engine_t engine, envoy_stream_t stream, envoy_headers headers, bool end_stream) { - return Envoy::EngineHandle::runOnEngineDispatcher( - engine, ([stream, headers, end_stream](auto& engine) -> void { - engine.httpClient().sendHeaders(stream, headers, end_stream); - })); + return runOnEngineDispatcher(engine, ([stream, headers, end_stream](auto& engine) -> void { + engine.httpClient().sendHeaders(stream, headers, end_stream); + })); } envoy_status_t read_data(envoy_engine_t engine, envoy_stream_t stream, size_t bytes_to_read) { - return Envoy::EngineHandle::runOnEngineDispatcher( - engine, [stream, bytes_to_read](auto& engine) -> void { - engine.httpClient().readData(stream, bytes_to_read); - }); + return runOnEngineDispatcher(engine, [stream, bytes_to_read](auto& engine) -> void { + engine.httpClient().readData(stream, bytes_to_read); + }); } envoy_status_t send_data(envoy_engine_t engine, envoy_stream_t stream, envoy_data data, bool end_stream) { - return Envoy::EngineHandle::runOnEngineDispatcher( - engine, [stream, data, end_stream](auto& engine) -> void { - engine.httpClient().sendData(stream, data, end_stream); - }); + return runOnEngineDispatcher(engine, [stream, data, end_stream](auto& engine) -> void { + engine.httpClient().sendData(stream, data, end_stream); + }); } // TODO: implement. @@ -57,21 +61,20 @@ envoy_status_t send_metadata(envoy_engine_t, envoy_stream_t, envoy_headers) { } envoy_status_t send_trailers(envoy_engine_t engine, envoy_stream_t stream, envoy_headers trailers) { - return Envoy::EngineHandle::runOnEngineDispatcher( - engine, [stream, trailers](auto& engine) -> void { - engine.httpClient().sendTrailers(stream, trailers); - }); + return runOnEngineDispatcher(engine, [stream, trailers](auto& engine) -> void { + engine.httpClient().sendTrailers(stream, trailers); + }); } envoy_status_t reset_stream(envoy_engine_t engine, envoy_stream_t stream) { - return Envoy::EngineHandle::runOnEngineDispatcher( + return runOnEngineDispatcher( engine, [stream](auto& engine) -> void { engine.httpClient().cancelStream(stream); }); } envoy_status_t set_preferred_network(envoy_engine_t engine, envoy_network_t network) { envoy_netconf_t configuration_key = Envoy::Network::ConnectivityManagerImpl::setPreferredNetwork(network); - Envoy::EngineHandle::runOnEngineDispatcher(engine, [configuration_key](auto& engine) -> void { + runOnEngineDispatcher(engine, [configuration_key](auto& engine) -> void { engine.networkConnectivityManager().refreshDns(configuration_key, true); }); // TODO(snowp): Should this return failure ever? @@ -79,7 +82,7 @@ envoy_status_t set_preferred_network(envoy_engine_t engine, envoy_network_t netw } envoy_status_t set_proxy_settings(envoy_engine_t e, const char* host, const uint16_t port) { - return Envoy::EngineHandle::runOnEngineDispatcher( + return runOnEngineDispatcher( e, [proxy_settings = Envoy::Network::ProxySettings::parseHostAndPort(host, port)](auto& engine) -> void { engine.networkConnectivityManager().setProxySettings(proxy_settings); }); @@ -87,30 +90,26 @@ envoy_status_t set_proxy_settings(envoy_engine_t e, const char* host, const uint envoy_status_t record_counter_inc(envoy_engine_t e, const char* elements, envoy_stats_tags tags, uint64_t count) { - return Envoy::EngineHandle::runOnEngineDispatcher( - e, [name = std::string(elements), tags, count](auto& engine) -> void { - engine.recordCounterInc(name, tags, count); - }); + return runOnEngineDispatcher(e, + [name = std::string(elements), tags, count](auto& engine) -> void { + engine.recordCounterInc(name, tags, count); + }); } envoy_status_t dump_stats(envoy_engine_t engine, envoy_data* out) { absl::Notification stats_received; - if (Envoy::EngineHandle::runOnEngineDispatcher( - engine, ([out, &stats_received](auto& engine) -> void { - Envoy::Buffer::OwnedImpl dumped_stats = engine.dumpStats(); - *out = Envoy::Data::Utility::toBridgeData(dumped_stats, 1024 * 1024 * 100); - stats_received.Notify(); - })) == ENVOY_FAILURE) { + if (runOnEngineDispatcher(engine, ([out, &stats_received](auto& engine) -> void { + Envoy::Buffer::OwnedImpl dumped_stats = engine.dumpStats(); + *out = Envoy::Data::Utility::toBridgeData(dumped_stats, + 1024 * 1024 * 100); + stats_received.Notify(); + })) == ENVOY_FAILURE) { return ENVOY_FAILURE; } stats_received.WaitForNotification(); return ENVOY_SUCCESS; } -void flush_stats(envoy_engine_t e) { - Envoy::EngineHandle::runOnEngineDispatcher(e, [](auto& engine) { engine.flushStats(); }); -} - envoy_status_t register_platform_api(const char* name, void* api) { Envoy::Api::External::registerApi(std::string(name), api); return ENVOY_SUCCESS; @@ -118,18 +117,29 @@ envoy_status_t register_platform_api(const char* name, void* api) { envoy_engine_t init_engine(envoy_engine_callbacks callbacks, envoy_logger logger, envoy_event_tracker event_tracker) { - return Envoy::EngineHandle::initEngine(callbacks, logger, event_tracker); + auto engine = new Envoy::Engine(callbacks, logger, event_tracker); + return reinterpret_cast(engine); } -envoy_status_t run_engine(envoy_engine_t engine, const char* config, const char* log_level) { - return Envoy::EngineHandle::runEngine(engine, config, log_level); +envoy_status_t run_engine(envoy_engine_t handle, const char* config, const char* log_level) { + if (auto engine = reinterpret_cast(handle)) { + engine->run(config, log_level); + return ENVOY_SUCCESS; + } + return ENVOY_FAILURE; } -envoy_status_t terminate_engine(envoy_engine_t engine, bool release) { - return Envoy::EngineHandle::terminateEngine(engine, release); +envoy_status_t terminate_engine(envoy_engine_t handle, bool release) { + auto engine = reinterpret_cast(handle); + envoy_status_t ret = engine->terminate(); + if (release) { + // TODO(jpsim): Always delete engine to avoid leaking it + delete engine; + } + return ret; } envoy_status_t reset_connectivity_state(envoy_engine_t e) { - return Envoy::EngineHandle::runOnEngineDispatcher( + return runOnEngineDispatcher( e, [](auto& engine) { engine.networkConnectivityManager().resetConnectivityState(); }); } diff --git a/mobile/library/common/main_interface.h b/mobile/library/common/main_interface.h index cd4f763c6571..7fe67f42be96 100644 --- a/mobile/library/common/main_interface.h +++ b/mobile/library/common/main_interface.h @@ -26,13 +26,10 @@ envoy_stream_t init_stream(envoy_engine_t engine); * @param stream, handle to the stream to be started. * @param callbacks, the callbacks that will run the stream callbacks. * @param explicit_flow_control, whether to enable explicit flow control on the response stream. - * @param min_delivery_size, if non-zero, indicates the smallest non-terminal number of bytes which - * should be delivered via on_data callbacks. * @return envoy_stream, with a stream handle and a success status, or a failure status. */ envoy_status_t start_stream(envoy_engine_t engine, envoy_stream_t stream, - envoy_http_callbacks callbacks, bool explicit_flow_control, - uint64_t min_delivery_size); + envoy_http_callbacks callbacks, bool explicit_flow_control); /** * Send headers over an open HTTP stream. This method can be invoked once and needs to be called diff --git a/mobile/library/common/network/BUILD b/mobile/library/common/network/BUILD index 82728955e042..332b4149265e 100644 --- a/mobile/library/common/network/BUILD +++ b/mobile/library/common/network/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "connectivity_manager_lib", diff --git a/mobile/library/common/network/connectivity_manager.cc b/mobile/library/common/network/connectivity_manager.cc index 1612af96db4e..bc75129fb289 100644 --- a/mobile/library/common/network/connectivity_manager.cc +++ b/mobile/library/common/network/connectivity_manager.cc @@ -442,12 +442,12 @@ ConnectivityManagerImpl::enumerateInterfaces([[maybe_unused]] unsigned short fam } ConnectivityManagerSharedPtr ConnectivityManagerFactory::get() { - return context_.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(connectivity_manager), [&] { - Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory{ - context_}; - return std::make_shared(context_.clusterManager(), - cache_manager_factory.get()); + return context_.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(connectivity_manager), [this] { + Envoy::Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl + cache_manager_factory{context_}; + return std::make_shared( + context_.serverFactoryContext().clusterManager(), cache_manager_factory.get()); }); } diff --git a/mobile/library/common/network/connectivity_manager.h b/mobile/library/common/network/connectivity_manager.h index 74f5c14a11dc..8d1b2c3cae11 100644 --- a/mobile/library/common/network/connectivity_manager.h +++ b/mobile/library/common/network/connectivity_manager.h @@ -263,7 +263,7 @@ using ConnectivityManagerSharedPtr = std::shared_ptr; */ class ConnectivityManagerFactory { public: - ConnectivityManagerFactory(Server::Configuration::CommonFactoryContext& context) + ConnectivityManagerFactory(Server::Configuration::GenericFactoryContext& context) : context_(context) {} /** @@ -272,7 +272,7 @@ class ConnectivityManagerFactory { ConnectivityManagerSharedPtr get(); private: - Server::Configuration::CommonFactoryContext& context_; + Server::GenericFactoryContextImpl context_; }; /** diff --git a/mobile/library/common/network/socket_tag_socket_option_impl.cc b/mobile/library/common/network/socket_tag_socket_option_impl.cc index 9e0942c5e3e2..31aa2c3c8b66 100644 --- a/mobile/library/common/network/socket_tag_socket_option_impl.cc +++ b/mobile/library/common/network/socket_tag_socket_option_impl.cc @@ -27,7 +27,7 @@ bool SocketTagSocketOptionImpl::setOption( // Further, this only works for sockets which have a raw fd and will be a no-op // otherwise. int fd = socket.ioHandle().fdDoNotUse(); - tag_socket(fd, uid_, traffic_stats_tag_); + JNI::tagSocket(fd, uid_, traffic_stats_tag_); return true; } diff --git a/mobile/library/common/stats/BUILD b/mobile/library/common/stats/BUILD index 3547804d6307..046ee434ca81 100644 --- a/mobile/library/common/stats/BUILD +++ b/mobile/library/common/stats/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "utility_lib", diff --git a/mobile/library/common/stream_info/BUILD b/mobile/library/common/stream_info/BUILD index 565a2b10189d..1a6c2a8c8090 100644 --- a/mobile/library/common/stream_info/BUILD +++ b/mobile/library/common/stream_info/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "extra_stream_info_lib", diff --git a/mobile/library/common/thread/BUILD b/mobile/library/common/thread/BUILD index 1db56c838f6c..24840332db66 100644 --- a/mobile/library/common/thread/BUILD +++ b/mobile/library/common/thread/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "lock_guard_lib", diff --git a/mobile/library/common/types/BUILD b/mobile/library/common/types/BUILD index 9445600c8d0c..1a3fb867a506 100644 --- a/mobile/library/common/types/BUILD +++ b/mobile/library/common/types/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_library( name = "c_types_lib", diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java index d20497ea3b4e..1d2f1b66a3a1 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/AndroidEngineImpl.java @@ -34,9 +34,8 @@ public AndroidEngineImpl(Context context, EnvoyOnEngineRunning runningCallback, } @Override - public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl, - long minDeliverySize) { - return envoyEngine.startStream(callbacks, explicitFlowControl, minDeliverySize); + public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl) { + return envoyEngine.startStream(callbacks, explicitFlowControl); } @Override @@ -59,11 +58,6 @@ public void terminate() { envoyEngine.terminate(); } - @Override - public void flushStats() { - envoyEngine.flushStats(); - } - @Override public String dumpStats() { return envoyEngine.dumpStats(); diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD b/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD index c23318d4c075..a0027ccd1777 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/BUILD @@ -1,8 +1,11 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@rules_java//java:defs.bzl", "java_library") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_library( name = "envoy_engine_lib", srcs = [ @@ -52,6 +55,7 @@ java_library( deps = [ "//library/java/io/envoyproxy/envoymobile/engine/types:envoy_c_types_lib", "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_protobuf_protobuf_javalite", ], ) diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java index 520f08f95f38..c9a7c853d457 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java @@ -1,19 +1,16 @@ package io.envoyproxy.envoymobile.engine; +import com.google.protobuf.Struct; import java.util.Collections; import java.util.List; import java.util.ArrayList; import java.util.Map; import java.util.HashMap; import java.util.regex.Pattern; -import java.util.regex.Matcher; -import java.lang.StringBuilder; -import javax.annotation.Nullable; import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory; import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor; import io.envoyproxy.envoymobile.engine.types.EnvoyKeyValueStore; -import io.envoyproxy.envoymobile.engine.JniLibrary; /* Typed configuration that may be used for starting Envoy. */ public class EnvoyConfiguration { @@ -28,7 +25,6 @@ public enum TrustChainVerification { ACCEPT_UNTRUSTED; } - public final String grpcStatsDomain; public final Integer connectTimeoutSeconds; public final Integer dnsRefreshSeconds; public final Integer dnsFailureRefreshSecondsBase; @@ -43,6 +39,7 @@ public enum TrustChainVerification { public final String http3ConnectionOptions; public final String http3ClientConnectionOptions; public final Map quicHints; + public final List quicCanonicalSuffixes; public final Boolean enableGzipDecompression; public final Boolean enableBrotliDecompression; public final Boolean enableSocketTagging; @@ -51,7 +48,6 @@ public enum TrustChainVerification { public final Integer h2ConnectionKeepaliveTimeoutSeconds; public final Integer maxConnectionsPerHost; public final List httpPlatformFilterFactories; - public final Integer statsFlushSeconds; public final Integer streamIdleTimeoutSeconds; public final Integer perTryIdleTimeoutSeconds; public final String appVersion; @@ -60,23 +56,19 @@ public enum TrustChainVerification { public final List nativeFilterChain; public final Map stringAccessors; public final Map keyValueStores; - public final List statSinks; public final Map runtimeGuards; public final Boolean enablePlatformCertificatesValidation; public final String rtdsResourceName; public final Integer rtdsTimeoutSeconds; public final String xdsAddress; public final Integer xdsPort; - public final String xdsAuthHeader; - public final String xdsAuthToken; - public final String xdsJwtToken; - public final Integer xdsJwtTokenLifetime; + public final Map xdsGrpcInitialMetadata; public final String xdsRootCerts; - public final String xdsSni; public final String nodeId; public final String nodeRegion; public final String nodeZone; public final String nodeSubZone; + public final Struct nodeMetadata; public final String cdsResourcesLocator; public final Integer cdsTimeoutSeconds; public final Boolean enableCds; @@ -86,7 +78,6 @@ public enum TrustChainVerification { /** * Create a new instance of the configuration. * - * @param grpcStatsDomain the domain to flush stats to. * @param connectTimeoutSeconds timeout for new network connections to * hosts in * the cluster. @@ -113,12 +104,12 @@ public enum TrustChainVerification { * HTTP/3. * @param quicHints A list of host port pairs that's known * to speak QUIC. + * @param quicCanonicalSuffixes A list of canonical suffixes that are + * known to speak QUIC. * @param enableGzipDecompression whether to enable response gzip * decompression. - * compression. * @param enableBrotliDecompression whether to enable response brotli * decompression. - * compression. * @param enableSocketTagging whether to enable socket tagging. * @param enableInterfaceBinding whether to allow interface binding. * @param h2ConnectionKeepaliveIdleIntervalMilliseconds rate in milliseconds seconds to send h2 @@ -126,7 +117,6 @@ public enum TrustChainVerification { * @param h2ConnectionKeepaliveTimeoutSeconds rate in seconds to timeout h2 pings. * @param maxConnectionsPerHost maximum number of connections to open to a * single host. - * @param statsFlushSeconds interval at which to flush Envoy stats. * @param streamIdleTimeoutSeconds idle timeout for HTTP streams. * @param perTryIdleTimeoutSeconds per try idle timeout for HTTP streams. * @param appVersion the App Version of the App using this @@ -144,52 +134,44 @@ public enum TrustChainVerification { * @param rtdsTimeoutSeconds the timeout for RTDS fetches. * @param xdsAddress the address for the xDS management server. * @param xdsPort the port for the xDS server. - * @param xdsAuthHeader the HTTP header to use for sending the - * authentication token to the xDS server. - * @param xdsAuthToken the token to send as the authentication - * header value to authenticate with the - * xDS server. - * @param xdsJwtToken the JWT token to use for authenticating - * with the xDS server. - * @param xdsTokenLifetime the lifetime of the JWT token. + * @param xdsGrpcInitialMetadata The Headers (as key/value pairs) that must + * be included in the xDs gRPC stream's + * initial metadata (as HTTP headers). * @param xdsRootCerts the root certificates to use for the TLS * handshake during connection establishment * with the xDS management server. - * @param xdsSni the SNI (server name identification) to - * use for the TLS handshake. * @param nodeId the node ID in the Node metadata. * @param nodeRegion the node region in the Node metadata. * @param nodeZone the node zone in the Node metadata. * @param nodeSubZone the node sub-zone in the Node metadata. + * @param nodeMetadata the node metadata. * @param cdsResourcesLocator the resources locator for CDS. * @param cdsTimeoutSeconds the timeout for CDS fetches. * @param enableCds enables CDS, used because all CDS params * could be empty. */ public EnvoyConfiguration( - String grpcStatsDomain, int connectTimeoutSeconds, int dnsRefreshSeconds, - int dnsFailureRefreshSecondsBase, int dnsFailureRefreshSecondsMax, int dnsQueryTimeoutSeconds, - int dnsMinRefreshSeconds, List dnsPreresolveHostnames, boolean enableDNSCache, - int dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh, boolean enableHttp3, - String http3ConnectionOptions, String http3ClientConnectionOptions, - Map quicHints, boolean enableGzipDecompression, + int connectTimeoutSeconds, int dnsRefreshSeconds, int dnsFailureRefreshSecondsBase, + int dnsFailureRefreshSecondsMax, int dnsQueryTimeoutSeconds, int dnsMinRefreshSeconds, + List dnsPreresolveHostnames, boolean enableDNSCache, int dnsCacheSaveIntervalSeconds, + boolean enableDrainPostDnsRefresh, boolean enableHttp3, String http3ConnectionOptions, + String http3ClientConnectionOptions, Map quicHints, + List quicCanonicalSuffixes, boolean enableGzipDecompression, boolean enableBrotliDecompression, boolean enableSocketTagging, boolean enableInterfaceBinding, int h2ConnectionKeepaliveIdleIntervalMilliseconds, - int h2ConnectionKeepaliveTimeoutSeconds, int maxConnectionsPerHost, int statsFlushSeconds, + int h2ConnectionKeepaliveTimeoutSeconds, int maxConnectionsPerHost, int streamIdleTimeoutSeconds, int perTryIdleTimeoutSeconds, String appVersion, String appId, TrustChainVerification trustChainVerification, List nativeFilterChain, List httpPlatformFilterFactories, Map stringAccessors, - Map keyValueStores, List statSinks, - Map runtimeGuards, boolean enablePlatformCertificatesValidation, - String rtdsResourceName, Integer rtdsTimeoutSeconds, String xdsAddress, Integer xdsPort, - String xdsAuthHeader, String xdsAuthToken, String xdsJwtToken, Integer xdsJwtTokenLifetime, - String xdsRootCerts, String xdsSni, String nodeId, String nodeRegion, String nodeZone, - String nodeSubZone, String cdsResourcesLocator, Integer cdsTimeoutSeconds, - boolean enableCds) { + Map keyValueStores, Map runtimeGuards, + boolean enablePlatformCertificatesValidation, String rtdsResourceName, + Integer rtdsTimeoutSeconds, String xdsAddress, Integer xdsPort, + Map xdsGrpcInitialMetadata, String xdsRootCerts, String nodeId, + String nodeRegion, String nodeZone, String nodeSubZone, Struct nodeMetadata, + String cdsResourcesLocator, Integer cdsTimeoutSeconds, boolean enableCds) { JniLibrary.load(); - this.grpcStatsDomain = grpcStatsDomain; this.connectTimeoutSeconds = connectTimeoutSeconds; this.dnsRefreshSeconds = dnsRefreshSeconds; this.dnsFailureRefreshSecondsBase = dnsFailureRefreshSecondsBase; @@ -207,6 +189,7 @@ public EnvoyConfiguration( for (Map.Entry hostAndPort : quicHints.entrySet()) { this.quicHints.put(hostAndPort.getKey(), String.valueOf(hostAndPort.getValue())); } + this.quicCanonicalSuffixes = quicCanonicalSuffixes; this.enableGzipDecompression = enableGzipDecompression; this.enableBrotliDecompression = enableBrotliDecompression; this.enableSocketTagging = enableSocketTagging; @@ -215,7 +198,6 @@ public EnvoyConfiguration( h2ConnectionKeepaliveIdleIntervalMilliseconds; this.h2ConnectionKeepaliveTimeoutSeconds = h2ConnectionKeepaliveTimeoutSeconds; this.maxConnectionsPerHost = maxConnectionsPerHost; - this.statsFlushSeconds = statsFlushSeconds; this.streamIdleTimeoutSeconds = streamIdleTimeoutSeconds; this.perTryIdleTimeoutSeconds = perTryIdleTimeoutSeconds; this.appVersion = appVersion; @@ -236,7 +218,6 @@ public EnvoyConfiguration( this.httpPlatformFilterFactories = httpPlatformFilterFactories; this.stringAccessors = stringAccessors; this.keyValueStores = keyValueStores; - this.statSinks = statSinks; this.runtimeGuards = new HashMap(); for (Map.Entry guardAndValue : runtimeGuards.entrySet()) { @@ -247,16 +228,13 @@ public EnvoyConfiguration( this.rtdsTimeoutSeconds = rtdsTimeoutSeconds; this.xdsAddress = xdsAddress; this.xdsPort = xdsPort; - this.xdsAuthHeader = xdsAuthHeader; - this.xdsAuthToken = xdsAuthToken; - this.xdsJwtToken = xdsJwtToken; - this.xdsJwtTokenLifetime = xdsJwtTokenLifetime; + this.xdsGrpcInitialMetadata = new HashMap<>(xdsGrpcInitialMetadata); this.xdsRootCerts = xdsRootCerts; - this.xdsSni = xdsSni; this.nodeId = nodeId; this.nodeRegion = nodeRegion; this.nodeZone = nodeZone; this.nodeSubZone = nodeSubZone; + this.nodeMetadata = nodeMetadata; this.cdsResourcesLocator = cdsResourcesLocator; this.cdsTimeoutSeconds = cdsTimeoutSeconds; this.enableCds = enableCds; @@ -268,25 +246,25 @@ public long createBootstrap() { List reverseFilterChain = new ArrayList<>(nativeFilterChain); Collections.reverse(reverseFilterChain); - byte[][] filter_chain = JniBridgeUtility.toJniBytes(reverseFilterChain); - byte[][] stats_sinks = JniBridgeUtility.stringsToJniBytes(statSinks); - byte[][] dns_preresolve = JniBridgeUtility.stringsToJniBytes(dnsPreresolveHostnames); - byte[][] runtime_guards = JniBridgeUtility.mapToJniBytes(runtimeGuards); - byte[][] quic_hints = JniBridgeUtility.mapToJniBytes(quicHints); + byte[][] filterChain = JniBridgeUtility.toJniBytes(reverseFilterChain); + byte[][] dnsPreresolve = JniBridgeUtility.stringsToJniBytes(dnsPreresolveHostnames); + byte[][] runtimeGuards = JniBridgeUtility.mapToJniBytes(this.runtimeGuards); + byte[][] quicHints = JniBridgeUtility.mapToJniBytes(this.quicHints); + byte[][] quicSuffixes = JniBridgeUtility.stringsToJniBytes(quicCanonicalSuffixes); + byte[][] xdsGrpcInitialMetadata = JniBridgeUtility.mapToJniBytes(this.xdsGrpcInitialMetadata); return JniLibrary.createBootstrap( - grpcStatsDomain, connectTimeoutSeconds, dnsRefreshSeconds, dnsFailureRefreshSecondsBase, - dnsFailureRefreshSecondsMax, dnsQueryTimeoutSeconds, dnsMinRefreshSeconds, dns_preresolve, + connectTimeoutSeconds, dnsRefreshSeconds, dnsFailureRefreshSecondsBase, + dnsFailureRefreshSecondsMax, dnsQueryTimeoutSeconds, dnsMinRefreshSeconds, dnsPreresolve, enableDNSCache, dnsCacheSaveIntervalSeconds, enableDrainPostDnsRefresh, enableHttp3, - http3ConnectionOptions, http3ClientConnectionOptions, quic_hints, enableGzipDecompression, - enableBrotliDecompression, enableSocketTagging, enableInterfaceBinding, - h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, - maxConnectionsPerHost, statsFlushSeconds, streamIdleTimeoutSeconds, - perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, filter_chain, - stats_sinks, enablePlatformCertificatesValidation, runtime_guards, rtdsResourceName, - rtdsTimeoutSeconds, xdsAddress, xdsPort, xdsAuthHeader, xdsAuthToken, xdsJwtToken, - xdsJwtTokenLifetime, xdsRootCerts, xdsSni, nodeId, nodeRegion, nodeZone, nodeSubZone, - cdsResourcesLocator, cdsTimeoutSeconds, enableCds); + http3ConnectionOptions, http3ClientConnectionOptions, quicHints, quicSuffixes, + enableGzipDecompression, enableBrotliDecompression, enableSocketTagging, + enableInterfaceBinding, h2ConnectionKeepaliveIdleIntervalMilliseconds, + h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, streamIdleTimeoutSeconds, + perTryIdleTimeoutSeconds, appVersion, appId, enforceTrustChainVerification, filterChain, + enablePlatformCertificatesValidation, runtimeGuards, rtdsResourceName, rtdsTimeoutSeconds, + xdsAddress, xdsPort, xdsGrpcInitialMetadata, xdsRootCerts, nodeId, nodeRegion, nodeZone, + nodeSubZone, nodeMetadata.toByteArray(), cdsResourcesLocator, cdsTimeoutSeconds, enableCds); } static class ConfigurationException extends RuntimeException { diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java index 627f35bf6561..90521cd65bb7 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngine.java @@ -14,12 +14,9 @@ public interface EnvoyEngine { * * @param callbacks The callbacks for receiving callbacks from the stream. * @param explicitFlowControl Whether explicit flow control will be enabled for this stream. - * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which - * should be delivered sans end stream. * @return A stream that may be used for sending data. */ - EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl, - long minDeliverySize); + EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl); /** * Terminates the running engine. @@ -69,13 +66,6 @@ EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowCo int registerStringAccessor(String accessor_name, EnvoyStringAccessor accessor); - /** - * Flush the stats sinks outside of a flushing interval. - * Note: stat flushing is done asynchronously, this function will never block. - * This is a noop if called before the underlying EnvoyEngine has started. - */ - void flushStats(); - String dumpStats(); /** diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java index f0d82f9d6866..1a9d529fcfbb 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyEngineImpl.java @@ -35,16 +35,13 @@ public EnvoyEngineImpl(EnvoyOnEngineRunning runningCallback, EnvoyLogger logger, * * @param callbacks The callbacks for the stream. * @param explicitFlowControl Whether explicit flow control will be enabled for this stream. - * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which - * should be delivered sans end stream. * @return A stream that may be used for sending data. */ @Override - public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl, - long minDeliverySize) { + public EnvoyHTTPStream startStream(EnvoyHTTPCallbacks callbacks, boolean explicitFlowControl) { long streamHandle = JniLibrary.initStream(engineHandle); - EnvoyHTTPStream stream = new EnvoyHTTPStream(engineHandle, streamHandle, callbacks, - explicitFlowControl, minDeliverySize); + EnvoyHTTPStream stream = + new EnvoyHTTPStream(engineHandle, streamHandle, callbacks, explicitFlowControl); stream.start(); return stream; } @@ -54,11 +51,6 @@ public void terminate() { JniLibrary.terminateEngine(engineHandle); } - @Override - public void flushStats() { - JniLibrary.flushStats(engineHandle); - } - @Override public String dumpStats() { return JniLibrary.dumpStats(engineHandle); @@ -113,14 +105,10 @@ public EnvoyStatus runWithYaml(String configurationYAML, String logLevel) { @Override public EnvoyStatus runWithConfig(EnvoyConfiguration envoyConfiguration, String logLevel) { performRegistration(envoyConfiguration); - try { - int status = JniLibrary.runEngine(this.engineHandle, "", envoyConfiguration.createBootstrap(), - logLevel); - if (status == 0) { - return EnvoyStatus.ENVOY_SUCCESS; - } - } catch (Throwable throwable) { - // TODO: Need to have a way to log the exception somewhere. + int status = + JniLibrary.runEngine(this.engineHandle, "", envoyConfiguration.createBootstrap(), logLevel); + if (status == 0) { + return EnvoyStatus.ENVOY_SUCCESS; } return EnvoyStatus.ENVOY_FAILURE; } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java index 952144f67341..dfa3e2c371e4 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyHTTPStream.java @@ -12,15 +12,13 @@ public class EnvoyHTTPStream { private final long engineHandle; private final long streamHandle; private final boolean explicitFlowControl; - private final long minDeliverySize; private final JvmCallbackContext callbacksContext; /** * Start the stream via the JNI library. */ void start() { - JniLibrary.startStream(engineHandle, streamHandle, callbacksContext, explicitFlowControl, - minDeliverySize); + JniLibrary.startStream(engineHandle, streamHandle, callbacksContext, explicitFlowControl); } /** @@ -29,15 +27,12 @@ void start() { * @param streamHandle Underlying handle of the HTTP stream owned by an Envoy engine. * @param callbacks The callbacks for the stream. * @param explicitFlowControl Whether explicit flow control will be enabled for this stream. - * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which - * should be delivered sans end stream. */ public EnvoyHTTPStream(long engineHandle, long streamHandle, EnvoyHTTPCallbacks callbacks, - boolean explicitFlowControl, long minDeliverySize) { + boolean explicitFlowControl) { this.engineHandle = engineHandle; this.streamHandle = streamHandle; this.explicitFlowControl = explicitFlowControl; - this.minDeliverySize = minDeliverySize; callbacksContext = new JvmCallbackContext(callbacks); } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java index 4bda63f72edc..aa2b82c8e4ed 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java @@ -3,7 +3,6 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyEventTracker; import io.envoyproxy.envoymobile.engine.types.EnvoyLogger; import io.envoyproxy.envoymobile.engine.types.EnvoyOnEngineRunning; - import java.nio.ByteBuffer; public class JniLibrary { @@ -63,13 +62,11 @@ private static class JavaLoader { * callbacks. * @param explicitFlowControl, whether explicit flow control should be enabled * for the stream. - * @param minDeliverySize If nonzero, indicates the smallest number of response body bytes which - * should be delivered sans end stream. * @return envoy_stream, with a stream handle and a success status, or a failure * status. */ protected static native int startStream(long engine, long stream, JvmCallbackContext context, - boolean explicitFlowControl, long minDeliverySize); + boolean explicitFlowControl); /** * Send headers over an open HTTP stream. This method can be invoked once and @@ -203,15 +200,6 @@ protected static native int runEngine(long engine, String config, long bootstrap protected static native int recordCounterInc(long engine, String elements, byte[][] tags, int count); - /** - * Flush the stats sinks outside of a flushing interval. - * Note: stat flushing is done asynchronously, this function will never block. - * This is a noop if called before the underlying EnvoyEngine has started. - * - * @param engine engine whose stats should be flushed. - */ - protected static native int flushStats(long engine); - /** * Retrieve the value of all active stats. Note that this function may block for some time. * @param engine, handle to the engine that owns the counter. @@ -307,20 +295,18 @@ public static native Object callCertificateVerificationFromNative(byte[][] certC * */ public static native long createBootstrap( - String grpcStatsDomain, long connectTimeoutSeconds, long dnsRefreshSeconds, - long dnsFailureRefreshSecondsBase, long dnsFailureRefreshSecondsMax, - long dnsQueryTimeoutSeconds, long dnsMinRefreshSeconds, byte[][] dnsPreresolveHostnames, - boolean enableDNSCache, long dnsCacheSaveIntervalSeconds, boolean enableDrainPostDnsRefresh, - boolean enableHttp3, String http3ConnectionOptions, String http3ClientConnectionOptions, - byte[][] quicHints, boolean enableGzipDecompression, boolean enableBrotliDecompression, + long connectTimeoutSeconds, long dnsRefreshSeconds, long dnsFailureRefreshSecondsBase, + long dnsFailureRefreshSecondsMax, long dnsQueryTimeoutSeconds, long dnsMinRefreshSeconds, + byte[][] dnsPreresolveHostnames, boolean enableDNSCache, long dnsCacheSaveIntervalSeconds, + boolean enableDrainPostDnsRefresh, boolean enableHttp3, String http3ConnectionOptions, + String http3ClientConnectionOptions, byte[][] quicHints, byte[][] quicCanonicalSuffixes, + boolean enableGzipDecompression, boolean enableBrotliDecompression, boolean enableSocketTagging, boolean enableInterfaceBinding, long h2ConnectionKeepaliveIdleIntervalMilliseconds, long h2ConnectionKeepaliveTimeoutSeconds, - long maxConnectionsPerHost, long statsFlushSeconds, long streamIdleTimeoutSeconds, - long perTryIdleTimeoutSeconds, String appVersion, String appId, - boolean trustChainVerification, byte[][] filterChain, byte[][] statSinks, + long maxConnectionsPerHost, long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds, + String appVersion, String appId, boolean trustChainVerification, byte[][] filterChain, boolean enablePlatformCertificatesValidation, byte[][] runtimeGuards, String rtdsResourceName, - long rtdsTimeoutSeconds, String xdsAddress, long xdsPort, String xdsAuthenticationHeader, - String xdsAuthenticationToken, String xdsJwtToken, long xdsJwtTokenLifetime, - String xdsRootCerts, String xdsSni, String nodeId, String nodeRegion, String nodeZone, - String nodeSubZone, String cdsResourcesLocator, long cdsTimeoutSeconds, boolean enableCds); + long rtdsTimeoutSeconds, String xdsAddress, long xdsPort, byte[][] xdsGrpcInitialMetadata, + String xdsRootCerts, String nodeId, String nodeRegion, String nodeZone, String nodeSubZone, + byte[] nodeMetadata, String cdsResourcesLocator, long cdsTimeoutSeconds, boolean enableCds); } diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmStringAccessorContext.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmStringAccessorContext.java index 6dc2c3b0cc14..0f148283a6a5 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmStringAccessorContext.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmStringAccessorContext.java @@ -9,7 +9,7 @@ class JvmStringAccessorContext { public JvmStringAccessorContext(EnvoyStringAccessor accessor) { this.accessor = accessor; } /** - * Invokes getEnvoyString callback. This method signature is used within the jni_interface.cc. + * Invokes getEnvoyString callback. This method signature is used within the jni_impl.cc. * Changing naming of this class or methods will likely require an audit across the jni usages * and proguard rules. * diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/types/BUILD b/mobile/library/java/io/envoyproxy/envoymobile/engine/types/BUILD index 34ed3a103cd5..c5d4639d10ff 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/types/BUILD +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/types/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@rules_java//java:defs.bzl", "java_library") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + java_library( name = "envoy_c_types_lib", srcs = [ diff --git a/mobile/library/java/io/envoyproxy/envoymobile/utilities/BUILD b/mobile/library/java/io/envoyproxy/envoymobile/utilities/BUILD index 5e010064fdf0..912ad351beb2 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/utilities/BUILD +++ b/mobile/library/java/io/envoyproxy/envoymobile/utilities/BUILD @@ -1,8 +1,11 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_library( name = "utilities", srcs = glob(["*.java"]), diff --git a/mobile/library/java/org/chromium/net/BUILD b/mobile/library/java/org/chromium/net/BUILD index 0e2168671f86..685986ae7130 100644 --- a/mobile/library/java/org/chromium/net/BUILD +++ b/mobile/library/java/org/chromium/net/BUILD @@ -1,8 +1,11 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + # Cronet API fork # Copy of the original Cronet API @@ -29,5 +32,6 @@ android_binary( deps = [ ":net", "//library/java/org/chromium/net/impl:cronvoy", + artifact("com.google.protobuf:protobuf-javalite"), ], ) diff --git a/mobile/library/java/org/chromium/net/impl/BUILD b/mobile/library/java/org/chromium/net/impl/BUILD index 1bde5cf9677d..b2f56777628d 100644 --- a/mobile/library/java/org/chromium/net/impl/BUILD +++ b/mobile/library/java/org/chromium/net/impl/BUILD @@ -1,8 +1,11 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + # Android libraries for cronvoy # An http client backed by Envoy-Mobile. @@ -51,6 +54,7 @@ android_library( "//library/java/io/envoyproxy/envoymobile/utilities", "//library/java/org/chromium/net", "//library/java/org/chromium/net/urlconnection", + "@maven//:com_google_protobuf_protobuf_javalite", artifact("androidx.annotation:annotation"), ], ) diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java index 054e6f8e8050..01cb9ad32dc9 100644 --- a/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyBidirectionalStream.java @@ -319,8 +319,7 @@ public void start() { public void run() { try { mStream.setStream(mRequestContext.getEnvoyEngine().startStream( - CronvoyBidirectionalStream.this, /* explicitFlowCrontrol= */ true, - /* minDeliverySize */ 0)); + CronvoyBidirectionalStream.this, /* explicitFlowCrontrol= */ true)); if (!mDelayRequestHeadersUntilFirstFlush) { mStream.sendHeaders(mEnvoyRequestHeaders, mReadOnly); } diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java index 5a58ce912e7a..7813bb95592a 100644 --- a/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyEngineBuilderImpl.java @@ -50,6 +50,7 @@ final static class Pkp { // See setters below for verbose descriptions. private final Context mApplicationContext; private final Map mQuicHints = new HashMap<>(); + private final List mQuicCanonicalSuffixes = new LinkedList<>(); private final List mPkps = new LinkedList<>(); private boolean mPublicKeyPinningBypassForLocalTrustAnchorsEnabled; private String mUserAgent; @@ -227,6 +228,13 @@ public CronvoyEngineBuilderImpl addQuicHint(String host, int port, int alternate Map quicHints() { return mQuicHints; } + public CronvoyEngineBuilderImpl addQuicCanonicalSuffix(String suffix) { + mQuicCanonicalSuffixes.add(suffix); + return this; + } + + List quicCanonicalSuffixes() { return mQuicCanonicalSuffixes; } + @Override public CronvoyEngineBuilderImpl addPublicKeyPins(String hostName, Set pinsSha256, boolean includeSubdomains, Date expirationDate) { diff --git a/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java index 63919abd6041..daa743f2a3b5 100644 --- a/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java +++ b/mobile/library/java/org/chromium/net/impl/CronvoyUrlRequest.java @@ -540,8 +540,7 @@ private void fireOpenConnection() { mCurrentUrl, mRequestContext.getBuilder().quicEnabled()); mCronvoyCallbacks = new CronvoyHttpCallbacks(); mStream.set(mRequestContext.getEnvoyEngine().startStream(mCronvoyCallbacks, - /* explicitFlowCrontrol */ true, - /* minDeliverySize */ 0)); + /* explicitFlowCrontrol= */ true)); mStream.get().sendHeaders(envoyRequestHeaders, mUploadDataStream == null); if (mUploadDataStream != null && mUrlChain.size() == 1) { mUploadDataStream.initializeWithRequest(); @@ -995,7 +994,7 @@ public void onSendWindowAvailable(EnvoyStreamIntel streamIntel) { return; } - mUploadDataStream.readDataReady(); // Have the next request body delivery to be sent. + mUploadDataStream.readDataReady(); // Have the next request body chunk to be sent. } @Override diff --git a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java index 68943535c077..8efa2325ca3e 100644 --- a/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java +++ b/mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java @@ -4,6 +4,7 @@ import android.content.Context; import androidx.annotation.VisibleForTesting; +import com.google.protobuf.Struct; import io.envoyproxy.envoymobile.engine.AndroidEngineImpl; import io.envoyproxy.envoymobile.engine.AndroidJniLibrary; import io.envoyproxy.envoymobile.engine.AndroidNetworkMonitor; @@ -32,7 +33,6 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl { // TODO(refactor) move unshared variables into their specific methods. private final List nativeFilterChain = new ArrayList<>(); private final EnvoyEventTracker mEnvoyEventTracker = null; - private String mGrpcStatsDomain = null; private int mConnectTimeoutSeconds = 30; private int mDnsRefreshSeconds = 60; private int mDnsFailureRefreshSecondsBase = 2; @@ -53,7 +53,6 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl { private int mH2ConnectionKeepaliveIdleIntervalMilliseconds = 1; private int mH2ConnectionKeepaliveTimeoutSeconds = 10; private int mMaxConnectionsPerHost = 7; - private int mStatsFlushSeconds = 60; private int mStreamIdleTimeoutSeconds = 15; private int mPerTryIdleTimeoutSeconds = 15; private String mAppVersion = "unspecified"; @@ -118,24 +117,23 @@ private EnvoyConfiguration createEnvoyConfiguration() { List platformFilterChain = Collections.emptyList(); Map stringAccessors = Collections.emptyMap(); Map keyValueStores = Collections.emptyMap(); - List statSinks = Collections.emptyList(); Map runtimeGuards = Collections.emptyMap(); return new EnvoyConfiguration( - mGrpcStatsDomain, mConnectTimeoutSeconds, mDnsRefreshSeconds, mDnsFailureRefreshSecondsBase, + mConnectTimeoutSeconds, mDnsRefreshSeconds, mDnsFailureRefreshSecondsBase, mDnsFailureRefreshSecondsMax, mDnsQueryTimeoutSeconds, mDnsMinRefreshSeconds, mDnsPreresolveHostnames, mEnableDNSCache, mDnsCacheSaveIntervalSeconds, mEnableDrainPostDnsRefresh, quicEnabled(), quicConnectionOptions(), - quicClientConnectionOptions(), quicHints(), mEnableGzipDecompression, brotliEnabled(), - mEnableSocketTag, mEnableInterfaceBinding, mH2ConnectionKeepaliveIdleIntervalMilliseconds, - mH2ConnectionKeepaliveTimeoutSeconds, mMaxConnectionsPerHost, mStatsFlushSeconds, - mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, - mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors, - keyValueStores, statSinks, runtimeGuards, mEnablePlatformCertificatesValidation, + quicClientConnectionOptions(), quicHints(), quicCanonicalSuffixes(), + mEnableGzipDecompression, brotliEnabled(), mEnableSocketTag, mEnableInterfaceBinding, + mH2ConnectionKeepaliveIdleIntervalMilliseconds, mH2ConnectionKeepaliveTimeoutSeconds, + mMaxConnectionsPerHost, mStreamIdleTimeoutSeconds, mPerTryIdleTimeoutSeconds, mAppVersion, + mAppId, mTrustChainVerification, nativeFilterChain, platformFilterChain, stringAccessors, + keyValueStores, runtimeGuards, mEnablePlatformCertificatesValidation, /*rtdsResourceName=*/"", /*rtdsTimeoutSeconds=*/0, /*xdsAddress=*/"", - /*xdsPort=*/0, /*xdsAuthenticationHeader=*/"", /*xdsAuthenticationToken=*/"", - /*xdsJwtToken=*/"", /*xdsJwtTokenLifetime=*/0, /*xdsSslRootCerts=*/"", - /*xdsSni=*/"", mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, /*cdsResourcesLocator=*/"", - /*cdsTimeoutSeconds=*/0, /*enableCds=*/false); + /*xdsPort=*/0, /*xdsGrpcInitialMetadata=*/Collections.emptyMap(), + /*xdsSslRootCerts=*/"", mNodeId, mNodeRegion, mNodeZone, mNodeSubZone, + Struct.getDefaultInstance(), /*cdsResourcesLocator=*/"", /*cdsTimeoutSeconds=*/0, + /*enableCds=*/false); } } diff --git a/mobile/library/java/org/chromium/net/urlconnection/BUILD b/mobile/library/java/org/chromium/net/urlconnection/BUILD index 2340d91645a7..4849b5f8082a 100644 --- a/mobile/library/java/org/chromium/net/urlconnection/BUILD +++ b/mobile/library/java/org/chromium/net/urlconnection/BUILD @@ -1,8 +1,11 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + # Android libraries for urlconnection # A URL connection interface backed by Envoy-Mobile. diff --git a/mobile/library/java_proto_proguard.txt b/mobile/library/java_proto_proguard.txt new file mode 100644 index 000000000000..c4905d5ba39e --- /dev/null +++ b/mobile/library/java_proto_proguard.txt @@ -0,0 +1,15 @@ +-dontwarn com.google.protobuf.** + +-dontnote com.google.protobuf.** + +-keep class com.google.protobuf.** { + *; +} + +-keep class com.google.protobuf.MessageLite { + *; +} + +-keep class com.google.protobuf.MessageLite$Builder { + *; +} diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/AndroidEngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/AndroidEngineBuilder.kt index 58145ec68f5f..9d07422f1729 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/AndroidEngineBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/AndroidEngineBuilder.kt @@ -3,14 +3,11 @@ package io.envoyproxy.envoymobile import android.content.Context import io.envoyproxy.envoymobile.engine.AndroidEngineImpl - -/** - * The engine builder to use to create Envoy engine on Android. - */ -class AndroidEngineBuilder @JvmOverloads constructor( - context: Context, - baseConfiguration: BaseConfiguration = Standard() -) : EngineBuilder(baseConfiguration) { +/** The engine builder to use to create Envoy engine on Android. */ +class AndroidEngineBuilder +@JvmOverloads +constructor(context: Context, baseConfiguration: BaseConfiguration = Standard()) : + EngineBuilder(baseConfiguration) { init { addEngineType { AndroidEngineImpl(context, onEngineRunning, logger, eventTracker, enableProxying) diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD b/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD index a4ea5c339633..190c67a33492 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_select_enable_http3", "envoy_select_envoy_mobile_request_compression") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:android_artifacts.bzl", "android_artifacts") load("@envoy_mobile//bazel:kotlin_lib.bzl", "envoy_mobile_kt_library") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") @@ -6,6 +6,8 @@ load("@rules_detekt//detekt:defs.bzl", "detekt") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_artifacts( name = "envoy_aar", android_library = ":envoy_lib", @@ -16,6 +18,32 @@ android_artifacts( "//conditions:default": ["//library/common/jni:libenvoy_jni.so"], }), proguard_rules = "//library:proguard_rules", + substitutions = { + "{pom_artifact_id}": "envoy", + "{pom_extra_dependencies}": "", + }, + visibility = ["//visibility:public"], +) + +android_artifacts( + name = "envoy_xds_aar", + android_library = ":envoy_lib", + archive_name = "envoy_xds", + manifest = "EnvoyManifest.xml", + native_deps = select({ + "@envoy//bazel:opt_build": ["//library/common/jni:libenvoy_jni.so.debug_info"], + "//conditions:default": ["//library/common/jni:libenvoy_jni.so"], + }), + proguard_rules = "//library:proguard_rules", + substitutions = { + "{pom_artifact_id}": "envoy-xds", + "{pom_extra_dependencies}": """ + + com.google.protobuf + protobuf-javalite + 3.24.4 + """, + }, visibility = ["//visibility:public"], ) @@ -75,16 +103,7 @@ envoy_mobile_kt_library( "grpc/*.kt", "mocks/*.kt", "stats/*.kt", - ]) + envoy_select_envoy_mobile_request_compression( - [ - "CompressionAlgorithm.kt", - "RequestHeadersBuilderCompressionUtil.kt", - ], - "@envoy", - ) + envoy_select_enable_http3( - ["EngineBuilderHTTP3Util.kt"], - "@envoy", - ), + ]), visibility = ["//visibility:public"], deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/CompressionAlgorithm.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/CompressionAlgorithm.kt deleted file mode 100644 index 84e1a4e9a5c4..000000000000 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/CompressionAlgorithm.kt +++ /dev/null @@ -1,11 +0,0 @@ -package io.envoyproxy.envoymobile - -/** - * Available algorithms to compress requests. - * - * @param stringValue string representation of a given compression algorithm. - */ -enum class CompressionAlgorithm(internal val stringValue: String) { - GZIP("gzip"), - BROTLI("brotli"), -} diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/Engine.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/Engine.kt index 18dcab9d5ae2..1f75449b1678 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/Engine.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/Engine.kt @@ -6,36 +6,22 @@ package io.envoyproxy.envoymobile */ interface Engine { - /** - * @return a {@link StreamClient} for opening and managing HTTP streams. - */ + /** @return a {@link StreamClient} for opening and managing HTTP streams. */ fun streamClient(): StreamClient - /** - * @return a {@link PulseClient} for recording time series metrics. - */ + /** @return a {@link PulseClient} for recording time series metrics. */ fun pulseClient(): PulseClient - /** - * Terminates the running engine. - */ + /** Terminates the running engine. */ fun terminate() - /** - * Flush the stats sinks outside of a flushing interval. - * Note: stat flushing is done asynchronously, this function will never block. - * This is a noop if called before the underlying EnvoyEngine has started. - */ - fun flushStats() - /** * Retrieve the value of all active stats. Note that this function may block for some time. + * * @return The list of active stats and their values, or empty string of the operation failed */ fun dumpStats(): String - /** - * Refresh DNS, and drain connections owned by this Engine. - */ + /** Refresh DNS, and drain connections owned by this Engine. */ fun resetConnectivityState() } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt index 34e903bbcccd..71775630f3fd 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt @@ -1,5 +1,6 @@ package io.envoyproxy.envoymobile +import com.google.protobuf.Struct import io.envoyproxy.envoymobile.engine.EnvoyConfiguration import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification import io.envoyproxy.envoymobile.engine.EnvoyEngine @@ -10,14 +11,10 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyKeyValueStore import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor import java.util.UUID -/** - * Envoy engine configuration. - */ +/** Envoy engine configuration. */ sealed class BaseConfiguration -/** - * The standard configuration. - */ +/** The standard configuration. */ class Standard : BaseConfiguration() /** @@ -28,27 +25,19 @@ class Standard : BaseConfiguration() class Custom(val yaml: String) : BaseConfiguration() /** - * Builder for generating the xDS configuration for the Envoy Mobile engine. - * xDS is a protocol for dynamic configuration of Envoy instances, more information can be found in - * https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol. - * - * This class is typically used as input to the EngineBuilder's setXds() method. + * Builder for generating the xDS configuration for the Envoy Mobile engine. xDS is a protocol for + * dynamic configuration of Envoy instances, more information can be found in + * https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol. + * + * This class is typically used as input to the EngineBuilder's setXds() method. */ -open class XdsBuilder ( - internal val xdsServerAddress: String, - internal val xdsServerPort: Int -) { +open class XdsBuilder(internal val xdsServerAddress: String, internal val xdsServerPort: Int) { companion object { - private const val DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS: Int = 60 * 60 * 24 * 90 // 90 days private const val DEFAULT_XDS_TIMEOUT_IN_SECONDS: Int = 5 } - internal var authHeader: String? = null - internal var authToken: String? = null - internal var jwtToken: String? = null - internal var jwtTokenLifetimeInSeconds: Int = DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS + internal var grpcInitialMetadata = mutableMapOf() internal var sslRootCerts: String? = null - internal var sni: String? = null internal var rtdsResourceName: String? = null internal var rtdsTimeoutInSeconds: Int = DEFAULT_XDS_TIMEOUT_IN_SECONDS internal var enableCds: Boolean = false @@ -56,41 +45,23 @@ open class XdsBuilder ( internal var cdsTimeoutInSeconds: Int = DEFAULT_XDS_TIMEOUT_IN_SECONDS /** - * Sets the authentication HTTP header and token value for authenticating with the xDS - * management server. - * - * @param header The HTTP authentication header. - * @param token The authentication token to be sent in the header. + * Adds a header to the initial HTTP metadata headers sent on the gRPC stream. * - * @return this builder. - */ - fun setAuthenticationToken( - header: String, - token: String - ): XdsBuilder { - this.authHeader = header - this.authToken = token - return this - } - - /** - * Sets JWT as the authentication method to the xDS management server, using the given token. + * A common use for the initial metadata headers is for authentication to the xDS management + * server. * - * @param token The JWT token used to authenticate the client to the xDS management server. - * @param tokenLifetimeInSeconds The lifetime of the JWT token, in seconds. If none - * (or 0) is specified, then defaultJwtTokenLifetimeSeconds is - * used. + * For example, if using API keys to authenticate to Traffic Director on GCP (see + * https://cloud.google.com/docs/authentication/api-keys for details), invoke: + * builder.addInitialStreamHeader("x-goog-api-key", apiKeyToken) + * .addInitialStreamHeader("X-Android-Package", appPackageName) + * .addInitialStreamHeader("X-Android-Cert", sha1KeyFingerprint) * + * @param header The HTTP header name to add to the initial gRPC stream's metadata. + * @param value The HTTP header value to add to the initial gRPC stream's metadata. * @return this builder. */ - fun setJwtAuthenticationToken( - token: String, - tokenLifetimeInSeconds: Int = DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS - ): XdsBuilder { - this.jwtToken = token - this.jwtTokenLifetimeInSeconds = if (tokenLifetimeInSeconds > 0) - tokenLifetimeInSeconds else - DEFAULT_JWT_TOKEN_LIFETIME_IN_SECONDS + fun addInitialStreamHeader(header: String, value: String): XdsBuilder { + this.grpcInitialMetadata.put(header, value) return this } @@ -99,7 +70,6 @@ open class XdsBuilder ( * connection. If no root certs are specified, the operating system defaults are used. * * @param rootCerts The PEM-encoded server root certificates. - * * @return this builder. */ fun setSslRootCerts(rootCerts: String): XdsBuilder { @@ -108,30 +78,15 @@ open class XdsBuilder ( } /** - * Sets the SNI (https://datatracker.ietf.org/doc/html/rfc6066#section-3) on the TLS handshake - * and the authority HTTP header. If not set, the SNI is set by default to the xDS server address - * and the authority HTTP header is not set. - * - * @param sni The SNI value. - * - * @return this builder. - */ - fun setSni(sni: String): XdsBuilder { - this.sni = sni - return this - } - - /** - * Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration, - * to retrieve dynamic runtime configuration via the xDS management server. + * Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration, to + * retrieve dynamic runtime configuration via the xDS management server. * * @param resourceName The runtime config resource to subscribe to. * @param timeoutInSeconds specifies the `initial_fetch_timeout` field on the - * api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch - * timeout value of 5s, to prevent mobile app initialization from stalling. The default - * parameter value may change through the course of experimentation and no assumptions should - * be made of its exact value. - * + * api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch + * timeout value of 5s, to prevent mobile app initialization from stalling. The default + * parameter value may change through the course of experimentation and no assumptions should be + * made of its exact value. * @return this builder. */ fun addRuntimeDiscoveryService( @@ -144,18 +99,17 @@ open class XdsBuilder ( } /** - * Adds the Cluster Discovery Service (CDS) configuration for retrieving dynamic cluster - * resources via the xDS management server. + * Adds the Cluster Discovery Service (CDS) configuration for retrieving dynamic cluster resources + * via the xDS management server. * * @param cdsResourcesLocator the xdstp:// URI for subscribing to the cluster - * resources. If not using xdstp, then `cds_resources_locator` should be set to the empty - * string. + * resources. If not using xdstp, then `cds_resources_locator` should be set to the empty + * string. * @param timeoutInSeconds specifies the `initial_fetch_timeout` field on the - * api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch - * timeout value of 5s, to prevent mobile app initialization from stalling. The default - * parameter value may change through the course of experimentation and no assumptions should - * be made of its exact value. - * + * api.v3.core.ConfigSource. Unlike the ConfigSource default of 15s, we set a default fetch + * timeout value of 5s, to prevent mobile app initialization from stalling. The default + * parameter value may change through the course of experimentation and no assumptions should be + * made of its exact value. * @return this builder. */ public fun addClusterDiscoveryService( @@ -173,12 +127,8 @@ open class XdsBuilder ( } } -/** - * Builder used for creating and running a new `Engine` instance. - */ -open class EngineBuilder( - private val configuration: BaseConfiguration = Standard() -) { +/** Builder used for creating and running a new `Engine` instance. */ +open class EngineBuilder(private val configuration: BaseConfiguration = Standard()) { protected var onEngineRunning: (() -> Unit) = {} protected var logger: ((String) -> Unit)? = null protected var eventTracker: ((Map) -> Unit)? = null @@ -188,7 +138,6 @@ open class EngineBuilder( EnvoyEngineImpl(onEngineRunning, logger, eventTracker) } private var logLevel = LogLevel.INFO - private var grpcStatsDomain: String? = null private var connectTimeoutSeconds = 30 private var dnsRefreshSeconds = 60 private var dnsFailureRefreshSecondsBase = 2 @@ -203,6 +152,7 @@ open class EngineBuilder( private var http3ConnectionOptions = "" private var http3ClientConnectionOptions = "" private var quicHints = mutableMapOf() + private var quicCanonicalSuffixes = mutableListOf() private var enableGzipDecompression = true private var enableBrotliDecompression = false private var enableSocketTagging = false @@ -210,7 +160,6 @@ open class EngineBuilder( private var h2ConnectionKeepaliveIdleIntervalMilliseconds = 1 private var h2ConnectionKeepaliveTimeoutSeconds = 10 private var maxConnectionsPerHost = 7 - private var statsFlushSeconds = 60 private var streamIdleTimeoutSeconds = 15 private var perTryIdleTimeoutSeconds = 15 private var appVersion = "unspecified" @@ -220,19 +169,18 @@ open class EngineBuilder( private var nativeFilterChain = mutableListOf() private var stringAccessors = mutableMapOf() private var keyValueStores = mutableMapOf() - private var statsSinks = listOf() private var enablePlatformCertificatesValidation = false private var nodeId: String = "" private var nodeRegion: String = "" private var nodeZone: String = "" private var nodeSubZone: String = "" + private var nodeMetadata: Struct = Struct.getDefaultInstance() private var xdsBuilder: XdsBuilder? = null /** * Add a log level to use with Envoy. * * @param logLevel the log level to use with Envoy. - * * @return this builder. */ fun addLogLevel(logLevel: LogLevel): EngineBuilder { @@ -240,41 +188,10 @@ open class EngineBuilder( return this } - /** - * Specifies the domain (e.g. `example.com`) to use in the default gRPC stat sink to flush - * stats. - * - * Setting this value enables the gRPC stat sink, which periodically flushes stats via the gRPC - * MetricsService API. The flush interval is specified via addStatsFlushSeconds. - * - * @param grpcStatsDomain The domain to use for the gRPC stats sink. - * - * @return this builder. - */ - fun addGrpcStatsDomain(grpcStatsDomain: String?): EngineBuilder { - this.grpcStatsDomain = grpcStatsDomain - return this - } - - /** - * Adds additional stats sinks, in the form of the raw YAML/JSON configuration. - * Sinks added in this fashion will be included in addition to the gRPC stats sink - * that may be enabled via addGrpcStatsDomain. - * - * @param statsSinks Configurations of stat sinks to add. - * - * @return this builder. - */ - fun addStatsSinks(statsSinks: List): EngineBuilder { - this.statsSinks = statsSinks - return this - } - /** * Add a timeout for new network connections to hosts in the cluster. * * @param connectTimeoutSeconds timeout for new network connections to hosts in the cluster. - * * @return this builder. */ fun addConnectTimeoutSeconds(connectTimeoutSeconds: Int): EngineBuilder { @@ -286,7 +203,6 @@ open class EngineBuilder( * Add a default rate at which to refresh DNS. * * @param dnsRefreshSeconds default rate in seconds at which to refresh DNS. - * * @return this builder. */ fun addDNSRefreshSeconds(dnsRefreshSeconds: Int): EngineBuilder { @@ -299,7 +215,6 @@ open class EngineBuilder( * * @param base rate in seconds. * @param max rate in seconds. - * * @return this builder. */ fun addDNSFailureRefreshSeconds(base: Int, max: Int): EngineBuilder { @@ -312,7 +227,6 @@ open class EngineBuilder( * Add a rate at which to timeout DNS queries. * * @param dnsQueryTimeoutSeconds rate in seconds to timeout DNS queries. - * * @return this builder. */ fun addDNSQueryTimeoutSeconds(dnsQueryTimeoutSeconds: Int): EngineBuilder { @@ -325,7 +239,6 @@ open class EngineBuilder( * will be respected, subject to this minimum. Defaults to 60 seconds. * * @param dnsMinRefreshSeconds minimum rate in seconds at which to refresh DNS. - * * @return this builder. */ fun addDNSMinRefreshSeconds(dnsMinRefreshSeconds: Int): EngineBuilder { @@ -337,7 +250,6 @@ open class EngineBuilder( * Add a list of hostnames to preresolve on Engine startup. * * @param dnsPreresolveHostnames hostnames to preresolve. - * * @return this builder. */ fun addDNSPreresolveHostnames(dnsPreresolveHostnames: List): EngineBuilder { @@ -352,7 +264,6 @@ open class EngineBuilder( * establish new connections for any further requests. * * @param enableDrainPostDnsRefresh whether to drain connections after soft DNS refresh. - * * @return This builder. */ fun enableDrainPostDnsRefresh(enableDrainPostDnsRefresh: Boolean): EngineBuilder { @@ -363,12 +274,10 @@ open class EngineBuilder( /** * Specify whether to enable DNS cache. * - * Note that DNS cache requires an addition of a key value store named - * 'reserved.platform_store'. + * Note that DNS cache requires an addition of a key value store named 'reserved.platform_store'. * * @param enableDNSCache whether to enable DNS cache. Disabled by default. - * @param saveInterval the interval at which to save results to the configured key value store. - * + * @param saveInterval the interval at which to save results to the configured key value store. * @return This builder. */ fun enableDNSCache(enableDNSCache: Boolean, saveInterval: Int = 1): EngineBuilder { @@ -378,10 +287,9 @@ open class EngineBuilder( } /** - * Specify whether to do gzip response decompression or not. Defaults to true. + * Specify whether to do gzip response decompression or not. Defaults to true. * * @param enableGzipDecompression whether or not to gunzip responses. - * * @return This builder. */ fun enableGzipDecompression(enableGzipDecompression: Boolean): EngineBuilder { @@ -390,10 +298,20 @@ open class EngineBuilder( } /** - * Specify whether to do brotli response decompression or not. Defaults to false. + * Specify whether to enable HTTP3. Defaults to true. * - * @param enableBrotliDecompression whether or not to brotli decompress responses. + * @param enableHttp3 whether or not to enable HTTP3. + * @return This builder. + */ + fun enableHttp3(enableHttp3: Boolean): EngineBuilder { + this.enableHttp3 = enableHttp3 + return this + } + + /** + * Specify whether to do brotli response decompression or not. Defaults to false. * + * @param enableBrotliDecompression whether or not to brotli decompress responses. * @return This builder. */ fun enableBrotliDecompression(enableBrotliDecompression: Boolean): EngineBuilder { @@ -405,7 +323,6 @@ open class EngineBuilder( * Specify whether to support socket tagging or not. Defaults to false. * * @param enableSocketTagging whether or not support socket tagging. - * * @return This builder. */ fun enableSocketTagging(enableSocketTagging: Boolean): EngineBuilder { @@ -418,7 +335,6 @@ open class EngineBuilder( * conditions. * * @param enableInterfaceBinding whether to allow interface binding. - * * @return This builder. */ fun enableInterfaceBinding(enableInterfaceBinding: Boolean): EngineBuilder { @@ -427,16 +343,15 @@ open class EngineBuilder( } /** - * Specify whether system proxy settings should be respected. If yes, Envoy Mobile will - * use Android APIs to query Android Proxy settings configured on a device and will - * respect these settings when establishing connections with remote services. + * Specify whether system proxy settings should be respected. If yes, Envoy Mobile will use + * Android APIs to query Android Proxy settings configured on a device and will respect these + * settings when establishing connections with remote services. * - * The method is introduced for experimentation purposes and as a safety guard against - * critical issues in the implementation of the proxying feature. It's intended to be removed - * after it's confirmed that proxies on Android work as expected. + * The method is introduced for experimentation purposes and as a safety guard against critical + * issues in the implementation of the proxying feature. It's intended to be removed after it's + * confirmed that proxies on Android work as expected. * * @param enableProxying whether to enable Envoy's support for proxies. - * * @return This builder. */ fun enableProxying(enableProxying: Boolean): EngineBuilder { @@ -445,13 +360,12 @@ open class EngineBuilder( } /** - * Add a rate at which to ping h2 connections on new stream creation if the connection has - * sat idle. Defaults to 1 millisecond which effectively enables h2 ping functionality - * and results in a connection ping on every new stream creation. Set it to - * 100000000 milliseconds to effectively disable the ping. + * Add a rate at which to ping h2 connections on new stream creation if the connection has sat + * idle. Defaults to 1 millisecond which effectively enables h2 ping functionality and results in + * a connection ping on every new stream creation. Set it to 100000000 milliseconds to effectively + * disable the ping. * * @param idleIntervalMs rate in milliseconds. - * * @return this builder. */ fun addH2ConnectionKeepaliveIdleIntervalMilliseconds(idleIntervalMs: Int): EngineBuilder { @@ -463,7 +377,6 @@ open class EngineBuilder( * Add a rate at which to timeout h2 pings. * * @param timeoutSeconds rate in seconds to timeout h2 pings. - * * @return this builder. */ fun addH2ConnectionKeepaliveTimeoutSeconds(timeoutSeconds: Int): EngineBuilder { @@ -475,7 +388,6 @@ open class EngineBuilder( * Set the maximum number of connections to open to a single host. Default is 7. * * @param maxConnectionsPerHost the maximum number of connections per host. - * * @return this builder. */ fun setMaxConnectionsPerHost(maxConnectionsPerHost: Int): EngineBuilder { @@ -483,23 +395,10 @@ open class EngineBuilder( return this } - /** - * Add an interval at which to flush Envoy stats. - * - * @param statsFlushSeconds interval at which to flush Envoy stats. - * - * @return this builder. - */ - fun addStatsFlushSeconds(statsFlushSeconds: Int): EngineBuilder { - this.statsFlushSeconds = statsFlushSeconds - return this - } - /** * Add a custom idle timeout for HTTP streams. Defaults to 15 seconds. * * @param streamIdleTimeoutSeconds idle timeout for HTTP streams. - * * @return this builder. */ fun addStreamIdleTimeoutSeconds(streamIdleTimeoutSeconds: Int): EngineBuilder { @@ -511,7 +410,6 @@ open class EngineBuilder( * Add a custom per try idle timeout for HTTP streams. Defaults to 15 seconds. * * @param perTryIdleTimeoutSeconds per try idle timeout for HTTP streams. - * * @return this builder. */ fun addPerTryIdleTimeoutSeconds(perTryIdleTimeoutSeconds: Int): EngineBuilder { @@ -522,53 +420,47 @@ open class EngineBuilder( /** * Add an HTTP filter factory used to create platform filters for streams sent by this client. * - * @param name Custom name to use for this filter factory. Useful for having - * more meaningful trace logs, but not required. Should be unique - * per factory registered. + * @param name Custom name to use for this filter factory. Useful for having more meaningful trace + * logs, but not required. Should be unique per factory registered. * @param factory closure returning an instantiated filter. - * * @return this builder. */ - fun addPlatformFilter(name: String, factory: () -> Filter): - EngineBuilder { - this.platformFilterChain.add(FilterFactory(name, factory)) - return this - } + fun addPlatformFilter(name: String, factory: () -> Filter): EngineBuilder { + this.platformFilterChain.add(FilterFactory(name, factory)) + return this + } /** * Add an HTTP filter factory used to create platform filters for streams sent by this client. * * @param factory closure returning an instantiated filter. - * * @return this builder. */ - fun addPlatformFilter(factory: () -> Filter): - EngineBuilder { - this.platformFilterChain.add(FilterFactory(UUID.randomUUID().toString(), factory)) - return this - } + fun addPlatformFilter(factory: () -> Filter): EngineBuilder { + this.platformFilterChain.add(FilterFactory(UUID.randomUUID().toString(), factory)) + return this + } /** * Add an HTTP filter config used to create native filters for streams sent by this client. * - * @param name Custom name to use for this filter factory. Useful for having - * more meaningful trace logs, but not required. Should be unique - * per filter. + * @param name Custom name to use for this filter factory. Useful for having more meaningful trace + * logs, but not required. Should be unique per filter. * @param typedConfig config string for the filter. - * * @return this builder. */ - fun addNativeFilter(name: String = UUID.randomUUID().toString(), typedConfig: String): - EngineBuilder { - this.nativeFilterChain.add(EnvoyNativeFilterConfig(name, typedConfig)) - return this - } + fun addNativeFilter( + name: String = UUID.randomUUID().toString(), + typedConfig: String + ): EngineBuilder { + this.nativeFilterChain.add(EnvoyNativeFilterConfig(name, typedConfig)) + return this + } /** * Set a closure to be called when the engine finishes its async startup and begins running. * * @param closure the closure to be called. - * * @return this builder. */ fun setOnEngineRunning(closure: () -> Unit): EngineBuilder { @@ -578,8 +470,8 @@ open class EngineBuilder( /** * Set a closure to be called when the engine's logger logs. - * @param closure: The closure to be called. * + * @param closure: The closure to be called. * @return This builder. */ fun setLogger(closure: (String) -> Unit): EngineBuilder { @@ -587,9 +479,7 @@ open class EngineBuilder( return this } - /** - * Set event tracker for the engine to call when it emits an event. - */ + /** Set event tracker for the engine to call when it emits an event. */ fun setEventTracker(eventTracker: (Map) -> Unit): EngineBuilder { this.eventTracker = eventTracker return this @@ -600,7 +490,6 @@ open class EngineBuilder( * * @param name the name of the accessor. * @param accessor the string accessor. - * * @return this builder. */ fun addStringAccessor(name: String, accessor: () -> String): EngineBuilder { @@ -613,7 +502,6 @@ open class EngineBuilder( * * @param name the name of the KV store. * @param keyValueStore the KV store implementation. - * * @return this builder. */ fun addKeyValueStore(name: String, keyValueStore: KeyValueStore): EngineBuilder { @@ -625,7 +513,6 @@ open class EngineBuilder( * Add the App Version of the App using this Envoy Client. * * @param appVersion the version. - * * @return this builder. */ fun addAppVersion(appVersion: String): EngineBuilder { @@ -637,7 +524,6 @@ open class EngineBuilder( * Add the App ID of the App using this Envoy Client. * * @param appId the ID. - * * @return this builder. */ fun addAppId(appId: String): EngineBuilder { @@ -649,7 +535,6 @@ open class EngineBuilder( * Set how the TrustChainVerification must be handled. * * @param trustChainVerification whether to mute TLS Cert verification - intended for testing - * * @return this builder. */ fun setTrustChainVerification(trustChainVerification: TrustChainVerification): EngineBuilder { @@ -661,7 +546,6 @@ open class EngineBuilder( * Sets the node.id field in the Bootstrap configuration. * * @param nodeId the node ID. - * * @return this builder. */ fun setNodeId(nodeId: String): EngineBuilder { @@ -675,7 +559,6 @@ open class EngineBuilder( * @param region the region of the node locality. * @param zone the zone of the node locality. * @param subZone the sub-zone of the node locality. - * * @return this builder. */ fun setNodeLocality(region: String, zone: String, subZone: String): EngineBuilder { @@ -685,11 +568,21 @@ open class EngineBuilder( return this } + /** + * Sets the node.metadata field in the Bootstrap configuration. + * + * @param metadata the metadata of the node. + * @return this builder. + */ + fun setNodeMetadata(metadata: Struct): EngineBuilder { + this.nodeMetadata = metadata + return this + } + /** * Sets the xDS configuration for the Envoy Mobile engine. * * @param xdsBuilder The XdsBuilder instance from which to construct the xDS configuration. - * * @return this builder. */ fun setXds(xdsBuilder: XdsBuilder): EngineBuilder { @@ -702,7 +595,6 @@ open class EngineBuilder( * * @param name the name of the runtime guard, e.g. test_feature_false. * @param value the value for the runtime guard. - * * @return This builder. */ fun setRuntimeGuard(name: String, value: Boolean): EngineBuilder { @@ -715,13 +607,23 @@ open class EngineBuilder( * * @param host the host's name. * @param port the port number. - * * @return This builder. */ - fun addQuicHint(host: String, port: Int): EngineBuilder { + fun addQuicHint(host: String, port: Int): EngineBuilder { this.quicHints.put(host, port) return this - } + } + + /** + * Add a host suffix that's known to speak QUIC. + * + * @param suffix the suffix string. + * @return This builder. + */ + fun addQuicCanonicalSuffix(suffix: String): EngineBuilder { + this.quicCanonicalSuffixes.add(suffix) + return this + } /** * Builds and runs a new Engine instance with the provided configuration. @@ -730,77 +632,63 @@ open class EngineBuilder( */ @Suppress("LongMethod") fun build(): Engine { - val engineConfiguration = EnvoyConfiguration( - grpcStatsDomain, - connectTimeoutSeconds, - dnsRefreshSeconds, - dnsFailureRefreshSecondsBase, - dnsFailureRefreshSecondsMax, - dnsQueryTimeoutSeconds, - dnsMinRefreshSeconds, - dnsPreresolveHostnames, - enableDNSCache, - dnsCacheSaveIntervalSeconds, - enableDrainPostDnsRefresh, - enableHttp3, - http3ConnectionOptions, - http3ClientConnectionOptions, - quicHints, - enableGzipDecompression, - enableBrotliDecompression, - enableSocketTagging, - enableInterfaceBinding, - h2ConnectionKeepaliveIdleIntervalMilliseconds, - h2ConnectionKeepaliveTimeoutSeconds, - maxConnectionsPerHost, - statsFlushSeconds, - streamIdleTimeoutSeconds, - perTryIdleTimeoutSeconds, - appVersion, - appId, - trustChainVerification, - nativeFilterChain, - platformFilterChain, - stringAccessors, - keyValueStores, - statsSinks, - runtimeGuards, - enablePlatformCertificatesValidation, - xdsBuilder?.rtdsResourceName, - xdsBuilder?.rtdsTimeoutInSeconds ?: 0, - xdsBuilder?.xdsServerAddress, - xdsBuilder?.xdsServerPort ?: 0, - xdsBuilder?.authHeader, - xdsBuilder?.authToken, - xdsBuilder?.jwtToken, - xdsBuilder?.jwtTokenLifetimeInSeconds ?: 0, - xdsBuilder?.sslRootCerts, - xdsBuilder?.sni, - nodeId, - nodeRegion, - nodeZone, - nodeSubZone, - xdsBuilder?.cdsResourcesLocator, - xdsBuilder?.cdsTimeoutInSeconds ?: 0, - xdsBuilder?.enableCds ?: false, - ) - + val engineConfiguration = + EnvoyConfiguration( + connectTimeoutSeconds, + dnsRefreshSeconds, + dnsFailureRefreshSecondsBase, + dnsFailureRefreshSecondsMax, + dnsQueryTimeoutSeconds, + dnsMinRefreshSeconds, + dnsPreresolveHostnames, + enableDNSCache, + dnsCacheSaveIntervalSeconds, + enableDrainPostDnsRefresh, + enableHttp3, + http3ConnectionOptions, + http3ClientConnectionOptions, + quicHints, + quicCanonicalSuffixes, + enableGzipDecompression, + enableBrotliDecompression, + enableSocketTagging, + enableInterfaceBinding, + h2ConnectionKeepaliveIdleIntervalMilliseconds, + h2ConnectionKeepaliveTimeoutSeconds, + maxConnectionsPerHost, + streamIdleTimeoutSeconds, + perTryIdleTimeoutSeconds, + appVersion, + appId, + trustChainVerification, + nativeFilterChain, + platformFilterChain, + stringAccessors, + keyValueStores, + runtimeGuards, + enablePlatformCertificatesValidation, + xdsBuilder?.rtdsResourceName, + xdsBuilder?.rtdsTimeoutInSeconds ?: 0, + xdsBuilder?.xdsServerAddress, + xdsBuilder?.xdsServerPort ?: 0, + xdsBuilder?.grpcInitialMetadata ?: mapOf(), + xdsBuilder?.sslRootCerts, + nodeId, + nodeRegion, + nodeZone, + nodeSubZone, + nodeMetadata, + xdsBuilder?.cdsResourcesLocator, + xdsBuilder?.cdsTimeoutInSeconds ?: 0, + xdsBuilder?.enableCds ?: false, + ) return when (configuration) { is Custom -> { - EngineImpl( - engineType(), - engineConfiguration, - configuration.yaml, - logLevel - ) + EngineImpl(engineType(), engineConfiguration, configuration.yaml, logLevel) } is Standard -> { - EngineImpl( - engineType(), - engineConfiguration, - logLevel - ) + EngineImpl(engineType(), engineConfiguration, logLevel) } } } @@ -820,11 +708,11 @@ open class EngineBuilder( * validation logic. Defaults to false. * * @param enablePlatformCertificatesValidation true if using platform APIs is desired. - * * @return This builder. */ - fun enablePlatformCertificatesValidation(enablePlatformCertificatesValidation: Boolean): - EngineBuilder { + fun enablePlatformCertificatesValidation( + enablePlatformCertificatesValidation: Boolean + ): EngineBuilder { this.enablePlatformCertificatesValidation = enablePlatformCertificatesValidation return this } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderHTTP3Util.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderHTTP3Util.kt deleted file mode 100644 index 564c2f13be81..000000000000 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilderHTTP3Util.kt +++ /dev/null @@ -1,20 +0,0 @@ -package io.envoyproxy.envoymobile - -/** - * Utility to enable HTTP/3. - */ -object EngineBuilderHTTP3Util { - /** - * Specify whether to enable experimental HTTP/3 (QUIC) support. Note the actual protocol will - * be negotiated with the upstream endpoint and so upstream support is still required for HTTP/3 - * to be utilized. - * - * @param enableHttp3 whether to enable HTTP/3. - * - * @return This builder. - */ - fun EngineBuilder.enableHttp3(enableHttp3: Boolean): EngineBuilder { - this.enableHttp3 = enableHttp3 - return this - } -} diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineImpl.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineImpl.kt index 066ac6794a65..9ed7451106c6 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineImpl.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EngineImpl.kt @@ -2,11 +2,11 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.EnvoyConfiguration import io.envoyproxy.envoymobile.engine.EnvoyEngine +import io.envoyproxy.envoymobile.engine.types.EnvoyStatus -/** - * An implementation of {@link Engine}. - */ -class EngineImpl constructor( +/** An implementation of {@link Engine}. */ +class EngineImpl +constructor( internal val envoyEngine: EnvoyEngine, internal val envoyConfiguration: EnvoyConfiguration, internal val configurationYAML: String?, @@ -25,11 +25,15 @@ class EngineImpl constructor( init { streamClient = StreamClientImpl(envoyEngine) pulseClient = PulseClientImpl(envoyEngine) - if (configurationYAML != null) { - envoyEngine.performRegistration(envoyConfiguration) - envoyEngine.runWithYaml(configurationYAML, logLevel.level) - } else { - envoyEngine.runWithConfig(envoyConfiguration, logLevel.level) + val envoyStatus = + if (configurationYAML != null) { + envoyEngine.performRegistration(envoyConfiguration) + envoyEngine.runWithYaml(configurationYAML, logLevel.level) + } else { + envoyEngine.runWithConfig(envoyConfiguration, logLevel.level) + } + if (envoyStatus == EnvoyStatus.ENVOY_FAILURE) { + throw IllegalStateException("Unable to start Envoy.") } } @@ -45,10 +49,6 @@ class EngineImpl constructor( envoyEngine.terminate() } - override fun flushStats() { - envoyEngine.flushStats() - } - override fun dumpStats(): String { return envoyEngine.dumpStats() } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/EnvoyError.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/EnvoyError.kt index 6f2163b610b7..9f2363e7fe84 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/EnvoyError.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/EnvoyError.kt @@ -5,11 +5,12 @@ package io.envoyproxy.envoymobile * * @param errorCode internal error code associated with the exception that occurred. * @param message a description of what exception that occurred. - * @param attemptCount an optional number of times an operation was attempted before firing - * this error. + * @param attemptCount an optional number of times an operation was attempted before firing this + * error. * @param cause an optional cause for the exception. */ -class EnvoyError constructor( +class EnvoyError +constructor( val errorCode: Int, val message: String, val attemptCount: Int? = null, diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/FinalStreamIntel.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/FinalStreamIntel.kt index 5f0a6e9689a9..884bc7476b6d 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/FinalStreamIntel.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/FinalStreamIntel.kt @@ -7,35 +7,37 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyStreamIntel * Exposes one time HTTP stream metrics, context, and other details. * * Note: a timestamp field (ends with "Ms") with a value of -1 indicates that it is absent. + * * @param streamId The stream identifier. * @param connectionId The connection identifier. * @param attemptCount The number of attempts used to perform a given request. * @param streamStartMs The time the stream started (a.k.a. request started), in ms since the epoch. * @param dnsStartMs The time the DNS resolution for this request started, in ms since the epoch. * @param dnsEndMs The time the DNS resolution for this request completed, in ms since the epoch. - * @param connectStartMs The time the upstream connection started, in ms since the epoch. - * This may not be set if socketReused is false. - * @param connectEndMs The time the upstream connection completed, in ms since the epoch. - * This may not be set if socketReused is false. - * @param sslStartMs The time the SSL handshake started, in ms since the epoch. - * This may not be set if socketReused is false. - * @param sslEndMs The time the SSL handshake completed, in ms since the epoch. - * This may not be set if socketReused is false. - * @param sendingStartMs The time the first byte of the request was sent upstream, - * in ms since the epoch. + * @param connectStartMs The time the upstream connection started, in ms since the epoch. This may + * not be set if socketReused is false. + * @param connectEndMs The time the upstream connection completed, in ms since the epoch. This may + * not be set if socketReused is false. + * @param sslStartMs The time the SSL handshake started, in ms since the epoch. This may not be set + * if socketReused is false. + * @param sslEndMs The time the SSL handshake completed, in ms since the epoch. This may not be set + * if socketReused is false. + * @param sendingStartMs The time the first byte of the request was sent upstream, in ms since the + * epoch. * @param sendingEndMs The time the last byte of the request was sent upstream, in ms since the - * epoch. + * epoch. * @param responseStartMs The time the first byte of the response was received, in ms since the - * epoch. - * @param streamEndMs The time when the stream reached a final state (Error, Cancel, Success), - * in ms since the epoch. + * epoch. + * @param streamEndMs The time when the stream reached a final state (Error, Cancel, Success), in ms + * since the epoch. * @param socketReused True if the upstream socket had been used previously. * @param sentByteCount The number of bytes sent upstream. * @param receivedByteCount The number of bytes received from upstream. * @param responseFlags The response flags for the stream. */ @Suppress("LongParameterList") -class FinalStreamIntel constructor( +class FinalStreamIntel +constructor( streamId: Long, connectionId: Long, attemptCount: Long, @@ -55,15 +57,27 @@ class FinalStreamIntel constructor( val receivedByteCount: Long, val responseFlags: Long ) : StreamIntel(streamId, connectionId, attemptCount) { - constructor(superBase: EnvoyStreamIntel, base: EnvoyFinalStreamIntel) : this( - superBase.streamId, superBase.connectionId, superBase.attemptCount, - base.streamStartMs, base.dnsStartMs, - base.dnsEndMs, base.connectStartMs, - base.connectEndMs, base.sslStartMs, - base.sslEndMs, base.sendingStartMs, + constructor( + superBase: EnvoyStreamIntel, + base: EnvoyFinalStreamIntel + ) : this( + superBase.streamId, + superBase.connectionId, + superBase.attemptCount, + base.streamStartMs, + base.dnsStartMs, + base.dnsEndMs, + base.connectStartMs, + base.connectEndMs, + base.sslStartMs, + base.sslEndMs, + base.sendingStartMs, base.sendingEndMs, - base.responseStartMs, base.streamEndMs, - base.socketReused, base.sentByteCount, - base.receivedByteCount, base.responseFlags + base.responseStartMs, + base.streamEndMs, + base.socketReused, + base.sentByteCount, + base.receivedByteCount, + base.responseFlags ) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/Headers.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/Headers.kt index 305ae118142c..e32274bca3a1 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/Headers.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/Headers.kt @@ -1,8 +1,8 @@ package io.envoyproxy.envoymobile /** - * Base class that is used to represent header/trailer data structures. - * To instantiate new instances, see `{Request|Response}HeadersBuilder`. + * Base class that is used to represent header/trailer data structures. To instantiate new + * instances, see `{Request|Response}HeadersBuilder`. */ open class Headers { internal val container: HeadersContainer @@ -17,13 +17,11 @@ open class Headers { } /** - * Get the value for the provided header name. It's discouraged - * to use this dictionary for equality key-based lookups as this - * may lead to issues with headers that do not follow expected + * Get the value for the provided header name. It's discouraged to use this dictionary for + * equality key-based lookups as this may lead to issues with headers that do not follow expected * casing i.e., "Content-Length" instead of "content-length". * * @param name: Header name for which to get the current value. - * * @return The current headers specified for the provided name. */ fun value(name: String): List? { diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersBuilder.kt index 293f4e8d31e5..e77e13ae03c4 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersBuilder.kt @@ -1,8 +1,8 @@ package io.envoyproxy.envoymobile /** - * Base builder class used to construct `Headers` instances. - * See `{Request|Response}HeadersBuilder` for usage. + * Base builder class used to construct `Headers` instances. See `{Request|Response}HeadersBuilder` + * for usage. */ open class HeadersBuilder { protected val container: HeadersContainer @@ -19,9 +19,8 @@ open class HeadersBuilder { /** * Append a value to the header name. * - * @param name: The header name. + * @param name: The header name. * @param value: The value associated to the header name. - * * @return HeadersBuilder, This builder. */ open fun add(name: String, value: String): HeadersBuilder { @@ -37,7 +36,6 @@ open class HeadersBuilder { * * @param name: The header name. * @param value: The value associated to the header name. - * * @return HeadersBuilder, This builder. */ open fun set(name: String, value: MutableList): HeadersBuilder { @@ -52,7 +50,6 @@ open class HeadersBuilder { * Remove all headers with this name. * * @param name: The header name to remove. - * * @return HeadersBuilder, This builder. */ open fun remove(name: String): HeadersBuilder { @@ -68,7 +65,6 @@ open class HeadersBuilder { * * @param name: The header name. * @param value: The value associated to the header name. - * * @return HeadersBuilder, This builder. */ internal open fun internalSet(name: String, value: MutableList): HeadersBuilder { @@ -76,6 +72,8 @@ open class HeadersBuilder { return this } - private fun isRestrictedHeader(name: String) = name.startsWith(":") || - name.startsWith("x-envoy-mobile", ignoreCase = true) || name.equals("host", ignoreCase = true) + private fun isRestrictedHeader(name: String) = + name.startsWith(":") || + name.startsWith("x-envoy-mobile", ignoreCase = true) || + name.equals("host", ignoreCase = true) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersContainer.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersContainer.kt index 90dabac23ce9..dffac831201a 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersContainer.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/HeadersContainer.kt @@ -1,19 +1,18 @@ package io.envoyproxy.envoymobile /** - * The container that manages the underlying headers map. - * It maintains the original casing of passed header names. - * It treats headers names as case-insensitive for the purpose - * of header lookups and header name conflict resolutions. + * The container that manages the underlying headers map. It maintains the original casing of passed + * header names. It treats headers names as case-insensitive for the purpose of header lookups and + * header name conflict resolutions. */ open class HeadersContainer { protected val headers: MutableMap /** - * Represents a header name together with all of its values. - * It preserves the original casing of the header name. + * Represents a header name together with all of its values. It preserves the original casing of + * the header name. * - * @param name The name of the header. Its casing is preserved. + * @param name The name of the header. Its casing is preserved. * @param value The value associated with a given header. */ data class Header(val name: String, var value: MutableList) { @@ -46,15 +45,15 @@ open class HeadersContainer { internal constructor(headers: Map>) { var underlyingHeaders = mutableMapOf() /** - * Dictionaries are unordered collections. Process headers with names - * that are the same when lowercased in an alphabetical order to avoid a situation - * in which the result of the initialization is non-derministic i.e., we want - * mapOf("A" to listOf("1"), "a" to listOf("2")) headers to be always converted to - * mapOf("A" to listOf("1", "2")) and never to mapOf("a" to listOf("2", "1")). + * Dictionaries are unordered collections. Process headers with names that are the same when + * lowercased in an alphabetical order to avoid a situation in which the result of the + * initialization is non-derministic i.e., we want mapOf("A" to listOf("1"), "a" to listOf("2")) + * headers to be always converted to mapOf("A" to listOf("1", "2")) and never to mapOf("a" to + * listOf("2", "1")). * - * If a given header name already exists in the processed headers map, check - * if the currently processed header name is before the existing header name as - * determined by an alphabetical order. + * If a given header name already exists in the processed headers map, check if the currently + * processed header name is before the existing header name as determined by an alphabetical + * order. */ headers.forEach { val lowercased = it.key.lowercase() @@ -74,13 +73,12 @@ open class HeadersContainer { companion object { /** - * Create a new instance of the receiver using a provider headers map. - * Not implemented as a constructor due to conflicting JVM signatures with - * other constructors. + * Create a new instance of the receiver using a provider headers map. Not implemented as a + * constructor due to conflicting JVM signatures with other constructors. * * @param headers The headers to create the container with. */ - fun create(headers: Map>) : HeadersContainer { + fun create(headers: Map>): HeadersContainer { return HeadersContainer(headers.mapValues { it.value.toMutableList() }) } } @@ -88,23 +86,20 @@ open class HeadersContainer { /** * Add a value to a header with a given name. * - * @param name The name of the header. For the purpose of headers lookup - * and header name conflict resolution, the name of the header - * is considered to be case-insensitive. + * @param name The name of the header. For the purpose of headers lookup and header name conflict + * resolution, the name of the header is considered to be case-insensitive. * @param value The value to add. */ fun add(name: String, value: String) { val lowercased = name.lowercase() - headers[lowercased]?.let { it.add(value) } ?: run { - headers.put(lowercased, Header(name, mutableListOf(value))) - } + headers[lowercased]?.let { it.add(value) } + ?: run { headers.put(lowercased, Header(name, mutableListOf(value))) } } - /** * Set the value of a given header. * - * @param name The name of the header. + * @param name The name of the header. * @param value The value to set the header value to. */ fun set(name: String, value: List) { @@ -123,8 +118,7 @@ open class HeadersContainer { /** * Get the value for the provided header name. * - * @param name The case-insensitive header name for which to - * get the current value. + * @param name The case-insensitive header name for which to get the current value. * @return The value associated with a given header. */ fun value(name: String): List? { @@ -132,16 +126,14 @@ open class HeadersContainer { } /** - * Accessor for all underlying case-sensitive headers. When possible, - * use case-insensitive accessors instead. + * Accessor for all underlying case-sensitive headers. When possible, use case-insensitive + * accessors instead. * * @return The underlying headers. */ fun caseSensitiveHeaders(): Map> { var caseSensitiveHeaders = mutableMapOf>() - headers.forEach { - caseSensitiveHeaders.put(it.value.name, it.value.value) - } + headers.forEach { caseSensitiveHeaders.put(it.value.name, it.value.value) } return caseSensitiveHeaders } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/LogLevel.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/LogLevel.kt index 763ffc77277a..63887409a580 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/LogLevel.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/LogLevel.kt @@ -13,5 +13,5 @@ enum class LogLevel(internal val level: String, val levelInt: Int) { WARN("warn", 3), ERROR("error", 4), CRITICAL("critical", 5), - OFF("off", -1); + OFF("off", -1) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClient.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClient.kt index 463b5275fdab..20957778e318 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClient.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClient.kt @@ -8,13 +8,9 @@ package io.envoyproxy.envoymobile */ interface PulseClient { - /** - * @return A counter based on the joined elements. - */ + /** @return A counter based on the joined elements. */ fun counter(vararg elements: Element): Counter - /** - * @return A counter based on the joined elements with tags. - */ + /** @return A counter based on the joined elements with tags. */ fun counter(vararg elements: Element, tags: Tags): Counter } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClientImpl.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClientImpl.kt index 43a739ffe8c9..66c7bd7909c6 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClientImpl.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/PulseClientImpl.kt @@ -2,12 +2,8 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.EnvoyEngine -/** - * Envoy implementation of `PulseClient`. - */ -internal class PulseClientImpl constructor( - internal val engine: EnvoyEngine -) : PulseClient { +/** Envoy implementation of `PulseClient`. */ +internal class PulseClientImpl constructor(internal val engine: EnvoyEngine) : PulseClient { override fun counter(vararg elements: Element): Counter { return CounterImpl(engine, elements.asList()) diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeaders.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeaders.kt index 4feeac437aac..185041c02994 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeaders.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeaders.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Headers representing an outbound request. - */ +/** Headers representing an outbound request. */ open class RequestHeaders : Headers { /** * Internal constructor used by builders. @@ -11,31 +9,21 @@ open class RequestHeaders : Headers { */ internal constructor(headers: Map>) : super(HeadersContainer.create(headers)) - internal constructor(container: HeadersContainer): super(container) + internal constructor(container: HeadersContainer) : super(container) - /** - * Method for the request. - */ + /** Method for the request. */ val method: RequestMethod by lazy { RequestMethod.enumValue(value(":method")?.first()!!) } - /** - * The URL scheme for the request (i.e., "https"). - */ + /** The URL scheme for the request (i.e., "https"). */ val scheme: String by lazy { value(":scheme")?.first()!! } - /** - * The URL authority for the request (i.e., "api.foo.com"). - */ + /** The URL authority for the request (i.e., "api.foo.com"). */ val authority: String by lazy { value(":authority")?.first()!! } - /** - * The URL path for the request (i.e., "/foo"). - */ + /** The URL path for the request (i.e., "/foo"). */ val path: String by lazy { value(":path")?.first()!! } - /** - * Retry policy to use for this request. - */ + /** Retry policy to use for this request. */ val retryPolicy: RetryPolicy? by lazy { RetryPolicy.from(this) } /** diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilder.kt index be9dd7cc3b15..89bf4ed4b91a 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilder.kt @@ -1,24 +1,22 @@ package io.envoyproxy.envoymobile -/** - * Builder used for constructing instances of RequestHeaders`. - */ +/** Builder used for constructing instances of RequestHeaders`. */ class RequestHeadersBuilder : HeadersBuilder { /** * Initialize a new instance of the builder. * - * @param method: Method for the request. - * @param scheme: The URL scheme for the request (i.e., "https"). + * @param method: Method for the request. + * @param scheme: The URL scheme for the request (i.e., "https"). * @param authority: The URL authority for the request (i.e., "api.foo.com"). - * @param path: The URL path for the request (i.e., "/foo"). + * @param path: The URL path for the request (i.e., "/foo"). */ constructor( method: RequestMethod, scheme: String = "https", authority: String, path: String - ) : - super(HeadersContainer( + ) : super( + HeadersContainer( mapOf( ":authority" to mutableListOf(authority), ":method" to mutableListOf(method.stringValue), @@ -26,7 +24,7 @@ class RequestHeadersBuilder : HeadersBuilder { ":scheme" to mutableListOf(scheme) ) ) - ) + ) /** * Instantiate a new builder. Used only by RequestHeaders to convert back to @@ -67,7 +65,6 @@ class RequestHeadersBuilder : HeadersBuilder { * Add a retry policy to be used with this request. * * @param retryPolicy: The retry policy to use. - * * @return RequestHeadersBuilder, This builder. */ fun addRetryPolicy(retryPolicy: RetryPolicy): RequestHeadersBuilder { @@ -84,21 +81,17 @@ class RequestHeadersBuilder : HeadersBuilder { * @param uid: Traffic stats UID to be applied. * @param tag: Traffic stats tag to be applied. * - * See: https://source.android.com/devices/tech/datausage/tags-explained - * See: https://developer.android.com/reference/android/net/TrafficStats#setThreadStatsTag(int) - * See: https://developer.android.com/reference/android/net/TrafficStats#setThreadStatsUid(int) - * See: https://developer.android.com/reference/android/net/TrafficStats#tagSocket(java.net.Socket) + * See: https://source.android.com/devices/tech/datausage/tags-explained See: + * https://developer.android.com/reference/android/net/TrafficStats#setThreadStatsTag(int) See: + * https://developer.android.com/reference/android/net/TrafficStats#setThreadStatsUid(int) See: + * https://developer.android.com/reference/android/net/TrafficStats#tagSocket(java.net.Socket) * * @return RequestHeadersBuilder, This builder. */ - fun addSocketTag(uid: Int, tag: Int): - RequestHeadersBuilder { - internalSet( - "x-envoy-mobile-socket-tag", - mutableListOf(uid.toString() + "," + tag.toString()) - ) - return this - } + fun addSocketTag(uid: Int, tag: Int): RequestHeadersBuilder { + internalSet("x-envoy-mobile-socket-tag", mutableListOf(uid.toString() + "," + tag.toString())) + return this + } /** * Build the request headers using the current builder. diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderCompressionUtil.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderCompressionUtil.kt deleted file mode 100644 index db7d14dbdc74..000000000000 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderCompressionUtil.kt +++ /dev/null @@ -1,23 +0,0 @@ -package io.envoyproxy.envoymobile - -/** - * Utility to enable request compression. - */ -object RequestHeadersBuilderCompressionUtil { - /** - * Compress this request's body using the specified algorithm. - * Will only apply if the content length exceeds 30 bytes. - * - * @param algorithm: The compression algorithm to use to compress this request. - * - * @return RequestHeadersBuilder, This builder. - */ - fun RequestHeadersBuilder.enableRequestCompression(algorithm: CompressionAlgorithm): - RequestHeadersBuilder { - internalSet( - "x-envoy-mobile-compression", - mutableListOf(algorithm.stringValue) - ) - return this - } -} diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestMethod.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestMethod.kt index 200931048b87..6ce182849778 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestMethod.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestMethod.kt @@ -2,9 +2,7 @@ package io.envoyproxy.envoymobile import java.lang.IllegalArgumentException -/** - * Represents an HTTP request method. - */ +/** Represents an HTTP request method. */ enum class RequestMethod(internal val stringValue: String) { DELETE("DELETE"), GET("GET"), diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailers.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailers.kt index 6c2157622bbd..bb7a0efe233b 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailers.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailers.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Trailers representing an outbound request. - */ +/** Trailers representing an outbound request. */ @Suppress("EmptyClassBlock") class RequestTrailers : Trailers { /** diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailersBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailersBuilder.kt index 085a1bc77578..665aa07ee0c4 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailersBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RequestTrailersBuilder.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Builder used for constructing instances of `RequestTrailers`. - */ +/** Builder used for constructing instances of `RequestTrailers`. */ class RequestTrailersBuilder : HeadersBuilder { /* * Instantiate a new builder. @@ -22,8 +20,9 @@ class RequestTrailersBuilder : HeadersBuilder { * * @param trailers: The trailers to start with. */ - internal constructor(trailers: MutableMap>) - : super(HeadersContainer(trailers)) + internal constructor( + trailers: MutableMap> + ) : super(HeadersContainer(trailers)) override fun add(name: String, value: String): RequestTrailersBuilder { super.add(name, value) diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeaders.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeaders.kt index 9710b08a4426..3ff33add6646 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeaders.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeaders.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Headers representing an inbound response. - */ +/** Headers representing an inbound response. */ class ResponseHeaders : Headers { /** * Internal constructor used by builders. @@ -13,12 +11,8 @@ class ResponseHeaders : Headers { internal constructor(container: HeadersContainer) : super(container) - /** - * HTTP status code received with the response. - */ - val httpStatus: Int? by lazy { - value(":status")?.first()?.toIntOrNull()?.takeIf { it >= 0 } - } + /** HTTP status code received with the response. */ + val httpStatus: Int? by lazy { value(":status")?.first()?.toIntOrNull()?.takeIf { it >= 0 } } /** * Convert the headers back to a builder for mutation. diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeadersBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeadersBuilder.kt index 93254aaac8f7..50cc7b860d3b 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeadersBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseHeadersBuilder.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Builder used for constructing instances of `ResponseHeaders`. - */ +/** Builder used for constructing instances of `ResponseHeaders`. */ class ResponseHeadersBuilder : HeadersBuilder { /* @@ -16,8 +14,9 @@ class ResponseHeadersBuilder : HeadersBuilder { * * @param headers: The headers to start with. */ - internal constructor(headers: MutableMap>) - : super(HeadersContainer(headers)) + internal constructor( + headers: MutableMap> + ) : super(HeadersContainer(headers)) /* * Instantiate a new builder. @@ -50,7 +49,6 @@ class ResponseHeadersBuilder : HeadersBuilder { * Add an HTTP status to the response headers. Must be a positive integer. * * @param status: The HTTP status to add. - * * @return ResponseHeadersBuilder, This builder. */ fun addHttpStatus(status: Int): ResponseHeadersBuilder { diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailers.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailers.kt index 579059535b93..0b95723e4955 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailers.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailers.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Trailers representing an inbound response. - */ +/** Trailers representing an inbound response. */ @Suppress("EmptyClassBlock") class ResponseTrailers : Trailers { /** diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailersBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailersBuilder.kt index f67635a28a16..ef05f9f12763 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailersBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/ResponseTrailersBuilder.kt @@ -1,12 +1,8 @@ package io.envoyproxy.envoymobile -/** - * Builder used for constructing instances of `ResponseTrailers`. - */ +/** Builder used for constructing instances of `ResponseTrailers`. */ class ResponseTrailersBuilder : HeadersBuilder { - /** - * Initialize a new instance of the builder. - */ + /** Initialize a new instance of the builder. */ constructor() : super(HeadersContainer(mapOf())) /** @@ -15,8 +11,9 @@ class ResponseTrailersBuilder : HeadersBuilder { * * @param trailers: The trailers to start with. */ - internal constructor(trailers: MutableMap>) - : super(HeadersContainer(trailers)) + internal constructor( + trailers: MutableMap> + ) : super(HeadersContainer(trailers)) internal constructor(container: HeadersContainer) : super(container) diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicy.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicy.kt index ef3dfacc4a61..1610022dfa49 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicy.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicy.kt @@ -8,11 +8,11 @@ import java.lang.IllegalArgumentException * @param maxRetryCount Maximum number of retries that a request may be performed. * @param retryOn Rules checked for retrying. * @param retryStatusCodes Additional list of status codes that should be retried. - * @param perRetryTimeoutMS Timeout (in milliseconds) to apply to each retry. - * Must be <= `totalUpstreamTimeoutMS` if it's a positive number. - * @param totalUpstreamTimeoutMS Total timeout (in milliseconds) that includes all retries. - * Spans the point at which the entire downstream request has been processed and when the - * upstream response has been completely processed. Null or 0 may be specified to disable it. + * @param perRetryTimeoutMS Timeout (in milliseconds) to apply to each retry. Must be <= + * `totalUpstreamTimeoutMS` if it's a positive number. + * @param totalUpstreamTimeoutMS Total timeout (in milliseconds) that includes all retries. Spans + * the point at which the entire downstream request has been processed and when the upstream + * response has been completely processed. Null or 0 may be specified to disable it. */ data class RetryPolicy( val maxRetryCount: Int, @@ -22,8 +22,11 @@ data class RetryPolicy( val totalUpstreamTimeoutMS: Long? = 15000 ) { init { - if (perRetryTimeoutMS != null && totalUpstreamTimeoutMS != null && - perRetryTimeoutMS > totalUpstreamTimeoutMS && totalUpstreamTimeoutMS != 0L + if ( + perRetryTimeoutMS != null && + totalUpstreamTimeoutMS != null && + perRetryTimeoutMS > totalUpstreamTimeoutMS && + totalUpstreamTimeoutMS != 0L ) { throw IllegalArgumentException("Per-retry timeout cannot be less than total timeout") } @@ -43,11 +46,15 @@ data class RetryPolicy( // Envoy internally coalesces multiple x-envoy header values into one comma-delimited value. // These flatMap transformations split those values up to correctly map back to // Kotlin enums. - headers.value("x-envoy-retry-on") - ?.flatMap { it.split(",") }?.map { retryOn -> RetryRule.enumValue(retryOn) } + headers + .value("x-envoy-retry-on") + ?.flatMap { it.split(",") } + ?.map { retryOn -> RetryRule.enumValue(retryOn) } ?.filterNotNull() ?: emptyList(), - headers.value("x-envoy-retriable-status-codes") - ?.flatMap { it.split(",") }?.map { statusCode -> statusCode.toIntOrNull() } + headers + .value("x-envoy-retriable-status-codes") + ?.flatMap { it.split(",") } + ?.map { statusCode -> statusCode.toIntOrNull() } ?.filterNotNull() ?: emptyList(), headers.value("x-envoy-upstream-rq-per-try-timeout-ms")?.firstOrNull()?.toLongOrNull(), headers.value("x-envoy-upstream-rq-timeout-ms")?.firstOrNull()?.toLongOrNull() @@ -57,8 +64,8 @@ data class RetryPolicy( } /** - * Rules that may be used with `RetryPolicy`. - * See the `x-envoy-retry-on` Envoy header for documentation. + * Rules that may be used with `RetryPolicy`. See the `x-envoy-retry-on` Envoy header for + * documentation. */ enum class RetryRule(internal val stringValue: String) { STATUS_5XX("5xx"), diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapper.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapper.kt index ba07bb904c65..d72e85009526 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapper.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapper.kt @@ -7,10 +7,11 @@ package io.envoyproxy.envoymobile */ internal fun RetryPolicy.outboundHeaders(): Map> { val upstreamTimeoutMS = totalUpstreamTimeoutMS ?: 0L - val headers = mutableMapOf( - "x-envoy-max-retries" to listOf("$maxRetryCount"), - "x-envoy-upstream-rq-timeout-ms" to listOf("$upstreamTimeoutMS") - ) + val headers = + mutableMapOf( + "x-envoy-max-retries" to listOf("$maxRetryCount"), + "x-envoy-upstream-rq-timeout-ms" to listOf("$upstreamTimeoutMS") + ) if (perRetryTimeoutMS != null) { headers["x-envoy-upstream-rq-per-try-timeout-ms"] = listOf("$perRetryTimeoutMS") diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/Stream.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/Stream.kt index 5c86dc26580b..056df5a52d9a 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/Stream.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/Stream.kt @@ -37,11 +37,11 @@ open class Stream( /** * For sending data to an associated stream. By default, the length sent is the - * **[ByteBuffer.capacity]**. However, the length will rather be **[ByteBuffer.position]** - * if the Stream was configured to do so - see **[StreamPrototype.useByteBufferPosition]**. + * **[ByteBuffer.capacity]**. However, the length will rather be **[ByteBuffer.position]** if the + * Stream was configured to do so - see **[StreamPrototype.useByteBufferPosition]**. * - * Note: the provided ByteBuffer won't be mutated in any case. On the other hand, until the - * stream is closed, any further mutations may lead to an unpredictable outcome. + * Note: the provided ByteBuffer won't be mutated in any case. On the other hand, until the stream + * is closed, any further mutations may lead to an unpredictable outcome. * * @param data Data to send over the stream. * @return This stream, for chaining syntax. @@ -63,11 +63,11 @@ open class Stream( /** * Close the stream with a data frame. By default, the length sent is the - * **[ByteBuffer.capacity]**. However, the length will rather be **[ByteBuffer.position]** - * if the Stream was configured to do so - see **[StreamPrototype.useByteBufferPosition]**. + * **[ByteBuffer.capacity]**. However, the length will rather be **[ByteBuffer.position]** if the + * Stream was configured to do so - see **[StreamPrototype.useByteBufferPosition]**. * - * Note: the provided ByteBuffer won't be mutated in any case. On the other hand, until the - * stream is closed, any further mutations may lead to an unpredictable outcome. + * Note: the provided ByteBuffer won't be mutated in any case. On the other hand, until the stream + * is closed, any further mutations may lead to an unpredictable outcome. * * @param data Data with which to close the stream. */ @@ -76,9 +76,7 @@ open class Stream( underlyingStream.sendData(data, length, true) } - /** - * Cancel the stream. - */ + /** Cancel the stream. */ open fun cancel() { underlyingStream.cancel() } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamCallbacks.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamCallbacks.kt index 9bd34c3579e3..e5668eb28b49 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamCallbacks.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamCallbacks.kt @@ -7,28 +7,26 @@ import java.nio.ByteBuffer import java.util.concurrent.Executor /** - * A collection of platform-level callbacks that are specified by consumers - * who wish to interact with streams. + * A collection of platform-level callbacks that are specified by consumers who wish to interact + * with streams. * * `StreamCallbacks` are bridged through to `EnvoyHTTPCallbacks` to communicate with the engine. */ internal class StreamCallbacks { - var onHeaders: ( - (headers: ResponseHeaders, endStream: Boolean, streamIntel: StreamIntel) -> Unit - )? = null + var onHeaders: + ((headers: ResponseHeaders, endStream: Boolean, streamIntel: StreamIntel) -> Unit)? = + null var onData: ((data: ByteBuffer, endStream: Boolean, streamIntel: StreamIntel) -> Unit)? = null var onTrailers: ((trailers: ResponseTrailers, streamIntel: StreamIntel) -> Unit)? = null var onCancel: ((finalStreamIntel: FinalStreamIntel) -> Unit)? = null - var onError: ( - (error: EnvoyError, finalStreamIntel: FinalStreamIntel) -> Unit - )? = null + var onError: ((error: EnvoyError, finalStreamIntel: FinalStreamIntel) -> Unit)? = null var onSendWindowAvailable: ((streamIntel: StreamIntel) -> Unit)? = null var onComplete: ((finalStreamIntel: FinalStreamIntel) -> Unit)? = null } /** - * Class responsible for bridging between the platform-level `StreamCallbacks` and the - * engine's `EnvoyHTTPCallbacks`. + * Class responsible for bridging between the platform-level `StreamCallbacks` and the engine's + * `EnvoyHTTPCallbacks`. */ internal class EnvoyHTTPCallbacksAdapter( private val executor: Executor, diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClient.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClient.kt index 497c4a8f91d8..bf4026d74901 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClient.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClient.kt @@ -1,8 +1,6 @@ package io.envoyproxy.envoymobile -/** - * Client used to create HTTP streams. - */ +/** Client used to create HTTP streams. */ interface StreamClient { /** * Create a new stream prototype which can be used to start streams. diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClientImpl.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClientImpl.kt index ebc582b47c79..ad83cfad32ca 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClientImpl.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamClientImpl.kt @@ -2,12 +2,8 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.EnvoyEngine -/** - * Envoy implementation of `StreamClient`. - */ -internal class StreamClientImpl constructor( - internal val engine: EnvoyEngine -) : StreamClient { +/** Envoy implementation of `StreamClient`. */ +internal class StreamClientImpl constructor(internal val engine: EnvoyEngine) : StreamClient { override fun newStreamPrototype() = StreamPrototype(engine) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamIntel.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamIntel.kt index 29238d361bc0..eb8266ce3e51 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamIntel.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamIntel.kt @@ -4,15 +4,13 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyStreamIntel /** * Exposes internal HTTP stream metrics, context, and other details. + * * @param streamId An internal identifier for the stream. -1 if not set. * @param connectionId An internal identifier for the connection carrying the stream. -1 if not set. - * @param attemptCount The number of internal attempts to carry out a request/operation. 0 if - * not set. + * @param attemptCount The number of internal attempts to carry out a request/operation. 0 if not + * set. */ -open class StreamIntel constructor( - val streamId: Long, - val connectionId: Long, - val attemptCount: Long -) { +open class StreamIntel +constructor(val streamId: Long, val connectionId: Long, val attemptCount: Long) { constructor(base: EnvoyStreamIntel) : this(base.streamId, base.connectionId, base.attemptCount) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt index 15f4f68b225f..887c21cede41 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StreamPrototype.kt @@ -8,15 +8,14 @@ import java.util.concurrent.Executors /** * A type representing a stream that has not yet been started. * - * Constructed via `StreamClient`, and used to assign response callbacks - * prior to starting an `Stream` by calling `start()`. + * Constructed via `StreamClient`, and used to assign response callbacks prior to starting an + * `Stream` by calling `start()`. * * @param engine Engine to use for starting streams. */ open class StreamPrototype(private val engine: EnvoyEngine) { private val callbacks = StreamCallbacks() private var explicitFlowControl = false - private var minDeliverySize: Long = 0 private var useByteBufferPosition = false /** @@ -26,27 +25,10 @@ open class StreamPrototype(private val engine: EnvoyEngine) { * @return The new stream. */ open fun start(executor: Executor = Executors.newSingleThreadExecutor()): Stream { - val engineStream = engine.startStream( - createCallbacks(executor), - explicitFlowControl, - minDeliverySize - ) + val engineStream = engine.startStream(createCallbacks(executor), explicitFlowControl) return Stream(engineStream, useByteBufferPosition) } - /** - * Sets min delivery: data will be buffered in the C++ layer until the min - * delivery length or end stream is read. - * - * @param value set the minimum delivery size fo for this stream - * @return This stream, for chaining syntax. - */ - fun setMinDeliverySize(value: Long): StreamPrototype { - this.minDeliverySize = value - return this - } - - /** * Allows explicit flow control to be enabled. When flow control is enabled, the owner of a stream * is responsible for providing a buffer to receive response body data. If the buffer is smaller @@ -75,11 +57,11 @@ open class StreamPrototype(private val engine: EnvoyEngine) { } /** - * Specify a callback for when response headers are received by the stream. - * If `endStream` is `true`, the stream is complete, pending an onComplete callback. + * Specify a callback for when response headers are received by the stream. If `endStream` is + * `true`, the stream is complete, pending an onComplete callback. * - * @param closure Closure which will receive the headers and flag indicating if the stream - * is headers-only. + * @param closure Closure which will receive the headers and flag indicating if the stream is + * headers-only. * @return This stream, for chaining syntax. */ fun setOnResponseHeaders( @@ -90,11 +72,11 @@ open class StreamPrototype(private val engine: EnvoyEngine) { } /** - * Specify a callback for when a data frame is received by the stream. - * If `endStream` is `true`, the stream is complete, pending an onComplete callback. + * Specify a callback for when a data frame is received by the stream. If `endStream` is `true`, + * the stream is complete, pending an onComplete callback. * - * @param closure Closure which will receive the data and flag indicating whether this - * is the last data frame. + * @param closure Closure which will receive the data and flag indicating whether this is the last + * data frame. * @return This stream, for chaining syntax. */ fun setOnResponseData( @@ -105,8 +87,8 @@ open class StreamPrototype(private val engine: EnvoyEngine) { } /** - * Specify a callback for when trailers are received by the stream. - * If the closure is called, the stream is complete, pending an onComplete callback. + * Specify a callback for when trailers are received by the stream. If the closure is called, the + * stream is complete, pending an onComplete callback. * * @param closure Closure which will receive the trailers. * @return This stream, for chaining syntax. @@ -119,62 +101,52 @@ open class StreamPrototype(private val engine: EnvoyEngine) { } /** - * Specify a callback for when an internal Envoy exception occurs with the stream. - * If the closure is called, the stream is complete. + * Specify a callback for when an internal Envoy exception occurs with the stream. If the closure + * is called, the stream is complete. * * @param closure Closure which will be called when an error occurs. * @return This stream, for chaining syntax. */ fun setOnError( - closure: ( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) -> Unit + closure: (error: EnvoyError, finalStreamIntel: FinalStreamIntel) -> Unit ): StreamPrototype { callbacks.onError = closure return this } -/** - * Specify a callback for when a stream is complete. - * If the closure is called, the stream is complete. + /** + * Specify a callback for when a stream is complete. If the closure is called, the stream is + * complete. * * @param closure Closure which will be called when an error occurs. * @return This stream, for chaining syntax. */ - fun setOnComplete( - closure: (finalStreamIntel: FinalStreamIntel) -> Unit - ): StreamPrototype { + fun setOnComplete(closure: (finalStreamIntel: FinalStreamIntel) -> Unit): StreamPrototype { callbacks.onComplete = closure return this } /** - * Specify a callback for when the stream is canceled. - * If the closure is called, the stream is complete. + * Specify a callback for when the stream is canceled. If the closure is called, the stream is + * complete. * * @param closure Closure which will be called when the stream is canceled. * @return This stream, for chaining syntax. */ - fun setOnCancel( - closure: (finalStreamIntel: FinalStreamIntel) -> Unit - ): StreamPrototype { + fun setOnCancel(closure: (finalStreamIntel: FinalStreamIntel) -> Unit): StreamPrototype { callbacks.onCancel = closure return this } /** - * Specify a callback for when additional send window becomes available. - * This is only ever called when the library is in explicit flow control mode. When enabled, - * the issuer should wait for this callback after calling sendData, before making another call - * to sendData. + * Specify a callback for when additional send window becomes available. This is only ever called + * when the library is in explicit flow control mode. When enabled, the issuer should wait for + * this callback after calling sendData, before making another call to sendData. * * @param closure Closure which will be called when additional send window becomes available. * @return This stream, for chaining syntax. */ - fun setOnSendWindowAvailable( - closure: (streamIntel: StreamIntel) -> Unit - ): StreamPrototype { + fun setOnSendWindowAvailable(closure: (streamIntel: StreamIntel) -> Unit): StreamPrototype { callbacks.onSendWindowAvailable = closure return this } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/StringAccessor.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/StringAccessor.kt index 0336681bd38a..8c24e50a8776 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/StringAccessor.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/StringAccessor.kt @@ -2,23 +2,19 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor -/** - * `StringAccessor` is bridged through to `EnvoyStringAccessor` to communicate with the engine. - */ -class StringAccessor constructor ( - /** - * Accessor for a string exposed by a platform. - */ +/** `StringAccessor` is bridged through to `EnvoyStringAccessor` to communicate with the engine. */ +class StringAccessor +constructor( + /** Accessor for a string exposed by a platform. */ val getEnvoyString: (() -> String) ) /** - * Class responsible for bridging between the platform-level `StringAccessor` and the - * engine's `EnvoyStringAccessor`. + * Class responsible for bridging between the platform-level `StringAccessor` and the engine's + * `EnvoyStringAccessor`. */ -internal class EnvoyStringAccessorAdapter( - private val callbacks: StringAccessor -) : EnvoyStringAccessor { +internal class EnvoyStringAccessorAdapter(private val callbacks: StringAccessor) : + EnvoyStringAccessor { override fun getEnvoyString(): String { return callbacks.getEnvoyString() } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/Trailers.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/Trailers.kt index 6742fdaebb06..b056c8aadbef 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/Trailers.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/Trailers.kt @@ -1,8 +1,8 @@ package io.envoyproxy.envoymobile /** - * Base class representing trailers data structures. - * To instantiate new instances see `{Request|Response}TrailersBuilder`. + * Base class representing trailers data structures. To instantiate new instances see + * `{Request|Response}TrailersBuilder`. */ open class Trailers : Headers { /** @@ -10,8 +10,9 @@ open class Trailers : Headers { * * @param trailers: Trailers to set. */ - protected constructor(trailers: Map>) - : super(HeadersContainer.create(trailers)) + protected constructor( + trailers: Map> + ) : super(HeadersContainer.create(trailers)) protected constructor(container: HeadersContainer) : super(container) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/android/SharedPreferencesStore.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/android/SharedPreferencesStore.kt index 783f12f1de0a..7873a204e387 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/android/SharedPreferencesStore.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/android/SharedPreferencesStore.kt @@ -1,12 +1,9 @@ package io.envoyproxy.envoymobile.android import android.content.SharedPreferences - import io.envoyproxy.envoymobile.KeyValueStore -/** - * Simple implementation of a `KeyValueStore` leveraging `SharedPreferences` for persistence. - */ +/** Simple implementation of a `KeyValueStore` leveraging `SharedPreferences` for persistence. */ class SharedPreferencesStore(sharedPreferences: SharedPreferences) : KeyValueStore { private val preferences = sharedPreferences private val editor = sharedPreferences.edit() diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncRequestFilter.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncRequestFilter.kt index 70f52b029602..abfc8d237a71 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncRequestFilter.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncRequestFilter.kt @@ -16,11 +16,11 @@ interface AsyncRequestFilter : RequestFilter { /** * Invoked explicitly in response to an asynchronous `resumeRequest()` callback when filter - * iteration has been stopped. The parameters passed to this invocation will be a snapshot - * of any stream state that has not yet been forwarded along the filter chain. + * iteration has been stopped. The parameters passed to this invocation will be a snapshot of any + * stream state that has not yet been forwarded along the filter chain. * - * As with other filter invocations, this will be called on Envoy's main thread, and thus - * no additional synchronization is required between this and other invocations. + * As with other filter invocations, this will be called on Envoy's main thread, and thus no + * additional synchronization is required between this and other invocations. * * @param headers: Headers, if `StopIteration` was returned from `onRequestHeaders`. * @param data: Any data that has been buffered where `StopIterationAndBuffer` was returned. diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncResponseFilter.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncResponseFilter.kt index b135ce7739a3..2765511449bb 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncResponseFilter.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/AsyncResponseFilter.kt @@ -16,11 +16,11 @@ interface AsyncResponseFilter : ResponseFilter { /** * Invoked explicitly in response to an asynchronous `resumeResponse()` callback when filter - * iteration has been stopped. The parameters passed to this invocation will be a snapshot - * of any stream state that has not yet been forwarded along the filter chain. + * iteration has been stopped. The parameters passed to this invocation will be a snapshot of any + * stream state that has not yet been forwarded along the filter chain. * - * As with other filter invocations, this will be called on Envoy's main thread, and thus - * no additional synchronization is required between this and other invocations. + * As with other filter invocations, this will be called on Envoy's main thread, and thus no + * additional synchronization is required between this and other invocations. * * @param headers: Headers, if `StopIteration` was returned from `onResponseHeaders`. * @param data: Any data that has been buffered where `StopIterationAndBuffer` was returned. diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/Filter.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/Filter.kt index ba9d419e2928..fe76f098a41b 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/Filter.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/Filter.kt @@ -10,99 +10,161 @@ import java.nio.ByteBuffer /* * Interface representing a filter. See `RequestFilter` and `ResponseFilter` for more details. */ -@Suppress("EmptyClassBlock") -interface Filter +@Suppress("EmptyClassBlock") interface Filter -internal class FilterFactory( - private val filterName: String, - private val factory: () -> Filter -) : EnvoyHTTPFilterFactory { +internal class FilterFactory(private val filterName: String, private val factory: () -> Filter) : + EnvoyHTTPFilterFactory { override fun getFilterName(): String { return filterName } - override fun create(): EnvoyHTTPFilter { return EnvoyHTTPFilterAdapter(factory()) } + override fun create(): EnvoyHTTPFilter { + return EnvoyHTTPFilterAdapter(factory()) + } } -internal class EnvoyHTTPFilterAdapter( - private val filter: Filter -) : EnvoyHTTPFilter { +internal class EnvoyHTTPFilterAdapter(private val filter: Filter) : EnvoyHTTPFilter { - override fun onRequestHeaders(headers: Map>, endStream: Boolean, streamIntel: EnvoyStreamIntel): Array { + override fun onRequestHeaders( + headers: Map>, + endStream: Boolean, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? RequestFilter)?.let { requestFilter -> - val result = requestFilter.onRequestHeaders(RequestHeaders(headers), endStream, StreamIntel(streamIntel)) + val result = + requestFilter.onRequestHeaders(RequestHeaders(headers), endStream, StreamIntel(streamIntel)) return when (result) { - is FilterHeadersStatus.Continue -> arrayOf(result.status, result.headers.caseSensitiveHeaders()) - is FilterHeadersStatus.StopIteration -> arrayOf(result.status, emptyMap>()) + is FilterHeadersStatus.Continue -> + arrayOf(result.status, result.headers.caseSensitiveHeaders()) + is FilterHeadersStatus.StopIteration -> + arrayOf(result.status, emptyMap>()) } } return arrayOf(0, headers) } - override fun onResponseHeaders(headers: Map>, endStream: Boolean, streamIntel: EnvoyStreamIntel): Array { + override fun onResponseHeaders( + headers: Map>, + endStream: Boolean, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? ResponseFilter)?.let { responseFilter -> - val result = responseFilter.onResponseHeaders(ResponseHeaders(headers), endStream, StreamIntel(streamIntel)) + val result = + responseFilter.onResponseHeaders( + ResponseHeaders(headers), + endStream, + StreamIntel(streamIntel) + ) return when (result) { - is FilterHeadersStatus.Continue -> arrayOf(result.status, result.headers.caseSensitiveHeaders()) - is FilterHeadersStatus.StopIteration -> arrayOf(result.status, emptyMap>()) + is FilterHeadersStatus.Continue -> + arrayOf(result.status, result.headers.caseSensitiveHeaders()) + is FilterHeadersStatus.StopIteration -> + arrayOf(result.status, emptyMap>()) } } return arrayOf(0, headers) } - override fun onRequestData(data: ByteBuffer, endStream: Boolean, streamIntel: EnvoyStreamIntel): Array { + override fun onRequestData( + data: ByteBuffer, + endStream: Boolean, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? RequestFilter)?.let { requestFilter -> val result = requestFilter.onRequestData(data, endStream, StreamIntel(streamIntel)) return when (result) { is FilterDataStatus.Continue<*> -> arrayOf(result.status, result.data) - is FilterDataStatus.StopIterationAndBuffer<*> -> arrayOf(result.status, ByteBuffer.allocate(0)) - is FilterDataStatus.StopIterationNoBuffer<*> -> arrayOf(result.status, ByteBuffer.allocate(0)) - is FilterDataStatus.ResumeIteration<*> -> arrayOf(result.status, result.data, result.headers?.caseSensitiveHeaders()) + is FilterDataStatus.StopIterationAndBuffer<*> -> + arrayOf(result.status, ByteBuffer.allocate(0)) + is FilterDataStatus.StopIterationNoBuffer<*> -> + arrayOf(result.status, ByteBuffer.allocate(0)) + is FilterDataStatus.ResumeIteration<*> -> + arrayOf(result.status, result.data, result.headers?.caseSensitiveHeaders()) } } return arrayOf(0, data) } - override fun onResponseData(data: ByteBuffer, endStream: Boolean, streamIntel: EnvoyStreamIntel): Array { + override fun onResponseData( + data: ByteBuffer, + endStream: Boolean, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? ResponseFilter)?.let { responseFilter -> val result = responseFilter.onResponseData(data, endStream, StreamIntel(streamIntel)) return when (result) { is FilterDataStatus.Continue<*> -> arrayOf(result.status, result.data) - is FilterDataStatus.StopIterationAndBuffer<*> -> arrayOf(result.status, ByteBuffer.allocate(0)) - is FilterDataStatus.StopIterationNoBuffer<*> -> arrayOf(result.status, ByteBuffer.allocate(0)) - is FilterDataStatus.ResumeIteration<*> -> arrayOf(result.status, result.data, result.headers?.caseSensitiveHeaders()) + is FilterDataStatus.StopIterationAndBuffer<*> -> + arrayOf(result.status, ByteBuffer.allocate(0)) + is FilterDataStatus.StopIterationNoBuffer<*> -> + arrayOf(result.status, ByteBuffer.allocate(0)) + is FilterDataStatus.ResumeIteration<*> -> + arrayOf(result.status, result.data, result.headers?.caseSensitiveHeaders()) } } return arrayOf(0, data) } - override fun onRequestTrailers(trailers: Map>, streamIntel: EnvoyStreamIntel): Array { + override fun onRequestTrailers( + trailers: Map>, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? RequestFilter)?.let { requestFilter -> - val result = requestFilter.onRequestTrailers(RequestTrailers(trailers), StreamIntel(streamIntel)) + val result = + requestFilter.onRequestTrailers(RequestTrailers(trailers), StreamIntel(streamIntel)) return when (result) { - is FilterTrailersStatus.Continue<*, *> -> arrayOf(result.status, result.trailers.caseSensitiveHeaders()) - is FilterTrailersStatus.StopIteration<*, *> -> arrayOf(result.status, emptyMap>()) - is FilterTrailersStatus.ResumeIteration<*, *> -> arrayOf(result.status, result.trailers.caseSensitiveHeaders(), result.headers?.caseSensitiveHeaders(), result.data) + is FilterTrailersStatus.Continue<*, *> -> + arrayOf(result.status, result.trailers.caseSensitiveHeaders()) + is FilterTrailersStatus.StopIteration<*, *> -> + arrayOf(result.status, emptyMap>()) + is FilterTrailersStatus.ResumeIteration<*, *> -> + arrayOf( + result.status, + result.trailers.caseSensitiveHeaders(), + result.headers?.caseSensitiveHeaders(), + result.data + ) } } return arrayOf(0, trailers) } - override fun onResponseTrailers(trailers: Map>, streamIntel: EnvoyStreamIntel): Array { + override fun onResponseTrailers( + trailers: Map>, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? ResponseFilter)?.let { responseFilter -> - val result = responseFilter.onResponseTrailers(ResponseTrailers(trailers), StreamIntel(streamIntel)) + val result = + responseFilter.onResponseTrailers(ResponseTrailers(trailers), StreamIntel(streamIntel)) return when (result) { - is FilterTrailersStatus.Continue<*, *> -> arrayOf(result.status, result.trailers.caseSensitiveHeaders()) - is FilterTrailersStatus.StopIteration<*, *> -> arrayOf(result.status, emptyMap>()) - is FilterTrailersStatus.ResumeIteration<*, *> -> arrayOf(result.status, result.trailers.caseSensitiveHeaders(), result.headers?.caseSensitiveHeaders(), result.data) + is FilterTrailersStatus.Continue<*, *> -> + arrayOf(result.status, result.trailers.caseSensitiveHeaders()) + is FilterTrailersStatus.StopIteration<*, *> -> + arrayOf(result.status, emptyMap>()) + is FilterTrailersStatus.ResumeIteration<*, *> -> + arrayOf( + result.status, + result.trailers.caseSensitiveHeaders(), + result.headers?.caseSensitiveHeaders(), + result.data + ) } } return arrayOf(0, trailers) } - override fun onError(errorCode: Int, message: String, attemptCount: Int, streamIntel: EnvoyStreamIntel, finalStreamIntel: EnvoyFinalStreamIntel) { + override fun onError( + errorCode: Int, + message: String, + attemptCount: Int, + streamIntel: EnvoyStreamIntel, + finalStreamIntel: EnvoyFinalStreamIntel + ) { (filter as? ResponseFilter)?.let { responseFilter -> - responseFilter.onError(EnvoyError(errorCode, message, attemptCount), FinalStreamIntel(streamIntel, finalStreamIntel)) + responseFilter.onError( + EnvoyError(errorCode, message, attemptCount), + FinalStreamIntel(streamIntel, finalStreamIntel) + ) } } @@ -124,17 +186,30 @@ internal class EnvoyHTTPFilterAdapter( } } - override fun onResumeRequest(headers: Map>?, data: ByteBuffer?, trailers: Map>?, endStream: Boolean, streamIntel: EnvoyStreamIntel): Array { + override fun onResumeRequest( + headers: Map>?, + data: ByteBuffer?, + trailers: Map>?, + endStream: Boolean, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? AsyncRequestFilter)?.let { asyncRequestFilter -> - val result = asyncRequestFilter.onResumeRequest( - headers?.let(::RequestHeaders), - data, - trailers?.let(::RequestTrailers), - endStream, - StreamIntel(streamIntel) - ) + val result = + asyncRequestFilter.onResumeRequest( + headers?.let(::RequestHeaders), + data, + trailers?.let(::RequestTrailers), + endStream, + StreamIntel(streamIntel) + ) return when (result) { - is FilterResumeStatus.ResumeIteration<*, *> -> arrayOf(result.status, result.headers?.caseSensitiveHeaders(), result.data, result.trailers?.caseSensitiveHeaders()) + is FilterResumeStatus.ResumeIteration<*, *> -> + arrayOf( + result.status, + result.headers?.caseSensitiveHeaders(), + result.data, + result.trailers?.caseSensitiveHeaders() + ) } } return arrayOf(-1, headers, data, trailers) @@ -146,17 +221,30 @@ internal class EnvoyHTTPFilterAdapter( } } - override fun onResumeResponse(headers: Map>?, data: ByteBuffer?, trailers: Map>?, endStream: Boolean, streamIntel: EnvoyStreamIntel): Array { + override fun onResumeResponse( + headers: Map>?, + data: ByteBuffer?, + trailers: Map>?, + endStream: Boolean, + streamIntel: EnvoyStreamIntel + ): Array { (filter as? AsyncResponseFilter)?.let { asyncResponseFilter -> - val result = asyncResponseFilter.onResumeResponse( - headers?.let(::ResponseHeaders), - data, - trailers?.let(::ResponseTrailers), - endStream, - StreamIntel(streamIntel) - ) + val result = + asyncResponseFilter.onResumeResponse( + headers?.let(::ResponseHeaders), + data, + trailers?.let(::ResponseTrailers), + endStream, + StreamIntel(streamIntel) + ) return when (result) { - is FilterResumeStatus.ResumeIteration<*, *> -> arrayOf(result.status, result.headers?.caseSensitiveHeaders(), result.data, result.trailers?.caseSensitiveHeaders()) + is FilterResumeStatus.ResumeIteration<*, *> -> + arrayOf( + result.status, + result.headers?.caseSensitiveHeaders(), + result.data, + result.trailers?.caseSensitiveHeaders() + ) } } return arrayOf(-1, headers, data, trailers) diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterDataStatus.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterDataStatus.kt index 737671837fb7..fe864381e4d0 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterDataStatus.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterDataStatus.kt @@ -5,9 +5,7 @@ import java.nio.ByteBuffer /* * Status to be returned by filters when transmitting or receiving data. */ -sealed class FilterDataStatus( - val status: Int -) { +sealed class FilterDataStatus(val status: Int) { /** * Continue filter chain iteration. If headers have not yet been sent to the next filter, they * will be sent first via `onRequestHeaders()`/`onResponseHeaders()`. @@ -24,8 +22,7 @@ sealed class FilterDataStatus( * been buffered so far. * * Returning `ResumeIteration` from another filter invocation or calling - * `resumeRequest()`/`resumeResponse()` MUST be called when continued filter iteration is - * desired. + * `resumeRequest()`/`resumeResponse()` MUST be called when continued filter iteration is desired. * * This should be called by filters which must parse a larger block of the incoming data before * continuing processing. @@ -33,14 +30,12 @@ sealed class FilterDataStatus( class StopIterationAndBuffer : FilterDataStatus(1) /** - * Do not iterate to any of the remaining filters in the chain, and do not internally buffer - * data. + * Do not iterate to any of the remaining filters in the chain, and do not internally buffer data. * * `onData` will continue to be called with new chunks of data. * * Returning `ResumeIteration` from another filter invocation or calling - * `resumeRequest()`/`resumeResponse()` MUST be called when continued filter iteration is - * desired. + * `resumeRequest()`/`resumeResponse()` MUST be called when continued filter iteration is desired. * * This may be called by filters which must parse a larger block of the incoming data before * continuing processing, and will handle their own buffering. @@ -51,12 +46,13 @@ sealed class FilterDataStatus( * Resume previously-stopped iteration, possibly forwarding headers if iteration was stopped * during an on*Headers invocation. * - * It is an error to return `ResumeIteration` if iteration is not currently stopped, and it is - * an error to include headers if headers have already been forwarded to the next filter - * (i.e. iteration was stopped during an on*Data invocation instead of on*Headers). + * It is an error to return `ResumeIteration` if iteration is not currently stopped, and it is an + * error to include headers if headers have already been forwarded to the next filter (i.e. + * iteration was stopped during an on*Data invocation instead of on*Headers). * * @param headers: Headers to be forwarded (if needed). * @param data: Data to be forwarded. */ - class ResumeIteration(val headers: T?, val data: ByteBuffer) : FilterDataStatus(-1) + class ResumeIteration(val headers: T?, val data: ByteBuffer) : + FilterDataStatus(-1) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterHeadersStatus.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterHeadersStatus.kt index c50bca57e2e2..5b33e477da4e 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterHeadersStatus.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterHeadersStatus.kt @@ -3,9 +3,7 @@ package io.envoyproxy.envoymobile /* * Status to be returned by filters when transmitting or receiving headers. */ -sealed class FilterHeadersStatus( - val status: Int -) { +sealed class FilterHeadersStatus(val status: Int) { /** * Continue filter chain iteration, passing the provided headers through. * @@ -17,8 +15,7 @@ sealed class FilterHeadersStatus( * Do not iterate to any of the remaining filters in the chain with headers. * * Returning `ResumeIteration` from another filter invocation or calling - * `resumeRequest()`/`resumeResponse()` MUST occur when continued filter iteration is - * desired. + * `resumeRequest()`/`resumeResponse()` MUST occur when continued filter iteration is desired. */ class StopIteration : FilterHeadersStatus(1) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterResumeStatus.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterResumeStatus.kt index 933eed5ff8aa..2544a7a6fe56 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterResumeStatus.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterResumeStatus.kt @@ -5,18 +5,16 @@ import java.nio.ByteBuffer /* * Status to be returned by filters after resuming iteration asynchronously. */ -sealed class FilterResumeStatus( - val status: Int -) { +sealed class FilterResumeStatus(val status: Int) { /** - * Resume previously-stopped iteration, potentially forwarding headers, data, and/or trailers - * that have not yet been passed along the filter chain. + * Resume previously-stopped iteration, potentially forwarding headers, data, and/or trailers that + * have not yet been passed along the filter chain. * - * It is an error to return ResumeIteration if iteration is not currently stopped, and it is - * an error to include headers if headers have already been forwarded to the next filter - * (i.e. iteration was stopped during an on*Data invocation instead of on*Headers). It is also - * an error to include data or trailers if `endStream` was previously set or if trailers have - * already been forwarded. + * It is an error to return ResumeIteration if iteration is not currently stopped, and it is an + * error to include headers if headers have already been forwarded to the next filter (i.e. + * iteration was stopped during an on*Data invocation instead of on*Headers). It is also an error + * to include data or trailers if `endStream` was previously set or if trailers have already been + * forwarded. * * @param headers: Headers to be forwarded (if needed). * @param data: Data to be forwarded (if needed). diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterTrailersStatus.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterTrailersStatus.kt index e4a1fe93d793..011ae22d2ff5 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterTrailersStatus.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/FilterTrailersStatus.kt @@ -5,9 +5,7 @@ import java.nio.ByteBuffer /* * Status to be returned by filters when transmitting or receiving trailers. */ -sealed class FilterTrailersStatus( - val status: Int -) { +sealed class FilterTrailersStatus(val status: Int) { /** * Continue filter chain iteration, passing the provided trailers through. * @@ -20,8 +18,8 @@ sealed class FilterTrailersStatus( * * Because trailers are by definition the last HTTP entity of a request or response, only * asynchronous filters support resumption after returning `StopIteration` from on*Trailers. - * Calling `resumeRequest()`/`resumeResponse()` MUST occur if continued filter iteration - * is desired. + * Calling `resumeRequest()`/`resumeResponse()` MUST occur if continued filter iteration is + * desired. */ class StopIteration : FilterTrailersStatus(1) @@ -29,9 +27,9 @@ sealed class FilterTrailersStatus( * Resume previously-stopped iteration, possibly forwarding headers and data if iteration was * stopped during an on*Headers or on*Data invocation. * - * It is an error to return `ResumeIteration` if iteration is not currently stopped, and it is - * an error to include headers if headers have already been forwarded to the next filter - * (i.e. iteration was stopped during an on*Data invocation instead of on*Headers). + * It is an error to return `ResumeIteration` if iteration is not currently stopped, and it is an + * error to include headers if headers have already been forwarded to the next filter (i.e. + * iteration was stopped during an on*Data invocation instead of on*Headers). * * @param headers: Headers to be forwarded (if needed). * @param data: Data to be forwarded (if needed). diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilter.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilter.kt index ef02dc065fc2..ccbcfbea6aec 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilter.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilter.kt @@ -11,39 +11,47 @@ interface RequestFilter : Filter { * * Filters may mutate or delay the request headers. * - * @param headers: The current request headers. - * @param endStream: Whether this is a headers-only request. + * @param headers: The current request headers. + * @param endStream: Whether this is a headers-only request. * @param streamIntel: Internal HTTP stream metrics, context, and other details. * * @return: The header status containing headers with which to continue or buffer. */ - fun onRequestHeaders(headers: RequestHeaders, endStream: Boolean, streamIntel: StreamIntel): - FilterHeadersStatus + fun onRequestHeaders( + headers: RequestHeaders, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterHeadersStatus /** * Called any number of times whenever body data is sent. * * Filters may mutate or buffer (defer and concatenate) the data. * - * @param body: The outbound body data chunk. - * @param endStream: Whether this is the last data frame. + * @param body: The outbound body data chunk. + * @param endStream: Whether this is the last data frame. * @param streamIntel: Internal HTTP stream metrics, context, and other details. * * @return: The data status containing body with which to continue or buffer. */ - fun onRequestData(body: ByteBuffer, endStream: Boolean, streamIntel: StreamIntel): - FilterDataStatus + fun onRequestData( + body: ByteBuffer, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterDataStatus /** * Called at most once when the request is closed from the client with trailers. * * Filters may mutate or delay the trailers. Note trailers imply the stream has ended. * - * @param trailers: The outbound trailers. + * @param trailers: The outbound trailers. * @param streamIntel: Internal HTTP stream metrics, context, and other details. * * @return: The trailer status containing body with which to continue or buffer. */ - fun onRequestTrailers(trailers: RequestTrailers, streamIntel: StreamIntel): - FilterTrailersStatus + fun onRequestTrailers( + trailers: RequestTrailers, + streamIntel: StreamIntel + ): FilterTrailersStatus } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacks.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacks.kt index 47ef5c8e36fa..013ad19976e2 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacks.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacks.kt @@ -9,8 +9,7 @@ interface RequestFilterCallbacks { * * This will result in an `onResumeRequest()` callback on the RequestFilter. * - * If the request is not complete, the filter may receive further `onData()`/`onTrailers()` - * calls. + * If the request is not complete, the filter may receive further `onData()`/`onTrailers()` calls. */ fun resumeRequest() diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacksImpl.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacksImpl.kt index 823c8166cd82..f49f36736a84 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacksImpl.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/RequestFilterCallbacksImpl.kt @@ -2,12 +2,9 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterCallbacks -/** - * Envoy implementation of `RequestFilterCallbacks`. - */ -internal class RequestFilterCallbacksImpl constructor( - internal val callbacks: EnvoyHTTPFilterCallbacks -) : RequestFilterCallbacks { +/** Envoy implementation of `RequestFilterCallbacks`. */ +internal class RequestFilterCallbacksImpl +constructor(internal val callbacks: EnvoyHTTPFilterCallbacks) : RequestFilterCallbacks { override fun resumeRequest() { callbacks.resumeIteration() diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilter.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilter.kt index fa7ce9a5eeeb..e924f016cb8e 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilter.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilter.kt @@ -11,50 +11,57 @@ interface ResponseFilter : Filter { * * Filters may mutate or delay the response headers. * - * @param headers: The current response headers. - * @param endStream: Whether this is a headers-only response. + * @param headers: The current response headers. + * @param endStream: Whether this is a headers-only response. * @param streamIntel: Internal HTTP stream metrics, context, and other details. * * @return: The header status containing headers with which to continue or buffer. */ - fun onResponseHeaders(headers: ResponseHeaders, endStream: Boolean, streamIntel: StreamIntel): - FilterHeadersStatus + fun onResponseHeaders( + headers: ResponseHeaders, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterHeadersStatus /** * Called any number of times whenever body data is received. * * Filters may mutate or buffer (defer and concatenate) the data. * - * @param body: The inbound body data chunk. - * @param endStream: Whether this is the last data frame. + * @param body: The inbound body data chunk. + * @param endStream: Whether this is the last data frame. * @param streamIntel: Internal HTTP stream metrics, context, and other details. * * @return: The data status containing body with which to continue or buffer. */ - fun onResponseData(body: ByteBuffer, endStream: Boolean, streamIntel: StreamIntel): - FilterDataStatus + fun onResponseData( + body: ByteBuffer, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterDataStatus /** * Called at most once when the response is closed from the server with trailers. * * Filters may mutate or delay the trailers. Note trailers imply the stream has ended. * - * @param trailers: The inbound trailers. + * @param trailers: The inbound trailers. * @param streamIntel: Internal HTTP stream metrics, context, and other details. * * @return: The trailer status containing body with which to continue or buffer. */ - fun onResponseTrailers(trailers: ResponseTrailers, streamIntel: StreamIntel): - FilterTrailersStatus + fun onResponseTrailers( + trailers: ResponseTrailers, + streamIntel: StreamIntel + ): FilterTrailersStatus /** * Called at most once when an error within Envoy occurs. * - * Only one of onError, onCancel, or onComplete will be called per stream. - * This should be considered a terminal state, and invalidates any previous attempts to - * `stopIteration{...}`. + * Only one of onError, onCancel, or onComplete will be called per stream. This should be + * considered a terminal state, and invalidates any previous attempts to `stopIteration{...}`. * - * @param error: The error that occurred within Envoy. + * @param error: The error that occurred within Envoy. * @param finalStreamIntel: Final internal HTTP stream metrics, context, and other details. */ fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) @@ -62,20 +69,18 @@ interface ResponseFilter : Filter { /** * Called at most once when the client cancels the stream. * - * Only one of onError, onCancel, or onComplete will be called per stream. - * This should be considered a terminal state, and invalidates any previous attempts to - * `stopIteration{...}`. + * Only one of onError, onCancel, or onComplete will be called per stream. This should be + * considered a terminal state, and invalidates any previous attempts to `stopIteration{...}`. * * @param finalStreamIntel: Final internal HTTP stream metrics, context, and other details. */ fun onCancel(finalStreamIntel: FinalStreamIntel) -/** + /** * Called at most once when the stream completes gracefully. * - * Only one of onError, onCancel, or onComplete will be called per stream. - * This should be considered a terminal state, and invalidates any previous attempts to - * `stopIteration{...}`. + * Only one of onError, onCancel, or onComplete will be called per stream. This should be + * considered a terminal state, and invalidates any previous attempts to `stopIteration{...}`. * * @param finalStreamIntel: Final internal HTTP stream metrics, context, and other details. */ diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilterCallbacksImpl.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilterCallbacksImpl.kt index 95a52310daef..ea74711b1f9d 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilterCallbacksImpl.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/filters/ResponseFilterCallbacksImpl.kt @@ -2,12 +2,9 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterCallbacks -/** - * Envoy implementation of `ResponseFilterCallbacks`. - */ -internal class ResponseFilterCallbacksImpl constructor( - internal val callbacks: EnvoyHTTPFilterCallbacks -) : ResponseFilterCallbacks { +/** Envoy implementation of `ResponseFilterCallbacks`. */ +internal class ResponseFilterCallbacksImpl +constructor(internal val callbacks: EnvoyHTTPFilterCallbacks) : ResponseFilterCallbacks { override fun resumeResponse() { callbacks.resumeIteration() diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCClient.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCClient.kt index 7f4dc6b3f9d0..77f2dcd95523 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCClient.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCClient.kt @@ -8,15 +8,11 @@ internal const val GRPC_PREFIX_LENGTH = 5 * * @param streamClient The stream client to use for gRPC streams. */ -class GRPCClient( - private val streamClient: StreamClient -) { +class GRPCClient(private val streamClient: StreamClient) { /** * Create a new gRPC stream prototype which can be used to start streams. * * @return The new gRPC stream prototype. */ - fun newGRPCStreamPrototype() = GRPCStreamPrototype( - streamClient.newStreamPrototype() - ) + fun newGRPCStreamPrototype() = GRPCStreamPrototype(streamClient.newStreamPrototype()) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCRequestHeadersBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCRequestHeadersBuilder.kt index 51b53dee2272..8b658802ce4a 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCRequestHeadersBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCRequestHeadersBuilder.kt @@ -45,16 +45,21 @@ class GRPCRequestHeadersBuilder : HeadersBuilder { * @param authority The URL authority for the request (i.e., "api.foo.com"). * @param path Path for the RPC (i.e., `/pb.api.v1.Foo/GetBar`). */ - constructor(scheme: String, authority: String, path: String) : super(HeadersContainer( - mapOf>( - ":authority" to mutableListOf(authority), - ":method" to mutableListOf("POST"), - ":path" to mutableListOf(path), - ":scheme" to mutableListOf(scheme), - "content-type" to mutableListOf("application/grpc"), + constructor( + scheme: String, + authority: String, + path: String + ) : super( + HeadersContainer( + mapOf>( + ":authority" to mutableListOf(authority), + ":method" to mutableListOf("POST"), + ":path" to mutableListOf(path), + ":scheme" to mutableListOf(scheme), + "content-type" to mutableListOf("application/grpc"), + ) ) ) - ) /** * Add a specific timeout for the gRPC request. This will be sent in the `grpc-timeout` header. diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStream.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStream.kt index cfa212989f0a..6872c854866d 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStream.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStream.kt @@ -8,9 +8,7 @@ import java.nio.ByteOrder * * Constructed using `GRPCStreamPrototype`, and used to write to the network. */ -class GRPCStream( - private val underlyingStream: Stream -) { +class GRPCStream(private val underlyingStream: Stream) { /** * Send headers over the gRPC stream. * @@ -50,9 +48,7 @@ class GRPCStream( return this } - /** - * Cancel the stream forcefully regardless of whether the peer has more data to send. - */ + /** Cancel the stream forcefully regardless of whether the peer has more data to send. */ fun cancel() { underlyingStream.cancel() } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStreamPrototype.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStreamPrototype.kt index 7b35796c5229..420936572c55 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStreamPrototype.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/grpc/GRPCStreamPrototype.kt @@ -9,14 +9,13 @@ import java.util.concurrent.Executors /** * A type representing a gRPC stream that has not yet been started. * - * Constructed via `GRPCClient`, and used to assign response callbacks - * prior to starting a `GRPCStream` by calling `start()`. + * Constructed via `GRPCClient`, and used to assign response callbacks prior to starting a + * `GRPCStream` by calling `start()`. */ -class GRPCStreamPrototype( - private val underlyingStream: StreamPrototype -) { +class GRPCStreamPrototype(private val underlyingStream: StreamPrototype) { /** * Start a new gRPC stream. + * * @param executor Executor on which to receive callback events. * @return The new gRPC stream. */ @@ -28,7 +27,8 @@ class GRPCStreamPrototype( /** * Specify a callback for when response headers are received by the stream. * - * @param closure Closure which will receive the headers and flag indicating if the stream is headers-only. + * @param closure Closure which will receive the headers and flag indicating if the stream is + * headers-only. * @return This stream, for chaining syntax. */ fun setOnResponseHeaders( @@ -39,8 +39,8 @@ class GRPCStreamPrototype( } /** - * Specify a callback for when a new message has been received by the stream. - * If `endStream` is `true`, the stream is complete. + * Specify a callback for when a new message has been received by the stream. If `endStream` is + * `true`, the stream is complete. * * @param closure Closure which will receive messages on the stream. * @return This stream, for chaining syntax. @@ -50,26 +50,29 @@ class GRPCStreamPrototype( ): GRPCStreamPrototype { val byteBufferedOutputStream = ByteArrayOutputStream() val processor = GRPCMessageProcessor() - var processState: GRPCMessageProcessor.ProcessState = GRPCMessageProcessor.ProcessState.CompressionFlag + var processState: GRPCMessageProcessor.ProcessState = + GRPCMessageProcessor.ProcessState.CompressionFlag underlyingStream.setOnResponseData { byteBuffer, _, streamIntel -> - val byteBufferArray = if (byteBuffer.hasArray()) { - byteBuffer.array() - } else { - val array = ByteArray(byteBuffer.remaining()) - byteBuffer.get(array) - array - } + val byteBufferArray = + if (byteBuffer.hasArray()) { + byteBuffer.array() + } else { + val array = ByteArray(byteBuffer.remaining()) + byteBuffer.get(array) + array + } byteBufferedOutputStream.write(byteBufferArray) - processState = processor.processData(byteBufferedOutputStream, processState, streamIntel, closure) + processState = + processor.processData(byteBufferedOutputStream, processState, streamIntel, closure) } return this } /** - * Specify a callback for when trailers are received by the stream. - * If the closure is called, the stream is complete. + * Specify a callback for when trailers are received by the stream. If the closure is called, the + * stream is complete. * * @param closure Closure which will receive the trailers. * @return This stream, for chaining syntax. @@ -82,8 +85,8 @@ class GRPCStreamPrototype( } /** - * Specify a callback for when an internal Envoy exception occurs with the stream. - * If the closure is called, the stream is complete. + * Specify a callback for when an internal Envoy exception occurs with the stream. If the closure + * is called, the stream is complete. * * @param closure Closure which will be called when an error occurs. * @return This stream, for chaining syntax. @@ -96,24 +99,20 @@ class GRPCStreamPrototype( } /** - * Specify a callback for when the stream is canceled. - * If the closure is called, the stream is complete. + * Specify a callback for when the stream is canceled. If the closure is called, the stream is + * complete. * * @param closure Closure which will be called when the stream is canceled. * @return This stream, for chaining syntax. */ - fun setOnCancel( - closure: (finalStreamIntel: FinalStreamIntel) -> Unit - ): GRPCStreamPrototype { + fun setOnCancel(closure: (finalStreamIntel: FinalStreamIntel) -> Unit): GRPCStreamPrototype { underlyingStream.setOnCancel(closure) return this } } private class GRPCMessageProcessor { - /** - * Represents the process state of the response stream's body data. - */ + /** Represents the process state of the response stream's body data. */ sealed class ProcessState { // Awaiting a gRPC compression flag. object CompressionFlag : ProcessState() @@ -126,8 +125,8 @@ private class GRPCMessageProcessor { } /** - * Recursively processes a buffer of data, buffering it into messages based on state. - * When a message has been fully buffered, `onMessage` will be called with the message. + * Recursively processes a buffer of data, buffering it into messages based on state. When a + * message has been fully buffered, `onMessage` will be called with the message. * * @param bufferedStream The buffer of data from which to determine state and messages. * @param processState The current process state of the buffering. @@ -187,9 +186,7 @@ private class GRPCMessageProcessor { ) bufferedStream.reset() bufferedStream.write( - byteArray.sliceArray( - GRPC_PREFIX_LENGTH + processState.messageLength until byteArray.size - ) + byteArray.sliceArray(GRPC_PREFIX_LENGTH + processState.messageLength until byteArray.size) ) val remainingLength = GRPC_PREFIX_LENGTH + processState.messageLength until byteArray.size diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt index 554e0a81d882..dd949fa4927b 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyEngine.kt @@ -5,38 +5,40 @@ import io.envoyproxy.envoymobile.engine.EnvoyEngine import io.envoyproxy.envoymobile.engine.EnvoyHTTPStream import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPCallbacks import io.envoyproxy.envoymobile.engine.types.EnvoyNetworkType -import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor import io.envoyproxy.envoymobile.engine.types.EnvoyStatus +import io.envoyproxy.envoymobile.engine.types.EnvoyStringAccessor /** * Mock implementation of `EnvoyEngine`. Used internally for testing the bridging layer & mocking. */ internal class MockEnvoyEngine : EnvoyEngine { - override fun runWithConfig(envoyConfiguration: EnvoyConfiguration?, logLevel: String?): EnvoyStatus = EnvoyStatus.ENVOY_SUCCESS + override fun runWithConfig( + envoyConfiguration: EnvoyConfiguration?, + logLevel: String? + ): EnvoyStatus = EnvoyStatus.ENVOY_SUCCESS override fun performRegistration(envoyConfiguration: EnvoyConfiguration) = Unit - override fun runWithYaml( - configurationYAML: String, - logLevel: String - ): EnvoyStatus = EnvoyStatus.ENVOY_SUCCESS + override fun runWithYaml(configurationYAML: String, logLevel: String): EnvoyStatus = + EnvoyStatus.ENVOY_SUCCESS override fun startStream( callbacks: EnvoyHTTPCallbacks?, - explicitFlowControl: Boolean, - minDeliverySize: Long + explicitFlowControl: Boolean ): EnvoyHTTPStream { - return MockEnvoyHTTPStream(callbacks!!, explicitFlowControl, minDeliverySize) + return MockEnvoyHTTPStream(callbacks!!, explicitFlowControl) } override fun terminate() = Unit - override fun recordCounterInc(elements: String, tags: MutableMap, count: Int): Int = 0 + override fun recordCounterInc( + elements: String, + tags: MutableMap, + count: Int + ): Int = 0 override fun registerStringAccessor(accessorName: String, accessor: EnvoyStringAccessor): Int = 0 - override fun flushStats() = Unit - override fun dumpStats(): String = "" override fun resetConnectivityState() = Unit diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt index 978000716102..64bd7849e747 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockEnvoyHTTPStream.kt @@ -11,9 +11,8 @@ import java.nio.ByteBuffer */ internal class MockEnvoyHTTPStream( val callbacks: EnvoyHTTPCallbacks, - val explicitFlowControl: Boolean, - val minDeliverySize: Long -) : EnvoyHTTPStream(0, 0, callbacks, explicitFlowControl, minDeliverySize) { + val explicitFlowControl: Boolean +) : EnvoyHTTPStream(0, 0, callbacks, explicitFlowControl) { override fun sendHeaders(headers: MutableMap>?, endStream: Boolean) {} override fun sendData(data: ByteBuffer?, endStream: Boolean) {} diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStream.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStream.kt index 4f1f54ddf24f..c5ad9839b6ab 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStream.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStream.kt @@ -5,52 +5,105 @@ import io.envoyproxy.envoymobile.engine.types.EnvoyStreamIntel import java.nio.ByteBuffer /** - * Mock implementation of `Stream` that also provides an interface for sending - * mocked responses through to the stream's callbacks. Created via `MockStreamPrototype`. + * Mock implementation of `Stream` that also provides an interface for sending mocked responses + * through to the stream's callbacks. Created via `MockStreamPrototype`. */ -class MockStream internal constructor(underlyingStream: MockEnvoyHTTPStream) : Stream(underlyingStream, useByteBufferPosition = false) { +class MockStream internal constructor(underlyingStream: MockEnvoyHTTPStream) : + Stream(underlyingStream, useByteBufferPosition = false) { private val mockStream: MockEnvoyHTTPStream = underlyingStream - private val mockStreamIntel = object : EnvoyStreamIntel { - override fun getStreamId(): Long { return 0 } - override fun getConnectionId(): Long { return 0 } - override fun getAttemptCount(): Long { return 0 } - override fun getConsumedBytesFromResponse(): Long { return 0 } - } + private val mockStreamIntel = + object : EnvoyStreamIntel { + override fun getStreamId(): Long { + return 0 + } - private val mockFinalStreamIntel = object : EnvoyFinalStreamIntel { - override fun getStreamStartMs(): Long { return 0 } - override fun getDnsStartMs(): Long { return 0 } - override fun getDnsEndMs(): Long { return 0 } - override fun getConnectStartMs(): Long { return 0 } - override fun getConnectEndMs(): Long { return 0 } - override fun getSslStartMs(): Long { return 0 } - override fun getSslEndMs(): Long { return 0 } - override fun getSendingStartMs(): Long { return 0 } - override fun getSendingEndMs(): Long { return 0 } - override fun getResponseStartMs(): Long { return 0 } - override fun getStreamEndMs(): Long { return 0 } - override fun getSocketReused(): Boolean { return false } - override fun getSentByteCount(): Long { return 0 } - override fun getReceivedByteCount(): Long { return 0 } - override fun getResponseFlags(): Long { return 0 } - override fun getUpstreamProtocol(): Long { return 0 } - } - /** - * Closure that will be called when request headers are sent. - */ + override fun getConnectionId(): Long { + return 0 + } + + override fun getAttemptCount(): Long { + return 0 + } + + override fun getConsumedBytesFromResponse(): Long { + return 0 + } + } + + private val mockFinalStreamIntel = + object : EnvoyFinalStreamIntel { + override fun getStreamStartMs(): Long { + return 0 + } + + override fun getDnsStartMs(): Long { + return 0 + } + + override fun getDnsEndMs(): Long { + return 0 + } + + override fun getConnectStartMs(): Long { + return 0 + } + + override fun getConnectEndMs(): Long { + return 0 + } + + override fun getSslStartMs(): Long { + return 0 + } + + override fun getSslEndMs(): Long { + return 0 + } + + override fun getSendingStartMs(): Long { + return 0 + } + + override fun getSendingEndMs(): Long { + return 0 + } + + override fun getResponseStartMs(): Long { + return 0 + } + + override fun getStreamEndMs(): Long { + return 0 + } + + override fun getSocketReused(): Boolean { + return false + } + + override fun getSentByteCount(): Long { + return 0 + } + + override fun getReceivedByteCount(): Long { + return 0 + } + + override fun getResponseFlags(): Long { + return 0 + } + + override fun getUpstreamProtocol(): Long { + return 0 + } + } + /** Closure that will be called when request headers are sent. */ var onRequestHeaders: ((headers: RequestHeaders, endStream: Boolean) -> Unit)? = null - /** - * Closure that will be called when request data is sent. - */ + /** Closure that will be called when request data is sent. */ var onRequestData: ((data: ByteBuffer, endStream: Boolean) -> Unit)? = null - /** - * Closure that will be called when request trailers are sent. - */ + /** Closure that will be called when request trailers are sent. */ var onRequestTrailers: ((trailers: RequestTrailers) -> Unit)? = null - /** - * Closure that will be called when the stream is canceled by the client. - */ + /** Closure that will be called when the stream is canceled by the client. */ var onCancel: (() -> Unit)? = null override fun sendHeaders(headers: RequestHeaders, endStream: Boolean): Stream { @@ -104,9 +157,7 @@ class MockStream internal constructor(underlyingStream: MockEnvoyHTTPStream) : S mockStream.callbacks.onTrailers(trailers.caseSensitiveHeaders(), mockStreamIntel) } - /** - * Simulate the stream receiving a cancellation signal from Envoy. - */ + /** Simulate the stream receiving a cancellation signal from Envoy. */ fun receiveCancel() { mockStream.callbacks.onCancel(mockStreamIntel, mockFinalStreamIntel) } @@ -117,6 +168,12 @@ class MockStream internal constructor(underlyingStream: MockEnvoyHTTPStream) : S * @param error The error to receive. */ fun receiveError(error: EnvoyError) { - mockStream.callbacks.onError(error.errorCode, error.message, error.attemptCount ?: 0, mockStreamIntel, mockFinalStreamIntel) + mockStream.callbacks.onError( + error.errorCode, + error.message, + error.attemptCount ?: 0, + mockStreamIntel, + mockFinalStreamIntel + ) } } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamClient.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamClient.kt index 9df0c98b7b5d..a346770c30b3 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamClient.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamClient.kt @@ -3,9 +3,9 @@ package io.envoyproxy.envoymobile /** * Mock implementation of `StreamClient` which produces `MockStreamPrototype` values. * - * @param onStartStream Closure that may be set to observe the creation of new streams. - * It will be called each time `newStreamPrototype()` is executed. - * Typically, this is used to capture streams on creation before sending values through them. + * @param onStartStream Closure that may be set to observe the creation of new streams. It will be + * called each time `newStreamPrototype()` is executed. Typically, this is used to capture streams + * on creation before sending values through them. */ class MockStreamClient(var onStartStream: ((MockStream) -> Unit)?) : StreamClient { override fun newStreamPrototype(): StreamPrototype { diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt index 22188905f54a..5cbeb740ab5b 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/mocks/MockStreamPrototype.kt @@ -7,10 +7,12 @@ import java.util.concurrent.Executor * * @param onStart Closure that will be called each time a new stream is started from the prototype. */ -class MockStreamPrototype internal constructor(private val onStart: ((stream: MockStream) -> Unit)?) : StreamPrototype(MockEnvoyEngine()) { +class MockStreamPrototype +internal constructor(private val onStart: ((stream: MockStream) -> Unit)?) : + StreamPrototype(MockEnvoyEngine()) { override fun start(executor: Executor): Stream { val callbacks = createCallbacks(executor) - val stream = MockStream(MockEnvoyHTTPStream(callbacks, false, 0)) + val stream = MockStream(MockEnvoyHTTPStream(callbacks, false)) onStart?.invoke(stream) return stream } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Counter.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Counter.kt index 45731c0ffe81..829c520b07e5 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Counter.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Counter.kt @@ -1,17 +1,11 @@ package io.envoyproxy.envoymobile -/** - * A time series counter. - */ +/** A time series counter. */ interface Counter { - /** - * Increments the counter by the given count. - */ + /** Increments the counter by the given count. */ fun increment(count: Int = 1) - /** - * Increments the counter by the given count and tags. - */ + /** Increments the counter by the given count and tags. */ fun increment(tags: Tags = TagsBuilder().build(), count: Int = 1) } diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/CounterImpl.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/CounterImpl.kt index 959c31487010..b6a77380ab97 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/CounterImpl.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/CounterImpl.kt @@ -3,15 +3,17 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.EnvoyEngine import java.lang.ref.WeakReference -/** - * Envoy implementation of a `Counter`. - */ +/** Envoy implementation of a `Counter`. */ internal class CounterImpl : Counter { var envoyEngine: WeakReference var series: String var tags: Tags - internal constructor(engine: EnvoyEngine, elements: List, tags: Tags = TagsBuilder().build()) { + internal constructor( + engine: EnvoyEngine, + elements: List, + tags: Tags = TagsBuilder().build() + ) { this.envoyEngine = WeakReference(engine) this.series = elements.joinToString(separator = ".") { it.value } this.tags = tags diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Element.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Element.kt index 30c47406c775..8b91da91b066 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Element.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Element.kt @@ -7,7 +7,6 @@ import java.util.regex.Pattern * * Element values must conform to the [Element.ELEMENT_REGEX]. */ - class Element(internal val value: String) { init { require(ELEMENT_PATTERN.matcher(value).matches()) { diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Tags.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Tags.kt index de385a90a8e9..975f1b7839c3 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Tags.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/Tags.kt @@ -5,8 +5,7 @@ package io.envoyproxy.envoymobile * To instantiate new instances, see `TagsBuilder`. */ class Tags { - @Suppress("MemberNameEqualsClassName") - val tags: Map + @Suppress("MemberNameEqualsClassName") val tags: Map /** * Internal constructor used by builders. @@ -21,7 +20,6 @@ class Tags { * Get the value for the provided tag name. * * @param name: Tag name for which to get the current value. - * * @return String?, The current tags specified for the provided name. */ fun value(name: String): String? { diff --git a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/TagsBuilder.kt b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/TagsBuilder.kt index 202b8215c425..3480a0905a8e 100644 --- a/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/TagsBuilder.kt +++ b/mobile/library/kotlin/io/envoyproxy/envoymobile/stats/TagsBuilder.kt @@ -22,9 +22,8 @@ public class TagsBuilder { /** * Append a value to the Tag name. * - * @param name: The Tag name. + * @param name: The Tag name. * @param value: The value associated to the Tag name. - * * @return TagsBuilder, This builder. */ public fun add(name: String, value: String): TagsBuilder { @@ -37,7 +36,6 @@ public class TagsBuilder { * * @param name: The Tag name. * @param value: The value associated to the Tag name. - * * @return TagsBuilder, This builder. */ public fun set(name: String, value: String): TagsBuilder { @@ -49,7 +47,6 @@ public class TagsBuilder { * Remove all Tags with this name. * * @param name: The Tag name to remove. - * * @return TagsBuilder, This builder. */ public fun remove(name: String): TagsBuilder { @@ -61,7 +58,6 @@ public class TagsBuilder { * Adds all tags from map to this builder. * * @param tags: A map of tags. - * * @return TagsBuilder, This builder. */ public fun putAll(tags: Map): TagsBuilder { diff --git a/mobile/library/objective-c/BUILD b/mobile/library/objective-c/BUILD index ba4bc212fdef..44d83f5f69a4 100644 --- a/mobile/library/objective-c/BUILD +++ b/mobile/library/objective-c/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("//bazel:apple.bzl", "envoy_objc_library") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + exports_files([ "EnvoyEngine.h", ]) diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h index 0f9fb7d43cb5..d38a25d23a82 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.h +++ b/mobile/library/objective-c/EnvoyConfiguration.h @@ -13,7 +13,6 @@ NS_ASSUME_NONNULL_BEGIN /// Typed configuration that may be used for starting Envoy. @interface EnvoyConfiguration : NSObject -@property (nonatomic, strong, nullable) NSString *grpcStatsDomain; @property (nonatomic, assign) UInt32 connectTimeoutSeconds; @property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsBase; @property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsMax; @@ -25,6 +24,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, assign) UInt32 dnsCacheSaveIntervalSeconds; @property (nonatomic, assign) BOOL enableHttp3; @property (nonatomic, strong) NSDictionary *quicHints; +@property (nonatomic, strong) NSArray *quicCanonicalSuffixes; @property (nonatomic, assign) BOOL enableGzipDecompression; @property (nonatomic, assign) BOOL enableBrotliDecompression; @property (nonatomic, assign) BOOL enableInterfaceBinding; @@ -35,7 +35,6 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, assign) UInt32 h2ConnectionKeepaliveIdleIntervalMilliseconds; @property (nonatomic, assign) UInt32 h2ConnectionKeepaliveTimeoutSeconds; @property (nonatomic, assign) UInt32 maxConnectionsPerHost; -@property (nonatomic, assign) UInt32 statsFlushSeconds; @property (nonatomic, assign) UInt32 streamIdleTimeoutSeconds; @property (nonatomic, assign) UInt32 perTryIdleTimeoutSeconds; @property (nonatomic, strong, nullable) NSString *appVersion; @@ -45,19 +44,14 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic, strong) NSArray *httpPlatformFilterFactories; @property (nonatomic, strong) NSDictionary *stringAccessors; @property (nonatomic, strong) NSDictionary> *keyValueStores; -@property (nonatomic, strong) NSArray *statsSinks; @property (nonatomic, strong, nullable) NSString *nodeId; @property (nonatomic, strong, nullable) NSString *nodeRegion; @property (nonatomic, strong, nullable) NSString *nodeZone; @property (nonatomic, strong, nullable) NSString *nodeSubZone; @property (nonatomic, strong, nullable) NSString *xdsServerAddress; @property (nonatomic, assign) UInt32 xdsServerPort; -@property (nonatomic, strong, nullable) NSString *xdsAuthHeader; -@property (nonatomic, strong, nullable) NSString *xdsAuthToken; -@property (nonatomic, strong, nullable) NSString *xdsJwtToken; -@property (nonatomic, assign) UInt32 xdsJwtTokenLifetimeSeconds; +@property (nonatomic, strong) NSDictionary *xdsGrpcInitialMetadata; @property (nonatomic, strong, nullable) NSString *xdsSslRootCerts; -@property (nonatomic, strong, nullable) NSString *xdsSni; @property (nonatomic, strong, nullable) NSString *rtdsResourceName; @property (nonatomic, assign) UInt32 rtdsTimeoutSeconds; @property (nonatomic, assign) BOOL enableCds; @@ -68,8 +62,7 @@ NS_ASSUME_NONNULL_BEGIN /** Create a new instance of the configuration. */ -- (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain - connectTimeoutSeconds:(UInt32)connectTimeoutSeconds +- (instancetype)initWithConnectTimeoutSeconds:(UInt32)connectTimeoutSeconds dnsRefreshSeconds:(UInt32)dnsRefreshSeconds dnsFailureRefreshSecondsBase:(UInt32)dnsFailureRefreshSecondsBase dnsFailureRefreshSecondsMax:(UInt32)dnsFailureRefreshSecondsMax @@ -80,6 +73,7 @@ NS_ASSUME_NONNULL_BEGIN dnsCacheSaveIntervalSeconds:(UInt32)dnsCacheSaveIntervalSeconds enableHttp3:(BOOL)enableHttp3 quicHints:(NSDictionary *)quicHints + quicCanonicalSuffixes:(NSArray *)quicCanonicalSuffixes enableGzipDecompression:(BOOL)enableGzipDecompression enableBrotliDecompression:(BOOL)enableBrotliDecompression enableInterfaceBinding:(BOOL)enableInterfaceBinding @@ -91,7 +85,6 @@ NS_ASSUME_NONNULL_BEGIN (UInt32)h2ConnectionKeepaliveIdleIntervalMilliseconds h2ConnectionKeepaliveTimeoutSeconds:(UInt32)h2ConnectionKeepaliveTimeoutSeconds maxConnectionsPerHost:(UInt32)maxConnectionsPerHost - statsFlushSeconds:(UInt32)statsFlushSeconds streamIdleTimeoutSeconds:(UInt32)streamIdleTimeoutSeconds perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds appVersion:(NSString *)appVersion @@ -108,19 +101,15 @@ NS_ASSUME_NONNULL_BEGIN keyValueStores: (NSDictionary> *) keyValueStores - statsSinks:(NSArray *)statsSinks nodeId:(nullable NSString *)nodeId nodeRegion:(nullable NSString *)nodeRegion nodeZone:(nullable NSString *)nodeZone nodeSubZone:(nullable NSString *)nodeSubZone xdsServerAddress:(nullable NSString *)xdsServerAddress xdsServerPort:(UInt32)xdsServerPort - xdsAuthHeader:(nullable NSString *)xdsAuthHeader - xdsAuthToken:(nullable NSString *)xdsAuthToken - xdsJwtToken:(nullable NSString *)xdsJwtToken - xdsJwtTokenLifetimeSeconds:(UInt32)xdsJwtTokenLifetimeSeconds + xdsGrpcInitialMetadata: + (NSDictionary *)xdsGrpcInitialMetadata xdsSslRootCerts:(nullable NSString *)xdsSslRootCerts - xdsSni:(nullable NSString *)xdsSni rtdsResourceName:(nullable NSString *)rtdsResourceName rtdsTimeoutSeconds:(UInt32)rtdsTimeoutSeconds enableCds:(BOOL)enableCds diff --git a/mobile/library/objective-c/EnvoyConfiguration.mm b/mobile/library/objective-c/EnvoyConfiguration.mm index f52ac74a4425..00a60f6bb1cc 100644 --- a/mobile/library/objective-c/EnvoyConfiguration.mm +++ b/mobile/library/objective-c/EnvoyConfiguration.mm @@ -67,8 +67,7 @@ @implementation EMODirectResponse @implementation EnvoyConfiguration -- (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain - connectTimeoutSeconds:(UInt32)connectTimeoutSeconds +- (instancetype)initWithConnectTimeoutSeconds:(UInt32)connectTimeoutSeconds dnsRefreshSeconds:(UInt32)dnsRefreshSeconds dnsFailureRefreshSecondsBase:(UInt32)dnsFailureRefreshSecondsBase dnsFailureRefreshSecondsMax:(UInt32)dnsFailureRefreshSecondsMax @@ -79,6 +78,7 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain dnsCacheSaveIntervalSeconds:(UInt32)dnsCacheSaveIntervalSeconds enableHttp3:(BOOL)enableHttp3 quicHints:(NSDictionary *)quicHints + quicCanonicalSuffixes:(NSArray *)quicCanonicalSuffixes enableGzipDecompression:(BOOL)enableGzipDecompression enableBrotliDecompression:(BOOL)enableBrotliDecompression enableInterfaceBinding:(BOOL)enableInterfaceBinding @@ -90,7 +90,6 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain (UInt32)h2ConnectionKeepaliveIdleIntervalMilliseconds h2ConnectionKeepaliveTimeoutSeconds:(UInt32)h2ConnectionKeepaliveTimeoutSeconds maxConnectionsPerHost:(UInt32)maxConnectionsPerHost - statsFlushSeconds:(UInt32)statsFlushSeconds streamIdleTimeoutSeconds:(UInt32)streamIdleTimeoutSeconds perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds appVersion:(NSString *)appVersion @@ -107,19 +106,15 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain keyValueStores: (NSDictionary> *) keyValueStores - statsSinks:(NSArray *)statsSinks nodeId:(nullable NSString *)nodeId nodeRegion:(nullable NSString *)nodeRegion nodeZone:(nullable NSString *)nodeZone nodeSubZone:(nullable NSString *)nodeSubZone xdsServerAddress:(nullable NSString *)xdsServerAddress xdsServerPort:(UInt32)xdsServerPort - xdsAuthHeader:(nullable NSString *)xdsAuthHeader - xdsAuthToken:(nullable NSString *)xdsAuthToken - xdsJwtToken:(nullable NSString *)xdsJwtToken - xdsJwtTokenLifetimeSeconds:(UInt32)xdsJwtTokenLifetimeSeconds + xdsGrpcInitialMetadata: + (NSDictionary *)xdsGrpcInitialMetadata xdsSslRootCerts:(nullable NSString *)xdsSslRootCerts - xdsSni:(nullable NSString *)xdsSni rtdsResourceName:(nullable NSString *)rtdsResourceName rtdsTimeoutSeconds:(UInt32)rtdsTimeoutSeconds enableCds:(BOOL)enableCds @@ -130,7 +125,6 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain return nil; } - self.grpcStatsDomain = grpcStatsDomain; self.connectTimeoutSeconds = connectTimeoutSeconds; self.dnsRefreshSeconds = dnsRefreshSeconds; self.dnsFailureRefreshSecondsBase = dnsFailureRefreshSecondsBase; @@ -142,6 +136,7 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain self.dnsCacheSaveIntervalSeconds = dnsCacheSaveIntervalSeconds; self.enableHttp3 = enableHttp3; self.quicHints = quicHints; + self.quicCanonicalSuffixes = quicCanonicalSuffixes; self.enableGzipDecompression = enableGzipDecompression; self.enableBrotliDecompression = enableBrotliDecompression; self.enableInterfaceBinding = enableInterfaceBinding; @@ -153,7 +148,6 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain h2ConnectionKeepaliveIdleIntervalMilliseconds; self.h2ConnectionKeepaliveTimeoutSeconds = h2ConnectionKeepaliveTimeoutSeconds; self.maxConnectionsPerHost = maxConnectionsPerHost; - self.statsFlushSeconds = statsFlushSeconds; self.streamIdleTimeoutSeconds = streamIdleTimeoutSeconds; self.perTryIdleTimeoutSeconds = perTryIdleTimeoutSeconds; self.appVersion = appVersion; @@ -163,19 +157,14 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain self.httpPlatformFilterFactories = httpPlatformFilterFactories; self.stringAccessors = stringAccessors; self.keyValueStores = keyValueStores; - self.statsSinks = statsSinks; self.nodeId = nodeId; self.nodeRegion = nodeRegion; self.nodeZone = nodeZone; self.nodeSubZone = nodeSubZone; self.xdsServerAddress = xdsServerAddress; self.xdsServerPort = xdsServerPort; - self.xdsAuthHeader = xdsAuthHeader; - self.xdsAuthToken = xdsAuthToken; - self.xdsJwtToken = xdsJwtToken; - self.xdsJwtTokenLifetimeSeconds = xdsJwtTokenLifetimeSeconds; + self.xdsGrpcInitialMetadata = xdsGrpcInitialMetadata; self.xdsSslRootCerts = xdsSslRootCerts; - self.xdsSni = xdsSni; self.rtdsResourceName = rtdsResourceName; self.rtdsTimeoutSeconds = rtdsTimeoutSeconds; self.cdsResourcesLocator = cdsResourcesLocator; @@ -205,6 +194,9 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain for (NSString *host in self.quicHints) { builder.addQuicHint([host toCXXString], [[self.quicHints objectForKey:host] intValue]); } + for (NSString *suffix in self.quicCanonicalSuffixes) { + builder.addQuicCanonicalSuffix([suffix toCXXString]); + } #endif builder.enableGzipDecompression(self.enableGzipDecompression); @@ -247,19 +239,6 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain builder.enablePlatformCertificatesValidation(self.enablePlatformCertificateValidation); builder.enableDnsCache(self.enableDNSCache, self.dnsCacheSaveIntervalSeconds); -#ifdef ENVOY_MOBILE_STATS_REPORTING - if (self.statsSinks.count > 0) { - std::vector sinks; - sinks.reserve(self.statsSinks.count); - for (NSString *sink in self.statsSinks) { - sinks.push_back([sink toCXXString]); - } - builder.addStatsSinks(std::move(sinks)); - } - builder.addGrpcStatsDomain([self.grpcStatsDomain toCXXString]); - builder.addStatsFlushSeconds(self.statsFlushSeconds); -#endif - if (self.nodeRegion != nil) { builder.setNodeLocality([self.nodeRegion toCXXString], [self.nodeZone toCXXString], [self.nodeSubZone toCXXString]); @@ -268,23 +247,16 @@ - (instancetype)initWithGrpcStatsDomain:(nullable NSString *)grpcStatsDomain builder.setNodeId([self.nodeId toCXXString]); } -#ifdef ENVOY_GOOGLE_GRPC +#ifdef ENVOY_MOBILE_XDS if (self.xdsServerAddress != nil) { Envoy::Platform::XdsBuilder xdsBuilder([self.xdsServerAddress toCXXString], self.xdsServerPort); - if (self.xdsAuthHeader != nil) { - xdsBuilder.setAuthenticationToken([self.xdsAuthHeader toCXXString], - [self.xdsAuthToken toCXXString]); - } - if (self.xdsJwtToken != nil) { - xdsBuilder.setJwtAuthenticationToken([self.xdsJwtToken toCXXString], - self.xdsJwtTokenLifetimeSeconds); + for (NSString *header in self.xdsGrpcInitialMetadata) { + xdsBuilder.addInitialStreamHeader( + [header toCXXString], [[self.xdsGrpcInitialMetadata objectForKey:header] toCXXString]); } if (self.xdsSslRootCerts != nil) { xdsBuilder.setSslRootCerts([self.xdsSslRootCerts toCXXString]); } - if (self.xdsSni != nil) { - xdsBuilder.setSni([self.xdsSni toCXXString]); - } if (self.rtdsResourceName != nil) { xdsBuilder.addRuntimeDiscoveryService([self.rtdsResourceName toCXXString], self.rtdsTimeoutSeconds); diff --git a/mobile/library/objective-c/EnvoyEngine.h b/mobile/library/objective-c/EnvoyEngine.h index 9578f07988e2..80c47aae7c4c 100644 --- a/mobile/library/objective-c/EnvoyEngine.h +++ b/mobile/library/objective-c/EnvoyEngine.h @@ -77,11 +77,6 @@ NS_ASSUME_NONNULL_BEGIN */ - (int)recordCounterInc:(NSString *)elements tags:(EnvoyTags *)tags count:(NSUInteger)count; -/** - Attempt to trigger a stat flush. - */ -- (void)flushStats; - /** Retrieve the value of all active stats. Note that this function may block for some time. @return The list of active stats and their values, or empty string of the operation failed diff --git a/mobile/library/objective-c/EnvoyEngineImpl.mm b/mobile/library/objective-c/EnvoyEngineImpl.mm index 3cb922742fed..f51d160338b5 100644 --- a/mobile/library/objective-c/EnvoyEngineImpl.mm +++ b/mobile/library/objective-c/EnvoyEngineImpl.mm @@ -528,9 +528,9 @@ - (int)runWithConfig:(EnvoyConfiguration *)config logLevel:(NSString *)logLevel [self startObservingLifecycleNotifications]; @try { - auto options = std::make_unique(); + auto options = std::make_unique(); options->setConfigProto(std::move(bootstrap)); - options->setLogLevel(options->parseAndValidateLogLevel(logLevel.UTF8String)); + ENVOY_BUG(options->setLogLevel(logLevel.UTF8String).ok(), "invalid log level"); options->setConcurrency(1); return reinterpret_cast(_engineHandle)->run(std::move(options)); } @catch (NSException *exception) { @@ -566,10 +566,6 @@ - (int)recordCounterInc:(NSString *)elements tags:(EnvoyTags *)tags count:(NSUIn return record_counter_inc(_engineHandle, elements.UTF8String, toNativeStatsTags(tags), count); } -- (void)flushStats { - flush_stats(_engineHandle); -} - - (NSString *)dumpStats { envoy_data data; envoy_status_t status = dump_stats(_engineHandle, &data); diff --git a/mobile/library/objective-c/EnvoyHTTPStreamImpl.m b/mobile/library/objective-c/EnvoyHTTPStreamImpl.m index 3613164dd6e4..16d6f0af0f93 100644 --- a/mobile/library/objective-c/EnvoyHTTPStreamImpl.m +++ b/mobile/library/objective-c/EnvoyHTTPStreamImpl.m @@ -174,7 +174,7 @@ - (instancetype)initWithHandle:(envoy_stream_t)handle // start_stream could result in a reset that would release the native ref. _strongSelf = self; envoy_status_t result = - start_stream(engineHandle, _streamHandle, native_callbacks, explicitFlowControl, 0); + start_stream(engineHandle, _streamHandle, native_callbacks, explicitFlowControl); if (result != ENVOY_SUCCESS) { _strongSelf = nil; return nil; diff --git a/mobile/library/swift/BUILD b/mobile/library/swift/BUILD index 58b660e7b607..2aba9b460e7a 100644 --- a/mobile/library/swift/BUILD +++ b/mobile/library/swift/BUILD @@ -1,17 +1,18 @@ -load("//bazel:apple.bzl", "envoy_mobile_swift_copts") -load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") -load("//bazel:swift_header_collector.bzl", "swift_header_collector") load("@build_bazel_rules_apple//apple:apple.bzl", "apple_static_xcframework") load("@build_bazel_rules_apple//apple:ios.bzl", "ios_static_framework") load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") -load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_defines", "envoy_mobile_package") +load("//bazel:apple.bzl", "envoy_mobile_swift_copts") +load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") +load("//bazel:swift_header_collector.bzl", "swift_header_collector") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + swift_library( name = "ios_lib", srcs = [ - "CompressionAlgorithm.swift", "DirectResponse.swift", "Engine.swift", "EngineBuilder.swift", diff --git a/mobile/library/swift/CompressionAlgorithm.swift b/mobile/library/swift/CompressionAlgorithm.swift deleted file mode 100644 index fd6dbbb88105..000000000000 --- a/mobile/library/swift/CompressionAlgorithm.swift +++ /dev/null @@ -1,7 +0,0 @@ -#if ENVOY_MOBILE_REQUEST_COMPRESSION -/// Available algorithms to compress requests. -public enum CompressionAlgorithm: String { - case gzip - case brotli -} -#endif diff --git a/mobile/library/swift/Engine.swift b/mobile/library/swift/Engine.swift index 7360b2a46827..ccd682a9d123 100644 --- a/mobile/library/swift/Engine.swift +++ b/mobile/library/swift/Engine.swift @@ -10,11 +10,6 @@ public protocol Engine: AnyObject { /// - returns: A client for recording time series metrics. func pulseClient() -> PulseClient - /// Flush the stats sinks outside of a flushing interval. - /// Note: stat flushing is done asynchronously, this function will never block. - /// This is a noop if called before the underlying EnvoyEngine has started. - func flushStats() - func dumpStats() -> String /// Terminates the running engine. diff --git a/mobile/library/swift/EngineBuilder.swift b/mobile/library/swift/EngineBuilder.swift index 42132ceff7dd..130fda314563 100644 --- a/mobile/library/swift/EngineBuilder.swift +++ b/mobile/library/swift/EngineBuilder.swift @@ -6,7 +6,7 @@ import Foundation // swiftlint:disable file_length -#if ENVOY_GOOGLE_GRPC +#if ENVOY_MOBILE_XDS /// Builder for generating the xDS configuration for the Envoy Mobile engine. /// xDS is a protocol for dynamic configuration of Envoy instances, more information can be found in /// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol. @@ -14,17 +14,12 @@ import Foundation /// This class is typically used as input to the EngineBuilder's setXds() method. @objcMembers open class XdsBuilder: NSObject { - public static let defaultJwtTokenLifetimeInSeconds: UInt32 = 60 * 60 * 24 * 90 // 90 days public static let defaultXdsTimeoutInSeconds: UInt32 = 5 let xdsServerAddress: String let xdsServerPort: UInt32 - var authHeader: String? - var authToken: String? - var jwtToken: String? - var jwtTokenLifetimeInSeconds: UInt32 = XdsBuilder.defaultJwtTokenLifetimeInSeconds + var xdsGrpcInitialMetadata: [String: String] = [:] var sslRootCerts: String? - var sni: String? var rtdsResourceName: String? var rtdsTimeoutInSeconds: UInt32 = 0 var enableCds: Bool = false @@ -40,38 +35,26 @@ open class XdsBuilder: NSObject { self.xdsServerPort = xdsServerPort } - /// Sets the authentication HTTP header and token value for authentication with the xDS - /// management server. + /// Adds a header to the initial HTTP metadata headers sent on the gRPC stream. /// - /// - parameter header: The HTTP authentication header. - /// - parameter token: The authentication token to be sent in the header. + /// A common use for the initial metadata headers is for authentication to the xDS management + /// server. /// - /// - returns: This builder. - @discardableResult - public func setAuthenticationToken( - header: String, - token: String) -> Self { - self.authHeader = header - self.authToken = token - return self - } - - /// Sets JWT as the authentication method to the xDS management server, using the given token. + /// For example, if using API keys to authenticate to Traffic Director on GCP (see + /// https://cloud.google.com/docs/authentication/api-keys for details), invoke: + /// builder.addInitialStreamHeader("x-goog-api-key", apiKeyToken) + /// .addInitialStreamHeader("X-Android-Package", appPackageName) + /// .addInitialStreamHeader("X-Android-Cert", sha1KeyFingerprint); /// - /// - parameter token: The JWT token used to authenticate the client to the xDS - /// management server. - /// - parameter tokenLifetimeInSeconds: the lifetime of the JWT token, in seconds. If - /// none (or 0) is specified, then - /// defaultJwtTokenLifetimeSeconds is used. + /// - parameter header: The HTTP header to add on the gRPC stream's initial metadata. + /// - parameter value: The HTTP header value to add on the gRPC stream's initial metadata. /// /// - returns: This builder. @discardableResult - public func setJwtAuthenticationToken( - token: String, - tokenLifetimeInSeconds: UInt32 = XdsBuilder.defaultJwtTokenLifetimeInSeconds) -> Self { - self.jwtToken = token - self.jwtTokenLifetimeInSeconds = (tokenLifetimeInSeconds > 0) ? - tokenLifetimeInSeconds : XdsBuilder.defaultJwtTokenLifetimeInSeconds + public func addInitialStreamHeader( + header: String, + value: String) -> Self { + self.xdsGrpcInitialMetadata[header] = value return self } @@ -87,19 +70,6 @@ open class XdsBuilder: NSObject { return self } - /// Sets the SNI (https://datatracker.ietf.org/doc/html/rfc6066#section-3) on the TLS handshake - /// and the authority HTTP header. If not set, the SNI is set by default to the xDS server address - /// and the authority HTTP header is not set. - /// - /// - parameter sni: The SNI (server name identification) value. - /// - /// - returns: This builder. - @discardableResult - public func setSni(sni: String) -> Self { - self.sni = sni - return self - } - /// Adds Runtime Discovery Service (RTDS) to the Runtime layers of the Bootstrap configuration, /// to retrieve dynamic runtime configuration via the xDS management server. /// @@ -165,7 +135,6 @@ open class EngineBuilder: NSObject { case custom(String) } - private var grpcStatsDomain: String? private var connectTimeoutSeconds: UInt32 = 30 private var dnsFailureRefreshSecondsBase: UInt32 = 2 private var dnsFailureRefreshSecondsMax: UInt32 = 10 @@ -183,6 +152,7 @@ open class EngineBuilder: NSObject { private var enableHttp3: Bool = false #endif private var quicHints: [String: Int] = [:] + private var quicCanonicalSuffixes: [String] = [] private var enableInterfaceBinding: Bool = false private var enforceTrustChainVerification: Bool = true private var enablePlatformCertificateValidation: Bool = false @@ -191,7 +161,6 @@ open class EngineBuilder: NSObject { private var h2ConnectionKeepaliveIdleIntervalMilliseconds: UInt32 = 1 private var h2ConnectionKeepaliveTimeoutSeconds: UInt32 = 10 private var maxConnectionsPerHost: UInt32 = 7 - private var statsFlushSeconds: UInt32 = 60 private var streamIdleTimeoutSeconds: UInt32 = 15 private var perTryIdleTimeoutSeconds: UInt32 = 15 private var appVersion: String = "unspecified" @@ -205,12 +174,11 @@ open class EngineBuilder: NSObject { private var stringAccessors: [String: EnvoyStringAccessor] = [:] private var keyValueStores: [String: EnvoyKeyValueStore] = [:] private var runtimeGuards: [String: Bool] = [:] - private var statsSinks: [String] = [] private var nodeID: String? private var nodeRegion: String? private var nodeZone: String? private var nodeSubZone: String? -#if ENVOY_GOOGLE_GRPC +#if ENVOY_MOBILE_XDS private var xdsBuilder: XdsBuilder? #endif private var enableSwiftBootstrap = false @@ -230,33 +198,6 @@ open class EngineBuilder: NSObject { self.base = .custom(yaml) } -#if ENVOY_MOBILE_STATS_REPORTING - /// Add a stats domain for Envoy to flush stats to. - /// Passing nil disables stats emission. - /// - /// - parameter grpcStatsDomain: The domain to use for stats. - /// - /// - returns: This builder. - @discardableResult - public func addGrpcStatsDomain(_ grpcStatsDomain: String?) -> Self { - self.grpcStatsDomain = grpcStatsDomain - return self - } - - /// Adds additional stats sink, in the form of the raw YAML/JSON configuration. - /// Sinks added in this fashion will be included in addition to the gRPC stats sink - /// that may be enabled via addGrpcStatsDomain. - /// - /// - parameter statsSinks: Configurations of stat sinks to add. - /// - /// - returns: This builder. - @discardableResult - public func addStatsSinks(_ statsSinks: [String]) -> Self { - self.statsSinks = statsSinks - return self - } -#endif - /// Add a log level to use with Envoy. /// /// - parameter logLevel: The log level to use with Envoy. @@ -400,18 +341,18 @@ open class EngineBuilder: NSObject { self.quicHints[host] = port return self } -#endif - /// Add an interval at which to flush Envoy stats. + /// Add a host suffix that's known to support QUIC. /// - /// - parameter statsFlushSeconds: Interval at which to flush Envoy stats. + /// - parameter suffix: the string representation of the host suffix /// /// - returns: This builder. @discardableResult - public func addStatsFlushSeconds(_ statsFlushSeconds: UInt32) -> Self { - self.statsFlushSeconds = statsFlushSeconds + public func addQuicCanonicalSuffix(_ suffix: String) -> Self { + self.quicCanonicalSuffixes.append(suffix) return self } +#endif /// Specify whether sockets may attempt to bind to a specific interface, based on network /// conditions. @@ -712,7 +653,7 @@ open class EngineBuilder: NSObject { return self } -#if ENVOY_GOOGLE_GRPC +#if ENVOY_MOBILE_XDS /// Sets the xDS configuration for the Envoy Mobile engine. /// /// - parameter xdsBuilder: The XdsBuilder instance which specifies the xDS config options. @@ -785,27 +726,19 @@ open class EngineBuilder: NSObject { func makeConfig() -> EnvoyConfiguration { var xdsServerAddress: String? var xdsServerPort: UInt32 = 0 - var xdsAuthHeader: String? - var xdsAuthToken: String? - var xdsJwtToken: String? - var xdsJwtTokenLifetimeSeconds: UInt32 = 0 + var xdsGrpcInitialMetadata: [String: String] = [:] var xdsSslRootCerts: String? - var xdsSni: String? var rtdsResourceName: String? var rtdsTimeoutSeconds: UInt32 = 0 var enableCds: Bool = false var cdsResourcesLocator: String? var cdsTimeoutSeconds: UInt32 = 0 -#if ENVOY_GOOGLE_GRPC +#if ENVOY_MOBILE_XDS xdsServerAddress = self.xdsBuilder?.xdsServerAddress xdsServerPort = self.xdsBuilder?.xdsServerPort ?? 0 - xdsAuthHeader = self.xdsBuilder?.authHeader - xdsAuthToken = self.xdsBuilder?.authToken - xdsJwtToken = self.xdsBuilder?.jwtToken - xdsJwtTokenLifetimeSeconds = self.xdsBuilder?.jwtTokenLifetimeInSeconds ?? 0 + xdsGrpcInitialMetadata = self.xdsBuilder?.xdsGrpcInitialMetadata ?? [:] xdsSslRootCerts = self.xdsBuilder?.sslRootCerts - xdsSni = self.xdsBuilder?.sni rtdsResourceName = self.xdsBuilder?.rtdsResourceName rtdsTimeoutSeconds = self.xdsBuilder?.rtdsTimeoutInSeconds ?? 0 enableCds = self.xdsBuilder?.enableCds ?? false @@ -814,7 +747,6 @@ open class EngineBuilder: NSObject { #endif return EnvoyConfiguration( - grpcStatsDomain: self.grpcStatsDomain, connectTimeoutSeconds: self.connectTimeoutSeconds, dnsRefreshSeconds: self.dnsRefreshSeconds, dnsFailureRefreshSecondsBase: self.dnsFailureRefreshSecondsBase, @@ -826,6 +758,7 @@ open class EngineBuilder: NSObject { dnsCacheSaveIntervalSeconds: self.dnsCacheSaveIntervalSeconds, enableHttp3: self.enableHttp3, quicHints: self.quicHints.mapValues { NSNumber(value: $0) }, + quicCanonicalSuffixes: self.quicCanonicalSuffixes, enableGzipDecompression: self.enableGzipDecompression, enableBrotliDecompression: self.enableBrotliDecompression, enableInterfaceBinding: self.enableInterfaceBinding, @@ -837,7 +770,6 @@ open class EngineBuilder: NSObject { self.h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds: self.h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost: self.maxConnectionsPerHost, - statsFlushSeconds: self.statsFlushSeconds, streamIdleTimeoutSeconds: self.streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds: self.perTryIdleTimeoutSeconds, appVersion: self.appVersion, @@ -847,19 +779,14 @@ open class EngineBuilder: NSObject { platformFilterChain: self.platformFilterChain, stringAccessors: self.stringAccessors, keyValueStores: self.keyValueStores, - statsSinks: self.statsSinks, nodeId: self.nodeID, nodeRegion: self.nodeRegion, nodeZone: self.nodeZone, nodeSubZone: self.nodeSubZone, xdsServerAddress: xdsServerAddress, xdsServerPort: xdsServerPort, - xdsAuthHeader: xdsAuthHeader, - xdsAuthToken: xdsAuthToken, - xdsJwtToken: xdsJwtToken, - xdsJwtTokenLifetimeSeconds: xdsJwtTokenLifetimeSeconds, + xdsGrpcInitialMetadata: xdsGrpcInitialMetadata, xdsSslRootCerts: xdsSslRootCerts, - xdsSni: xdsSni, rtdsResourceName: rtdsResourceName, rtdsTimeoutSeconds: rtdsTimeoutSeconds, enableCds: enableCds, @@ -885,9 +812,6 @@ private extension EngineBuilder { func generateBootstrap() -> Bootstrap { var cxxBuilder = Envoy.Platform.EngineBuilder() cxxBuilder.addLogLevel(self.logLevel.toCXX()) - if let grpcStatsDomain = self.grpcStatsDomain { - cxxBuilder.addGrpcStatsDomain(grpcStatsDomain.toCXX()) - } cxxBuilder.addConnectTimeoutSeconds(Int32(self.connectTimeoutSeconds)) cxxBuilder.addDnsRefreshSeconds(Int32(self.dnsRefreshSeconds)) @@ -902,6 +826,9 @@ private extension EngineBuilder { for (host, port) in self.quicHints { cxxBuilder.addQuicHint(host.toCXX(), Int32(port)) } + for (suffix) in self.quicCanonicalSuffixes { + cxxBuilder.addQuicCanonicalSuffix(suffix.toCXX()) + } #endif cxxBuilder.enableGzipDecompression(self.enableGzipDecompression) cxxBuilder.enableBrotliDecompression(self.enableBrotliDecompression) @@ -917,7 +844,6 @@ private extension EngineBuilder { Int32(self.h2ConnectionKeepaliveTimeoutSeconds) ) cxxBuilder.addMaxConnectionsPerHost(Int32(self.maxConnectionsPerHost)) - cxxBuilder.addStatsFlushSeconds(Int32(self.statsFlushSeconds)) cxxBuilder.setStreamIdleTimeoutSeconds(Int32(self.streamIdleTimeoutSeconds)) cxxBuilder.setPerTryIdleTimeoutSeconds(Int32(self.perTryIdleTimeoutSeconds)) cxxBuilder.setAppVersion(self.appVersion.toCXX()) @@ -936,8 +862,6 @@ private extension EngineBuilder { cxxBuilder.addPlatformFilter(filter.filterName.toCXX()) } - cxxBuilder.addStatsSinks(self.statsSinks.toCXX()) - if let nodeRegion = self.nodeRegion, let nodeZone = self.nodeZone, @@ -956,24 +880,16 @@ private extension EngineBuilder { } private func generateXds(_ cxxBuilder: inout Envoy.Platform.EngineBuilder) { -#if ENVOY_GOOGLE_GRPC +#if ENVOY_MOBILE_XDS if let xdsBuilder = self.xdsBuilder { var cxxXdsBuilder = Envoy.Platform.XdsBuilder(xdsBuilder.xdsServerAddress.toCXX(), - Int32(xdsBuilder.xdsServerPort)) - if let xdsAuthHeader = xdsBuilder.authHeader { - cxxXdsBuilder.setAuthenticationToken(xdsAuthHeader.toCXX(), - xdsBuilder.authToken?.toCXX() ?? "".toCXX()) - } - if let xdsJwtToken = xdsBuilder.jwtToken { - cxxXdsBuilder.setJwtAuthenticationToken(xdsJwtToken.toCXX(), - Int32(xdsBuilder.jwtTokenLifetimeInSeconds)) + xdsBuilder.xdsServerPort) + for (header, value) in xdsBuilder.xdsGrpcInitialMetadata { + cxxXdsBuilder.addInitialStreamHeader(header.toCXX(), value.toCXX()) } if let xdsSslRootCerts = xdsBuilder.sslRootCerts { cxxXdsBuilder.setSslRootCerts(xdsSslRootCerts.toCXX()) } - if let xdsSni = xdsBuilder.sni { - cxxXdsBuilder.setSni(xdsSni.toCXX()) - } if let rtdsResourceName = xdsBuilder.rtdsResourceName { cxxXdsBuilder.addRuntimeDiscoveryService(rtdsResourceName.toCXX(), Int32(xdsBuilder.rtdsTimeoutInSeconds)) diff --git a/mobile/library/swift/EngineImpl.swift b/mobile/library/swift/EngineImpl.swift index 2482a43fedc6..d23b06db8f68 100644 --- a/mobile/library/swift/EngineImpl.swift +++ b/mobile/library/swift/EngineImpl.swift @@ -58,10 +58,6 @@ extension EngineImpl: Engine { return self.pulseClientImpl } - func flushStats() { - self.engine.flushStats() - } - func dumpStats() -> String { self.engine.dumpStats() } diff --git a/mobile/library/swift/EnvoyCxxSwiftInterop/BUILD b/mobile/library/swift/EnvoyCxxSwiftInterop/BUILD index bed4212e421f..21f99fe24222 100644 --- a/mobile/library/swift/EnvoyCxxSwiftInterop/BUILD +++ b/mobile/library/swift/EnvoyCxxSwiftInterop/BUILD @@ -1,8 +1,10 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_c_module") -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_mobile_package") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_cc_library( name = "cxx_swift_interop_lib", srcs = ["cxx_swift_interop.cc"], diff --git a/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.cc b/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.cc index b4786e86f01d..3f72d4764ac9 100644 --- a/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.cc +++ b/mobile/library/swift/EnvoyCxxSwiftInterop/cxx_swift_interop.cc @@ -1,6 +1,6 @@ #include "cxx_swift_interop.h" -#include "source/server/options_impl.h" +#include "source/server/options_impl_base.h" #include "library/common/engine.h" @@ -8,7 +8,7 @@ namespace Envoy { namespace CxxSwift { void run(BootstrapPtr bootstrap_ptr, Platform::LogLevel log_level, envoy_engine_t engine_handle) { - auto options = std::make_unique(); + auto options = std::make_unique(); options->setConfigProto(bootstrapFromPtr(bootstrap_ptr)); options->setLogLevel(static_cast(log_level)); options->setConcurrency(1); diff --git a/mobile/library/swift/RequestHeadersBuilder.swift b/mobile/library/swift/RequestHeadersBuilder.swift index 848f86bf6460..95724ee45a25 100644 --- a/mobile/library/swift/RequestHeadersBuilder.swift +++ b/mobile/library/swift/RequestHeadersBuilder.swift @@ -34,23 +34,6 @@ public final class RequestHeadersBuilder: HeadersBuilder { return self } -#if ENVOY_MOBILE_REQUEST_COMPRESSION - /// Compress this request's body using the specified algorithm. - /// - /// - note: Will only apply if the content length exceeds 30 bytes. - /// - /// - parameter algorithm: The compression algorithm to use to compress this request. - /// - /// - returns: This builder. - @discardableResult - public func enableRequestCompression(using algorithm: CompressionAlgorithm) - -> RequestHeadersBuilder - { - self.internalSet(name: "x-envoy-mobile-compression", value: [algorithm.rawValue]) - return self - } -#endif - /// Build the request headers using the current builder. /// /// - returns: New instance of request headers. diff --git a/mobile/library/swift/mocks/MockEnvoyEngine.swift b/mobile/library/swift/mocks/MockEnvoyEngine.swift index fa5e2c53ba92..43c81d6744ce 100644 --- a/mobile/library/swift/mocks/MockEnvoyEngine.swift +++ b/mobile/library/swift/mocks/MockEnvoyEngine.swift @@ -18,7 +18,6 @@ final class MockEnvoyEngine: NSObject { /// Closure called when `recordCounterInc(_:tags:count:)` is called. static var onRecordCounter: ( (_ elements: String, _ tags: [String: String], _ count: UInt) -> Void)? - static var onFlushStats: (() -> Void)? } extension MockEnvoyEngine: EnvoyEngine { @@ -45,10 +44,6 @@ extension MockEnvoyEngine: EnvoyEngine { return kEnvoySuccess } - func flushStats() { - MockEnvoyEngine.onFlushStats?() - } - func dumpStats() -> String { return "" } diff --git a/mobile/test/cc/integration/BUILD b/mobile/test/cc/integration/BUILD index c592a1359736..4c9444c9c7e6 100644 --- a/mobile/test/cc/integration/BUILD +++ b/mobile/test/cc/integration/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") load("@envoy//bazel:envoy_select.bzl", "envoy_select_enable_yaml") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "send_headers_test", diff --git a/mobile/test/cc/unit/BUILD b/mobile/test/cc/unit/BUILD index 6fa4f0f3a3e1..2e67c30eea75 100644 --- a/mobile/test/cc/unit/BUILD +++ b/mobile/test/cc/unit/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") load("@envoy//bazel:envoy_select.bzl", "envoy_select_enable_yaml") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "envoy_config_test", @@ -15,6 +15,7 @@ envoy_cc_test( deps = [ "//library/cc:engine_builder_lib", "//library/cc:envoy_engine_cc_lib_no_stamp", + "@envoy_api//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg_cc_proto", "@envoy_build_config//:extension_registry", "@envoy_build_config//:test_extensions", ], diff --git a/mobile/test/cc/unit/envoy_config_test.cc b/mobile/test/cc/unit/envoy_config_test.cc index 8c57c8acd1c0..414cf2b0af79 100644 --- a/mobile/test/cc/unit/envoy_config_test.cc +++ b/mobile/test/cc/unit/envoy_config_test.cc @@ -1,8 +1,11 @@ #include #include +#include "envoy/extensions/clusters/dynamic_forward_proxy/v3/cluster.pb.h" + #include "test/test_common/utility.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "absl/synchronization/notification.h" #include "gtest/gtest.h" @@ -15,29 +18,39 @@ #include "source/extensions/network/dns_resolver/apple/apple_dns_impl.h" #endif +namespace Envoy { +namespace { + +using namespace Platform; + using envoy::config::bootstrap::v3::Bootstrap; +using DfpClusterConfig = ::envoy::extensions::clusters::dynamic_forward_proxy::v3::ClusterConfig; using testing::HasSubstr; using testing::IsEmpty; using testing::Not; using testing::SizeIs; -namespace Envoy { -namespace { - -using namespace Platform; +DfpClusterConfig getDfpClusterConfig(const Bootstrap& bootstrap) { + DfpClusterConfig cluster_config; + const auto& clusters = bootstrap.static_resources().clusters(); + for (const auto& cluster : clusters) { + if (cluster.name() == "base") { + MessageUtil::unpackTo(cluster.cluster_type().typed_config(), cluster_config); + } + } + return cluster_config; +} TEST(TestConfig, ConfigIsApplied) { EngineBuilder engine_builder; engine_builder -#ifdef ENVOY_MOBILE_STATS_REPORTING - .addGrpcStatsDomain("asdf.fake.website") - .addStatsFlushSeconds(654) -#endif #ifdef ENVOY_ENABLE_QUIC .setHttp3ConnectionOptions("5RTO") .setHttp3ClientConnectionOptions("MPQC") .addQuicHint("www.abc.com", 443) .addQuicHint("www.def.com", 443) + .addQuicCanonicalSuffix(".opq.com") + .addQuicCanonicalSuffix(".xyz.com") #endif .addConnectTimeoutSeconds(123) .addDnsRefreshSeconds(456) @@ -65,15 +78,13 @@ TEST(TestConfig, ConfigIsApplied) { "dns_failure_refresh_rate { base_interval { seconds: 789 } max_interval { seconds: 987 } }", "connection_idle_interval { nanos: 222000000 }", "connection_keepalive { timeout { seconds: 333 }", -#ifdef ENVOY_MOBILE_STATS_REPORTING - "asdf.fake.website", - "stats_flush_interval { seconds: 654 }", -#endif #ifdef ENVOY_ENABLE_QUIC "connection_options: \"5RTO\"", "client_connection_options: \"MPQC\"", "hostname: \"www.abc.com\"", "hostname: \"www.def.com\"", + "canonical_suffixes: \".opq.com\"", + "canonical_suffixes: \".xyz.com\"", #endif "key: \"dns_persistent_cache\" save_interval { seconds: 101 }", "key: \"always_use_v6\" value { bool_value: true }", @@ -228,27 +239,34 @@ TEST(TestConfig, AddMaxConnectionsPerHost) { EXPECT_THAT(bootstrap->ShortDebugString(), HasSubstr("max_connections { value: 16 }")); } -#ifdef ENVOY_MOBILE_STATS_REPORTING -std::string statsdSinkConfig(int port) { - std::string config = R"({ name: envoy.stat_sinks.statsd, - typed_config: { - "@type": type.googleapis.com/envoy.config.metrics.v3.StatsdSink, - address: { socket_address: { address: 127.0.0.1, port_value: )" + - fmt::format("{}", port) + " } } } }"; - return config; -} - -TEST(TestConfig, AddStatsSinks) { +TEST(TestConfig, AddDnsPreresolveHostnames) { EngineBuilder engine_builder; - + engine_builder.addDnsPreresolveHostnames({"google.com", "lyft.com"}); std::unique_ptr bootstrap = engine_builder.generateBootstrap(); - EXPECT_EQ(bootstrap->stats_sinks_size(), 0); - engine_builder.addStatsSinks({statsdSinkConfig(1), statsdSinkConfig(2)}); + Protobuf::RepeatedPtrField + expected_dns_preresolve_hostnames; + auto& host_addr1 = *expected_dns_preresolve_hostnames.Add(); + host_addr1.set_address("google.com"); + host_addr1.set_port_value(443); + auto& host_addr2 = *expected_dns_preresolve_hostnames.Add(); + host_addr2.set_address("lyft.com"); + host_addr2.set_port_value(443); + EXPECT_TRUE(TestUtility::repeatedPtrFieldEqual( + getDfpClusterConfig(*bootstrap).dns_cache_config().preresolve_hostnames(), + expected_dns_preresolve_hostnames)); + + // Resetting the DNS preresolve hostnames with just "google.com" now. + engine_builder.addDnsPreresolveHostnames({"google.com"}); bootstrap = engine_builder.generateBootstrap(); - EXPECT_EQ(bootstrap->stats_sinks_size(), 2); + expected_dns_preresolve_hostnames.Clear(); + auto& host_addr3 = *expected_dns_preresolve_hostnames.Add(); + host_addr3.set_address("google.com"); + host_addr3.set_port_value(443); + EXPECT_TRUE(TestUtility::repeatedPtrFieldEqual( + getDfpClusterConfig(*bootstrap).dns_cache_config().preresolve_hostnames(), + expected_dns_preresolve_hostnames)); } -#endif TEST(TestConfig, DisableHttp3) { EngineBuilder engine_builder; @@ -273,96 +291,49 @@ TEST(TestConfig, DisableHttp3) { #endif } -#ifdef ENVOY_GOOGLE_GRPC +#ifdef ENVOY_MOBILE_XDS TEST(TestConfig, XdsConfig) { EngineBuilder engine_builder; - XdsBuilder xds_builder(/*xds_server_address=*/"fake-td.googleapis.com", - /*xds_server_port=*/12345); + const std::string host = "fake-td.googleapis.com"; + const uint32_t port = 12345; + const std::string authority = absl::StrCat(host, ":", port); + + XdsBuilder xds_builder(/*xds_server_address=*/host, + /*xds_server_port=*/port); engine_builder.setXds(std::move(xds_builder)); std::unique_ptr bootstrap = engine_builder.generateBootstrap(); + auto& ads_config = bootstrap->dynamic_resources().ads_config(); EXPECT_EQ(ads_config.api_type(), envoy::config::core::v3::ApiConfigSource::GRPC); - EXPECT_EQ(ads_config.grpc_services(0).google_grpc().target_uri(), "fake-td.googleapis.com:12345"); - EXPECT_EQ(ads_config.grpc_services(0).google_grpc().stat_prefix(), "ads"); - EXPECT_THAT(ads_config.grpc_services(0) - .google_grpc() - .channel_credentials() - .ssl_credentials() - .root_certs() - .inline_string(), - IsEmpty()); - EXPECT_THAT(ads_config.grpc_services(0).google_grpc().call_credentials(), SizeIs(0)); - - // With authentication credentials. - xds_builder = - XdsBuilder(/*xds_server_address=*/"fake-td.googleapis.com", /*xds_server_port=*/12345); - xds_builder.setAuthenticationToken(/*header=*/"x-goog-api-key", /*token=*/"A1B2C3"); - xds_builder.setSslRootCerts(/*root_certs=*/"my_root_cert"); - xds_builder.setSni(/*sni=*/"fake-td.googleapis.com"); + EXPECT_EQ(ads_config.grpc_services(0).envoy_grpc().cluster_name(), "base"); + EXPECT_EQ(ads_config.grpc_services(0).envoy_grpc().authority(), authority); + + Protobuf::RepeatedPtrField + expected_dns_preresolve_hostnames; + auto& host_addr1 = *expected_dns_preresolve_hostnames.Add(); + host_addr1.set_address(host); + host_addr1.set_port_value(port); + EXPECT_TRUE(TestUtility::repeatedPtrFieldEqual( + getDfpClusterConfig(*bootstrap).dns_cache_config().preresolve_hostnames(), + expected_dns_preresolve_hostnames)); + + // With initial gRPC metadata. + xds_builder = XdsBuilder(/*xds_server_address=*/host, /*xds_server_port=*/port); + xds_builder.addInitialStreamHeader(/*header=*/"x-goog-api-key", /*value=*/"A1B2C3") + .addInitialStreamHeader(/*header=*/"x-android-package", + /*value=*/"com.google.envoymobile.io.myapp"); engine_builder.setXds(std::move(xds_builder)); bootstrap = engine_builder.generateBootstrap(); - auto& ads_config_with_tokens = bootstrap->dynamic_resources().ads_config(); - EXPECT_EQ(ads_config_with_tokens.api_type(), envoy::config::core::v3::ApiConfigSource::GRPC); - EXPECT_EQ(ads_config_with_tokens.grpc_services(0).google_grpc().target_uri(), - "fake-td.googleapis.com:12345"); - EXPECT_EQ(ads_config_with_tokens.grpc_services(0).google_grpc().stat_prefix(), "ads"); - EXPECT_EQ(ads_config_with_tokens.grpc_services(0) - .google_grpc() - .channel_credentials() - .ssl_credentials() - .root_certs() - .inline_string(), - "my_root_cert"); - EXPECT_EQ(ads_config_with_tokens.grpc_services(0).initial_metadata(0).key(), "x-goog-api-key"); - EXPECT_EQ(ads_config_with_tokens.grpc_services(0).initial_metadata(0).value(), "A1B2C3"); - EXPECT_EQ(ads_config_with_tokens.grpc_services(0) - .google_grpc() - .channel_args() - .args() - .at("grpc.default_authority") - .string_value(), - "fake-td.googleapis.com"); - - // With JWT security credentials. - xds_builder = - XdsBuilder(/*xds_server_address=*/"fake-td.googleapis.com", /*xds_server_port=*/12345); - xds_builder.setJwtAuthenticationToken(/*token=*/"my_jwt_token", - /*token_lifetime_in_seconds=*/500); - xds_builder.setSslRootCerts(/*root_certs=*/"my_root_cert"); - xds_builder.setSni(/*sni=*/"fake-td.googleapis.com"); - engine_builder.setXds(std::move(xds_builder)); - bootstrap = engine_builder.generateBootstrap(); - auto& ads_config_with_jwt_tokens = bootstrap->dynamic_resources().ads_config(); - EXPECT_EQ(ads_config_with_jwt_tokens.api_type(), envoy::config::core::v3::ApiConfigSource::GRPC); - EXPECT_EQ(ads_config_with_jwt_tokens.grpc_services(0).google_grpc().target_uri(), - "fake-td.googleapis.com:12345"); - EXPECT_EQ(ads_config_with_jwt_tokens.grpc_services(0).google_grpc().stat_prefix(), "ads"); - EXPECT_EQ(ads_config_with_jwt_tokens.grpc_services(0) - .google_grpc() - .channel_credentials() - .ssl_credentials() - .root_certs() - .inline_string(), - "my_root_cert"); - EXPECT_EQ(ads_config_with_jwt_tokens.grpc_services(0) - .google_grpc() - .call_credentials(0) - .service_account_jwt_access() - .json_key(), - "my_jwt_token"); - EXPECT_EQ(ads_config_with_jwt_tokens.grpc_services(0) - .google_grpc() - .call_credentials(0) - .service_account_jwt_access() - .token_lifetime_seconds(), - 500); - EXPECT_EQ(ads_config_with_jwt_tokens.grpc_services(0) - .google_grpc() - .channel_args() - .args() - .at("grpc.default_authority") - .string_value(), - "fake-td.googleapis.com"); + auto& ads_config_with_metadata = bootstrap->dynamic_resources().ads_config(); + EXPECT_EQ(ads_config_with_metadata.api_type(), envoy::config::core::v3::ApiConfigSource::GRPC); + EXPECT_EQ(ads_config_with_metadata.grpc_services(0).envoy_grpc().cluster_name(), "base"); + EXPECT_EQ(ads_config_with_metadata.grpc_services(0).envoy_grpc().authority(), authority); + EXPECT_EQ(ads_config_with_metadata.grpc_services(0).initial_metadata(0).key(), "x-goog-api-key"); + EXPECT_EQ(ads_config_with_metadata.grpc_services(0).initial_metadata(0).value(), "A1B2C3"); + EXPECT_EQ(ads_config_with_metadata.grpc_services(0).initial_metadata(1).key(), + "x-android-package"); + EXPECT_EQ(ads_config_with_metadata.grpc_services(0).initial_metadata(1).value(), + "com.google.envoymobile.io.myapp"); } TEST(TestConfig, CopyConstructor) { @@ -503,7 +474,20 @@ TEST(TestConfig, SetNodeLocality) { EXPECT_EQ(bootstrap->node().locality().sub_zone(), sub_zone); } -#ifdef ENVOY_GOOGLE_GRPC +TEST(TestConfig, SetNodeMetadata) { + ProtobufWkt::Struct node_metadata; + (*node_metadata.mutable_fields())["string_field"].set_string_value("some_string"); + (*node_metadata.mutable_fields())["bool_field"].set_bool_value(true); + (*node_metadata.mutable_fields())["number_field"].set_number_value(3.14); + EngineBuilder engine_builder; + engine_builder.setNodeMetadata(node_metadata); + std::unique_ptr bootstrap = engine_builder.generateBootstrap(); + EXPECT_EQ(bootstrap->node().metadata().fields().at("string_field").string_value(), "some_string"); + EXPECT_EQ(bootstrap->node().metadata().fields().at("bool_field").bool_value(), true); + EXPECT_EQ(bootstrap->node().metadata().fields().at("number_field").number_value(), 3.14); +} + +#ifdef ENVOY_MOBILE_XDS TEST(TestConfig, AddCdsLayer) { XdsBuilder xds_builder(/*xds_server_address=*/"fake-xds-server", /*xds_server_port=*/12345); xds_builder.addClusterDiscoveryService(); diff --git a/mobile/test/common/BUILD b/mobile/test/common/BUILD index 1a20e9a5c3af..2751206ca5ba 100644 --- a/mobile/test/common/BUILD +++ b/mobile/test/common/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") load("@envoy//bazel:envoy_select.bzl", "envoy_select_enable_yaml") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "engine_common_test", diff --git a/mobile/test/common/bridge/BUILD b/mobile/test/common/bridge/BUILD index b2f6ef1f1821..b8c7f6b6023c 100644 --- a/mobile/test/common/bridge/BUILD +++ b/mobile/test/common/bridge/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "utility_test", diff --git a/mobile/test/common/buffer/BUILD b/mobile/test/common/buffer/BUILD index 9ba085082197..8e04157c00ad 100644 --- a/mobile/test/common/buffer/BUILD +++ b/mobile/test/common/buffer/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "bridge_fragment_test", diff --git a/mobile/test/common/common/BUILD b/mobile/test/common/common/BUILD index 60cbc7171ee5..55f2dcb7f3ec 100644 --- a/mobile/test/common/common/BUILD +++ b/mobile/test/common/common/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "lambda_logger_delegate_test", diff --git a/mobile/test/common/data/BUILD b/mobile/test/common/data/BUILD index 5a73f31f3607..a282d96774f7 100644 --- a/mobile/test/common/data/BUILD +++ b/mobile/test/common/data/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "utility_test", diff --git a/mobile/test/common/engine_common_test.cc b/mobile/test/common/engine_common_test.cc index 3b400df0231f..ffb511fa51a6 100644 --- a/mobile/test/common/engine_common_test.cc +++ b/mobile/test/common/engine_common_test.cc @@ -7,7 +7,7 @@ namespace Envoy { TEST(EngineCommonTest, SignalHandlingFalse) { ExtensionRegistry::registerFactories(); - auto options = std::make_unique(); + auto options = std::make_unique(); Platform::EngineBuilder builder; options->setConfigProto(builder.generateBootstrap()); diff --git a/mobile/test/common/engine_test.cc b/mobile/test/common/engine_test.cc index 1b0715bb2ac1..fff42154c5df 100644 --- a/mobile/test/common/engine_test.cc +++ b/mobile/test/common/engine_test.cc @@ -2,7 +2,6 @@ #include "gtest/gtest.h" #include "library/cc/engine_builder.h" #include "library/common/engine.h" -#include "library/common/engine_handle.h" #include "library/common/main_interface.h" namespace Envoy { @@ -58,7 +57,7 @@ TEST_F(EngineTest, EarlyExit) { ASSERT_EQ(engine_->terminate(), ENVOY_SUCCESS); ASSERT_TRUE(test_context.on_exit.WaitForNotificationWithTimeout(absl::Seconds(10))); - start_stream(handle, 0, {}, false, 0); + start_stream(handle, 0, {}, false); engine_.reset(); } @@ -79,21 +78,15 @@ TEST_F(EngineTest, AccessEngineAfterInitialization) { ASSERT_TRUE(test_context.on_engine_running.WaitForNotificationWithTimeout(absl::Seconds(10))); absl::Notification getClusterManagerInvoked; - // Scheduling on the dispatcher should work, the engine is running. - EXPECT_EQ(ENVOY_SUCCESS, EngineHandle::runOnEngineDispatcher( - handle, [&getClusterManagerInvoked](Envoy::Engine& engine) { - engine.getClusterManager(); - getClusterManagerInvoked.Notify(); - })); - - // Validate that we actually invoked the function. - EXPECT_TRUE(getClusterManagerInvoked.WaitForNotificationWithTimeout(absl::Seconds(1))); + envoy_data stats_data; + // Running engine functions should work because the engine is running + EXPECT_EQ(ENVOY_SUCCESS, dump_stats(handle, &stats_data)); + release_envoy_data(stats_data); engine_->terminate(); // Now that the engine has been shut down, we no longer expect scheduling to work. - EXPECT_EQ(ENVOY_FAILURE, EngineHandle::runOnEngineDispatcher( - handle, [](Envoy::Engine& engine) { engine.getClusterManager(); })); + EXPECT_EQ(ENVOY_FAILURE, dump_stats(handle, &stats_data)); engine_.reset(); } diff --git a/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD b/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD index 270bdd75a91e..143630841677 100644 --- a/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD +++ b/mobile/test/common/extensions/cert_validator/platform_bridge/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load( "@envoy//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -6,7 +6,7 @@ load( licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_extension_cc_test( name = "platform_bridge_cert_validator_test", diff --git a/mobile/test/common/extensions/filters/http/network_configuration/BUILD b/mobile/test/common/extensions/filters/http/network_configuration/BUILD index 9f98add3a0d9..9630ee6652bb 100644 --- a/mobile/test/common/extensions/filters/http/network_configuration/BUILD +++ b/mobile/test/common/extensions/filters/http/network_configuration/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load( "@envoy//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -6,7 +6,7 @@ load( licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_extension_cc_test( name = "network_configuration_filter_test", diff --git a/mobile/test/common/extensions/filters/http/platform_bridge/BUILD b/mobile/test/common/extensions/filters/http/platform_bridge/BUILD index 0093f2409b5b..1402a3626687 100644 --- a/mobile/test/common/extensions/filters/http/platform_bridge/BUILD +++ b/mobile/test/common/extensions/filters/http/platform_bridge/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load( "@envoy//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -6,7 +6,7 @@ load( licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_extension_cc_test( name = "platform_bridge_filter_test", diff --git a/mobile/test/common/extensions/filters/http/platform_bridge/platform_bridge_filter_test.cc b/mobile/test/common/extensions/filters/http/platform_bridge/platform_bridge_filter_test.cc index 8b85c6441b97..e6fc3b7375a3 100644 --- a/mobile/test/common/extensions/filters/http/platform_bridge/platform_bridge_filter_test.cc +++ b/mobile/test/common/extensions/filters/http/platform_bridge/platform_bridge_filter_test.cc @@ -40,9 +40,9 @@ envoy_headers make_envoy_headers(std::vector class PlatformBridgeFilterTest : public testing::Test { public: - void setUpFilter(std::string&& yaml, envoy_http_filter* platform_filter) { + void setUpFilter(std::string name, envoy_http_filter* platform_filter) { envoymobile::extensions::filters::http::platform_bridge::PlatformBridge config; - TestUtility::loadFromYaml(yaml, config); + config.set_platform_filter_name(name); Api::External::registerApi(config.platform_filter_name(), platform_filter); config_ = std::make_shared(context_, config); @@ -110,7 +110,7 @@ class PlatformBridgeFilterTest : public testing::Test { TEST_F(PlatformBridgeFilterTest, NullImplementation) { envoy_http_filter* null_filter = static_cast(safe_calloc(1, sizeof(envoy_http_filter))); - setUpFilter("platform_filter_name: NullImplementation\n", null_filter); + setUpFilter("NullImplementation", null_filter); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); @@ -155,7 +155,7 @@ TEST_F(PlatformBridgeFilterTest, PartialNullImplementation) { filter_invocations* invocations = static_cast(const_cast(context)); invocations->release_filter_calls++; }; - setUpFilter("platform_filter_name: PartialNullImplementation\n", noop_filter); + setUpFilter("PartialNullImplementation", noop_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -205,10 +205,7 @@ TEST_F(PlatformBridgeFilterTest, BasicContinueOnRequestHeaders) { return {kEnvoyFilterHeadersStatusContinue, c_headers}; }; - setUpFilter(R"EOF( -platform_filter_name: BasicContinueOnRequestHeaders -)EOF", - &platform_filter); + setUpFilter("BasicContinueOnRequestHeaders", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -252,10 +249,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenResumeOnData) { return {kEnvoyFilterDataStatusResumeIteration, c_data, modified_headers}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenResumeOnData -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenResumeOnData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -317,10 +311,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenResumeOnResumeDecoding) return {kEnvoyFilterResumeStatusResumeIteration, modified_headers, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenResumeOnResumeDecoding -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenResumeOnResumeDecoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -390,10 +381,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenResumeOnResumeDecodingW return {kEnvoyFilterResumeStatusResumeIteration, modified_headers, modified_data, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenResumeOnResumeDecoding -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenResumeOnResumeDecoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -464,10 +452,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenResumeOnResumeDecodingW return {kEnvoyFilterResumeStatusResumeIteration, modified_headers, nullptr, modified_trailers}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenResumeOnResumeDecoding -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenResumeOnResumeDecoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -538,10 +523,7 @@ TEST_F(PlatformBridgeFilterTest, AsyncResumeDecodingIsNoopAfterPreviousResume) { return {kEnvoyFilterResumeStatusResumeIteration, nullptr, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: AsyncResumeDecodingIsNoopAfterPreviousResume -)EOF", - &platform_filter); + setUpFilter("AsyncResumeDecodingIsNoopAfterPreviousResume", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -590,10 +572,7 @@ TEST_F(PlatformBridgeFilterTest, BasicContinueOnRequestData) { return {kEnvoyFilterDataStatusContinue, c_data, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: BasicContinueOnRequestData -)EOF", - &platform_filter); + setUpFilter("BasicContinueOnRequestData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl request_data = Buffer::OwnedImpl("request body"); @@ -634,10 +613,7 @@ TEST_F(PlatformBridgeFilterTest, StopAndBufferOnRequestData) { callback(decoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopAndBufferOnRequestData -)EOF", - &platform_filter); + setUpFilter("StopAndBufferOnRequestData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl first_chunk = Buffer::OwnedImpl("A"); @@ -696,10 +672,7 @@ TEST_F(PlatformBridgeFilterTest, BasicError) { release_envoy_error(c_error); }; - setUpFilter(R"EOF( -platform_filter_name: BasicError -)EOF", - &platform_filter); + setUpFilter("BasicError", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{ @@ -766,10 +739,7 @@ TEST_F(PlatformBridgeFilterTest, StopAndBufferThenResumeOnRequestData) { callback(decoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopAndBufferThenResumeOnRequestData -)EOF", - &platform_filter); + setUpFilter("StopAndBufferThenResumeOnRequestData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl first_chunk = Buffer::OwnedImpl("A"); @@ -852,10 +822,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenBufferThenResumeOnData) callback(decoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenBufferThenResumeOnData -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenBufferThenResumeOnData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -907,10 +874,7 @@ TEST_F(PlatformBridgeFilterTest, StopNoBufferOnRequestData) { return {kEnvoyFilterDataStatusStopIterationNoBuffer, envoy_nodata, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: StopNoBufferOnRequestData -)EOF", - &platform_filter); + setUpFilter("StopNoBufferOnRequestData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl first_chunk = Buffer::OwnedImpl("A"); @@ -948,10 +912,7 @@ TEST_F(PlatformBridgeFilterTest, BasicContinueOnRequestTrailers) { return {kEnvoyFilterTrailersStatusContinue, c_trailers, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: BasicContinueOnRequestTrailers -)EOF", - &platform_filter); + setUpFilter("BasicContinueOnRequestTrailers", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestTrailerMapImpl request_trailers{{"x-test-trailer", "test trailer"}}; @@ -1022,10 +983,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenBufferThenResumeOnTrail callback(decoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenBufferThenResumeOnTrailers -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenBufferThenResumeOnTrailers", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -1150,10 +1108,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenBufferThenResumeOnResum callback(decoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenBufferThenResumeOnResumeDecoding -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenBufferThenResumeOnResumeDecoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -1281,10 +1236,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenBufferThenDontResumeOnR callback(decoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenBufferThenResumeOnResumeDecoding -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenBufferThenResumeOnResumeDecoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -1346,10 +1298,7 @@ TEST_F(PlatformBridgeFilterTest, BasicContinueOnResponseHeaders) { return {kEnvoyFilterHeadersStatusContinue, c_headers}; }; - setUpFilter(R"EOF( -platform_filter_name: BasicContinueOnResponseHeaders -)EOF", - &platform_filter); + setUpFilter("BasicContinueOnResponseHeaders", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -1393,10 +1342,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnResponseHeadersThenResumeOnData) { return {kEnvoyFilterDataStatusResumeIteration, c_data, modified_headers}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnResponseHeadersThenResumeOnData -)EOF", - &platform_filter); + setUpFilter("StopOnResponseHeadersThenResumeOnData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -1458,10 +1404,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnResponseHeadersThenResumeOnResumeEncoding return {kEnvoyFilterResumeStatusResumeIteration, modified_headers, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnResponseHeadersThenResumeOnResumeEncoding -)EOF", - &platform_filter); + setUpFilter("StopOnResponseHeadersThenResumeOnResumeEncoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -1526,10 +1469,7 @@ TEST_F(PlatformBridgeFilterTest, AsyncResumeEncodingIsNoopAfterPreviousResume) { return {kEnvoyFilterResumeStatusResumeIteration, nullptr, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: AsyncResumeEncodingIsNoopAfterPreviousResume -)EOF", - &platform_filter); + setUpFilter("AsyncResumeEncodingIsNoopAfterPreviousResume", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -1592,10 +1532,7 @@ TEST_F(PlatformBridgeFilterTest, AsyncResumeEncodingIsNoopAfterFilterIsPendingDe invocations->release_filter_calls++; }; - setUpFilter(R"EOF( -platform_filter_name: AsyncResumeEncodingIsNoopAfterFilterIsPendingDestruction -)EOF", - &platform_filter); + setUpFilter("AsyncResumeEncodingIsNoopAfterFilterIsPendingDestruction", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -1645,10 +1582,7 @@ TEST_F(PlatformBridgeFilterTest, BasicContinueOnResponseData) { return {kEnvoyFilterDataStatusContinue, c_data, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: BasicContinueOnResponseData -)EOF", - &platform_filter); + setUpFilter("BasicContinueOnResponseData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl response_data = Buffer::OwnedImpl("response body"); @@ -1689,10 +1623,7 @@ TEST_F(PlatformBridgeFilterTest, StopAndBufferOnResponseData) { callback(encoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopAndBufferOnResponseData -)EOF", - &platform_filter); + setUpFilter("StopAndBufferOnResponseData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl first_chunk = Buffer::OwnedImpl("A"); @@ -1763,10 +1694,7 @@ TEST_F(PlatformBridgeFilterTest, StopAndBufferThenResumeOnResponseData) { callback(encoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopAndBufferThenResumeOnResponseData -)EOF", - &platform_filter); + setUpFilter("StopAndBufferThenResumeOnResponseData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl first_chunk = Buffer::OwnedImpl("A"); @@ -1848,10 +1776,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnResponseHeadersThenBufferThenResumeOnData callback(encoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnResponseHeadersThenBufferThenResumeOnData -)EOF", - &platform_filter); + setUpFilter("StopOnResponseHeadersThenBufferThenResumeOnData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -1903,10 +1828,7 @@ TEST_F(PlatformBridgeFilterTest, StopNoBufferOnResponseData) { return {kEnvoyFilterDataStatusStopIterationNoBuffer, envoy_nodata, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: StopNoBufferOnResponseData -)EOF", - &platform_filter); + setUpFilter("StopNoBufferOnResponseData", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Buffer::OwnedImpl first_chunk = Buffer::OwnedImpl("A"); @@ -1944,10 +1866,7 @@ TEST_F(PlatformBridgeFilterTest, BasicContinueOnResponseTrailers) { return {kEnvoyFilterTrailersStatusContinue, c_trailers, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: BasicContinueOnResponseTrailers -)EOF", - &platform_filter); + setUpFilter("BasicContinueOnResponseTrailers", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseTrailerMapImpl response_trailers{{"x-test-trailer", "test trailer"}}; @@ -2018,10 +1937,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnResponseHeadersThenBufferThenResumeOnTrai callback(encoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnResponseHeadersThenBufferThenResumeOnTrailers -)EOF", - &platform_filter); + setUpFilter("StopOnResponseHeadersThenBufferThenResumeOnTrailers", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -2146,10 +2062,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnResponseHeadersThenBufferThenResumeOnResu callback(encoding_buffer); })); - setUpFilter(R"EOF( -platform_filter_name: StopOnResponseHeadersThenBufferThenResumeOnResumeEncoding -)EOF", - &platform_filter); + setUpFilter("StopOnResponseHeadersThenBufferThenResumeOnResumeEncoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -2233,10 +2146,7 @@ TEST_F(PlatformBridgeFilterTest, StopOnRequestHeadersThenResumeOnResumeDecodingP return {kEnvoyFilterResumeStatusResumeIteration, pending_headers, nullptr, nullptr}; }; - setUpFilter(R"EOF( -platform_filter_name: StopOnRequestHeadersThenResumeOnResumeDecoding -)EOF", - &platform_filter); + setUpFilter("StopOnRequestHeadersThenResumeOnResumeDecoding", &platform_filter); EXPECT_EQ(invocations.init_filter_calls, 1); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; diff --git a/mobile/test/common/extensions/key_value/platform/BUILD b/mobile/test/common/extensions/key_value/platform/BUILD index 830f173196c7..d27d31067f7d 100644 --- a/mobile/test/common/extensions/key_value/platform/BUILD +++ b/mobile/test/common/extensions/key_value/platform/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load( "@envoy//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -6,7 +6,7 @@ load( licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_extension_cc_test( name = "platform_store_test", diff --git a/mobile/test/common/extensions/retry/options/network_configuration/BUILD b/mobile/test/common/extensions/retry/options/network_configuration/BUILD index 64dfe4de2c95..280096f93904 100644 --- a/mobile/test/common/extensions/retry/options/network_configuration/BUILD +++ b/mobile/test/common/extensions/retry/options/network_configuration/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load( "@envoy//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -6,7 +6,7 @@ load( licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_extension_cc_test( name = "network_configuration_retry_options_test", diff --git a/mobile/test/common/extensions/retry/options/network_configuration/predicate_test.cc b/mobile/test/common/extensions/retry/options/network_configuration/predicate_test.cc index ec043a7b257e..4bc683ab7c2d 100644 --- a/mobile/test/common/extensions/retry/options/network_configuration/predicate_test.cc +++ b/mobile/test/common/extensions/retry/options/network_configuration/predicate_test.cc @@ -25,7 +25,7 @@ TEST(NetworkConfigurationRetryOptionsPredicateTest, PredicateTest) { NiceMock mock_factory_context; NiceMock mock_stream_info; Upstream::RetryExtensionFactoryContextImpl retry_extension_factory_context{ - *mock_factory_context.singleton_manager_}; + *mock_factory_context.server_factory_context_.singleton_manager_}; auto connectivity_manager = Network::ConnectivityManagerFactory(mock_factory_context).get(); ASSERT_NE(nullptr, connectivity_manager); @@ -45,7 +45,7 @@ TEST(NetworkConfigurationRetryOptionsPredicateTest, PredicateTestWithoutConnecti ExtensionRegistry::registerFactories(); NiceMock mock_factory_context; Upstream::RetryExtensionFactoryContextImpl retry_extension_factory_context{ - *mock_factory_context.singleton_manager_}; + *mock_factory_context.server_factory_context_.singleton_manager_}; auto factory = Registry::FactoryRegistry::getFactory( "envoy.retry_options_predicates.network_configuration"); diff --git a/mobile/test/common/extensions/stat_sinks/metrics_service/BUILD b/mobile/test/common/extensions/stat_sinks/metrics_service/BUILD index a327e6893c86..aa4a795148a8 100644 --- a/mobile/test/common/extensions/stat_sinks/metrics_service/BUILD +++ b/mobile/test/common/extensions/stat_sinks/metrics_service/BUILD @@ -1,4 +1,4 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load( "@envoy//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -6,7 +6,7 @@ load( licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_extension_cc_test( name = "mobile_grpc_streamer_test", diff --git a/mobile/test/common/http/BUILD b/mobile/test/common/http/BUILD index 8a76a97f4b32..53d060abd8d0 100644 --- a/mobile/test/common/http/BUILD +++ b/mobile/test/common/http/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "client_test", diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index 925634345929..742fec0456d2 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -148,7 +148,7 @@ class ClientTest : public testing::TestWithParam { response_encoder_ = &encoder; return std::make_unique(*request_decoder_); })); - http_client_.startStream(stream_, bridge_callbacks_, explicit_flow_control_, 0); + http_client_.startStream(stream_, bridge_callbacks_, explicit_flow_control_); } void resumeDataIfExplicitFlowControl(int32_t bytes) { @@ -442,7 +442,7 @@ TEST_P(ClientTest, MultipleStreams) { response_encoder2 = &encoder; return std::make_unique(request_decoder2); })); - http_client_.startStream(stream2, bridge_callbacks_2, explicit_flow_control_, 0); + http_client_.startStream(stream2, bridge_callbacks_2, explicit_flow_control_); // Send request headers. EXPECT_CALL(dispatcher_, pushTrackedObject(_)); @@ -703,7 +703,7 @@ TEST_P(ClientTest, NullAccessors) { response_encoder_ = &encoder; return std::make_unique(*request_decoder_); })); - http_client_.startStream(stream, bridge_callbacks, explicit_flow_control_, 0); + http_client_.startStream(stream, bridge_callbacks, explicit_flow_control_); EXPECT_FALSE(response_encoder_->http1StreamEncoderOptions().has_value()); EXPECT_FALSE(response_encoder_->streamErrorOnInvalidHttpMessage()); diff --git a/mobile/test/common/http/filters/assertion/BUILD b/mobile/test/common/http/filters/assertion/BUILD index 30e054af6339..505a99756d41 100644 --- a/mobile/test/common/http/filters/assertion/BUILD +++ b/mobile/test/common/http/filters/assertion/BUILD @@ -2,13 +2,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_cc_test", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/assertion/assertion_filter_test.cc b/mobile/test/common/http/filters/assertion/assertion_filter_test.cc index 9623ccc6787e..ec02f9c3c7ea 100644 --- a/mobile/test/common/http/filters/assertion/assertion_filter_test.cc +++ b/mobile/test/common/http/filters/assertion/assertion_filter_test.cc @@ -18,9 +18,9 @@ namespace { class AssertionFilterTest : public testing::Test { public: - void setUpFilter(std::string&& yaml) { + void setUpFilter(std::string&& proto_str) { envoymobile::extensions::filters::http::assertion::Assertion config; - TestUtility::loadFromYaml(yaml, config); + Protobuf::TextFormat::ParseFromString(proto_str, &config); config_ = std::make_shared(config); filter_ = std::make_unique(config_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); @@ -35,11 +35,14 @@ class AssertionFilterTest : public testing::Test { TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code +match_config { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } +} )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -49,11 +52,14 @@ TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndStream) { TEST_F(AssertionFilterTest, RequestHeadersMatch) { setUpFilter(R"EOF( -match_config: - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code + match_config { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } + } )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -63,11 +69,14 @@ TEST_F(AssertionFilterTest, RequestHeadersMatch) { TEST_F(AssertionFilterTest, RequestHeadersNoMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code + match_config { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } + } )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "no.match"}}; @@ -81,11 +90,14 @@ TEST_F(AssertionFilterTest, RequestHeadersNoMatchWithEndStream) { TEST_F(AssertionFilterTest, RequestHeadersNoMatch) { setUpFilter(R"EOF( -match_config: - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code + match_config { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } + } )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "no.match"}}; @@ -99,16 +111,25 @@ TEST_F(AssertionFilterTest, RequestHeadersNoMatch) { TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndstreamAndDataMissing) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code - - http_request_generic_body_match: - patterns: - - string_match: match_me +match_config { + and_match { + rules { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } + } + rules { + http_request_generic_body_match { + patterns { + string_match: "match_me" + } + } + } + } +} )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -122,17 +143,26 @@ TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndstreamAndDataMissing) { TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndstreamAndTrailersMissing) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code - - http_request_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + and_match { + rules { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } + } + rules { + http_request_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } + } + } +} )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -146,10 +176,13 @@ TEST_F(AssertionFilterTest, RequestHeadersMatchWithEndstreamAndTrailersMissing) TEST_F(AssertionFilterTest, RequestDataMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_request_generic_body_match: - patterns: - - string_match: match_me +match_config { + http_request_generic_body_match { + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; @@ -159,10 +192,13 @@ TEST_F(AssertionFilterTest, RequestDataMatchWithEndStream) { TEST_F(AssertionFilterTest, RequestDataMatch) { setUpFilter(R"EOF( -match_config: - http_request_generic_body_match: - patterns: - - string_match: match_me +match_config { + http_request_generic_body_match { + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; @@ -172,11 +208,14 @@ TEST_F(AssertionFilterTest, RequestDataMatch) { TEST_F(AssertionFilterTest, RequestDataNoMatchFastPath) { setUpFilter(R"EOF( -match_config: - http_request_generic_body_match: +match_config { + http_request_generic_body_match { bytes_limit: 1 - patterns: - - string_match: match_me + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; @@ -189,10 +228,13 @@ TEST_F(AssertionFilterTest, RequestDataNoMatchFastPath) { TEST_F(AssertionFilterTest, RequestDataNoMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_request_generic_body_match: - patterns: - - string_match: match_me +match_config { + http_request_generic_body_match { + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; @@ -205,16 +247,25 @@ TEST_F(AssertionFilterTest, RequestDataNoMatchWithEndStream) { TEST_F(AssertionFilterTest, RequestDataMatchWithEndStreamAndTrailersMissing) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_request_generic_body_match: - patterns: - - string_match: match_me - - http_request_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + and_match { + rules { + http_request_generic_body_match { + patterns { + string_match: "match_me" + } + } + } + rules { + http_request_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; @@ -227,16 +278,25 @@ TEST_F(AssertionFilterTest, RequestDataMatchWithEndStreamAndTrailersMissing) { TEST_F(AssertionFilterTest, RequestDataNoMatchAfterTrailers) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_request_headers_match: - headers: - - name: ":authority" - exact_match: test.code - - http_request_generic_body_match: - patterns: - - string_match: match_me +match_config { + and_match { + rules { + http_request_headers_match { + headers { + name: ":authority" + exact_match: "test.code" + } + } + } + rules { + http_request_generic_body_match { + patterns { + string_match: "match_me" + } + } + } + } +} )EOF"); Http::TestRequestHeaderMapImpl request_headers{{":authority", "test.code"}}; @@ -253,11 +313,14 @@ TEST_F(AssertionFilterTest, RequestDataNoMatchAfterTrailers) { TEST_F(AssertionFilterTest, RequestTrailersMatch) { setUpFilter(R"EOF( -match_config: - http_request_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code + match_config { + http_request_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } +} )EOF"); Http::TestRequestTrailerMapImpl request_trailers{{"test-trailer", "test.code"}}; @@ -267,11 +330,14 @@ TEST_F(AssertionFilterTest, RequestTrailersMatch) { TEST_F(AssertionFilterTest, RequestTrailersNoMatch) { setUpFilter(R"EOF( -match_config: - http_request_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + http_request_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } +} )EOF"); Http::TestRequestTrailerMapImpl request_trailers{{"test-trailer", "no.match"}}; @@ -284,11 +350,14 @@ TEST_F(AssertionFilterTest, RequestTrailersNoMatch) { TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code +match_config { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -298,11 +367,14 @@ TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndStream) { TEST_F(AssertionFilterTest, ResponseHeadersMatch) { setUpFilter(R"EOF( -match_config: - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code + match_config { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -312,11 +384,14 @@ TEST_F(AssertionFilterTest, ResponseHeadersMatch) { TEST_F(AssertionFilterTest, ResponseHeadersNoMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code +match_config { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "no.match"}}; @@ -330,11 +405,14 @@ TEST_F(AssertionFilterTest, ResponseHeadersNoMatchWithEndStream) { TEST_F(AssertionFilterTest, ResponseHeadersNoMatch) { setUpFilter(R"EOF( -match_config: - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code + match_config { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "no.match"}}; @@ -348,16 +426,25 @@ TEST_F(AssertionFilterTest, ResponseHeadersNoMatch) { TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndstreamAndDataMissing) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code - - http_response_generic_body_match: - patterns: - - string_match: match_me +match_config { + and_match { + rules { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } + } + rules { + http_response_generic_body_match { + patterns { + string_match: "match_me" + } + } + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -371,17 +458,26 @@ TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndstreamAndDataMissing) { TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndstreamAndTrailersMissing) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code - - http_response_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + and_match { + rules { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } + } + rules { + http_response_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -395,10 +491,13 @@ TEST_F(AssertionFilterTest, ResponseHeadersMatchWithEndstreamAndTrailersMissing) TEST_F(AssertionFilterTest, ResponseDataMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_response_generic_body_match: - patterns: - - string_match: match_me +match_config { + http_response_generic_body_match { + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; @@ -408,10 +507,13 @@ TEST_F(AssertionFilterTest, ResponseDataMatchWithEndStream) { TEST_F(AssertionFilterTest, ResponseDataMatch) { setUpFilter(R"EOF( -match_config: - http_response_generic_body_match: - patterns: - - string_match: match_me +match_config { + http_response_generic_body_match { + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; @@ -421,11 +523,14 @@ TEST_F(AssertionFilterTest, ResponseDataMatch) { TEST_F(AssertionFilterTest, ResponseDataNoMatchFastPath) { setUpFilter(R"EOF( -match_config: - http_response_generic_body_match: +match_config { + http_response_generic_body_match { bytes_limit: 1 - patterns: - - string_match: match_me + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; @@ -438,10 +543,13 @@ TEST_F(AssertionFilterTest, ResponseDataNoMatchFastPath) { TEST_F(AssertionFilterTest, ResponseDataNoMatchWithEndStream) { setUpFilter(R"EOF( -match_config: - http_response_generic_body_match: - patterns: - - string_match: match_me +match_config { + http_response_generic_body_match { + patterns { + string_match: "match_me" + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("garbage")}; @@ -454,16 +562,25 @@ TEST_F(AssertionFilterTest, ResponseDataNoMatchWithEndStream) { TEST_F(AssertionFilterTest, ResponseDataMatchWithEndStreamAndTrailersMissing) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_response_generic_body_match: - patterns: - - string_match: match_me - - http_response_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + and_match { + rules { + http_response_generic_body_match { + patterns { + string_match: "match_me" + } + } + } + rules { + http_response_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } + } + } +} )EOF"); Buffer::InstancePtr body{new Buffer::OwnedImpl("match_me")}; @@ -476,16 +593,25 @@ TEST_F(AssertionFilterTest, ResponseDataMatchWithEndStreamAndTrailersMissing) { TEST_F(AssertionFilterTest, ResponseDataNoMatchAfterTrailers) { setUpFilter(R"EOF( -match_config: - and_match: - rules: - - http_response_headers_match: - headers: - - name: ":status" - exact_match: test.code - - http_response_generic_body_match: - patterns: - - string_match: match_me +match_config { + and_match { + rules { + http_response_headers_match { + headers { + name: ":status" + exact_match: "test.code" + } + } + } + rules { + http_response_generic_body_match { + patterns { + string_match: "match_me" + } + } + } + } +} )EOF"); Http::TestResponseHeaderMapImpl response_headers{{":status", "test.code"}}; @@ -502,11 +628,14 @@ TEST_F(AssertionFilterTest, ResponseDataNoMatchAfterTrailers) { TEST_F(AssertionFilterTest, ResponseTrailersMatch) { setUpFilter(R"EOF( -match_config: - http_response_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + http_response_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } +} )EOF"); Http::TestResponseTrailerMapImpl response_trailers{{"test-trailer", "test.code"}}; @@ -516,11 +645,14 @@ TEST_F(AssertionFilterTest, ResponseTrailersMatch) { TEST_F(AssertionFilterTest, ResponseTrailersNoMatch) { setUpFilter(R"EOF( -match_config: - http_response_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + http_response_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } +} )EOF"); Http::TestResponseTrailerMapImpl response_trailers{{"test-trailer", "no.match"}}; @@ -540,16 +672,20 @@ TEST(AssertionFilterFactoryTest, Config) { factory.createEmptyConfigProto().get()); std::string config_str = R"EOF( -match_config: - http_response_trailers_match: - headers: - - name: "test-trailer" - exact_match: test.code +match_config { + http_response_trailers_match { + headers { + name: "test-trailer" + exact_match: "test.code" + } + } +} )EOF"; - TestUtility::loadFromYamlAndValidate(config_str, proto_config); + Protobuf::TextFormat::ParseFromString(config_str, &proto_config); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "test", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "test", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamFilter(_)); cb(filter_callbacks); diff --git a/mobile/test/common/http/filters/route_cache_reset/BUILD b/mobile/test/common/http/filters/route_cache_reset/BUILD index 866454f1e06b..cb8c18b62049 100644 --- a/mobile/test/common/http/filters/route_cache_reset/BUILD +++ b/mobile/test/common/http/filters/route_cache_reset/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/test_accessor/BUILD b/mobile/test/common/http/filters/test_accessor/BUILD index 90c55313131d..c83de3ceeba4 100644 --- a/mobile/test/common/http/filters/test_accessor/BUILD +++ b/mobile/test/common/http/filters/test_accessor/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/test_event_tracker/BUILD b/mobile/test/common/http/filters/test_event_tracker/BUILD index 9c1315425c81..49472189a20f 100644 --- a/mobile/test/common/http/filters/test_event_tracker/BUILD +++ b/mobile/test/common/http/filters/test_event_tracker/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/test_kv_store/BUILD b/mobile/test/common/http/filters/test_kv_store/BUILD index 076bbb8fa330..f2fcbeff6d50 100644 --- a/mobile/test/common/http/filters/test_kv_store/BUILD +++ b/mobile/test/common/http/filters/test_kv_store/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/test_logger/BUILD b/mobile/test/common/http/filters/test_logger/BUILD index 93203f0d3a31..7be3019c250a 100644 --- a/mobile/test/common/http/filters/test_logger/BUILD +++ b/mobile/test/common/http/filters/test_logger/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/test_read/BUILD b/mobile/test/common/http/filters/test_read/BUILD index 96743b875e87..474dfd5bcdd1 100644 --- a/mobile/test/common/http/filters/test_read/BUILD +++ b/mobile/test/common/http/filters/test_read/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/http/filters/test_read/filter.cc b/mobile/test/common/http/filters/test_read/filter.cc index 50b20cdf3a06..cec0323b2685 100644 --- a/mobile/test/common/http/filters/test_read/filter.cc +++ b/mobile/test/common/http/filters/test_read/filter.cc @@ -9,17 +9,17 @@ namespace TestRead { Http::FilterHeadersStatus TestReadFilter::decodeHeaders(Http::RequestHeaderMap& request_headers, bool) { // sample path is /failed?error=0x10000 - Http::Utility::QueryParams query_parameters = - Http::Utility::parseQueryString(request_headers.Path()->value().getStringView()); - auto error = query_parameters.find("error"); + auto query_parameters = Http::Utility::QueryParamsMulti::parseQueryString( + request_headers.Path()->value().getStringView()); + auto error = query_parameters.getFirstValue("error"); uint64_t response_flag; - if (error != query_parameters.end() && absl::SimpleAtoi(error->second, &response_flag)) { + if (error.has_value() && absl::SimpleAtoi(error.value(), &response_flag)) { // set response error code StreamInfo::StreamInfo& stream_info = decoder_callbacks_->streamInfo(); stream_info.setResponseFlag(TestReadFilter::mapErrorToResponseFlag(response_flag)); // check if we want a quic server error: sample path is /failed?quic=1&error=0x10000 - if (query_parameters.find("quic") != query_parameters.end()) { + if (query_parameters.getFirstValue("quic").has_value()) { stream_info.setUpstreamInfo(std::make_shared()); stream_info.upstreamInfo()->setUpstreamProtocol(Http::Protocol::Http3); } diff --git a/mobile/test/common/http/filters/test_remote_response/BUILD b/mobile/test/common/http/filters/test_remote_response/BUILD index 94c9f6df790b..ef75ddc881a9 100644 --- a/mobile/test/common/http/filters/test_remote_response/BUILD +++ b/mobile/test/common/http/filters/test_remote_response/BUILD @@ -1,13 +1,13 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_package", + "envoy_mobile_package", "envoy_proto_library", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_proto_library( name = "filter", diff --git a/mobile/test/common/integration/BUILD b/mobile/test/common/integration/BUILD index 1304b3311976..cb7bb176b4b7 100644 --- a/mobile/test/common/integration/BUILD +++ b/mobile/test/common/integration/BUILD @@ -2,14 +2,14 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_cc_test_library", - "envoy_package", - "envoy_select_google_grpc", + "envoy_mobile_package", + "envoy_select_envoy_mobile_xds", "envoy_select_signal_trace", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "client_integration_test", @@ -19,19 +19,30 @@ envoy_cc_test( "sandboxNetwork": "standard", }, repository = "@envoy", + shard_count = 4, deps = [ ":base_client_integration_test_lib", "//test/common/mocks/common:common_mocks", + "@envoy//source/common/quic:active_quic_listener_lib", + "@envoy//source/common/quic:client_connection_factory_lib", + "@envoy//source/common/quic:quic_server_factory_lib", + "@envoy//source/common/quic:quic_server_transport_socket_factory_lib", + "@envoy//source/common/quic:quic_transport_socket_factory_lib", + "@envoy//source/common/quic:udp_gso_batch_writer_lib", + "@envoy//source/extensions/udp_packet_writer/gso:config", + "@envoy//test/test_common:test_random_generator_lib", ], ) envoy_cc_test( name = "rtds_integration_test", - # The test relies on the Google gRPC library. - srcs = envoy_select_google_grpc( + srcs = envoy_select_envoy_mobile_xds( ["rtds_integration_test.cc"], "@envoy", ), + data = [ + "@envoy//test/config/integration/certs", + ], exec_properties = { # TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. "sandboxNetwork": "standard", @@ -42,6 +53,8 @@ envoy_cc_test( repository = "@envoy", deps = [ ":xds_integration_test_lib", + "@envoy//test/test_common:environment_lib", + "@envoy//test/test_common:test_runtime_lib", "@envoy//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/service/runtime/v3:pkg_cc_proto", @@ -50,11 +63,13 @@ envoy_cc_test( envoy_cc_test( name = "cds_integration_test", - # The test relies on the Google gRPC library. - srcs = envoy_select_google_grpc( + srcs = envoy_select_envoy_mobile_xds( ["cds_integration_test.cc"], "@envoy", ), + data = [ + "@envoy//test/config/integration/certs", + ], exec_properties = { # TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. "sandboxNetwork": "standard", @@ -65,6 +80,7 @@ envoy_cc_test( repository = "@envoy", deps = [ ":xds_integration_test_lib", + "@envoy//test/test_common:environment_lib", "@envoy//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/service/runtime/v3:pkg_cc_proto", @@ -73,8 +89,7 @@ envoy_cc_test( envoy_cc_test( name = "sds_integration_test", - # The test relies on the Google gRPC library. - srcs = envoy_select_google_grpc( + srcs = envoy_select_envoy_mobile_xds( ["sds_integration_test.cc"], "@envoy", ), @@ -92,12 +107,12 @@ envoy_cc_test( "@envoy//source/extensions/transport_sockets/tls:config", "@envoy//source/extensions/transport_sockets/tls:context_config_lib", "@envoy//source/extensions/transport_sockets/tls:context_lib", + "@envoy//test/test_common:environment_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", "@envoy_api//envoy/service/secret/v3:pkg_cc_proto", - "@envoy_build_config//:extension_registry", ], ) @@ -133,10 +148,10 @@ envoy_cc_test_library( deps = [ ":base_client_integration_test_lib", "@envoy//source/common/config:api_version_lib", - "@envoy//source/common/grpc:google_grpc_creds_lib", - "@envoy//test/common/grpc:grpc_client_integration_lib", + "@envoy//test/test_common:environment_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_build_config//:extension_registry", "@envoy_build_config//:test_extensions", ], ) @@ -168,3 +183,33 @@ envoy_cc_test_library( "@envoy", ), ) + +envoy_cc_test_library( + name = "xds_test_server_interface_lib", + srcs = [ + "xds_test_server.cc", + "xds_test_server_interface.cc", + ], + hdrs = [ + "xds_test_server.h", + "xds_test_server_interface.h", + ], + repository = "@envoy", + deps = [ + ":base_client_integration_test_lib", + "@envoy//source/common/event:libevent_lib", + "@envoy//source/exe:process_wide_lib", + "@envoy//source/extensions/transport_sockets/tls:context_config_lib", + "@envoy//source/extensions/transport_sockets/tls:context_lib", + "@envoy//source/extensions/transport_sockets/tls:ssl_socket_lib", + "@envoy//test/integration:autonomous_upstream_lib", + "@envoy//test/integration:utility_lib", + "@envoy//test/mocks/server:transport_socket_factory_context_mocks", + "@envoy//test/test_common:environment_lib", + "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", + "@envoy_build_config//:extension_registry", + ] + envoy_select_signal_trace( + ["@envoy//source/common/signal:sigaction_lib"], + "@envoy", + ), +) diff --git a/mobile/test/common/integration/base_client_integration_test.cc b/mobile/test/common/integration/base_client_integration_test.cc index b4a932b62946..9d5fbc34fb69 100644 --- a/mobile/test/common/integration/base_client_integration_test.cc +++ b/mobile/test/common/integration/base_client_integration_test.cc @@ -10,7 +10,6 @@ #include "library/cc/bridge_utility.h" #include "library/cc/log_level.h" #include "library/common/engine.h" -#include "library/common/engine_handle.h" #include "library/common/http/header_utility.h" #include "spdlog/spdlog.h" @@ -106,7 +105,10 @@ BaseClientIntegrationTest::BaseClientIntegrationTest(Network::Address::IpVersion void BaseClientIntegrationTest::initialize() { BaseIntegrationTest::initialize(); - stream_prototype_ = engine_->streamClient()->newStreamPrototype(); + { + absl::MutexLock l(&engine_lock_); + stream_prototype_ = engine_->streamClient()->newStreamPrototype(); + } stream_prototype_->setOnHeaders( [this](Platform::ResponseHeadersSharedPtr headers, bool, envoy_stream_intel intel) { @@ -139,7 +141,7 @@ void BaseClientIntegrationTest::initialize() { cc_.terminal_callback->setReady(); }); - stream_ = (*stream_prototype_).start(explicit_flow_control_, min_delivery_size_); + stream_ = (*stream_prototype_).start(explicit_flow_control_); HttpTestUtility::addDefaultHeaders(default_request_headers_); default_request_headers_.setHost(fake_upstreams_[0]->localAddress()->asStringView()); } @@ -172,7 +174,10 @@ std::shared_ptr BaseClientIntegrationTest::envoyToMobi void BaseClientIntegrationTest::threadRoutine(absl::Notification& engine_running) { builder_.setOnEngineRunning([&]() { engine_running.Notify(); }); - engine_ = builder_.build(); + { + absl::MutexLock l(&engine_lock_); + engine_ = builder_.build(); + } full_dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -182,9 +187,12 @@ void BaseClientIntegrationTest::TearDown() { } test_server_.reset(); fake_upstreams_.clear(); - if (engine_) { - engine_->terminate(); - engine_.reset(); + { + absl::MutexLock l(&engine_lock_); + if (engine_) { + engine_->terminate(); + engine_.reset(); + } } stream_.reset(); stream_prototype_.reset(); @@ -213,16 +221,15 @@ uint64_t BaseClientIntegrationTest::getCounterValue(const std::string& name) { uint64_t counter_value = 0UL; uint64_t* counter_value_ptr = &counter_value; absl::Notification counter_value_set; - EXPECT_EQ(ENVOY_SUCCESS, - EngineHandle::runOnEngineDispatcher( - rawEngine(), [counter_value_ptr, &name, &counter_value_set](Envoy::Engine& engine) { - Stats::CounterSharedPtr counter = - TestUtility::findCounter(engine.getStatsStore(), name); - if (counter != nullptr) { - *counter_value_ptr = counter->value(); - } - counter_value_set.Notify(); - })); + auto engine = reinterpret_cast(rawEngine()); + engine->dispatcher().post([&] { + Stats::CounterSharedPtr counter = TestUtility::findCounter(engine->getStatsStore(), name); + if (counter != nullptr) { + *counter_value_ptr = counter->value(); + } + counter_value_set.Notify(); + }); + EXPECT_TRUE(counter_value_set.WaitForNotificationWithTimeout(absl::Seconds(5))); return counter_value; } @@ -245,16 +252,14 @@ uint64_t BaseClientIntegrationTest::getGaugeValue(const std::string& name) { uint64_t gauge_value = 0UL; uint64_t* gauge_value_ptr = &gauge_value; absl::Notification gauge_value_set; - EXPECT_EQ(ENVOY_SUCCESS, - EngineHandle::runOnEngineDispatcher( - rawEngine(), [gauge_value_ptr, &name, &gauge_value_set](Envoy::Engine& engine) { - Stats::GaugeSharedPtr gauge = - TestUtility::findGauge(engine.getStatsStore(), name); - if (gauge != nullptr) { - *gauge_value_ptr = gauge->value(); - } - gauge_value_set.Notify(); - })); + auto engine = reinterpret_cast(rawEngine()); + engine->dispatcher().post([&] { + Stats::GaugeSharedPtr gauge = TestUtility::findGauge(engine->getStatsStore(), name); + if (gauge != nullptr) { + *gauge_value_ptr = gauge->value(); + } + gauge_value_set.Notify(); + }); EXPECT_TRUE(gauge_value_set.WaitForNotificationWithTimeout(absl::Seconds(5))); return gauge_value; } diff --git a/mobile/test/common/integration/base_client_integration_test.h b/mobile/test/common/integration/base_client_integration_test.h index b741ede4af2f..8e2eb70941de 100644 --- a/mobile/test/common/integration/base_client_integration_test.h +++ b/mobile/test/common/integration/base_client_integration_test.h @@ -46,7 +46,10 @@ class BaseClientIntegrationTest : public BaseIntegrationTest { void TearDown(); protected: - envoy_engine_t& rawEngine() { return engine_->engine_; } + envoy_engine_t& rawEngine() { + absl::MutexLock l(&engine_lock_); + return engine_->engine_; + } virtual void initialize() override; void createEnvoy() override; void threadRoutine(absl::Notification& engine_running); @@ -72,10 +75,10 @@ class BaseClientIntegrationTest : public BaseIntegrationTest { Event::DispatcherPtr full_dispatcher_; Platform::StreamPrototypeSharedPtr stream_prototype_; Platform::StreamSharedPtr stream_; - Platform::EngineSharedPtr engine_; + absl::Mutex engine_lock_; + Platform::EngineSharedPtr engine_ ABSL_GUARDED_BY(engine_lock_); Thread::ThreadPtr envoy_thread_; bool explicit_flow_control_ = false; - uint64_t min_delivery_size_ = 10; bool expect_dns_ = true; // True if data plane requests are expected in the test; false otherwise. bool expect_data_streams_ = true; diff --git a/mobile/test/common/integration/cds_integration_test.cc b/mobile/test/common/integration/cds_integration_test.cc index 639c108f6283..68f1d6efc137 100644 --- a/mobile/test/common/integration/cds_integration_test.cc +++ b/mobile/test/common/integration/cds_integration_test.cc @@ -2,6 +2,7 @@ #include "envoy/service/runtime/v3/rtds.pb.h" #include "test/common/integration/xds_integration_test.h" +#include "test/test_common/environment.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -9,11 +10,17 @@ namespace Envoy { namespace { +using envoy::config::cluster::v3::Cluster; + class CdsIntegrationTest : public XdsIntegrationTest { public: - CdsIntegrationTest() { - use_lds_ = false; + void initialize() override { + setUpstreamProtocol(Http::CodecType::HTTP1); + + XdsIntegrationTest::initialize(); + default_request_headers_.setScheme("http"); + initializeXdsStream(); } void createEnvoy() override { @@ -25,36 +32,72 @@ class CdsIntegrationTest : public XdsIntegrationTest { cds_namespace_ = "xdstp://" + target_uri + "/envoy.config.cluster.v3.Cluster"; cds_resources_locator = cds_namespace_ + "/*"; } - xds_builder.addClusterDiscoveryService(cds_resources_locator, /*timeout_in_seconds=*/1); + xds_builder.addClusterDiscoveryService(cds_resources_locator, /*timeout_in_seconds=*/1) + .setSslRootCerts(getUpstreamCert()); builder_.setXds(std::move(xds_builder)); XdsIntegrationTest::createEnvoy(); } - void SetUp() override { setUpstreamProtocol(Http::CodecType::HTTP1); } + void SetUp() override { initialize(); } protected: - void executeCdsRequestsAndVerify() { - initialize(); + Cluster createCluster() { const std::string cluster_name = use_xdstp_ ? cds_namespace_ + "/my_cluster?xds.node.cluster=envoy-mobile" : "my_cluster"; - envoy::config::cluster::v3::Cluster cluster1 = ConfigHelper::buildStaticCluster( - cluster_name, fake_upstreams_[0]->localAddress()->ip()->port(), - Network::Test::getLoopbackAddressString(ipVersion())); - initializeXdsStream(); - int cluster_count = getGaugeValue("cluster_manager.active_clusters"); - // Do the initial compareDiscoveryRequest / sendDiscoveryResponse for cluster_1. + return ConfigHelper::buildStaticCluster(cluster_name, + fake_upstreams_[0]->localAddress()->ip()->port(), + Network::Test::getLoopbackAddressString(ipVersion())); + } + + std::vector getExpectedResources() { std::vector expected_resources; if (use_xdstp_) { expected_resources.push_back(cds_namespace_ + "/*"); } + return expected_resources; + } + + void sendInitialCdsResponseAndVerify(const std::string& version) { + const int cluster_count = getGaugeValue("cluster_manager.active_clusters"); + const std::vector expected_resources = getExpectedResources(); + + // Envoy sends the initial DiscoveryRequest. EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", expected_resources, {}, - {}, true)); - sendDiscoveryResponse(Config::TypeUrl::get().Cluster, - {cluster1}, {cluster1}, {}, "55"); - // Wait for cluster to be added - ASSERT_TRUE(waitForCounterGe("cluster_manager.cluster_added", 1)); - ASSERT_TRUE(waitForGaugeGe("cluster_manager.active_clusters", cluster_count + 1)); + {}, /*expect_node=*/true)); + + Cluster cluster = createCluster(); + // Server sends back the initial DiscoveryResponse. + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster}, {cluster}, {}, + version); + + // Wait for cluster to be added. + EXPECT_TRUE(waitForCounterGe("cluster_manager.cluster_added", 1)); + EXPECT_TRUE(waitForGaugeGe("cluster_manager.active_clusters", cluster_count + 1)); + + // ACK of the initial version. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, version, expected_resources, + {}, {}, /*expect_node=*/false)); + + EXPECT_TRUE(waitForGaugeGe("cluster_manager.cluster_removed", 0)); + } + + void sendUpdatedCdsResponseAndVerify(const std::string& version) { + const int cluster_count = getGaugeValue("cluster_manager.active_clusters"); + const std::vector expected_resources = getExpectedResources(); + + // Server sends an updated DiscoveryResponse over the xDS stream. + Cluster cluster = createCluster(); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster}, {cluster}, {}, + version); + + // ACK of the cluster update at the new version. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, version, expected_resources, + {}, {}, /*expect_node=*/false)); + + // Cluster count should stay the same. + EXPECT_TRUE(waitForGaugeGe("cluster_manager.active_clusters", cluster_count)); + EXPECT_TRUE(waitForGaugeGe("cluster_manager.cluster_removed", 0)); } bool use_xdstp_{false}; @@ -68,11 +111,17 @@ INSTANTIATE_TEST_SUITE_P( // Envoy Mobile's xDS APIs only support state-of-the-world, not delta. testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::UnifiedSotw))); -TEST_P(CdsIntegrationTest, Basic) { executeCdsRequestsAndVerify(); } +TEST_P(CdsIntegrationTest, Basic) { sendInitialCdsResponseAndVerify(/*version=*/"55"); } TEST_P(CdsIntegrationTest, BasicWithXdstp) { use_xdstp_ = true; - executeCdsRequestsAndVerify(); + sendInitialCdsResponseAndVerify(/*version=*/"55"); +} + +TEST_P(CdsIntegrationTest, ClusterUpdates) { + use_xdstp_ = true; + sendInitialCdsResponseAndVerify(/*version=*/"55"); + sendUpdatedCdsResponseAndVerify(/*version=*/"56"); } } // namespace diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index a92dfb19126d..14a49af1b02e 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -1,59 +1,193 @@ +#include "source/common/quic/quic_server_transport_socket_factory.h" +#include "source/common/quic/server_codec_impl.h" #include "source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h" +#include "source/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator_config.h" +#include "source/extensions/quic/crypto_stream/envoy_quic_crypto_server_stream.h" +#include "source/extensions/quic/proof_source/envoy_quic_proof_source_factory_impl.h" +#include "source/extensions/transport_sockets/tls/cert_validator/default_validator.h" +#include "source/extensions/udp_packet_writer/default/config.h" #include "test/common/integration/base_client_integration_test.h" #include "test/common/mocks/common/mocks.h" #include "test/integration/autonomous_upstream.h" +#include "test/test_common/test_random_generator.h" +#include "extension_registry.h" #include "library/common/data/utility.h" #include "library/common/main_interface.h" #include "library/common/network/proxy_settings.h" #include "library/common/types/c_types.h" using testing::_; +using testing::AnyNumber; using testing::Return; using testing::ReturnRef; namespace Envoy { namespace { -class ClientIntegrationTest : public BaseClientIntegrationTest, - public testing::TestWithParam { +// The only thing this TestKeyValueStore does is return value_ when asked for +// initial loaded contents. +// In this case the TestKeyValueStore will be used for DNS and value will map +// www.lyft.com -> fake test upstream. +class TestKeyValueStore : public Envoy::Platform::KeyValueStore { public: - ClientIntegrationTest() : BaseClientIntegrationTest(/*ip_version=*/GetParam()) {} + absl::optional read(const std::string&) override { + ASSERT(!value_.empty()); + return value_; + } + void save(std::string, std::string) override {} + void remove(const std::string&) override {} + void addOrUpdate(absl::string_view, absl::string_view, absl::optional) {} + absl::optional get(absl::string_view) { return {}; } + void flush() {} + void iterate(::Envoy::KeyValueStore::ConstIterateCb) const {} + void setValue(std::string value) { value_ = value; } + +protected: + std::string value_; +}; + +class ClientIntegrationTest + : public BaseClientIntegrationTest, + public testing::TestWithParam> { +public: + static void SetUpTestCase() { test_key_value_store_ = std::make_shared(); } + static void TearDownTestCase() { test_key_value_store_.reset(); } + + Http::CodecType getCodecType() { return std::get<1>(GetParam()); } + + ClientIntegrationTest() : BaseClientIntegrationTest(/*ip_version=*/std::get<0>(GetParam())) { + // For H3 tests. + Network::forceRegisterUdpDefaultWriterFactoryFactory(); + Quic::forceRegisterEnvoyQuicCryptoServerStreamFactoryImpl(); + Quic::forceRegisterQuicHttpServerConnectionFactoryImpl(); + Quic::forceRegisterQuicServerTransportSocketConfigFactory(); + Quic::forceRegisterEnvoyQuicProofSourceFactoryImpl(); + Quic::forceRegisterEnvoyDeterministicConnectionIdGeneratorConfigFactory(); + // For H2 tests. + Extensions::TransportSockets::Tls::forceRegisterDefaultCertValidatorFactory(); + } + + void initialize() override { + if (getCodecType() == Http::CodecType::HTTP3) { + setUpstreamProtocol(Http::CodecType::HTTP3); + builder_.enablePlatformCertificatesValidation(true); + // Create a k-v store for DNS lookup which createEnvoy() will use to point + // www.lyft.com -> fake H3 backend. + builder_.addKeyValueStore("reserved.platform_store", test_key_value_store_); + builder_.enableDnsCache(true, /* save_interval_seconds */ 1); + upstream_tls_ = true; + add_quic_hints_ = true; + } else if (getCodecType() == Http::CodecType::HTTP2) { + setUpstreamProtocol(Http::CodecType::HTTP2); + builder_.enablePlatformCertificatesValidation(true); + upstream_tls_ = true; + } + + BaseClientIntegrationTest::initialize(); + + if (getCodecType() == Http::CodecType::HTTP3) { + auto address = fake_upstreams_[0]->localAddress(); + auto upstream_port = fake_upstreams_[0]->localAddress()->ip()->port(); + default_request_headers_.setHost(fmt::format("www.lyft.com:{}", upstream_port)); + default_request_headers_.setScheme("https"); + } else if (getCodecType() == Http::CodecType::HTTP2) { + default_request_headers_.setScheme("https"); + } + } void SetUp() override { setUpstreamCount(config_helper_.bootstrap().static_resources().clusters_size()); - // TODO(abeyad): Add paramaterized tests for HTTP1, HTTP2, and HTTP3. helper_handle_ = test::SystemHelperPeer::replaceSystemHelper(); EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)) .WillRepeatedly(Return(true)); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)).Times(AnyNumber()); + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()) + .Times(AnyNumber()); } void createEnvoy() override { - BaseClientIntegrationTest::createEnvoy(); // Allow last minute addition of QUIC hints. This is done lazily as it must be done after // upstreams are created. if (add_quic_hints_) { auto address = fake_upstreams_[0]->localAddress(); - builder_.addQuicHint(address->ip()->addressAsString(), address->ip()->port()); + auto upstream_port = fake_upstreams_[0]->localAddress()->ip()->port(); + // With canonical suffix, having a quic hint of foo.lyft.com will make + // www.lyft.com being recognized as QUIC ready. + builder_.addQuicCanonicalSuffix(".lyft.com"); + builder_.addQuicHint("foo.lyft.com", upstream_port); + + // Force www.lyft.com to resolve to the fake upstream. It's the only domain + // name the certs work for so we want that in the request, but we need to + // fake resolution to not result in a request to the real www.lyft.com + std::string host = fmt::format("www.lyft.com:{}", upstream_port); + std::string cache_file_value_contents = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", + fake_upstreams_[0]->localAddress()->ip()->port(), "|1000000|0"); + test_key_value_store_->setValue(absl::StrCat(host.length(), "\n", host, + cache_file_value_contents.length(), "\n", + cache_file_value_contents)); } + BaseClientIntegrationTest::createEnvoy(); } - void TearDown() override { BaseClientIntegrationTest::TearDown(); } + void TearDown() override { + if (upstream_connection_) { + ASSERT_TRUE(upstream_connection_->close()); + ASSERT_TRUE(upstream_connection_->waitForDisconnect()); + upstream_connection_.reset(); + } + BaseClientIntegrationTest::TearDown(); + } void basicTest(); void trickleTest(); + void explicitFlowControlWithCancels(uint32_t body_size = 1000, bool terminate_engine = false); + + static std::string protocolToString(Http::CodecType type) { + if (type == Http::CodecType::HTTP3) { + return "Http3Upstream"; + } + if (type == Http::CodecType::HTTP2) { + return "Http2Upstream"; + } + return "Http1Upstream"; + } + + static std::string testParamsToString( + const testing::TestParamInfo> + params) { + return fmt::format( + "{}_{}", + TestUtility::ipTestParamsToString(testing::TestParamInfo( + std::get<0>(params.param), params.index)), + protocolToString(std::get<1>(params.param))); + } protected: std::unique_ptr helper_handle_; bool add_quic_hints_ = false; + static std::shared_ptr test_key_value_store_; + FakeHttpConnectionPtr upstream_connection_; + FakeStreamPtr upstream_request_; }; -INSTANTIATE_TEST_SUITE_P(IpVersions, ClientIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); +std::shared_ptr ClientIntegrationTest::test_key_value_store_{}; + +INSTANTIATE_TEST_SUITE_P( + IpVersions, ClientIntegrationTest, + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + testing::ValuesIn({Http::CodecType::HTTP1, Http::CodecType::HTTP2, + Http::CodecType::HTTP3})), + ClientIntegrationTest::testParamsToString); void ClientIntegrationTest::basicTest() { + if (getCodecType() != Http::CodecType::HTTP1) { + EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).Times(0); + EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); + EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); + } Buffer::OwnedImpl request_data = Buffer::OwnedImpl("request body"); default_request_headers_.addCopy(AutonomousStream::EXPECT_REQUEST_SIZE_BYTES, std::to_string(request_data.length())); @@ -80,19 +214,30 @@ void ClientIntegrationTest::basicTest() { ASSERT_EQ(cc_.on_headers_calls, 1); ASSERT_EQ(cc_.status, "200"); - ASSERT_EQ(cc_.on_data_calls, 2); + ASSERT_GE(cc_.on_data_calls, 1); ASSERT_EQ(cc_.on_complete_calls, 1); if (upstreamProtocol() == Http::CodecType::HTTP1) { ASSERT_EQ(cc_.on_header_consumed_bytes_from_response, 27); + // HTTP/1 + ASSERT_EQ(1, last_stream_final_intel_.upstream_protocol); + } else if (upstreamProtocol() == Http::CodecType::HTTP2) { + ASSERT_EQ(2, last_stream_final_intel_.upstream_protocol); + } else { + // This verifies the H3 attempt was made due to the quic hints + absl::MutexLock l(&engine_lock_); + std::string stats = engine_->dumpStats(); + EXPECT_TRUE((absl::StrContains(stats, "cluster.base.upstream_cx_http3_total: 1"))) << stats; + // Make sure the client reported protocol was also HTTP/3. + ASSERT_EQ(3, last_stream_final_intel_.upstream_protocol); } } TEST_P(ClientIntegrationTest, Basic) { initialize(); basicTest(); - ASSERT_EQ(cc_.on_complete_received_byte_count, 67); - // HTTP/1 - ASSERT_EQ(1, last_stream_final_intel_.upstream_protocol); + if (upstreamProtocol() == Http::CodecType::HTTP1) { + ASSERT_EQ(cc_.on_complete_received_byte_count, 67); + } } TEST_P(ClientIntegrationTest, LargeResponse) { @@ -100,7 +245,11 @@ TEST_P(ClientIntegrationTest, LargeResponse) { std::string data(1024 * 32, 'a'); reinterpret_cast(fake_upstreams_.front().get())->setResponseBody(data); basicTest(); - ASSERT_EQ(cc_.on_complete_received_byte_count, 32828); + if (upstreamProtocol() == Http::CodecType::HTTP1) { + ASSERT_EQ(cc_.on_complete_received_byte_count, 32828); + } else { + ASSERT_GE(cc_.on_complete_received_byte_count, 32000); + } } void ClientIntegrationTest::trickleTest() { @@ -129,46 +278,171 @@ void ClientIntegrationTest::trickleTest() { std::make_shared(builder.build()); stream_->close(trailers); - FakeHttpConnectionPtr upstream_connection; - FakeStreamPtr upstream_request; ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, - upstream_connection)); + upstream_connection_)); ASSERT_TRUE( - upstream_connection->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request)); + upstream_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForEndStream(*BaseIntegrationTest::dispatcher_)); - upstream_request->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); for (int i = 0; i < 10; ++i) { - upstream_request->encodeData(1, i == 9); + upstream_request_->encodeData(1, i == 9); } terminal_callback_.waitReady(); } -TEST_P(ClientIntegrationTest, TrickleNoMinDelivery) { - min_delivery_size_ = 0; +TEST_P(ClientIntegrationTest, Trickle) { trickleTest(); ASSERT_LE(cc_.on_data_calls, 11); } -TEST_P(ClientIntegrationTest, TrickleNoNoMinDeliveryExplicitFlowControl) { - min_delivery_size_ = 0; +TEST_P(ClientIntegrationTest, TrickleExplicitFlowControl) { explicit_flow_control_ = true; trickleTest(); ASSERT_LE(cc_.on_data_calls, 11); } -TEST_P(ClientIntegrationTest, TrickleMinDelivery) { - trickleTest(); - ASSERT_EQ(cc_.on_data_calls, 2); +TEST_P(ClientIntegrationTest, ManyStreamExplicitFlowControl) { + explicit_flow_control_ = true; + initialize(); + + default_request_headers_.addCopy(AutonomousStream::RESPONSE_SIZE_BYTES, std::to_string(1000)); + + uint32_t num_requests = 100; + std::vector prototype_streams; + std::vector streams; + + for (uint32_t i = 0; i < num_requests; ++i) { + Platform::StreamPrototypeSharedPtr stream_prototype; + { + absl::MutexLock l(&engine_lock_); + stream_prototype = engine_->streamClient()->newStreamPrototype(); + } + Platform::StreamSharedPtr stream = (*stream_prototype).start(explicit_flow_control_); + stream_prototype->setOnComplete( + [this, &num_requests](envoy_stream_intel, envoy_final_stream_intel) { + cc_.on_complete_calls++; + if (cc_.on_complete_calls == num_requests) { + cc_.terminal_callback->setReady(); + } + }); + + stream_prototype->setOnData([stream](envoy_data c_data, bool) { + // Allow reading up to 10 bytes. + stream->readData(100); + release_envoy_data(c_data); + }); + stream->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + stream->readData(100); + prototype_streams.push_back(stream_prototype); + streams.push_back(stream); + } + ASSERT(streams.size() == num_requests); + ASSERT(prototype_streams.size() == num_requests); + + terminal_callback_.waitReady(); + ASSERT_EQ(num_requests, cc_.on_complete_calls); } -TEST_P(ClientIntegrationTest, TrickleNoMinDeliveryExplicitFlowControl) { +void ClientIntegrationTest::explicitFlowControlWithCancels(const uint32_t body_size, + const bool terminate_engine) { + default_request_headers_.addCopy(AutonomousStream::RESPONSE_SIZE_BYTES, + std::to_string(body_size)); + + uint32_t num_requests = 100; + std::vector prototype_streams; + std::vector streams; + + // Randomly select which request number to terminate the engine on. + uint32_t request_for_engine_termination = 0; + if (terminate_engine) { + TestRandomGenerator rand; + request_for_engine_termination = rand.random() % (num_requests / 2); + } + + for (uint32_t i = 0; i < num_requests; ++i) { + Platform::StreamPrototypeSharedPtr stream_prototype; + { + absl::MutexLock l(&engine_lock_); + stream_prototype = engine_->streamClient()->newStreamPrototype(); + } + Platform::StreamSharedPtr stream = (*stream_prototype).start(explicit_flow_control_); + stream_prototype->setOnComplete( + [this, &num_requests](envoy_stream_intel, envoy_final_stream_intel) { + cc_.on_complete_calls++; + if (cc_.on_complete_calls + cc_.on_cancel_calls == num_requests) { + cc_.terminal_callback->setReady(); + } + }); + stream_prototype->setOnCancel( + [this, &num_requests](envoy_stream_intel, envoy_final_stream_intel) { + cc_.on_cancel_calls++; + if (cc_.on_complete_calls + cc_.on_cancel_calls == num_requests) { + cc_.terminal_callback->setReady(); + } + }); + stream_prototype->setOnData([stream](envoy_data c_data, bool) { + // Allow reading up to 10 bytes. + stream->readData(100); + release_envoy_data(c_data); + }); + stream_prototype_->setOnError( + [](Platform::EnvoyErrorSharedPtr, envoy_stream_intel, envoy_final_stream_intel) { + RELEASE_ASSERT(0, "unexpected"); + }); + + stream->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + prototype_streams.push_back(stream_prototype); + streams.push_back(stream); + if (i % 2 == 0) { + stream->cancel(); + } else { + stream->readData(100); + } + + if (terminate_engine && request_for_engine_termination == i) { + absl::MutexLock l(&engine_lock_); + ASSERT_EQ(engine_->terminate(), ENVOY_SUCCESS); + engine_.reset(); + break; + } + } + + if (terminate_engine) { + // Only the cancel calls are guaranteed to have completed when engine->terminate() is called. + EXPECT_GE(cc_.on_cancel_calls, request_for_engine_termination / 2); + } else { + ASSERT(streams.size() == num_requests); + ASSERT(prototype_streams.size() == num_requests); + terminal_callback_.waitReady(); + EXPECT_EQ(num_requests / 2, cc_.on_complete_calls); + EXPECT_EQ(num_requests / 2, cc_.on_cancel_calls); + } +} + +TEST_P(ClientIntegrationTest, ManyStreamExplicitFlowWithCancels) { explicit_flow_control_ = true; - trickleTest(); - ASSERT_EQ(cc_.on_data_calls, 2); + initialize(); + explicitFlowControlWithCancels(); +} + +TEST_P(ClientIntegrationTest, ManyStreamExplicitFlowWithCancelsAfterComplete) { + explicit_flow_control_ = true; + initialize(); + explicitFlowControlWithCancels(/*body_size=*/100); +} + +TEST_P(ClientIntegrationTest, ManyStreamExplicitFlowWithCancelsAfterCompleteEngineTermination) { + explicit_flow_control_ = true; + initialize(); + explicitFlowControlWithCancels(/*body_size=*/100, /*terminate_engine=*/true); } TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { + if (getCodecType() != Http::CodecType::HTTP1) { + return; + } EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).WillRepeatedly(Return(false)); expect_data_streams_ = false; @@ -196,43 +470,6 @@ TEST_P(ClientIntegrationTest, ClearTextNotPermitted) { ASSERT_EQ(cc_.on_complete_calls, 1); } -TEST_P(ClientIntegrationTest, BasicHttp2) { - EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).Times(0); - EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); - EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); - - setUpstreamProtocol(Http::CodecType::HTTP2); - builder_.enablePlatformCertificatesValidation(true); - - upstream_tls_ = true; - - initialize(); - - default_request_headers_.setScheme("https"); - - basicTest(); - // HTTP/2 - ASSERT_EQ(2, last_stream_final_intel_.upstream_protocol); -} - -// Do HTTP/3 without doing the alt-svc-over-HTTP/2 dance. -TEST_P(ClientIntegrationTest, DISABLED_Http3WithQuicHints) { - EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).Times(0); - EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); - EXPECT_CALL(helper_handle_->mock_helper(), cleanupAfterCertificateValidation()); - - setUpstreamProtocol(Http::CodecType::HTTP3); - builder_.enablePlatformCertificatesValidation(true); - upstream_tls_ = true; - add_quic_hints_ = true; - - initialize(); - default_request_headers_.setScheme("https"); - basicTest(); - // HTTP/3 - ASSERT_EQ(3, last_stream_final_intel_.upstream_protocol); -} - TEST_P(ClientIntegrationTest, BasicHttps) { EXPECT_CALL(helper_handle_->mock_helper(), isCleartextPermitted(_)).Times(0); EXPECT_CALL(helper_handle_->mock_helper(), validateCertificateChain(_, _)); @@ -273,10 +510,11 @@ TEST_P(ClientIntegrationTest, BasicHttps) { ASSERT_EQ(cc_.on_headers_calls, 1); ASSERT_EQ(cc_.status, "200"); - ASSERT_EQ(cc_.on_data_calls, 2); + ASSERT_GE(cc_.on_data_calls, 1); ASSERT_EQ(cc_.on_complete_calls, 1); - ASSERT_EQ(cc_.on_header_consumed_bytes_from_response, 27); - ASSERT_EQ(cc_.on_complete_received_byte_count, 67); + if (upstreamProtocol() == Http::CodecType::HTTP1) { + ASSERT_EQ(cc_.on_complete_received_byte_count, 67); + } } TEST_P(ClientIntegrationTest, BasicNon2xx) { @@ -285,7 +523,7 @@ TEST_P(ClientIntegrationTest, BasicNon2xx) { // Set response header status to be non-2xx to test that the correct stats get charged. reinterpret_cast(fake_upstreams_.front().get()) ->setResponseHeaders(std::make_unique( - Http::TestResponseHeaderMapImpl({{":status", "503"}, {"content-length", "0"}}))); + Http::TestResponseHeaderMapImpl({{":status", "503"}}))); stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); terminal_callback_.waitReady(); @@ -296,7 +534,18 @@ TEST_P(ClientIntegrationTest, BasicNon2xx) { ASSERT_EQ(cc_.on_complete_calls, 1); } -TEST_P(ClientIntegrationTest, BasicReset) { +TEST_P(ClientIntegrationTest, InvalidDomain) { + initialize(); + + default_request_headers_.setHost("www.doesnotexist.com"); + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_error_calls, 1); + ASSERT_EQ(cc_.on_headers_calls, 0); +} + +TEST_P(ClientIntegrationTest, BasicBeforeResponseHeaders) { initialize(); default_request_headers_.addCopy(AutonomousStream::RESET_AFTER_REQUEST, "yes"); @@ -308,7 +557,92 @@ TEST_P(ClientIntegrationTest, BasicReset) { ASSERT_EQ(cc_.on_headers_calls, 0); } -TEST_P(ClientIntegrationTest, BasicCancel) { +TEST_P(ClientIntegrationTest, ResetAfterResponseHeaders) { + autonomous_allow_incomplete_streams_ = true; + initialize(); + + default_request_headers_.addCopy(AutonomousStream::RESET_AFTER_RESPONSE_HEADERS, "yes"); + default_request_headers_.addCopy(AutonomousStream::RESPONSE_DATA_BLOCKS, "1"); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_error_calls, 1); +} + +TEST_P(ClientIntegrationTest, ResetAfterHeaderOnlyResponse) { + autonomous_allow_incomplete_streams_ = true; + initialize(); + + default_request_headers_.addCopy(AutonomousStream::RESET_AFTER_RESPONSE_HEADERS, "yes"); + default_request_headers_.addCopy(AutonomousStream::RESPONSE_DATA_BLOCKS, "0"); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), false); + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_error_calls, 1); +} + +TEST_P(ClientIntegrationTest, ResetBetweenDataChunks) { + autonomous_allow_incomplete_streams_ = true; + initialize(); + + default_request_headers_.addCopy(AutonomousStream::RESET_AFTER_RESPONSE_DATA, "yes"); + default_request_headers_.addCopy(AutonomousStream::RESPONSE_DATA_BLOCKS, "2"); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_error_calls, 1); +} + +TEST_P(ClientIntegrationTest, ResetAfterData) { + autonomous_allow_incomplete_streams_ = true; + initialize(); + + default_request_headers_.addCopy(AutonomousStream::RESET_AFTER_RESPONSE_DATA, "yes"); + default_request_headers_.addCopy(AutonomousStream::RESPONSE_DATA_BLOCKS, "1"); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_error_calls, 1); +} + +TEST_P(ClientIntegrationTest, CancelBeforeRequestHeadersSent) { + autonomous_upstream_ = false; + initialize(); + ConditionalInitializer headers_callback; + + stream_->cancel(); + + terminal_callback_.waitReady(); + + ASSERT_EQ(cc_.on_cancel_calls, 1); +} + +TEST_P(ClientIntegrationTest, CancelAfterRequestHeadersSent) { + initialize(); + + default_request_headers_.addCopy(AutonomousStream::RESPOND_AFTER_REQUEST_HEADERS, "yes"); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), false); + stream_->cancel(); + terminal_callback_.waitReady(); + ASSERT_EQ(cc_.on_cancel_calls, 1); +} + +TEST_P(ClientIntegrationTest, CancelAfterRequestComplete) { + autonomous_upstream_ = false; + initialize(); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + stream_->cancel(); + terminal_callback_.waitReady(); + ASSERT_EQ(cc_.on_cancel_calls, 1); +} + +TEST_P(ClientIntegrationTest, CancelDuringResponse) { autonomous_upstream_ = false; initialize(); ConditionalInitializer headers_callback; @@ -324,16 +658,12 @@ TEST_P(ClientIntegrationTest, BasicCancel) { stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); - Envoy::FakeRawConnectionPtr upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(upstream_connection)); - - std::string upstream_request; - EXPECT_TRUE(upstream_connection->waitForData(FakeRawConnection::waitForInexactMatch("GET /"), - &upstream_request)); - + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, + upstream_connection_)); + ASSERT_TRUE( + upstream_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request_)); // Send an incomplete response. - auto response = "HTTP/1.1 200 OK\r\nContent-Length: 15\r\n\r\n"; - ASSERT_TRUE(upstream_connection->write(response)); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); headers_callback.waitReady(); ASSERT_EQ(cc_.on_headers_calls, 1); @@ -351,6 +681,43 @@ TEST_P(ClientIntegrationTest, BasicCancel) { ASSERT_EQ(cc_.on_data_calls, 0); ASSERT_EQ(cc_.on_complete_calls, 0); ASSERT_EQ(cc_.on_cancel_calls, 1); + + if (upstreamProtocol() != Http::CodecType::HTTP1) { + ASSERT_TRUE(upstream_request_->waitForReset()); + } +} + +TEST_P(ClientIntegrationTest, BasicCancelWithCompleteStream) { + autonomous_upstream_ = false; + + initialize(); + ConditionalInitializer headers_callback; + + stream_prototype_->setOnHeaders( + [this, &headers_callback](Platform::ResponseHeadersSharedPtr headers, bool, + envoy_stream_intel) { + cc_.status = absl::StrCat(headers->httpStatus()); + cc_.on_headers_calls++; + headers_callback.setReady(); + return nullptr; + }); + + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, + upstream_connection_)); + ASSERT_TRUE( + upstream_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request_)); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + + terminal_callback_.waitReady(); + ASSERT_EQ(cc_.on_headers_calls, 1); + ASSERT_EQ(cc_.status, "200"); + ASSERT_EQ(cc_.on_complete_calls, 1); + + // Now cancel. As on_complete has been called cancel is a no-op but is + // non-problematic. + stream_->cancel(); } TEST_P(ClientIntegrationTest, CancelWithPartialStream) { @@ -370,22 +737,21 @@ TEST_P(ClientIntegrationTest, CancelWithPartialStream) { stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); - Envoy::FakeRawConnectionPtr upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(upstream_connection)); - - std::string upstream_request; - EXPECT_TRUE(upstream_connection->waitForData(FakeRawConnection::waitForInexactMatch("GET /"), - &upstream_request)); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, + upstream_connection_)); + ASSERT_TRUE( + upstream_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request_)); // Send a complete response with body. - auto response = "HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nasd"; - ASSERT_TRUE(upstream_connection->write(response)); - headers_callback.waitReady(); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request_->encodeData(1, true); + headers_callback.waitReady(); ASSERT_EQ(cc_.on_headers_calls, 1); ASSERT_EQ(cc_.status, "200"); ASSERT_EQ(cc_.on_data_calls, 0); ASSERT_EQ(cc_.on_complete_calls, 0); + // Due to explicit flow control, the upstream stream is complete, but the // callbacks will not be called for data and completion. Cancel the stream // and make sure the cancel is received. @@ -399,12 +765,11 @@ TEST_P(ClientIntegrationTest, CancelWithPartialStream) { ASSERT_EQ(cc_.on_cancel_calls, 1); } -// TODO(junr03): test with envoy local reply with local stream not closed, which causes a reset -// fired from the Http:ConnectionManager rather than the Http::Client. This cannot be done in -// unit tests because the Http::ConnectionManager is mocked using a mock response encoder. - // Test header key case sensitivity. TEST_P(ClientIntegrationTest, CaseSensitive) { + if (getCodecType() != Http::CodecType::HTTP1) { + return; + } autonomous_upstream_ = false; initialize(); @@ -456,18 +821,23 @@ TEST_P(ClientIntegrationTest, TimeoutOnRequestPath) { stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), false); - Envoy::FakeRawConnectionPtr upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(upstream_connection)); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, + upstream_connection_)); + ASSERT_TRUE( + upstream_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request_)); - std::string upstream_request; - EXPECT_TRUE(upstream_connection->waitForData(FakeRawConnection::waitForInexactMatch("GET /"), - &upstream_request)); terminal_callback_.waitReady(); ASSERT_EQ(cc_.on_headers_calls, 0); ASSERT_EQ(cc_.on_data_calls, 0); ASSERT_EQ(cc_.on_complete_calls, 0); ASSERT_EQ(cc_.on_error_calls, 1); + + if (getCodecType() != Http::CodecType::HTTP1) { + ASSERT_TRUE(upstream_request_->waitForReset()); + } else { + ASSERT_TRUE(upstream_connection_->waitForDisconnect()); + } } TEST_P(ClientIntegrationTest, TimeoutOnResponsePath) { @@ -477,17 +847,15 @@ TEST_P(ClientIntegrationTest, TimeoutOnResponsePath) { stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); - Envoy::FakeRawConnectionPtr upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(upstream_connection)); - - std::string upstream_request; - EXPECT_TRUE(upstream_connection->waitForData(FakeRawConnection::waitForInexactMatch("GET /"), - &upstream_request)); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*BaseIntegrationTest::dispatcher_, + upstream_connection_)); + ASSERT_TRUE( + upstream_connection_->waitForNewStream(*BaseIntegrationTest::dispatcher_, upstream_request_)); // Send response headers but no body. - auto response = "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nMy-ResponsE-Header: foo\r\n\r\n"; - ASSERT_TRUE(upstream_connection->write(response)); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + // Wait for timeout. terminal_callback_.waitReady(); ASSERT_EQ(cc_.on_headers_calls, 1); @@ -495,9 +863,16 @@ TEST_P(ClientIntegrationTest, TimeoutOnResponsePath) { ASSERT_EQ(cc_.on_data_calls, 0); ASSERT_EQ(cc_.on_complete_calls, 0); ASSERT_EQ(cc_.on_error_calls, 1); + + if (upstreamProtocol() != Http::CodecType::HTTP1) { + ASSERT_TRUE(upstream_request_->waitForReset()); + } } TEST_P(ClientIntegrationTest, Proxying) { + if (getCodecType() != Http::CodecType::HTTP1) { + return; + } builder_.addLogLevel(Platform::LogLevel::trace); initialize(); @@ -521,10 +896,6 @@ TEST_P(ClientIntegrationTest, Proxying) { TEST_P(ClientIntegrationTest, DirectResponse) { initialize(); - if (version_ == Network::Address::IpVersion::v6) { - // Localhost only resolves to an ipv4 address - alas no kernel happy eyeballs. - return; - } // Override to not validate stream intel. stream_prototype_->setOnComplete( @@ -561,8 +932,11 @@ TEST_P(ClientIntegrationTest, TestRuntimeSet) { TEST_P(ClientIntegrationTest, TestStats) { initialize(); - std::string stats = engine_->dumpStats(); - EXPECT_TRUE((absl::StrContains(stats, "runtime.load_success: 1"))) << stats; + { + absl::MutexLock l(&engine_lock_); + std::string stats = engine_->dumpStats(); + EXPECT_TRUE((absl::StrContains(stats, "runtime.load_success: 1"))) << stats; + } } } // namespace diff --git a/mobile/test/common/integration/proxy.proto b/mobile/test/common/integration/proxy.proto new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/mobile/test/common/integration/proxy.proto @@ -0,0 +1 @@ + diff --git a/mobile/test/common/integration/rtds_integration_test.cc b/mobile/test/common/integration/rtds_integration_test.cc index 91a3f2c89cfa..96cde1310d73 100644 --- a/mobile/test/common/integration/rtds_integration_test.cc +++ b/mobile/test/common/integration/rtds_integration_test.cc @@ -2,6 +2,8 @@ #include "envoy/service/runtime/v3/rtds.pb.h" #include "test/common/integration/xds_integration_test.h" +#include "test/test_common/environment.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -12,23 +14,80 @@ namespace { class RtdsIntegrationTest : public XdsIntegrationTest { public: void initialize() override { + // using http1 because the h1 cluster has a plaintext socket + setUpstreamProtocol(Http::CodecType::HTTP1); + XdsIntegrationTest::initialize(); default_request_headers_.setScheme("http"); initializeXdsStream(); } + void createEnvoy() override { Platform::XdsBuilder xds_builder( /*xds_server_address=*/Network::Test::getLoopbackAddressUrlString(ipVersion()), /*xds_server_port=*/fake_upstreams_[1]->localAddress()->ip()->port()); // Add the layered runtime config, which includes the RTDS layer. - xds_builder.addRuntimeDiscoveryService("some_rtds_resource", /*timeout_in_seconds=*/1); + xds_builder.addRuntimeDiscoveryService("some_rtds_resource", /*timeout_in_seconds=*/1) + .setSslRootCerts(getUpstreamCert()); builder_.setXds(std::move(xds_builder)); XdsIntegrationTest::createEnvoy(); } - // using http1 because the h1 cluster has a plaintext socket - void SetUp() override { setUpstreamProtocol(Http::CodecType::HTTP1); } + void SetUp() override { initialize(); } + + void runReloadTest() { + // Send a request on the data plane. + stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); + terminal_callback_.waitReady(); + + EXPECT_EQ(cc_.on_headers_calls, 1); + EXPECT_EQ(cc_.status, "200"); + EXPECT_EQ(cc_.on_data_calls, 2); + EXPECT_EQ(cc_.on_complete_calls, 1); + EXPECT_EQ(cc_.on_cancel_calls, 0); + EXPECT_EQ(cc_.on_error_calls, 0); + EXPECT_EQ(cc_.on_header_consumed_bytes_from_response, 27); + EXPECT_EQ(cc_.on_complete_received_byte_count, 67); + // Check that the Runtime config is from the static layer. + EXPECT_FALSE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); + + const std::string load_success_counter = "runtime.load_success"; + uint64_t load_success_value = getCounterValue(load_success_counter); + // Send a RTDS request and get back the RTDS response. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_resource"}, + {"some_rtds_resource"}, {}, true)); + + envoy::service::runtime::v3::Runtime some_rtds_resource; + some_rtds_resource.set_name("some_rtds_resource"); + auto* static_layer = some_rtds_resource.mutable_layer(); + (*static_layer->mutable_fields())["envoy.reloadable_features.test_feature_false"] + .set_bool_value(true); + + sendDiscoveryResponse( + Config::TypeUrl::get().Runtime, {some_rtds_resource}, {some_rtds_resource}, {}, "1"); + // Wait until the RTDS updates from the DiscoveryResponse have been applied. + ASSERT_TRUE(waitForCounterGe(load_success_counter, load_success_value + 1)); + + // Verify that the Runtime config values are from the RTDS response. + EXPECT_TRUE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); + + load_success_value = getCounterValue(load_success_counter); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_resource"}, + {"some_rtds_resource"}, {})); + (*static_layer->mutable_fields())["envoy.reloadable_features.test_feature_false"] + .set_bool_value(false); + + // Send another response with Resource wrapper. + sendDiscoveryResponse( + Config::TypeUrl::get().Runtime, {some_rtds_resource}, {some_rtds_resource}, {}, "2", + {{"test", ProtobufWkt::Any()}}); + // Wait until the RTDS updates from the DiscoveryResponse have been applied. + ASSERT_TRUE(waitForCounterGe(load_success_counter, load_success_value + 1)); + + // Verify that the Runtime config values are from the RTDS response. + EXPECT_FALSE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); + } }; INSTANTIATE_TEST_SUITE_P( @@ -38,59 +97,14 @@ INSTANTIATE_TEST_SUITE_P( // Envoy Mobile's xDS APIs only support state-of-the-world, not delta. testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::UnifiedSotw))); -TEST_P(RtdsIntegrationTest, RtdsReload) { - initialize(); - - // Send a request on the data plane. - stream_->sendHeaders(envoyToMobileHeaders(default_request_headers_), true); - terminal_callback_.waitReady(); - - EXPECT_EQ(cc_.on_headers_calls, 1); - EXPECT_EQ(cc_.status, "200"); - EXPECT_EQ(cc_.on_data_calls, 2); - EXPECT_EQ(cc_.on_complete_calls, 1); - EXPECT_EQ(cc_.on_cancel_calls, 0); - EXPECT_EQ(cc_.on_error_calls, 0); - EXPECT_EQ(cc_.on_header_consumed_bytes_from_response, 27); - EXPECT_EQ(cc_.on_complete_received_byte_count, 67); - // Check that the Runtime config is from the static layer. - EXPECT_FALSE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); - - const std::string load_success_counter = "runtime.load_success"; - uint64_t load_success_value = getCounterValue(load_success_counter); - // Send a RTDS request and get back the RTDS response. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_resource"}, - {"some_rtds_resource"}, {}, true)); - auto some_rtds_resource = TestUtility::parseYaml(R"EOF( - name: some_rtds_resource - layer: - envoy.reloadable_features.test_feature_false: True - )EOF"); - sendDiscoveryResponse( - Config::TypeUrl::get().Runtime, {some_rtds_resource}, {some_rtds_resource}, {}, "1"); - // Wait until the RTDS updates from the DiscoveryResponse have been applied. - ASSERT_TRUE(waitForCounterGe(load_success_counter, load_success_value + 1)); - - // Verify that the Runtime config values are from the RTDS response. - EXPECT_TRUE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); - - load_success_value = getCounterValue(load_success_counter); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_resource"}, - {"some_rtds_resource"}, {})); - some_rtds_resource = TestUtility::parseYaml(R"EOF( - name: some_rtds_resource - layer: - envoy.reloadable_features.test_feature_false: False - )EOF"); - // Send another response with Resource wrapper. - sendDiscoveryResponse( - Config::TypeUrl::get().Runtime, {some_rtds_resource}, {some_rtds_resource}, {}, "2", - {{"test", ProtobufWkt::Any()}}); - // Wait until the RTDS updates from the DiscoveryResponse have been applied. - ASSERT_TRUE(waitForCounterGe(load_success_counter, load_success_value + 1)); - - // Verify that the Runtime config values are from the RTDS response. - EXPECT_FALSE(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); +TEST_P(RtdsIntegrationTest, RtdsReloadWithDfpMixedScheme) { + TestScopedStaticReloadableFeaturesRuntime scoped_runtime({{"dfp_mixed_scheme", true}}); + runReloadTest(); +} + +TEST_P(RtdsIntegrationTest, RtdsReloadWithoutDfpMixedScheme) { + TestScopedStaticReloadableFeaturesRuntime scoped_runtime({{"dfp_mixed_scheme", false}}); + runReloadTest(); } } // namespace diff --git a/mobile/test/common/integration/sds_integration_test.cc b/mobile/test/common/integration/sds_integration_test.cc index 49b524f3557c..4000d0f24e46 100644 --- a/mobile/test/common/integration/sds_integration_test.cc +++ b/mobile/test/common/integration/sds_integration_test.cc @@ -5,8 +5,8 @@ #include "test/common/integration/xds_integration_test.h" #include "test/integration/ssl_utility.h" +#include "test/test_common/environment.h" -#include "extension_registry.h" #include "gtest/gtest.h" namespace Envoy { @@ -20,11 +20,6 @@ constexpr absl::string_view SECRET_NAME = "client_cert"; class SdsIntegrationTest : public XdsIntegrationTest { public: void initialize() override { - create_xds_upstream_ = true; - use_lds_ = false; - skip_tag_extraction_rule_check_ = true; - ExtensionRegistry::registerFactories(); - XdsIntegrationTest::initialize(); initializeXdsStream(); } @@ -33,11 +28,13 @@ class SdsIntegrationTest : public XdsIntegrationTest { const std::string target_uri = Network::Test::getLoopbackAddressUrlString(ipVersion()); Platform::XdsBuilder xds_builder(target_uri, fake_upstreams_.back()->localAddress()->ip()->port()); - xds_builder.addClusterDiscoveryService(); + xds_builder.addClusterDiscoveryService().setSslRootCerts(getUpstreamCert()); builder_.setXds(std::move(xds_builder)); XdsIntegrationTest::createEnvoy(); } + void SetUp() override { initialize(); } + protected: void sendCdsResponse() { auto cds_cluster = createSingleEndpointClusterConfig(std::string(XDS_CLUSTER_NAME)); @@ -93,8 +90,6 @@ INSTANTIATE_TEST_SUITE_P( // Note: Envoy Mobile does not have listener sockets, so we aren't including a downstream test. TEST_P(SdsIntegrationTest, SdsForUpstreamCluster) { - initialize(); - // Wait until the new cluster from CDS is added before sending the SDS response. sendCdsResponse(); ASSERT_TRUE(waitForCounterGe("cluster_manager.cluster_added", 1)); diff --git a/mobile/test/common/integration/test_server.cc b/mobile/test/common/integration/test_server.cc index de0847c1821a..f771ba80c6ff 100644 --- a/mobile/test/common/integration/test_server.cc +++ b/mobile/test/common/integration/test_server.cc @@ -1,8 +1,13 @@ #include "test_server.h" +#include "source/common/common/random_generator.h" +#include "source/common/stats/allocator_impl.h" +#include "source/common/stats/thread_local_store.h" +#include "source/common/thread_local/thread_local_impl.h" #include "source/extensions/transport_sockets/tls/context_config_impl.h" +#include "source/server/hot_restart_nop_impl.h" +#include "source/server/instance_impl.h" -#include "test/integration/server.h" #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" @@ -16,9 +21,9 @@ Network::DownstreamTransportSocketFactoryPtr TestServer::createQuicUpstreamTlsCo envoy::extensions::transport_sockets::tls::v3::TlsCertificate* certs = tls_context.mutable_common_tls_context()->add_tls_certificates(); certs->mutable_certificate_chain()->set_filename( - "../envoy/test/config/integration/certs/upstreamcert.pem"); + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcert.pem")); certs->mutable_private_key()->set_filename( - "../envoy/test/config/integration/certs/upstreamkey.pem"); + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamkey.pem")); envoy::extensions::transport_sockets::quic::v3::QuicDownstreamTransport quic_config; quic_config.mutable_downstream_tls_context()->MergeFrom(tls_context); @@ -36,12 +41,12 @@ Network::DownstreamTransportSocketFactoryPtr TestServer::createUpstreamTlsContex envoy::extensions::transport_sockets::tls::v3::TlsCertificate* certs = tls_context.mutable_common_tls_context()->add_tls_certificates(); certs->mutable_certificate_chain()->set_filename( - "../envoy/test/config/integration/certs/upstreamcert.pem"); + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcert.pem")); certs->mutable_private_key()->set_filename( - "../envoy/test/config/integration/certs/upstreamkey.pem"); + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamkey.pem")); auto* ctx = tls_context.mutable_common_tls_context()->mutable_validation_context(); ctx->mutable_trusted_ca()->set_filename( - "../envoy/test/config/integration/certs/upstreamcacert.pem"); + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); tls_context.mutable_common_tls_context()->add_alpn_protocols("h2"); auto cfg = std::make_unique( tls_context, factory_context); @@ -54,6 +59,13 @@ Network::DownstreamTransportSocketFactoryPtr TestServer::createUpstreamTlsContex TestServer::TestServer() : api_(Api::createApiForTest(stats_store_, time_system_)), version_(Network::Address::IpVersion::v4), upstream_config_(time_system_), port_(0) { + std::string runfiles_error; + runfiles_ = std::unique_ptr{ + bazel::tools::cpp::runfiles::Runfiles::CreateForTest(&runfiles_error)}; + RELEASE_ASSERT(TestEnvironment::getOptionalEnvVar("NORUNFILES").has_value() || + runfiles_ != nullptr, + runfiles_error); + TestEnvironment::setRunfiles(runfiles_.get()); ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); ON_CALL(factory_context_, statsScope()) .WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); @@ -81,6 +93,24 @@ void TestServer::startTestServer(TestServerType test_server_type) { upstream_config_.upstream_protocol_ = Http::CodecType::HTTP1; factory = Network::Test::createRawBufferDownstreamSocketFactory(); break; + case TestServerType::HTTP_PROXY: { + std::string config_path = + TestEnvironment::writeStringToFileForTest("config.yaml", http_proxy_config); + test_server_ = IntegrationTestServer::create(config_path, Network::Address::IpVersion::v4, + nullptr, nullptr, {}, time_system_, *api_); + test_server_->waitUntilListenersReady(); + ENVOY_LOG_MISC(debug, "Http proxy is now running"); + return; + } + case TestServerType::HTTPS_PROXY: { + std::string config_path = + TestEnvironment::writeStringToFileForTest("config.yaml", https_proxy_config); + test_server_ = IntegrationTestServer::create(config_path, Network::Address::IpVersion::v4, + nullptr, nullptr, {}, time_system_, *api_); + test_server_->waitUntilListenersReady(); + ENVOY_LOG_MISC(debug, "Https proxy is now running"); + return; + } } upstream_ = std::make_unique(std::move(factory), port_, version_, @@ -101,20 +131,177 @@ void TestServer::startTestServer(TestServerType test_server_type) { } void TestServer::shutdownTestServer() { - ASSERT(upstream_); + ASSERT(upstream_ || test_server_); upstream_.reset(); + test_server_.reset(); } int TestServer::getServerPort() { - ASSERT(upstream_); - return upstream_->localAddress()->ip()->port(); + ASSERT(upstream_ || test_server_); + if (upstream_) { + return upstream_->localAddress()->ip()->port(); + } + std::atomic port = 0; + absl::Notification port_set; + test_server_->server().dispatcher().post([&]() { + auto listeners = test_server_->server().listenerManager().listeners(); + auto listener_it = listeners.cbegin(); + auto socket_factory_it = listener_it->get().listenSocketFactories().begin(); + const auto listen_addr = (*socket_factory_it)->localAddress(); + port = listen_addr->ip()->port(); + port_set.Notify(); + }); + port_set.WaitForNotification(); + return port; } void TestServer::setHeadersAndData(absl::string_view header_key, absl::string_view header_value, absl::string_view response_body) { + ASSERT(upstream_); upstream_->setResponseHeaders( std::make_unique(Http::TestResponseHeaderMapImpl( {{std::string(header_key), std::string(header_value)}, {":status", "200"}}))); upstream_->setResponseBody(std::string(response_body)); } + +const std::string TestServer::http_proxy_config = R"EOF( +static_resources: + listeners: + - name: listener_proxy + address: + socket_address: { address: 127.0.0.1, port_value: 0 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: remote_hcm + route_config: + name: remote_route + virtual_hosts: + - name: remote_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: cluster_proxy } + response_headers_to_add: + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-proxy-response + value: 'true' + http_filters: + - name: envoy.filters.http.local_error + typed_config: + "@type": type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: &dns_cache_config + name: base_dns_cache + dns_lookup_family: ALL + host_ttl: 86400s + dns_min_refresh_rate: 20s + dns_refresh_rate: 60s + dns_failure_refresh_rate: + base_interval: 2s + max_interval: 10s + dns_query_timeout: 25s + typed_dns_resolver_config: + name: envoy.network.dns_resolver.getaddrinfo + typed_config: {"@type":"type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig"} + - name: envoy.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: cluster_proxy + connect_timeout: 30s + lb_policy: CLUSTER_PROVIDED + dns_lookup_family: ALL + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: *dns_cache_config +layered_runtime: + layers: + - name: static_layer_0 + static_layer: + envoy: + # This disables envoy bug stats, which are filtered out of our stats inclusion list anyway + # Global stats do not play well with engines with limited lifetimes + disallow_global_stats: true +)EOF"; + +const std::string TestServer::https_proxy_config = R"EOF( +static_resources: + listeners: + - name: listener_proxy + address: + socket_address: { address: 127.0.0.1, port_value: 0 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: remote_hcm + route_config: + name: remote_route + virtual_hosts: + - name: remote_service + domains: ["*"] + routes: + - match: { connect_matcher: {} } + route: + cluster: cluster_proxy + upgrade_configs: + - upgrade_type: CONNECT + connect_config: + response_headers_to_add: + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-response-header-that-should-be-stripped + value: 'true' + http_filters: + - name: envoy.filters.http.local_error + typed_config: + "@type": type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: &dns_cache_config + name: base_dns_cache + dns_lookup_family: ALL + host_ttl: 86400s + dns_min_refresh_rate: 20s + dns_refresh_rate: 60s + dns_failure_refresh_rate: + base_interval: 2s + max_interval: 10s + dns_query_timeout: 25s + typed_dns_resolver_config: + name: envoy.network.dns_resolver.getaddrinfo + typed_config: {"@type":"type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig"} + - name: envoy.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: cluster_proxy + connect_timeout: 30s + lb_policy: CLUSTER_PROVIDED + dns_lookup_family: ALL + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: *dns_cache_config +layered_runtime: + layers: + - name: static_layer_0 + static_layer: + envoy: + # This disables envoy bug stats, which are filtered out of our stats inclusion list anyway + # Global stats do not play well with engines with limited lifetimes + disallow_global_stats: true +)EOF"; + } // namespace Envoy diff --git a/mobile/test/common/integration/test_server.h b/mobile/test/common/integration/test_server.h index ff5411c7ad39..59cfa6bfc5df 100644 --- a/mobile/test/common/integration/test_server.h +++ b/mobile/test/common/integration/test_server.h @@ -4,10 +4,14 @@ // test_runner setups #include "source/exe/process_wide.h" +#include "source/server/listener_hooks.h" #include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" #include "test/integration/autonomous_upstream.h" #include "test/mocks/server/transport_socket_factory_context.h" +#include "test/integration/server.h" + +#include "tools/cpp/runfiles/runfiles.h" namespace Envoy { @@ -15,22 +19,29 @@ enum class TestServerType { HTTP1_WITHOUT_TLS, HTTP2_WITH_TLS, HTTP3, + HTTP_PROXY, + HTTPS_PROXY, }; -class TestServer { +class TestServer : public ListenerHooks { private: testing::NiceMock factory_context_; Stats::IsolatedStoreImpl stats_store_; Event::GlobalTimeSystem time_system_; Api::ApiPtr api_; Network::Address::IpVersion version_; - std::unique_ptr upstream_; FakeUpstreamConfig upstream_config_; int port_; Thread::SkipAsserts skip_asserts_; ProcessWide process_wide; Thread::MutexBasicLockable lock; Extensions::TransportSockets::Tls::ContextManagerImpl context_manager_{time_system_}; + std::unique_ptr runfiles_; + + // Either test_server_ will be set for test_server_type is a proxy, otherwise upstream_ will be + // used. + std::unique_ptr upstream_; + IntegrationTestServerPtr test_server_; Network::DownstreamTransportSocketFactoryPtr createQuicUpstreamTlsContext( testing::NiceMock&); @@ -38,13 +49,16 @@ class TestServer { Network::DownstreamTransportSocketFactoryPtr createUpstreamTlsContext( testing::NiceMock&); + static const std::string http_proxy_config; + static const std::string https_proxy_config; + public: TestServer(); /** * Starts the server. Can only have one server active per JVM. This is blocking until the port can * start accepting requests. - * test_server_type: selects between HTTP3, HTTP2, or HTTP1 without TLS + * test_server_type: selects between TestServerTypes */ void startTestServer(TestServerType test_server_type); @@ -65,6 +79,11 @@ class TestServer { */ void setHeadersAndData(absl::string_view header_key, absl::string_view header_value, absl::string_view response_body); + + // ListenerHooks + void onWorkerListenerAdded() override {} + void onWorkerListenerRemoved() override {} + void onWorkersStarted() override {} }; } // namespace Envoy diff --git a/mobile/test/common/integration/xds_integration_test.cc b/mobile/test/common/integration/xds_integration_test.cc index 408480c2fe4b..1ede54812e4b 100644 --- a/mobile/test/common/integration/xds_integration_test.cc +++ b/mobile/test/common/integration/xds_integration_test.cc @@ -3,17 +3,12 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/cluster/v3/cluster.pb.h" -#include "source/common/grpc/google_grpc_creds_impl.h" -#include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" -#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" - #include "test/common/grpc/grpc_client_integration.h" #include "test/common/integration/base_client_integration_test.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" +#include "extension_registry.h" #include "gtest/gtest.h" namespace Envoy { @@ -22,21 +17,16 @@ using ::testing::AssertionFailure; using ::testing::AssertionResult; using ::testing::AssertionSuccess; -XdsIntegrationTest::XdsIntegrationTest() : BaseClientIntegrationTest(ipVersion()) { - Grpc::forceRegisterDefaultGoogleGrpcCredentialsFactory(); - Config::forceRegisterAdsConfigSubscriptionFactory(); - Config::forceRegisterGrpcConfigSubscriptionFactory(); - Config::forceRegisterDeltaGrpcConfigSubscriptionFactory(); - Config::forceRegisterDeltaGrpcCollectionConfigSubscriptionFactory(); - Config::forceRegisterAggregatedGrpcCollectionConfigSubscriptionFactory(); - Config::forceRegisterAdsCollectionConfigSubscriptionFactory(); - Config::forceRegisterGrpcMuxFactory(); - Config::forceRegisterNewGrpcMuxFactory(); - - expect_dns_ = false; // doesn't use DFP. +XdsIntegrationTest::XdsIntegrationTest() : BaseClientIntegrationTest(ipVersion()) {} + +void XdsIntegrationTest::initialize() { create_xds_upstream_ = true; + tls_xds_upstream_ = true; sotw_or_delta_ = sotwOrDelta(); + // Register the extensions required for Envoy Mobile. + ExtensionRegistry::registerFactories(); + if (sotw_or_delta_ == Grpc::SotwOrDelta::UnifiedSotw || sotw_or_delta_ == Grpc::SotwOrDelta::UnifiedDelta) { config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); @@ -44,10 +34,9 @@ XdsIntegrationTest::XdsIntegrationTest() : BaseClientIntegrationTest(ipVersion() // xDS upstream is created separately in the test infra, and there's only one non-xDS cluster. setUpstreamCount(1); -} -void XdsIntegrationTest::initialize() { BaseClientIntegrationTest::initialize(); + default_request_headers_.setScheme("https"); } @@ -101,4 +90,9 @@ XdsIntegrationTest::createSingleEndpointClusterConfig(const std::string& cluster return config; } +std::string XdsIntegrationTest::getUpstreamCert() { + return TestEnvironment::readFileToStringForTest( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); +} + } // namespace Envoy diff --git a/mobile/test/common/integration/xds_integration_test.h b/mobile/test/common/integration/xds_integration_test.h index 0fd5465be114..701715b03b52 100644 --- a/mobile/test/common/integration/xds_integration_test.h +++ b/mobile/test/common/integration/xds_integration_test.h @@ -43,6 +43,9 @@ class XdsIntegrationTest : public BaseClientIntegrationTest, // a fake upstream on the loopback address. envoy::config::cluster::v3::Cluster createSingleEndpointClusterConfig(const std::string& cluster_name); + + // Gets the upstream cert for the xDS cluster's TLS over the `base` cluster. + std::string getUpstreamCert(); }; } // namespace Envoy diff --git a/mobile/test/common/integration/xds_test_server.cc b/mobile/test/common/integration/xds_test_server.cc new file mode 100644 index 000000000000..bec30c48c698 --- /dev/null +++ b/mobile/test/common/integration/xds_test_server.cc @@ -0,0 +1,109 @@ +#include "test/common/integration/xds_test_server.h" + +#include + +#include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" + +#include "source/common/event/libevent.h" +#include "source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" +#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" +#include "source/extensions/transport_sockets/tls/context_config_impl.h" +#include "source/extensions/transport_sockets/tls/ssl_socket.h" + +#include "test/integration/fake_upstream.h" +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/utility.h" + +namespace Envoy { + +XdsTestServer::XdsTestServer() + : api_(Api::createApiForTest(stats_store_, time_system_)), + version_(Network::Address::IpVersion::v4), + mock_buffer_factory_(new NiceMock), upstream_config_(time_system_) { + std::string runfiles_error; + runfiles_ = std::unique_ptr{ + bazel::tools::cpp::runfiles::Runfiles::Create("", &runfiles_error)}; + RELEASE_ASSERT(TestEnvironment::getOptionalEnvVar("NORUNFILES").has_value() || + runfiles_ != nullptr, + runfiles_error); + TestEnvironment::setRunfiles(runfiles_.get()); + + if (!Envoy::Event::Libevent::Global::initialized()) { + // Required by the Dispatcher. + Envoy::Event::Libevent::Global::initialize(); + } + dispatcher_ = + api_->allocateDispatcher("test_thread", Buffer::WatermarkFactoryPtr{mock_buffer_factory_}); + + ON_CALL(*mock_buffer_factory_, createBuffer_(_, _, _)) + .WillByDefault(Invoke([](std::function below_low, std::function above_high, + std::function above_overflow) -> Buffer::Instance* { + return new Buffer::WatermarkBuffer(std::move(below_low), std::move(above_high), + std::move(above_overflow)); + })); + ON_CALL(factory_context_.server_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(factory_context_, statsScope()) + .WillByDefault(testing::ReturnRef(*stats_store_.rootScope())); + Logger::Context logging_state(spdlog::level::level_enum::err, + "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v", lock_, false, false); + upstream_config_.upstream_protocol_ = Http::CodecType::HTTP2; + Config::forceRegisterAdsConfigSubscriptionFactory(); + Config::forceRegisterGrpcConfigSubscriptionFactory(); + Config::forceRegisterDeltaGrpcConfigSubscriptionFactory(); + Config::forceRegisterDeltaGrpcCollectionConfigSubscriptionFactory(); + Config::forceRegisterAggregatedGrpcCollectionConfigSubscriptionFactory(); + Config::forceRegisterAdsCollectionConfigSubscriptionFactory(); + Config::forceRegisterGrpcMuxFactory(); + Config::forceRegisterNewGrpcMuxFactory(); + + envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; + auto* common_tls_context = tls_context.mutable_common_tls_context(); + common_tls_context->add_alpn_protocols(Http::Utility::AlpnNames::get().Http2); + auto* tls_cert = common_tls_context->add_tls_certificates(); + tls_cert->mutable_certificate_chain()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcert.pem")); + tls_cert->mutable_private_key()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamkey.pem")); + auto cfg = std::make_unique( + tls_context, factory_context_); + auto context = std::make_unique( + std::move(cfg), context_manager_, *stats_store_.rootScope(), std::vector{}); + xds_upstream_ = std::make_unique(std::move(context), 0, version_, upstream_config_); +} + +std::string XdsTestServer::getHost() const { + return Network::Test::getLoopbackAddressUrlString(version_); +} + +int XdsTestServer::getPort() const { + ASSERT(xds_upstream_); + return xds_upstream_->localAddress()->ip()->port(); +} + +void XdsTestServer::start() { + AssertionResult result = xds_upstream_->waitForHttpConnection(*dispatcher_, xds_connection_); + RELEASE_ASSERT(result, result.message()); + result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); + RELEASE_ASSERT(result, result.message()); + xds_stream_->startGrpcStream(); +} + +void XdsTestServer::send(const envoy::service::discovery::v3::DiscoveryResponse& response) { + ASSERT(xds_stream_); + xds_stream_->sendGrpcMessage(response); +} + +void XdsTestServer::shutdown() { + if (xds_connection_ != nullptr) { + AssertionResult result = xds_connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = xds_connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + xds_connection_.reset(); + } +} + +} // namespace Envoy diff --git a/mobile/test/common/integration/xds_test_server.h b/mobile/test/common/integration/xds_test_server.h new file mode 100644 index 000000000000..95e5e4dbeed1 --- /dev/null +++ b/mobile/test/common/integration/xds_test_server.h @@ -0,0 +1,54 @@ +#pragma once + +#include "envoy/api/api.h" + +#include "source/common/stats/isolated_store_impl.h" +#include "source/extensions/transport_sockets/tls/context_manager_impl.h" + +#include "test/integration/fake_upstream.h" +#include "test/integration/server.h" +#include "test/mocks/server/transport_socket_factory_context.h" +#include "test/test_common/test_time.h" + +#include "tools/cpp/runfiles/runfiles.h" + +namespace Envoy { + +/** An xDS test server. */ +class XdsTestServer { +public: + XdsTestServer(); + + /** Starts the xDS server and returns the port number of the server. */ + void start(); + + /** Gets the xDS host. */ + std::string getHost() const; + + /** Gets the xDS port. */ + int getPort() const; + + /** Sends a `DiscoveryResponse` from the xDS server. */ + void send(const envoy::service::discovery::v3::DiscoveryResponse& response); + + /** Shuts down the xDS server. */ + void shutdown(); + +private: + testing::NiceMock factory_context_; + Stats::IsolatedStoreImpl stats_store_; + Event::GlobalTimeSystem time_system_; + Api::ApiPtr api_; + Network::Address::IpVersion version_; + MockBufferFactory* mock_buffer_factory_; + Event::DispatcherPtr dispatcher_; + FakeUpstreamConfig upstream_config_; + Thread::MutexBasicLockable lock_; + Extensions::TransportSockets::Tls::ContextManagerImpl context_manager_{time_system_}; + std::unique_ptr runfiles_; + std::unique_ptr xds_upstream_; + FakeHttpConnectionPtr xds_connection_; + FakeStreamPtr xds_stream_; +}; + +} // namespace Envoy diff --git a/mobile/test/common/integration/xds_test_server_interface.cc b/mobile/test/common/integration/xds_test_server_interface.cc new file mode 100644 index 000000000000..12152d5d9071 --- /dev/null +++ b/mobile/test/common/integration/xds_test_server_interface.cc @@ -0,0 +1,54 @@ +#include "test/common/integration/xds_test_server_interface.h" + +#include "test/common/integration/xds_test_server.h" + +#include "extension_registry.h" + +// NOLINT(namespace-envoy) + +static std::shared_ptr strong_test_server_; +static std::weak_ptr weak_test_server_; + +static std::shared_ptr testServer() { return weak_test_server_.lock(); } + +void initXdsServer() { + Envoy::ExtensionRegistry::registerFactories(); + strong_test_server_ = std::make_shared(); + weak_test_server_ = strong_test_server_; +} + +const char* getXdsServerHost() { + if (auto server = testServer()) { + const char* host = strdup(server->getHost().c_str()); + return host; + } + return ""; // failure +} + +int getXdsServerPort() { + if (auto server = testServer()) { + return server->getPort(); + } + return -1; // failure +} + +void startXdsServer() { + if (auto server = testServer()) { + server->start(); + } +} + +void sendDiscoveryResponse(const envoy::service::discovery::v3::DiscoveryResponse& response) { + if (auto server = testServer()) { + ASSERT(server); + server->send(response); + } +} + +void shutdownXdsServer() { + // Reset the primary handle to the test_server, + // but retain it long enough to synchronously shutdown. + auto server = strong_test_server_; + strong_test_server_.reset(); + server->shutdown(); +} diff --git a/mobile/test/common/integration/xds_test_server_interface.h b/mobile/test/common/integration/xds_test_server_interface.h new file mode 100644 index 000000000000..cebc5b18ae13 --- /dev/null +++ b/mobile/test/common/integration/xds_test_server_interface.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include "envoy/service/discovery/v3/discovery.pb.h" + +// NOLINT(namespace-envoy) + +#ifdef __cplusplus +extern "C" { // functions +#endif + +/** Initializes xDS server. */ +void initXdsServer(); + +/** Gets the xDS server host. `initXdsServer` must be called prior to calling this function. + */ +const char* getXdsServerHost(); + +/** + * Gets the xDS server port. `initXdsServer` must be called prior to calling this function. + */ +int getXdsServerPort(); + +/** + * Starts the xDS server. `initXdsServer` must be called prior to calling this function. + */ +void startXdsServer(); + +/** + * Sends the `DiscoveryResponse`. `startXdsServer` must be called prior to calling this function. + */ +void sendDiscoveryResponse(const envoy::service::discovery::v3::DiscoveryResponse& response); + +/** + * Shuts down the xDS server. `startXdsServer` must be called prior to calling this function. + */ +void shutdownXdsServer(); + +#ifdef __cplusplus +} // functions +#endif diff --git a/mobile/test/common/jni/BUILD b/mobile/test/common/jni/BUILD index 0607a3185b30..4ff3d508af14 100644 --- a/mobile/test/common/jni/BUILD +++ b/mobile/test/common/jni/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("//bazel:kotlin_lib.bzl", "envoy_mobile_so_to_jni_lib") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() # Library which contains all the JNI related targets and test extensions cc_library( @@ -21,7 +21,7 @@ cc_library( name = "server_envoy_jni_lib", testonly = True, srcs = [ - "test_jni_interface.cc", + "test_jni_impl.cc", ], linkopts = select({ "@envoy//bazel:dbg_build": ["-Wl,--build-id=sha1"], @@ -30,6 +30,7 @@ cc_library( deps = [ "//library/common/jni:envoy_jni_lib", "//test/common/integration:test_server_interface_lib", + "//test/common/integration:xds_test_server_interface_lib", ], # We need this to ensure that we link this into the .so even though there are no code references. alwayslink = True, @@ -63,7 +64,7 @@ cc_library( deps = [ ":server_envoy_jni_lib", "//library/common/jni:envoy_jni_lib", - "@envoy//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "@envoy//source/common/listener_manager:listener_manager_lib", ], ) @@ -82,3 +83,43 @@ envoy_mobile_so_to_jni_lib( testonly = True, native_dep = "libenvoy_jni_with_test_and_listener_extensions.so", ) + +cc_library( + name = "jni_helper_test_lib", + srcs = [ + "jni_helper_test.cc", + ], + deps = [ + "//library/common/jni:jni_helper_lib", + ], + alwayslink = True, +) + +cc_binary( + name = "libenvoy_jni_helper_test.so", + testonly = True, + linkshared = True, + deps = [ + ":jni_helper_test_lib", + ], +) + +cc_library( + name = "jni_utility_test_lib", + srcs = [ + "jni_utility_test.cc", + ], + deps = [ + "//library/common/jni:jni_utility_lib", + ], + alwayslink = True, +) + +cc_binary( + name = "libenvoy_jni_utility_test.so", + testonly = True, + linkshared = True, + deps = [ + ":jni_utility_test_lib", + ], +) diff --git a/mobile/test/common/jni/jni_helper_test.cc b/mobile/test/common/jni/jni_helper_test.cc new file mode 100644 index 000000000000..1ab5c3699dcd --- /dev/null +++ b/mobile/test/common/jni/jni_helper_test.cc @@ -0,0 +1,226 @@ +#include + +#include "library/common/jni/jni_helper.h" + +// NOLINT(namespace-envoy) + +// This file contains JNI implementation used by +// `test/java/io/envoyproxy/envoymobile/jni/JniHelperTest.java` unit tests. + +extern "C" JNIEXPORT void JNICALL Java_io_envoyproxy_envoymobile_jni_JniHelperTest_getMethodId( + JNIEnv* env, jclass, jclass clazz, jstring name, jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jni_helper.getMethodId(clazz, name_ptr.get(), sig_ptr.get()); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_getStaticMethodId(JNIEnv* env, jclass, + jclass clazz, jstring name, + jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jni_helper.getStaticMethodId(clazz, name_ptr.get(), sig_ptr.get()); +} + +extern "C" JNIEXPORT jclass JNICALL Java_io_envoyproxy_envoymobile_jni_JniHelperTest_findClass( + JNIEnv* env, jclass, jstring class_name) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr class_name_ptr = jni_helper.getStringUtfChars(class_name, nullptr); + Envoy::JNI::LocalRefUniquePtr clazz = jni_helper.findClass(class_name_ptr.get()); + return clazz.release(); +} + +extern "C" JNIEXPORT jclass JNICALL Java_io_envoyproxy_envoymobile_jni_JniHelperTest_getObjectClass( + JNIEnv* env, jclass, jobject object) { + Envoy::JNI::JniHelper jni_helper(env); + return jni_helper.getObjectClass(object).release(); +} + +extern "C" JNIEXPORT void JNICALL Java_io_envoyproxy_envoymobile_jni_JniHelperTest_throwNew( + JNIEnv* env, jclass, jstring class_name, jstring message) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr class_name_ptr = jni_helper.getStringUtfChars(class_name, nullptr); + Envoy::JNI::StringUtfUniquePtr message_ptr = jni_helper.getStringUtfChars(message, nullptr); + jni_helper.throwNew(class_name_ptr.get(), message_ptr.get()); +} + +extern "C" JNIEXPORT jobject JNICALL Java_io_envoyproxy_envoymobile_jni_JniHelperTest_newObject( + JNIEnv* env, jclass, jclass clazz, jstring name, jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jmethodID method_id = jni_helper.getMethodId(clazz, name_ptr.get(), sig_ptr.get()); + return jni_helper.newObject(clazz, method_id).release(); +} + +extern "C" JNIEXPORT jint JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_getArrayLength(JNIEnv* env, jclass, jarray array) { + Envoy::JNI::JniHelper jni_helper(env); + return jni_helper.getArrayLength(array); +} + +#define DEFINE_JNI_NEW_ARRAY(JAVA_TYPE, JNI_TYPE) \ + extern "C" JNIEXPORT JNI_TYPE JNICALL \ + Java_io_envoyproxy_envoymobile_jni_JniHelperTest_new##JAVA_TYPE##Array(JNIEnv* env, jclass, \ + jsize length) { \ + Envoy::JNI::JniHelper jni_helper(env); \ + return jni_helper.new##JAVA_TYPE##Array(length).release(); \ + } + +DEFINE_JNI_NEW_ARRAY(Byte, jbyteArray) +DEFINE_JNI_NEW_ARRAY(Char, jcharArray) +DEFINE_JNI_NEW_ARRAY(Short, jshortArray) +DEFINE_JNI_NEW_ARRAY(Int, jintArray) +DEFINE_JNI_NEW_ARRAY(Long, jlongArray) +DEFINE_JNI_NEW_ARRAY(Float, jfloatArray) +DEFINE_JNI_NEW_ARRAY(Double, jdoubleArray) +DEFINE_JNI_NEW_ARRAY(Boolean, jbooleanArray) + +extern "C" JNIEXPORT jobjectArray JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_newObjectArray(JNIEnv* env, jclass, jsize length, + jclass element_class, + jobject initial_element) { + Envoy::JNI::JniHelper jni_helper(env); + return jni_helper.newObjectArray(length, element_class, initial_element).release(); +} + +#define DEFINE_JNI_GET_ARRAY_ELEMENTS(JAVA_TYPE, JNI_TYPE, VALUE) \ + extern "C" JNIEXPORT JNI_TYPE JNICALL \ + Java_io_envoyproxy_envoymobile_jni_JniHelperTest_get##JAVA_TYPE##ArrayElements( \ + JNIEnv* env, jclass, JNI_TYPE array) { \ + Envoy::JNI::JniHelper jni_helper(env); \ + auto array_elements = jni_helper.get##JAVA_TYPE##ArrayElements(array, nullptr); \ + jsize length = jni_helper.getArrayLength(array); \ + for (size_t i = 0; i < length; i++) { \ + array_elements.get()[i] = VALUE; \ + } \ + return array; \ + } + +DEFINE_JNI_GET_ARRAY_ELEMENTS(Byte, jbyteArray, 123) +DEFINE_JNI_GET_ARRAY_ELEMENTS(Char, jcharArray, 'a') +DEFINE_JNI_GET_ARRAY_ELEMENTS(Short, jshortArray, 123) +DEFINE_JNI_GET_ARRAY_ELEMENTS(Int, jintArray, 123) +DEFINE_JNI_GET_ARRAY_ELEMENTS(Long, jlongArray, 123) +DEFINE_JNI_GET_ARRAY_ELEMENTS(Float, jfloatArray, 3.14) +DEFINE_JNI_GET_ARRAY_ELEMENTS(Double, jdoubleArray, 3.14) +DEFINE_JNI_GET_ARRAY_ELEMENTS(Boolean, jbooleanArray, true) + +extern "C" JNIEXPORT jobject JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_getObjectArrayElement(JNIEnv* env, jclass, + jobjectArray array, + jsize index) { + Envoy::JNI::JniHelper jni_helper(env); + return jni_helper.getObjectArrayElement(array, index).release(); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_setObjectArrayElement(JNIEnv* env, jclass, + jobjectArray array, + jsize index, jobject value) { + Envoy::JNI::JniHelper jni_helper(env); + jni_helper.setObjectArrayElement(array, index, value); +} + +#define DEFINE_JNI_SET_ARRAY_REGION(JAVA_TYPE, JNI_TYPE) \ + extern "C" JNIEXPORT void JNICALL \ + Java_io_envoyproxy_envoymobile_jni_JniHelperTest_set##JAVA_TYPE##ArrayRegion( \ + JNIEnv* env, jclass, JNI_TYPE array, jsize start, jsize length, JNI_TYPE buffer) { \ + Envoy::JNI::JniHelper jni_helper(env); \ + auto c_buffer = jni_helper.get##JAVA_TYPE##ArrayElements(buffer, nullptr); \ + env->Set##JAVA_TYPE##ArrayRegion(array, start, length, c_buffer.get()); \ + } + +DEFINE_JNI_SET_ARRAY_REGION(Byte, jbyteArray) +DEFINE_JNI_SET_ARRAY_REGION(Char, jcharArray) +DEFINE_JNI_SET_ARRAY_REGION(Short, jshortArray) +DEFINE_JNI_SET_ARRAY_REGION(Int, jintArray) +DEFINE_JNI_SET_ARRAY_REGION(Long, jlongArray) +DEFINE_JNI_SET_ARRAY_REGION(Float, jfloatArray) +DEFINE_JNI_SET_ARRAY_REGION(Double, jdoubleArray) +DEFINE_JNI_SET_ARRAY_REGION(Boolean, jbooleanArray) + +#define DEFINE_JNI_CALL_METHOD(JAVA_TYPE, JNI_TYPE) \ + extern "C" JNIEXPORT JNI_TYPE JNICALL \ + Java_io_envoyproxy_envoymobile_jni_JniHelperTest_call##JAVA_TYPE##Method( \ + JNIEnv* env, jclass, jclass clazz, jobject object, jstring name, jstring signature) { \ + Envoy::JNI::JniHelper jni_helper(env); \ + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); \ + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); \ + jmethodID method_id = jni_helper.getMethodId(clazz, name_ptr.get(), sig_ptr.get()); \ + return jni_helper.call##JAVA_TYPE##Method(object, method_id); \ + } + +DEFINE_JNI_CALL_METHOD(Byte, jbyte) +DEFINE_JNI_CALL_METHOD(Char, jchar) +DEFINE_JNI_CALL_METHOD(Short, jshort) +DEFINE_JNI_CALL_METHOD(Int, jint) +DEFINE_JNI_CALL_METHOD(Long, jlong) +DEFINE_JNI_CALL_METHOD(Float, jfloat) +DEFINE_JNI_CALL_METHOD(Double, jdouble) +DEFINE_JNI_CALL_METHOD(Boolean, jboolean) + +extern "C" JNIEXPORT void JNICALL Java_io_envoyproxy_envoymobile_jni_JniHelperTest_callVoidMethod( + JNIEnv* env, jclass, jclass clazz, jobject object, jstring name, jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jmethodID method_id = jni_helper.getMethodId(clazz, name_ptr.get(), sig_ptr.get()); + jni_helper.callVoidMethod(object, method_id); +} + +extern "C" JNIEXPORT jobject JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_callObjectMethod(JNIEnv* env, jclass, jclass clazz, + jobject object, jstring name, + jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jmethodID method_id = jni_helper.getMethodId(clazz, name_ptr.get(), sig_ptr.get()); + return jni_helper.callObjectMethod(object, method_id).release(); +} + +#define DEFINE_JNI_CALL_STATIC_METHOD(JAVA_TYPE, JNI_TYPE) \ + extern "C" JNIEXPORT JNI_TYPE JNICALL \ + Java_io_envoyproxy_envoymobile_jni_JniHelperTest_callStatic##JAVA_TYPE##Method( \ + JNIEnv* env, jclass, jclass clazz, jstring name, jstring signature) { \ + Envoy::JNI::JniHelper jni_helper(env); \ + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); \ + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); \ + jmethodID method_id = jni_helper.getStaticMethodId(clazz, name_ptr.get(), sig_ptr.get()); \ + return jni_helper.callStatic##JAVA_TYPE##Method(clazz, method_id); \ + } + +DEFINE_JNI_CALL_STATIC_METHOD(Byte, jbyte) +DEFINE_JNI_CALL_STATIC_METHOD(Char, jchar) +DEFINE_JNI_CALL_STATIC_METHOD(Short, jshort) +DEFINE_JNI_CALL_STATIC_METHOD(Int, jint) +DEFINE_JNI_CALL_STATIC_METHOD(Long, jlong) +DEFINE_JNI_CALL_STATIC_METHOD(Float, jfloat) +DEFINE_JNI_CALL_STATIC_METHOD(Double, jdouble) +DEFINE_JNI_CALL_STATIC_METHOD(Boolean, jboolean) + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_callStaticVoidMethod(JNIEnv* env, jclass, + jclass clazz, jstring name, + jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jmethodID method_id = jni_helper.getStaticMethodId(clazz, name_ptr.get(), sig_ptr.get()); + jni_helper.callStaticVoidMethod(clazz, method_id); +} + +extern "C" JNIEXPORT jobject JNICALL +Java_io_envoyproxy_envoymobile_jni_JniHelperTest_callStaticObjectMethod(JNIEnv* env, jclass, + jclass clazz, jstring name, + jstring signature) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::JNI::StringUtfUniquePtr name_ptr = jni_helper.getStringUtfChars(name, nullptr); + Envoy::JNI::StringUtfUniquePtr sig_ptr = jni_helper.getStringUtfChars(signature, nullptr); + jmethodID method_id = jni_helper.getStaticMethodId(clazz, name_ptr.get(), sig_ptr.get()); + return jni_helper.callStaticObjectMethod(clazz, method_id).release(); +} diff --git a/mobile/test/common/jni/jni_utility_test.cc b/mobile/test/common/jni/jni_utility_test.cc new file mode 100644 index 000000000000..e81dc108727b --- /dev/null +++ b/mobile/test/common/jni/jni_utility_test.cc @@ -0,0 +1,17 @@ +#include + +#include "library/common/jni/jni_utility.h" + +// NOLINT(namespace-envoy) + +// This file contains JNI implementation used by +// `test/java/io/envoyproxy/envoymobile/jni/JniUtilityTest.java` unit tests. + +extern "C" JNIEXPORT jbyteArray JNICALL +Java_io_envoyproxy_envoymobile_jni_JniUtilityTest_protoJavaByteArrayConversion(JNIEnv* env, jclass, + jbyteArray source) { + Envoy::JNI::JniHelper jni_helper(env); + Envoy::ProtobufWkt::Struct s; + Envoy::JNI::javaByteArrayToProto(jni_helper, source, &s); + return Envoy::JNI::protoToJavaByteArray(jni_helper, s).release(); +} diff --git a/mobile/test/common/jni/test_jni_impl.cc b/mobile/test/common/jni/test_jni_impl.cc new file mode 100644 index 000000000000..5f6e8ad135f9 --- /dev/null +++ b/mobile/test/common/jni/test_jni_impl.cc @@ -0,0 +1,116 @@ +#include + +#include "test/common/integration/test_server_interface.h" +#include "test/common/integration/xds_test_server_interface.h" +#include "test/test_common/utility.h" + +#include "library/common/jni/jni_support.h" + +// NOLINT(namespace-envoy) + +// Quic Test ServerJniLibrary + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttpProxyTestServer(JNIEnv* env, + jclass clazz) { + jni_log("[QTS]", "starting server"); + start_server(Envoy::TestServerType::HTTP_PROXY); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttpsProxyTestServer( + JNIEnv* env, jclass clazz) { + jni_log("[QTS]", "starting server"); + start_server(Envoy::TestServerType::HTTPS_PROXY); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttp3TestServer(JNIEnv* env, + jclass clazz) { + jni_log("[QTS]", "starting server"); + start_server(Envoy::TestServerType::HTTP3); +} + +extern "C" JNIEXPORT jint JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeGetServerPort(JNIEnv* env, + jclass clazz) { + jni_log("[QTS]", "getting server port"); + return get_server_port(); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttp2TestServer(JNIEnv* env, + jclass clazz) { + jni_log("[QTS]", "starting server"); + start_server(Envoy::TestServerType::HTTP2_WITH_TLS); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeShutdownTestServer(JNIEnv* env, + jclass clazz) { + jni_log("[QTS]", "shutting down server"); + shutdown_server(); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeInitXdsTestServer(JNIEnv* env, + jclass clazz) { + jni_log("[XTS]", "initializing xDS server"); + initXdsServer(); +} + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartXdsTestServer(JNIEnv* env, + jclass clazz) { + jni_log("[XTS]", "starting xDS server"); + startXdsServer(); +} + +extern "C" JNIEXPORT jstring JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeGetXdsTestServerHost(JNIEnv* env, + jclass clazz) { + jni_log("[XTS]", "getting xDS server host"); + return env->NewStringUTF(getXdsServerHost()); +} + +extern "C" JNIEXPORT jint JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeGetXdsTestServerPort(JNIEnv* env, + jclass clazz) { + jni_log("[XTS]", "getting xDS server port"); + return getXdsServerPort(); +} + +#ifdef ENVOY_ENABLE_YAML +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeSendDiscoveryResponse(JNIEnv* env, + jclass clazz, + jstring yaml) { + jni_log("[XTS]", "sending DiscoveryResponse from the xDS server"); + const char* yaml_chars = env->GetStringUTFChars(yaml, /* isCopy= */ nullptr); + // The yaml utilities have non-relevant thread asserts. + Envoy::Thread::SkipAsserts skip; + envoy::service::discovery::v3::DiscoveryResponse response; + Envoy::TestUtility::loadFromYaml(yaml_chars, response); + sendDiscoveryResponse(response); + env->ReleaseStringUTFChars(yaml, yaml_chars); +} +#endif + +extern "C" JNIEXPORT void JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeShutdownXdsTestServer(JNIEnv* env, + jclass clazz) { + jni_log("[XTS]", "shutting down xDS server"); + shutdownXdsServer(); +} + +#ifdef ENVOY_ENABLE_YAML +extern "C" JNIEXPORT jstring JNICALL +Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeCreateYaml(JNIEnv* env, jclass, + jlong bootstrap_ptr) { + Envoy::Thread::SkipAsserts skip_asserts; + std::unique_ptr bootstrap( + reinterpret_cast(bootstrap_ptr)); + std::string yaml = Envoy::MessageUtil::getYamlStringFromMessage(*bootstrap); + return env->NewStringUTF(yaml.c_str()); +} +#endif diff --git a/mobile/test/common/jni/test_jni_interface.cc b/mobile/test/common/jni/test_jni_interface.cc deleted file mode 100644 index 6cc0e31909cb..000000000000 --- a/mobile/test/common/jni/test_jni_interface.cc +++ /dev/null @@ -1,50 +0,0 @@ -#include - -#include "test/common/integration/test_server_interface.h" - -#include "library/common/jni/jni_support.h" -#include "library/common/jni/jni_utility.h" - -// NOLINT(namespace-envoy) - -// Quic Test ServerJniLibrary - -extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttp3TestServer(JNIEnv* env, - jclass clazz) { - jni_log("[QTS]", "starting server"); - start_server(Envoy::TestServerType::HTTP3); -} - -extern "C" JNIEXPORT jint JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeGetServerPort(JNIEnv* env, - jclass clazz) { - jni_log("[QTS]", "getting server port"); - return get_server_port(); -} - -extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeStartHttp2TestServer(JNIEnv* env, - jclass clazz) { - jni_log("[QTS]", "starting server"); - start_server(Envoy::TestServerType::HTTP2_WITH_TLS); -} - -extern "C" JNIEXPORT void JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeShutdownTestServer(JNIEnv* env, - jclass clazz) { - jni_log("[QTS]", "shutting down server"); - shutdown_server(); -} - -#ifdef ENVOY_ENABLE_YAML -extern "C" JNIEXPORT jstring JNICALL -Java_io_envoyproxy_envoymobile_engine_testing_TestJni_nativeCreateYaml(JNIEnv* env, jclass, - jlong bootstrap_ptr) { - Envoy::Thread::SkipAsserts skip_asserts; - std::unique_ptr bootstrap( - reinterpret_cast(bootstrap_ptr)); - std::string yaml = Envoy::MessageUtil::getYamlStringFromMessage(*bootstrap); - return env->NewStringUTF(yaml.c_str()); -} -#endif diff --git a/mobile/test/common/main_interface_test.cc b/mobile/test/common/main_interface_test.cc index a7a600b0e91c..38e3017abd4f 100644 --- a/mobile/test/common/main_interface_test.cc +++ b/mobile/test/common/main_interface_test.cc @@ -197,7 +197,7 @@ TEST_F(MainInterfaceTest, BasicStream) { envoy_stream_t stream = init_stream(engine_handle); - start_stream(engine_handle, stream, stream_cbs, false, 0); + start_stream(engine_handle, stream, stream_cbs, false); send_headers(engine_handle, stream, c_headers, false); send_data(engine_handle, stream, c_data, false); @@ -241,7 +241,7 @@ TEST_F(MainInterfaceTest, SendMetadata) { envoy_stream_t stream = init_stream(engine_handle); - start_stream(engine_handle, stream, stream_cbs, false, 0); + start_stream(engine_handle, stream, stream_cbs, false); EXPECT_EQ(ENVOY_FAILURE, send_metadata(engine_handle, stream, {})); @@ -289,7 +289,7 @@ TEST_F(MainInterfaceTest, ResetStream) { envoy_stream_t stream = init_stream(engine_handle); - start_stream(engine_handle, stream, stream_cbs, false, 0); + start_stream(engine_handle, stream, stream_cbs, false); reset_stream(engine_handle, stream); diff --git a/mobile/test/common/mocks/common/BUILD b/mobile/test/common/mocks/common/BUILD index 41d56371e474..e9f4f1c71750 100644 --- a/mobile/test/common/mocks/common/BUILD +++ b/mobile/test/common/mocks/common/BUILD @@ -1,12 +1,12 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_mock", - "envoy_package", + "envoy_mobile_package", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_mock( name = "common_mocks", diff --git a/mobile/test/common/mocks/event/BUILD b/mobile/test/common/mocks/event/BUILD index 0196deec39ab..ecb70f1aeca0 100644 --- a/mobile/test/common/mocks/event/BUILD +++ b/mobile/test/common/mocks/event/BUILD @@ -1,12 +1,12 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_mock", - "envoy_package", + "envoy_mobile_package", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_mock( name = "event_mocks", diff --git a/mobile/test/common/network/BUILD b/mobile/test/common/network/BUILD index 050e0386a5b2..b26d4b163bf1 100644 --- a/mobile/test/common/network/BUILD +++ b/mobile/test/common/network/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "connectivity_manager_test", diff --git a/mobile/test/common/stats/BUILD b/mobile/test/common/stats/BUILD index 0a9cfe5529f4..5a053f1a6a0f 100644 --- a/mobile/test/common/stats/BUILD +++ b/mobile/test/common/stats/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "utility_test", diff --git a/mobile/test/common/stream_info/BUILD b/mobile/test/common/stream_info/BUILD index 3962f637074e..528486c353e5 100644 --- a/mobile/test/common/stream_info/BUILD +++ b/mobile/test/common/stream_info/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "extra_stream_info_test", diff --git a/mobile/test/common/thread/BUILD b/mobile/test/common/thread/BUILD index a4d283d0ff72..2c1a905d98a6 100644 --- a/mobile/test/common/thread/BUILD +++ b/mobile/test/common/thread/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_test( name = "lock_guard_test", diff --git a/mobile/test/java/integration/BUILD b/mobile/test/java/integration/BUILD index 73b0ea69f7bc..898bcece51e5 100644 --- a/mobile/test/java/integration/BUILD +++ b/mobile/test/java/integration/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_mobile_android_test( name = "android_engine_start_test", @@ -16,8 +16,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", ], @@ -34,8 +39,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", "//test/java/io/envoyproxy/envoymobile/engine/testing", @@ -53,8 +63,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", "//test/java/io/envoyproxy/envoymobile/engine/testing", @@ -72,8 +87,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", "//test/java/io/envoyproxy/envoymobile/engine/testing", diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD index 071068ca711e..2b8fe20f6f5d 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test", "envoy_mobile_jni_kt_test", "envoy_mobile_kt_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_jni_kt_test( name = "envoy_configuration_test", srcs = [ @@ -9,11 +12,17 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//test/java/io/envoyproxy/envoymobile/engine/testing", + "@maven//:com_google_protobuf_protobuf_javalite", ], ) @@ -44,8 +53,13 @@ envoy_mobile_android_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt index 93cd14529d58..6c1e33581e51 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt @@ -1,18 +1,16 @@ package io.envoyproxy.envoymobile.engine +import com.google.protobuf.Struct +import com.google.protobuf.Value import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilter import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification -import io.envoyproxy.envoymobile.engine.JniLibrary -import io.envoyproxy.envoymobile.engine.HeaderMatchConfig -import io.envoyproxy.envoymobile.engine.HeaderMatchConfig.Type import io.envoyproxy.envoymobile.engine.types.EnvoyStreamIntel import io.envoyproxy.envoymobile.engine.types.EnvoyFinalStreamIntel import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterCallbacks import io.envoyproxy.envoymobile.engine.testing.TestJni import java.nio.ByteBuffer import org.assertj.core.api.Assertions.assertThat -import org.junit.Assert.fail import org.junit.Test import java.util.regex.Pattern @@ -68,7 +66,6 @@ class TestEnvoyHTTPFilterFactory(name : String) : EnvoyHTTPFilterFactory { class EnvoyConfigurationTest { fun buildTestEnvoyConfiguration( - grpcStatsDomain: String = "stats.example.com", connectTimeoutSeconds: Int = 123, dnsRefreshSeconds: Int = 234, dnsFailureRefreshSecondsBase: Int = 345, @@ -83,6 +80,7 @@ class EnvoyConfigurationTest { http3ConnectionOptions: String = "5RTO", http3ClientConnectionOptions: String = "MPQC", quicHints: Map = mapOf("www.abc.com" to 443, "www.def.com" to 443), + quicCanonicalSuffixes: MutableList = mutableListOf(".opq.com", ".xyz.com"), enableGzipDecompression: Boolean = true, enableBrotliDecompression: Boolean = false, enableSocketTagging: Boolean = false, @@ -90,7 +88,6 @@ class EnvoyConfigurationTest { h2ConnectionKeepaliveIdleIntervalMilliseconds: Int = 222, h2ConnectionKeepaliveTimeoutSeconds: Int = 333, maxConnectionsPerHost: Int = 543, - statsFlushSeconds: Int = 567, streamIdleTimeoutSeconds: Int = 678, perTryIdleTimeoutSeconds: Int = 910, appVersion: String = "v1.2.3", @@ -99,29 +96,24 @@ class EnvoyConfigurationTest { filterChain: MutableList = mutableListOf(EnvoyNativeFilterConfig("buffer_filter_1", "{'@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer'}"), EnvoyNativeFilterConfig("buffer_filter_2", "{'@type': 'type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer'}")), platformFilterFactories: MutableList = mutableListOf(TestEnvoyHTTPFilterFactory("name1"), TestEnvoyHTTPFilterFactory("name2")), runtimeGuards: Map = emptyMap(), - statSinks: MutableList = mutableListOf(), enablePlatformCertificatesValidation: Boolean = false, rtdsResourceName: String = "", rtdsTimeoutSeconds: Int = 0, xdsAddress: String = "", xdsPort: Int = 0, - xdsAuthHeader: String = "", - xdsAuthToken: String = "", - xdsJwtToken: String = "", - xdsJwtTokenLifetimeSeconds: Int = 0, + xdsGrpcInitialMetadata: Map = emptyMap(), xdsSslRootCerts: String = "", - xdsSni: String = "", nodeId: String = "", nodeRegion: String = "", nodeZone: String = "", nodeSubZone: String = "", + nodeMetadata: Struct = Struct.getDefaultInstance(), cdsResourcesLocator: String = "", cdsTimeoutSeconds: Int = 0, enableCds: Boolean = false, ): EnvoyConfiguration { return EnvoyConfiguration( - grpcStatsDomain, connectTimeoutSeconds, dnsRefreshSeconds, dnsFailureRefreshSecondsBase, @@ -136,6 +128,7 @@ class EnvoyConfigurationTest { http3ConnectionOptions, http3ClientConnectionOptions, quicHints, + quicCanonicalSuffixes, enableGzipDecompression, enableBrotliDecompression, enableSocketTagging, @@ -143,7 +136,6 @@ class EnvoyConfigurationTest { h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds, maxConnectionsPerHost, - statsFlushSeconds, streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion, @@ -153,31 +145,27 @@ class EnvoyConfigurationTest { platformFilterFactories, emptyMap(), emptyMap(), - statSinks, runtimeGuards, enablePlatformCertificatesValidation, rtdsResourceName, rtdsTimeoutSeconds, xdsAddress, xdsPort, - xdsAuthHeader, - xdsAuthToken, - xdsJwtToken, - xdsJwtTokenLifetimeSeconds, + xdsGrpcInitialMetadata, xdsSslRootCerts, - xdsSni, nodeId, nodeRegion, nodeZone, nodeSubZone, + nodeMetadata, cdsResourcesLocator, cdsTimeoutSeconds, enableCds ) } - fun isGoogleGrpcDisabled(): Boolean { - return System.getProperty("envoy_jni_google_grpc_disabled") != null; + fun isEnvoyMobileXdsDisabled(): Boolean { + return System.getProperty("envoy_jni_envoy_mobile_xds_disabled") != null; } @Test @@ -212,15 +200,12 @@ class EnvoyConfigurationTest { assertThat(resolvedTemplate).contains("hostname: www.abc.com"); assertThat(resolvedTemplate).contains("hostname: www.def.com"); assertThat(resolvedTemplate).contains("port: 443"); + assertThat(resolvedTemplate).contains("canonical_suffixes:"); + assertThat(resolvedTemplate).contains(".opq.com"); + assertThat(resolvedTemplate).contains(".xyz.com"); assertThat(resolvedTemplate).contains("connection_options: 5RTO"); assertThat(resolvedTemplate).contains("client_connection_options: MPQC"); - // Gzip - assertThat(resolvedTemplate).contains("type.googleapis.com/envoy.extensions.compression.gzip.decompressor.v3.Gzip"); - - // Brotli - assertThat(resolvedTemplate).doesNotContain("type.googleapis.com/envoy.extensions.compression.brotli.decompressor.v3.Brotli"); - // Per Host Limits assertThat(resolvedTemplate).contains("max_connections: 543") @@ -229,10 +214,6 @@ class EnvoyConfigurationTest { assertThat(resolvedTemplate).contains("app_version: v1.2.3") assertThat(resolvedTemplate).contains("app_id: com.example.myapp") - // Stats - assertThat(resolvedTemplate).contains("stats_flush_interval: 567s") - assertThat(resolvedTemplate).contains("stats.example.com"); - // Idle timeouts assertThat(resolvedTemplate).contains("stream_idle_timeout: 678s") assertThat(resolvedTemplate).contains("per_try_idle_timeout: 910s") @@ -260,7 +241,6 @@ class EnvoyConfigurationTest { fun `configuration resolves with alternate values`() { JniLibrary.loadTestLibrary() val envoyConfiguration = buildTestEnvoyConfiguration( - grpcStatsDomain = "", enableDrainPostDnsRefresh = true, enableDNSCache = true, dnsCacheSaveIntervalSeconds = 101, @@ -273,7 +253,6 @@ class EnvoyConfigurationTest { dnsPreresolveHostnames = mutableListOf(), filterChain = mutableListOf(), runtimeGuards = mapOf("test_feature_false" to true), - statSinks = mutableListOf("{ name: envoy.stat_sinks.statsd, typed_config: { '@type': type.googleapis.com/envoy.config.metrics.v3.StatsdSink, address: { socket_address: { address: 127.0.0.1, port_value: 123 } } } }"), trustChainVerification = TrustChainVerification.ACCEPT_UNTRUSTED ) @@ -296,22 +275,15 @@ class EnvoyConfigurationTest { // enableGzipDecompression = false assertThat(resolvedTemplate).doesNotContain("type.googleapis.com/envoy.extensions.compression.gzip.decompressor.v3.Gzip"); - assertThat(resolvedTemplate).contains("type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip"); - // enableBrotliDecompression = true assertThat(resolvedTemplate).contains("type.googleapis.com/envoy.extensions.compression.brotli.decompressor.v3.Brotli"); - assertThat(resolvedTemplate).contains("type.googleapis.com/envoy.extensions.compression.brotli.compressor.v3.Brotli"); - // enableInterfaceBinding = true assertThat(resolvedTemplate).contains("enable_interface_binding: true") // enablePlatformCertificatesValidation = true assertThat(resolvedTemplate).doesNotContain("trusted_ca:") - // statsSinks - assertThat(resolvedTemplate).contains("envoy.stat_sinks.statsd"); - // ADS and RTDS not included by default assertThat(resolvedTemplate).doesNotContain("rtds_layer:"); assertThat(resolvedTemplate).doesNotContain("ads_config:"); @@ -333,7 +305,7 @@ class EnvoyConfigurationTest { @Test fun `test adding RTDS`() { - if (isGoogleGrpcDisabled()) { + if (isEnvoyMobileXdsDisabled()) { return; } @@ -350,7 +322,7 @@ class EnvoyConfigurationTest { @Test fun `test adding RTDS and CDS`() { - if (isGoogleGrpcDisabled()) { + if (isEnvoyMobileXdsDisabled()) { return; } @@ -381,7 +353,7 @@ class EnvoyConfigurationTest { @Test fun `test enableCds with default string`() { - if (isGoogleGrpcDisabled()) { + if (isEnvoyMobileXdsDisabled()) { return; } @@ -398,7 +370,7 @@ class EnvoyConfigurationTest { @Test fun `test RTDS default timeout`() { - if (isGoogleGrpcDisabled()) { + if (isEnvoyMobileXdsDisabled()) { return; } @@ -413,18 +385,17 @@ class EnvoyConfigurationTest { } @Test - fun `test YAML loads with stats sinks and stats domain`() { + fun `test node metadata`() { JniLibrary.loadTestLibrary() val envoyConfiguration = buildTestEnvoyConfiguration( - grpcStatsDomain = "stats.example.com", - statSinks = mutableListOf("{ name: envoy.stat_sinks.statsd, typed_config: { '@type': type.googleapis.com/envoy.config.metrics.v3.StatsdSink, address: { socket_address: { address: 127.0.0.1, port_value: 123 } } } }"), - trustChainVerification = TrustChainVerification.ACCEPT_UNTRUSTED + nodeMetadata = Struct.newBuilder() + .putFields("metadata_field", Value.newBuilder().setStringValue("metadata_value").build()) + .build() ) val resolvedTemplate = TestJni.createYaml(envoyConfiguration) - // statsSinks - assertThat(resolvedTemplate).contains("envoy.stat_sinks.statsd"); - assertThat(resolvedTemplate).contains("stats.example.com"); - } + assertThat(resolvedTemplate).contains("metadata_field") + assertThat(resolvedTemplate).contains("metadata_value") + } } diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD index c25d9390608c..271ef49ae3d3 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD @@ -1,10 +1,10 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() android_library( name = "testing", @@ -32,8 +32,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ ":testing", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java index 177fc68d2155..1b2e7cc85f4e 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/TestJni.java @@ -2,7 +2,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import io.envoyproxy.envoymobile.engine.EnvoyConfiguration; -import io.envoyproxy.envoymobile.engine.JniLibrary; /** * Wrapper class for test JNI functions @@ -10,13 +9,50 @@ public final class TestJni { private static final AtomicBoolean sServerRunning = new AtomicBoolean(); + private static final AtomicBoolean xdsServerRunning = new AtomicBoolean(); + + /** + * Initializes an envoy server which will terminate cleartext CONNECT requests. + * + * @throws IllegalStateException if it's already started. + */ + public static void startHttpProxyTestServer() { + if (!sServerRunning.compareAndSet(false, true)) { + throw new IllegalStateException("Server is already running"); + } + nativeStartHttpProxyTestServer(); + } + + /** + * Initializes an envoy server which will terminate encrypted CONNECT requests. + * + * @throws IllegalStateException if it's already started. + */ + public static void startHttpsProxyTestServer() { + if (!sServerRunning.compareAndSet(false, true)) { + throw new IllegalStateException("Server is already running"); + } + nativeStartHttpsProxyTestServer(); + } + + /** + * Initializes the xDS test server. + * + * @throws IllegalStateException if it's already started. + */ + public static void initXdsTestServer() { + if (xdsServerRunning.get()) { + throw new IllegalStateException("xDS server is already running"); + } + nativeInitXdsTestServer(); + } /* * Starts the server. Throws an {@link IllegalStateException} if already started. */ public static void startHttp3TestServer() { if (!sServerRunning.compareAndSet(false, true)) { - throw new IllegalStateException("Http3 server is already running"); + throw new IllegalStateException("Server is already running"); } nativeStartHttp3TestServer(); } @@ -31,6 +67,28 @@ public static void startHttp2TestServer() { nativeStartHttp2TestServer(); } + /** + * Starts the xDS test server. + * + * @throws IllegalStateException if it's already started. + */ + public static void startXdsTestServer() { + if (!xdsServerRunning.compareAndSet(false, true)) { + throw new IllegalStateException("xDS server is already running"); + } + nativeStartXdsTestServer(); + } + + /** + * Sends the `DiscoveryResponse` message in the YAML format. + */ + public static void sendDiscoveryResponse(String yaml) { + if (!xdsServerRunning.get()) { + throw new IllegalStateException("xDS server is not running"); + } + nativeSendDiscoveryResponse(yaml); + } + /** * Shutdowns the server. No-op if the server is already shutdown. */ @@ -41,18 +99,38 @@ public static void shutdownTestServer() { nativeShutdownTestServer(); } + /** + * Shutdowns the xDS test server. No-op if the server is already shutdown. + */ + public static void shutdownXdsTestServer() { + if (!xdsServerRunning.compareAndSet(true, false)) { + return; + } + nativeShutdownXdsTestServer(); + } + public static String getServerURL() { return "https://" + getServerHost() + ":" + getServerPort(); } public static String getServerHost() { return "test.example.com"; } + /** + * Gets the xDS test server host. + */ + public static String getXdsTestServerHost() { return nativeGetXdsTestServerHost(); } + + /** + * Gets the xDS test server port. + */ + public static int getXdsTestServerPort() { return nativeGetXdsTestServerPort(); } + /** * Returns the server attributed port. Throws an {@link IllegalStateException} if not started. */ public static int getServerPort() { if (!sServerRunning.get()) { - throw new IllegalStateException("Quic server not started."); + throw new IllegalStateException("Server not started."); } return nativeGetServerPort(); } @@ -69,6 +147,22 @@ public static String createYaml(EnvoyConfiguration envoyConfiguration) { private static native int nativeGetServerPort(); + private static native void nativeStartHttpProxyTestServer(); + + private static native void nativeStartHttpsProxyTestServer(); + + private static native void nativeInitXdsTestServer(); + + private static native String nativeGetXdsTestServerHost(); + + private static native int nativeGetXdsTestServerPort(); + + private static native void nativeStartXdsTestServer(); + + private static native void nativeSendDiscoveryResponse(String yaml); + + private static native int nativeShutdownXdsTestServer(); + private static native String nativeCreateYaml(long bootstrap); private TestJni() {} diff --git a/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD new file mode 100644 index 000000000000..fa34649e5f64 --- /dev/null +++ b/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD @@ -0,0 +1,31 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") +load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") + +licenses(["notice"]) # Apache 2 + +envoy_mobile_package() + +envoy_mobile_android_test( + name = "jni_helper_test", + srcs = [ + "JniHelperTest.java", + ], + native_deps = [ + "//test/common/jni:libenvoy_jni_helper_test.so", + ], + native_lib_name = "envoy_jni_helper_test", +) + +envoy_mobile_android_test( + name = "jni_utility_test", + srcs = [ + "JniUtilityTest.java", + ], + native_deps = [ + "//test/common/jni:libenvoy_jni_utility_test.so", + ], + native_lib_name = "envoy_jni_utility_test", + deps = [ + "@maven//:com_google_protobuf_protobuf_javalite", + ], +) diff --git a/mobile/test/java/io/envoyproxy/envoymobile/jni/JniHelperTest.java b/mobile/test/java/io/envoyproxy/envoymobile/jni/JniHelperTest.java new file mode 100644 index 000000000000..804a3d99882e --- /dev/null +++ b/mobile/test/java/io/envoyproxy/envoymobile/jni/JniHelperTest.java @@ -0,0 +1,427 @@ +package io.envoyproxy.envoymobile.jni; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class JniHelperTest { + public JniHelperTest() { System.loadLibrary("envoy_jni_helper_test"); } + + //================================================================================ + // Native methods for testing. + //================================================================================ + public static native void getMethodId(Class clazz, String name, String signature); + public static native void getStaticMethodId(Class clazz, String name, String signature); + public static native Class findClass(String className); + public static native Class getObjectClass(Object object); + public static native Object newObject(Class clazz, String name, String signature); + public static native void throwNew(String className, String message); + public static native int getArrayLength(int[] array); + public static native byte[] newByteArray(int length); + public static native char[] newCharArray(int length); + public static native short[] newShortArray(int length); + public static native int[] newIntArray(int length); + public static native long[] newLongArray(int length); + public static native float[] newFloatArray(int length); + public static native double[] newDoubleArray(int length); + public static native boolean[] newBooleanArray(int length); + public static native Object[] newObjectArray(int length, Class elementClass, + Object initialElement); + public static native byte[] getByteArrayElements(byte[] array); + public static native char[] getCharArrayElements(char[] array); + public static native short[] getShortArrayElements(short[] array); + public static native int[] getIntArrayElements(int[] array); + public static native long[] getLongArrayElements(long[] array); + public static native float[] getFloatArrayElements(float[] array); + public static native double[] getDoubleArrayElements(double[] array); + public static native boolean[] getBooleanArrayElements(boolean[] array); + public static native Object getObjectArrayElement(Object[] array, int index); + public static native void setObjectArrayElement(Object[] array, int index, Object value); + public static native void setByteArrayRegion(byte[] array, int start, int index, byte[] buffer); + public static native void setCharArrayRegion(char[] array, int start, int index, char[] buffer); + public static native void setShortArrayRegion(short[] array, int start, int index, + short[] buffer); + public static native void setIntArrayRegion(int[] array, int start, int index, int[] buffer); + public static native void setLongArrayRegion(long[] array, int start, int index, long[] buffer); + public static native void setFloatArrayRegion(float[] array, int start, int index, + float[] buffer); + public static native void setDoubleArrayRegion(double[] array, int start, int index, + double[] buffer); + public static native void setBooleanArrayRegion(boolean[] array, int start, int index, + boolean[] buffer); + public static native byte callByteMethod(Class clazz, Object instance, String name, + String signature); + public static native char callCharMethod(Class clazz, Object instance, String name, + String signature); + public static native short callShortMethod(Class clazz, Object instance, String name, + String signature); + public static native int callIntMethod(Class clazz, Object instance, String name, + String signature); + public static native long callLongMethod(Class clazz, Object instance, String name, + String signature); + public static native float callFloatMethod(Class clazz, Object instance, String name, + String signature); + public static native double callDoubleMethod(Class clazz, Object instance, String name, + String signature); + public static native boolean callBooleanMethod(Class clazz, Object instance, String name, + String signature); + public static native void callVoidMethod(Class clazz, Object instance, String name, + String signature); + public static native Object callObjectMethod(Class clazz, Object instance, String name, + String signature); + public static native byte callStaticByteMethod(Class clazz, String name, String signature); + public static native char callStaticCharMethod(Class clazz, String name, String signature); + public static native short callStaticShortMethod(Class clazz, String name, String signature); + public static native int callStaticIntMethod(Class clazz, String name, String signature); + public static native long callStaticLongMethod(Class clazz, String name, String signature); + public static native float callStaticFloatMethod(Class clazz, String name, String signature); + public static native double callStaticDoubleMethod(Class clazz, String name, String signature); + public static native boolean callStaticBooleanMethod(Class clazz, String name, + String signature); + public static native void callStaticVoidMethod(Class clazz, String name, String signature); + public static native Object callStaticObjectMethod(Class clazz, String name, String signature); + + //================================================================================ + // Object methods used for CallMethod tests. + //================================================================================ + public byte byteMethod() { return 1; } + public char charMethod() { return 'a'; } + public short shortMethod() { return 1; } + public int intMethod() { return 1; } + public long longMethod() { return 1; } + public float floatMethod() { return 3.14f; } + public double doubleMethod() { return 3.14; } + public boolean booleanMethod() { return true; } + public void voidMethod() {} + public String objectMethod() { return "Hello"; } + + //================================================================================ + // Static methods used for CallStaticMethod tests. + //================================================================================ + public static byte staticByteMethod() { return 1; } + public static char staticCharMethod() { return 'a'; } + public static short staticShortMethod() { return 1; } + public static int staticIntMethod() { return 1; } + public static long staticLongMethod() { return 1; } + public static float staticFloatMethod() { return 3.14f; } + public static double staticDoubleMethod() { return 3.14; } + public static boolean staticBooleanMethod() { return true; } + public static void staticVoidMethod() {} + public static String staticObjectMethod() { return "Hello"; } + + static class Foo {} + + @Test + public void testMethodId() { + getMethodId(Foo.class, "", "()V"); + } + + @Test + public void testStaticMethodId() { + getStaticMethodId(JniHelperTest.class, "staticVoidMethod", "()V"); + } + + @Test + public void testFindClass() { + assertThat(findClass("java/lang/Exception")).isEqualTo(Exception.class); + } + + @Test + public void testGetObjectClass() { + String s = "Hello"; + assertThat(getObjectClass(s)).isEqualTo(String.class); + } + + @Test + public void testNewObject() { + assertThat(newObject(Foo.class, "", "()V")).isInstanceOf(Foo.class); + } + + @Test + public void testThrowNew() { + assertThatThrownBy(() -> throwNew("java/lang/RuntimeException", "Test")) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Test"); + } + + @Test + public void testGetArrayLength() { + assertThat(getArrayLength(new int[] {1, 2, 3})).isEqualTo(3); + } + + @Test + public void testNewCharArray() { + assertThat(newCharArray(3)).isEqualTo(new char[] {0, 0, 0}); + } + + @Test + public void testNewShortArray() { + assertThat(newShortArray(3)).isEqualTo(new short[] {0, 0, 0}); + } + + @Test + public void testNewIntArray() { + assertThat(newIntArray(3)).isEqualTo(new int[] {0, 0, 0}); + } + + @Test + public void testNewLongArray() { + assertThat(newLongArray(3)).isEqualTo(new long[] {0, 0, 0}); + } + + @Test + public void testNewFloatArray() { + assertThat(newFloatArray(3)).isEqualTo(new float[] {0, 0, 0}); + } + + @Test + public void testNewDoubleArray() { + assertThat(newDoubleArray(3)).isEqualTo(new double[] {0, 0, 0}); + } + + @Test + public void testNewBooleanArray() { + assertThat(newBooleanArray(3)).isEqualTo(new boolean[] {false, false, false}); + } + + @Test + public void testNewObjectArray() { + assertThat(newObjectArray(3, String.class, "foo")) + .isEqualTo(new String[] {"foo", "foo", "foo"}); + } + + @Test + public void testGetByteArrayElements() { + assertThat(getByteArrayElements(new byte[] {0, 0, 0})).isEqualTo(new byte[] {123, 123, 123}); + } + + @Test + public void testGetCharArrayElements() { + assertThat(getCharArrayElements(new char[] {' ', ' ', ' '})) + .isEqualTo(new char[] {'a', 'a', 'a'}); + } + + @Test + public void testGetShortArrayElements() { + assertThat(getShortArrayElements(new short[] {0, 0, 0})).isEqualTo(new short[] {123, 123, 123}); + } + + @Test + public void testGetIntArrayElements() { + assertThat(getIntArrayElements(new int[] {0, 0, 0})).isEqualTo(new int[] {123, 123, 123}); + } + + @Test + public void testGetLongArrayElements() { + assertThat(getLongArrayElements(new long[] {0, 0, 0})).isEqualTo(new long[] {123, 123, 123}); + } + + @Test + public void testGetFloatArrayElements() { + assertThat(getFloatArrayElements(new float[] {0, 0, 0})) + .isEqualTo(new float[] {3.14f, 3.14f, 3.14f}); + } + + @Test + public void testGetDoubleArrayElements() { + assertThat(getDoubleArrayElements(new double[] {0, 0, 0})) + .isEqualTo(new double[] {3.14, 3.14, 3.14}); + } + + @Test + public void testGetBooleanArrayElements() { + assertThat(getBooleanArrayElements(new boolean[] {false, false, false})) + .isEqualTo(new boolean[] {true, true, true}); + } + + @Test + public void testGetObjectArrayElement() { + Object[] array = new Object[] {1, 2, 3}; + assertThat(getObjectArrayElement(array, 1)).isEqualTo(2); + } + + @Test + public void testSetObjectArrayElement() { + Object[] array = new Object[] {1, 2, 3}; + setObjectArrayElement(array, 1, 200); + assertThat(array).isEqualTo(new Object[] {1, 200, 3}); + } + + @Test + public void testSetByteArrayRegion() { + byte[] array = new byte[] {1, 0, 0, 0, 5}; + byte[] buffer = new byte[] {2, 3, 4}; + setByteArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new byte[] {1, 2, 3, 4, 5}); + } + + @Test + public void testSetCharArrayRegion() { + char[] array = new char[] {'a', ' ', ' ', ' ', 'e'}; + char[] buffer = new char[] {'b', 'c', 'd'}; + setCharArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new char[] {'a', 'b', 'c', 'd', 'e'}); + } + + @Test + public void testSetShortArrayRegion() { + short[] array = new short[] {1, 0, 0, 0, 5}; + short[] buffer = new short[] {2, 3, 4}; + setShortArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new short[] {1, 2, 3, 4, 5}); + } + + @Test + public void testSetIntArrayRegion() { + int[] array = new int[] {1, 0, 0, 0, 5}; + int[] buffer = new int[] {2, 3, 4}; + setIntArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new int[] {1, 2, 3, 4, 5}); + } + + @Test + public void testSetLongArrayRegion() { + long[] array = new long[] {1, 0, 0, 0, 5}; + long[] buffer = new long[] {2, 3, 4}; + setLongArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new long[] {1, 2, 3, 4, 5}); + } + + @Test + public void testSetFloatArrayRegion() { + float[] array = new float[] {1, 0, 0, 0, 5}; + float[] buffer = new float[] {2, 3, 4}; + setFloatArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new float[] {1, 2, 3, 4, 5}); + } + + @Test + public void testSetDoubleArrayRegion() { + double[] array = new double[] {1, 0, 0, 0, 5}; + double[] buffer = new double[] {2, 3, 4}; + setDoubleArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new double[] {1, 2, 3, 4, 5}); + } + + @Test + public void testSetBooleanArrayRegion() { + boolean[] array = new boolean[] {true, false, false, false, true}; + boolean[] buffer = new boolean[] {true, true, true}; + setBooleanArrayRegion(array, 1, 3, buffer); + assertThat(array).isEqualTo(new boolean[] {true, true, true, true, true}); + } + + @Test + public void testCallByteMethod() { + assertThat(callByteMethod(JniHelperTest.class, this, "byteMethod", "()B")).isEqualTo((byte)1); + } + + @Test + public void testCallCharMethod() { + assertThat(callCharMethod(JniHelperTest.class, this, "charMethod", "()C")).isEqualTo('a'); + } + + @Test + public void testCallShortMethod() { + assertThat(callShortMethod(JniHelperTest.class, this, "shortMethod", "()S")) + .isEqualTo((short)1); + } + + @Test + public void testCallIntMethod() { + assertThat(callIntMethod(JniHelperTest.class, this, "intMethod", "()I")).isEqualTo(1); + } + + @Test + public void testCallLongMethod() { + assertThat(callLongMethod(JniHelperTest.class, this, "longMethod", "()J")).isEqualTo(1L); + } + + @Test + public void testCallFloatMethod() { + assertThat(callFloatMethod(JniHelperTest.class, this, "floatMethod", "()F")).isEqualTo(3.14f); + } + + @Test + public void testCallDoubleMethod() { + assertThat(callDoubleMethod(JniHelperTest.class, this, "doubleMethod", "()D")).isEqualTo(3.14); + } + + @Test + public void testCallBooleanMethod() { + assertThat(callBooleanMethod(JniHelperTest.class, this, "booleanMethod", "()Z")) + .isEqualTo(true); + } + + @Test + public void testCallVoidMethod() { + callVoidMethod(JniHelperTest.class, this, "voidMethod", "()V"); + } + + @Test + public void testCallObjectMethod() { + assertThat(callObjectMethod(JniHelperTest.class, this, "objectMethod", "()Ljava/lang/String;")) + .isEqualTo("Hello"); + } + + @Test + public void testCallStaticByteMethod() { + assertThat(callStaticByteMethod(JniHelperTest.class, "staticByteMethod", "()B")) + .isEqualTo((byte)1); + } + + @Test + public void testCallStaticCharMethod() { + assertThat(callStaticCharMethod(JniHelperTest.class, "staticCharMethod", "()C")).isEqualTo('a'); + } + + @Test + public void testCallStaticShortMethod() { + assertThat(callStaticShortMethod(JniHelperTest.class, "staticShortMethod", "()S")) + .isEqualTo((short)1); + } + + @Test + public void testCallStaticIntMethod() { + assertThat(callStaticIntMethod(JniHelperTest.class, "staticIntMethod", "()I")).isEqualTo(1); + } + + @Test + public void testCallStaticLongMethod() { + assertThat(callStaticLongMethod(JniHelperTest.class, "staticLongMethod", "()J")).isEqualTo(1L); + } + + @Test + public void testCallStaticFloatMethod() { + assertThat(callStaticFloatMethod(JniHelperTest.class, "staticFloatMethod", "()F")) + .isEqualTo(3.14f); + } + + @Test + public void testCallStaticDoubleMethod() { + assertThat(callStaticDoubleMethod(JniHelperTest.class, "staticDoubleMethod", "()D")) + .isEqualTo(3.14); + } + + @Test + public void testCallStaticBooleanMethod() { + assertThat(callStaticBooleanMethod(JniHelperTest.class, "staticBooleanMethod", "()Z")) + .isEqualTo(true); + } + + @Test + public void testCallStaticVoidMethod() { + callStaticVoidMethod(JniHelperTest.class, "staticVoidMethod", "()V"); + } + + @Test + public void testCallStaticObjectMethod() { + assertThat( + callStaticObjectMethod(JniHelperTest.class, "staticObjectMethod", "()Ljava/lang/String;")) + .isEqualTo("Hello"); + } +} diff --git a/mobile/test/java/io/envoyproxy/envoymobile/jni/JniUtilityTest.java b/mobile/test/java/io/envoyproxy/envoymobile/jni/JniUtilityTest.java new file mode 100644 index 000000000000..9deb8164422b --- /dev/null +++ b/mobile/test/java/io/envoyproxy/envoymobile/jni/JniUtilityTest.java @@ -0,0 +1,31 @@ +package io.envoyproxy.envoymobile.jni; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.protobuf.Struct; +import com.google.protobuf.Value; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class JniUtilityTest { + public JniUtilityTest() { System.loadLibrary("envoy_jni_utility_test"); } + + //================================================================================ + // Native methods for testing. + //================================================================================ + public static native byte[] protoJavaByteArrayConversion(byte[] source); + + @Test + public void testProtoJavaByteArrayConversion() throws Exception { + Struct source = + Struct.newBuilder() + .putFields("string_key", Value.newBuilder().setStringValue("string_value").build()) + .putFields("num_key", Value.newBuilder().setNumberValue(123).build()) + .putFields("bool_key", Value.newBuilder().setBoolValue(true).build()) + .build(); + Struct dest = Struct.parseFrom(protoJavaByteArrayConversion(source.toByteArray())); + assertThat(source).isEqualTo(dest); + } +} diff --git a/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD index 67bbba18c17e..4173f1d837e8 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_mobile_android_test( name = "certificate_verification_tests", @@ -12,8 +12,13 @@ envoy_mobile_android_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", diff --git a/mobile/test/java/org/chromium/net/BUILD b/mobile/test/java/org/chromium/net/BUILD index 6f67363de3dd..3f25190ac492 100644 --- a/mobile/test/java/org/chromium/net/BUILD +++ b/mobile/test/java/org/chromium/net/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_mobile_android_test( name = "net_tests", @@ -23,8 +23,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", @@ -48,8 +53,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", @@ -72,8 +82,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", @@ -96,8 +111,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java index 99d7de35ce27..1a32ccb6ba2d 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java @@ -1769,8 +1769,8 @@ public void testUploadFailsWithoutInitializingStream() throws Exception { callback.mError.getMessage()); } - private void throwOrCancel(FailureType failureType, ResponseStep failureStep, - boolean expectResponseInfo, boolean expectError) { + private TestUrlRequestCallback throwOrCancel(FailureType failureType, ResponseStep failureStep, + boolean expectError) { if (Log.isLoggable("TESTING", Log.VERBOSE)) { Log.v("TESTING", "Testing " + failureType + " during " + failureStep); } @@ -1790,7 +1790,6 @@ private void throwOrCancel(FailureType failureType, ResponseStep failureStep, assertEquals(ResponseStep.ON_FAILED, callback.mResponseStep); } assertTrue(urlRequest.isDone()); - assertEquals(expectResponseInfo, callback.mResponseInfo != null); assertEquals(expectError, callback.mError != null); assertEquals(expectError, callback.mOnErrorCalled); // When failureType is FailureType.CANCEL_ASYNC_WITHOUT_PAUSE and failureStep is @@ -1803,6 +1802,13 @@ private void throwOrCancel(FailureType failureType, ResponseStep failureStep, failureType == FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, callback.mOnCanceledCalled); } + return callback; + } + + private void throwOrCancel(FailureType failureType, ResponseStep failureStep, + boolean expectResponseInfo, boolean expectError) { + TestUrlRequestCallback callback = throwOrCancel(failureType, failureStep, expectError); + assertEquals(expectResponseInfo, callback.mResponseInfo != null); } @Test @@ -1811,20 +1817,17 @@ private void throwOrCancel(FailureType failureType, ResponseStep failureStep, public void testFailures() throws Exception { throwOrCancel(FailureType.CANCEL_SYNC, ResponseStep.ON_RECEIVED_REDIRECT, false, false); throwOrCancel(FailureType.CANCEL_ASYNC, ResponseStep.ON_RECEIVED_REDIRECT, false, false); - throwOrCancel(FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, ResponseStep.ON_RECEIVED_REDIRECT, false, - false); + throwOrCancel(FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, ResponseStep.ON_RECEIVED_REDIRECT, false); throwOrCancel(FailureType.THROW_SYNC, ResponseStep.ON_RECEIVED_REDIRECT, false, true); throwOrCancel(FailureType.CANCEL_SYNC, ResponseStep.ON_RESPONSE_STARTED, true, false); throwOrCancel(FailureType.CANCEL_ASYNC, ResponseStep.ON_RESPONSE_STARTED, true, false); - throwOrCancel(FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, ResponseStep.ON_RESPONSE_STARTED, true, - false); + throwOrCancel(FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, ResponseStep.ON_RESPONSE_STARTED, false); throwOrCancel(FailureType.THROW_SYNC, ResponseStep.ON_RESPONSE_STARTED, true, true); throwOrCancel(FailureType.CANCEL_SYNC, ResponseStep.ON_READ_COMPLETED, true, false); throwOrCancel(FailureType.CANCEL_ASYNC, ResponseStep.ON_READ_COMPLETED, true, false); - throwOrCancel(FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, ResponseStep.ON_READ_COMPLETED, true, - false); + throwOrCancel(FailureType.CANCEL_ASYNC_WITHOUT_PAUSE, ResponseStep.ON_READ_COMPLETED, false); throwOrCancel(FailureType.THROW_SYNC, ResponseStep.ON_READ_COMPLETED, true, true); } diff --git a/mobile/test/java/org/chromium/net/impl/BUILD b/mobile/test/java/org/chromium/net/impl/BUILD index 51189b5a5091..38137996f741 100644 --- a/mobile/test/java/org/chromium/net/impl/BUILD +++ b/mobile/test/java/org/chromium/net/impl/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_mobile_android_test( name = "cronvoy_test", @@ -21,8 +21,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", diff --git a/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java b/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java index 6debc9821f76..0e477322aa22 100644 --- a/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java +++ b/mobile/test/java/org/chromium/net/impl/CancelProofEnvoyStreamTest.java @@ -357,7 +357,7 @@ public void cancel_manyConcurrentStreamOperationsInFlight() throws Exception { private class MockedStream extends EnvoyHTTPStream { - private MockedStream() { super(0, 0, null, false, 0); } + private MockedStream() { super(0, 0, null, false); } @Override public void sendHeaders(Map> headers, boolean endStream) { diff --git a/mobile/test/java/org/chromium/net/testing/BUILD b/mobile/test/java/org/chromium/net/testing/BUILD index e17fc3b4c738..fded9b2def00 100644 --- a/mobile/test/java/org/chromium/net/testing/BUILD +++ b/mobile/test/java/org/chromium/net/testing/BUILD @@ -1,10 +1,10 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") load("@build_bazel_rules_android//android:rules.bzl", "android_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() android_library( name = "testing", @@ -62,8 +62,13 @@ envoy_mobile_android_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ ":testing", "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", @@ -85,8 +90,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ ":testing", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", @@ -104,8 +114,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ ":testing", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", diff --git a/mobile/test/java/org/chromium/net/testing/Http2TestServerTest.java b/mobile/test/java/org/chromium/net/testing/Http2TestServerTest.java index 411a17f4a9b9..cd79c3515d1e 100644 --- a/mobile/test/java/org/chromium/net/testing/Http2TestServerTest.java +++ b/mobile/test/java/org/chromium/net/testing/Http2TestServerTest.java @@ -8,7 +8,6 @@ import static org.junit.Assert.assertNotNull; import android.content.Context; import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; import io.envoyproxy.envoymobile.utilities.AndroidNetworkLibrary; import io.envoyproxy.envoymobile.AndroidEngineBuilder; import io.envoyproxy.envoymobile.Engine; @@ -29,17 +28,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import java.nio.charset.StandardCharsets; -import org.chromium.net.testing.CertTestUtil; import io.envoyproxy.envoymobile.utilities.FakeX509Util; @RunWith(RobolectricTestRunner.class) @@ -96,10 +91,10 @@ private void getSchemeIsHttps(boolean enablePlatformCertificatesValidation, Response response = sendRequest(requestScenario); + assertThat(response.getEnvoyError()).isNull(); assertThat(response.getHeaders().getHttpStatus()).isEqualTo(200); assertThat(response.getBodyAsString()).contains(":scheme: https"); assertThat(response.getHeaders().value("x-envoy-upstream-alpn")).containsExactly("h2"); - assertThat(response.getEnvoyError()).isNull(); } @Test @@ -107,6 +102,7 @@ public void testGetRequest() throws Exception { getSchemeIsHttps(false, TrustChainVerification.ACCEPT_UNTRUSTED); } + @Ignore @Test public void testGetRequestWithPlatformCertValidatorSuccess() throws Exception { getSchemeIsHttps(true, TrustChainVerification.VERIFY_TRUST_CHAIN); diff --git a/mobile/test/java/org/chromium/net/urlconnection/BUILD b/mobile/test/java/org/chromium/net/urlconnection/BUILD index 8d17d77fe40e..56c47a6b9b9f 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/BUILD +++ b/mobile/test/java/org/chromium/net/urlconnection/BUILD @@ -1,9 +1,9 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_mobile_android_test( name = "urlconnection_test", @@ -23,8 +23,13 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/java/io/envoyproxy/envoymobile/engine:envoy_base_engine_lib", "//library/java/io/envoyproxy/envoymobile/engine:envoy_engine_lib", diff --git a/mobile/test/kotlin/apps/baseline/AsyncDemoFilter.kt b/mobile/test/kotlin/apps/baseline/AsyncDemoFilter.kt index 1c0ae69a1a7a..dda47a7b3460 100644 --- a/mobile/test/kotlin/apps/baseline/AsyncDemoFilter.kt +++ b/mobile/test/kotlin/apps/baseline/AsyncDemoFilter.kt @@ -30,9 +30,7 @@ class AsyncDemoFilter : AsyncResponseFilter { ): FilterHeadersStatus { // If this is the end of the stream, asynchronously resume response processing via callback. if (endStream) { - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } } return FilterHeadersStatus.StopIteration() } @@ -44,9 +42,7 @@ class AsyncDemoFilter : AsyncResponseFilter { ): FilterDataStatus { // If this is the end of the stream, asynchronously resume response processing via callback. if (endStream) { - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } } return FilterDataStatus.StopIterationAndBuffer() } @@ -56,9 +52,7 @@ class AsyncDemoFilter : AsyncResponseFilter { streamIntel: StreamIntel ): FilterTrailersStatus { // Trailers imply end of stream, so asynchronously resume response processing via callbacka - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } return FilterTrailersStatus.StopIteration() } @@ -73,23 +67,14 @@ class AsyncDemoFilter : AsyncResponseFilter { endStream: Boolean, streamIntel: StreamIntel ): FilterResumeStatus { - val builder = headers!!.toResponseHeadersBuilder() - .add("async-filter-demo", "1") + val builder = headers!!.toResponseHeadersBuilder().add("async-filter-demo", "1") return FilterResumeStatus.ResumeIteration(builder.build(), data, trailers) } @Suppress("EmptyFunctionBlock") - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onCancel(finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/test/kotlin/apps/baseline/BUILD b/mobile/test/kotlin/apps/baseline/BUILD index 9357573b6836..2847d1ccf4f7 100644 --- a/mobile/test/kotlin/apps/baseline/BUILD +++ b/mobile/test/kotlin/apps/baseline/BUILD @@ -1,10 +1,13 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") load("@rules_detekt//detekt:defs.bzl", "detekt") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_binary( name = "hello_envoy_kt", custom_package = "io.envoyproxy.envoymobile.helloenvoybaselinetest", @@ -34,6 +37,7 @@ kt_android_library( artifact("androidx.recyclerview:recyclerview"), artifact("androidx.annotation:annotation"), artifact("com.google.code.findbugs:jsr305"), + artifact("com.google.protobuf:protobuf-javalite"), ], ) diff --git a/mobile/test/kotlin/apps/baseline/BufferDemoFilter.kt b/mobile/test/kotlin/apps/baseline/BufferDemoFilter.kt index 478a283f33b0..069a9c8c157f 100644 --- a/mobile/test/kotlin/apps/baseline/BufferDemoFilter.kt +++ b/mobile/test/kotlin/apps/baseline/BufferDemoFilter.kt @@ -13,8 +13,7 @@ import java.nio.ByteBuffer /** * Example of a more complex HTTP filter that pauses processing on the response filter chain, - * buffers until the response is complete, then resumes filter iteration while setting a new - * header. + * buffers until the response is complete, then resumes filter iteration while setting a new header. */ class BufferDemoFilter : ResponseFilter { private lateinit var headers: ResponseHeaders @@ -39,8 +38,7 @@ class BufferDemoFilter : ResponseFilter { // If this is the end of the stream, resume processing of the (now fully-buffered) response. if (endStream) { - val builder = headers.toResponseHeadersBuilder() - .add("buffer-filter-demo", "1") + val builder = headers.toResponseHeadersBuilder().add("buffer-filter-demo", "1") return FilterDataStatus.ResumeIteration(builder.build(), body) } return FilterDataStatus.StopIterationAndBuffer() @@ -51,23 +49,14 @@ class BufferDemoFilter : ResponseFilter { streamIntel: StreamIntel ): FilterTrailersStatus { // Trailers imply end of stream; resume processing of the (now fully-buffered) response. - val builder = headers.toResponseHeadersBuilder() - .add("buffer-filter-demo", "1") + val builder = headers.toResponseHeadersBuilder().add("buffer-filter-demo", "1") return FilterTrailersStatus.ResumeIteration(builder.build(), this.body, trailers) } @Suppress("EmptyFunctionBlock") - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onCancel(finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/test/kotlin/apps/baseline/DemoFilter.kt b/mobile/test/kotlin/apps/baseline/DemoFilter.kt index ab027dafe636..706b584b89ad 100644 --- a/mobile/test/kotlin/apps/baseline/DemoFilter.kt +++ b/mobile/test/kotlin/apps/baseline/DemoFilter.kt @@ -41,10 +41,7 @@ class DemoFilter : ResponseFilter { return FilterTrailersStatus.Continue(trailers) } - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) { Log.d("DemoFilter", "On error!") } @@ -52,7 +49,5 @@ class DemoFilter : ResponseFilter { Log.d("DemoFilter", "On cancel!") } - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/test/kotlin/apps/baseline/MainActivity.kt b/mobile/test/kotlin/apps/baseline/MainActivity.kt index 25bc30a4cf9b..ebadd21327c0 100644 --- a/mobile/test/kotlin/apps/baseline/MainActivity.kt +++ b/mobile/test/kotlin/apps/baseline/MainActivity.kt @@ -25,13 +25,14 @@ private const val REQUEST_HANDLER_THREAD_NAME = "hello_envoy_kt" private const val REQUEST_AUTHORITY = "api.lyft.com" private const val REQUEST_PATH = "/ping" private const val REQUEST_SCHEME = "http" -private val FILTERED_HEADERS = setOf( - "server", - "filter-demo", - "buffer-filter-demo", - "async-filter-demo", - "x-envoy-upstream-service-time" -) +private val FILTERED_HEADERS = + setOf( + "server", + "filter-demo", + "buffer-filter-demo", + "async-filter-demo", + "x-envoy-upstream-service-time" + ) class MainActivity : Activity() { private val thread = HandlerThread(REQUEST_HANDLER_THREAD_NAME) @@ -44,30 +45,31 @@ class MainActivity : Activity() { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - engine = AndroidEngineBuilder(application) - .addLogLevel(LogLevel.DEBUG) - .addPlatformFilter(::DemoFilter) - .addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}") - .addStringAccessor("demo-accessor", { "PlatformString" }) - .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") } - .setEventTracker({ - for (entry in it.entries) { - Log.d("MainActivity", "Event emitted: ${entry.key}, ${entry.value}") - } - }) - .setLogger { - Log.d("MainActivity", it) - } - .build() + engine = + AndroidEngineBuilder(application) + .addLogLevel(LogLevel.DEBUG) + .addPlatformFilter(::DemoFilter) + .addNativeFilter( + "envoy.filters.http.buffer", + "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}" + ) + .addStringAccessor("demo-accessor", { "PlatformString" }) + .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") } + .setEventTracker({ + for (entry in it.entries) { + Log.d("MainActivity", "Event emitted: ${entry.key}, ${entry.value}") + } + }) + .setLogger { Log.d("MainActivity", it) } + .build() recyclerView = findViewById(R.id.recycler_view) as RecyclerView recyclerView.layoutManager = LinearLayoutManager(this) viewAdapter = ResponseRecyclerViewAdapter() recyclerView.adapter = viewAdapter - val dividerItemDecoration = DividerItemDecoration( - recyclerView.context, DividerItemDecoration.VERTICAL - ) + val dividerItemDecoration = + DividerItemDecoration(recyclerView.context, DividerItemDecoration.VERTICAL) recyclerView.addItemDecoration(dividerItemDecoration) thread.start() val handler = Handler(thread.looper) @@ -100,10 +102,9 @@ class MainActivity : Activity() { // Note: this request will use an h2 stream for the upstream request. // The Java example uses http/1.1. This is done on purpose to test both paths in end-to-end // tests in CI. - val requestHeaders = RequestHeadersBuilder( - RequestMethod.GET, REQUEST_SCHEME, REQUEST_AUTHORITY, REQUEST_PATH - ) - .build() + val requestHeaders = + RequestHeadersBuilder(RequestMethod.GET, REQUEST_SCHEME, REQUEST_AUTHORITY, REQUEST_PATH) + .build() engine .streamClient() .newStreamPrototype() diff --git a/mobile/test/kotlin/apps/experimental/AsyncDemoFilter.kt b/mobile/test/kotlin/apps/experimental/AsyncDemoFilter.kt index 90c5c5991706..5bc6126bdde5 100644 --- a/mobile/test/kotlin/apps/experimental/AsyncDemoFilter.kt +++ b/mobile/test/kotlin/apps/experimental/AsyncDemoFilter.kt @@ -30,9 +30,7 @@ class AsyncDemoFilter : AsyncResponseFilter { ): FilterHeadersStatus { // If this is the end of the stream, asynchronously resume response processing via callback. if (endStream) { - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } } return FilterHeadersStatus.StopIteration() } @@ -44,9 +42,7 @@ class AsyncDemoFilter : AsyncResponseFilter { ): FilterDataStatus { // If this is the end of the stream, asynchronously resume response processing via callback. if (endStream) { - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } } return FilterDataStatus.StopIterationAndBuffer() } @@ -56,9 +52,7 @@ class AsyncDemoFilter : AsyncResponseFilter { streamIntel: StreamIntel ): FilterTrailersStatus { // Trailers imply end of stream, so asynchronously resume response processing via callbacka - Timer("AsyncResume", false).schedule(100) { - callbacks.resumeResponse() - } + Timer("AsyncResume", false).schedule(100) { callbacks.resumeResponse() } return FilterTrailersStatus.StopIteration() } @@ -73,23 +67,14 @@ class AsyncDemoFilter : AsyncResponseFilter { endStream: Boolean, streamIntel: StreamIntel ): FilterResumeStatus { - val builder = headers!!.toResponseHeadersBuilder() - .add("async-filter-demo", "1") + val builder = headers!!.toResponseHeadersBuilder().add("async-filter-demo", "1") return FilterResumeStatus.ResumeIteration(builder.build(), data, trailers) } @Suppress("EmptyFunctionBlock") - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onCancel(finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/test/kotlin/apps/experimental/BUILD b/mobile/test/kotlin/apps/experimental/BUILD index 8e0ec8fa02ba..2d99b5ba5099 100644 --- a/mobile/test/kotlin/apps/experimental/BUILD +++ b/mobile/test/kotlin/apps/experimental/BUILD @@ -1,10 +1,13 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") load("@rules_detekt//detekt:defs.bzl", "detekt") load("@rules_jvm_external//:defs.bzl", "artifact") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + android_binary( name = "hello_envoy_kt", custom_package = "io.envoyproxy.envoymobile.helloenvoyexperimentaltest", @@ -34,6 +37,7 @@ kt_android_library( artifact("androidx.recyclerview:recyclerview"), artifact("androidx.annotation:annotation"), artifact("com.google.code.findbugs:jsr305"), + artifact("com.google.protobuf:protobuf-javalite"), ], ) diff --git a/mobile/test/kotlin/apps/experimental/BufferDemoFilter.kt b/mobile/test/kotlin/apps/experimental/BufferDemoFilter.kt index 2c22504becbe..29f8f84b6fe0 100644 --- a/mobile/test/kotlin/apps/experimental/BufferDemoFilter.kt +++ b/mobile/test/kotlin/apps/experimental/BufferDemoFilter.kt @@ -13,8 +13,7 @@ import java.nio.ByteBuffer /** * Example of a more complex HTTP filter that pauses processing on the response filter chain, - * buffers until the response is complete, then resumes filter iteration while setting a new - * header. + * buffers until the response is complete, then resumes filter iteration while setting a new header. */ class BufferDemoFilter : ResponseFilter { private lateinit var headers: ResponseHeaders @@ -39,8 +38,7 @@ class BufferDemoFilter : ResponseFilter { // If this is the end of the stream, resume processing of the (now fully-buffered) response. if (endStream) { - val builder = headers.toResponseHeadersBuilder() - .add("buffer-filter-demo", "1") + val builder = headers.toResponseHeadersBuilder().add("buffer-filter-demo", "1") return FilterDataStatus.ResumeIteration(builder.build(), body) } return FilterDataStatus.StopIterationAndBuffer() @@ -51,23 +49,14 @@ class BufferDemoFilter : ResponseFilter { streamIntel: StreamIntel ): FilterTrailersStatus { // Trailers imply end of stream; resume processing of the (now fully-buffered) response. - val builder = headers.toResponseHeadersBuilder() - .add("buffer-filter-demo", "1") + val builder = headers.toResponseHeadersBuilder().add("buffer-filter-demo", "1") return FilterTrailersStatus.ResumeIteration(builder.build(), this.body, trailers) } @Suppress("EmptyFunctionBlock") - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onCancel(finalStreamIntel: FinalStreamIntel) {} - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/test/kotlin/apps/experimental/DemoFilter.kt b/mobile/test/kotlin/apps/experimental/DemoFilter.kt index 6790e22d3f55..3621722c95dd 100644 --- a/mobile/test/kotlin/apps/experimental/DemoFilter.kt +++ b/mobile/test/kotlin/apps/experimental/DemoFilter.kt @@ -41,10 +41,7 @@ class DemoFilter : ResponseFilter { return FilterTrailersStatus.Continue(trailers) } - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) { + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) { Log.d("DemoFilter", "On error!") } @@ -52,7 +49,5 @@ class DemoFilter : ResponseFilter { Log.d("DemoFilter", "On cancel!") } - @Suppress("EmptyFunctionBlock") - override fun onComplete(finalStreamIntel: FinalStreamIntel) { - } + @Suppress("EmptyFunctionBlock") override fun onComplete(finalStreamIntel: FinalStreamIntel) {} } diff --git a/mobile/test/kotlin/apps/experimental/MainActivity.kt b/mobile/test/kotlin/apps/experimental/MainActivity.kt index 6a76e94ae47a..a50c69a63577 100644 --- a/mobile/test/kotlin/apps/experimental/MainActivity.kt +++ b/mobile/test/kotlin/apps/experimental/MainActivity.kt @@ -9,15 +9,13 @@ import android.util.Log import androidx.recyclerview.widget.DividerItemDecoration import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView -import io.envoyproxy.envoymobile.android.SharedPreferencesStore import io.envoyproxy.envoymobile.AndroidEngineBuilder -import io.envoyproxy.envoymobile.CompressionAlgorithm import io.envoyproxy.envoymobile.Element import io.envoyproxy.envoymobile.Engine import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder -import io.envoyproxy.envoymobile.RequestHeadersBuilderCompressionUtil.enableRequestCompression import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.android.SharedPreferencesStore import io.envoyproxy.envoymobile.shared.Failure import io.envoyproxy.envoymobile.shared.ResponseRecyclerViewAdapter import io.envoyproxy.envoymobile.shared.Success @@ -30,13 +28,14 @@ private const val REQUEST_AUTHORITY = "api.lyft.com" private const val REQUEST_PATH = "/ping" private const val REQUEST_SCHEME = "https" private const val PERSISTENCE_KEY = "EnvoyMobilePersistenceKey" -private val FILTERED_HEADERS = setOf( - "server", - "filter-demo", - "buffer-filter-demo", - "async-filter-demo", - "x-envoy-upstream-service-time" -) +private val FILTERED_HEADERS = + setOf( + "server", + "filter-demo", + "buffer-filter-demo", + "async-filter-demo", + "x-envoy-upstream-service-time" + ) class MainActivity : Activity() { private val thread = HandlerThread(REQUEST_HANDLER_THREAD_NAME) @@ -51,40 +50,41 @@ class MainActivity : Activity() { val preferences = getSharedPreferences(PERSISTENCE_KEY, Context.MODE_PRIVATE) - engine = AndroidEngineBuilder(application) - .addLogLevel(LogLevel.DEBUG) - .addPlatformFilter(::DemoFilter) - .addPlatformFilter(::BufferDemoFilter) - .addPlatformFilter(::AsyncDemoFilter) - .enableDNSCache(true) - // required by DNS cache - .addKeyValueStore("reserved.platform_store", SharedPreferencesStore(preferences)) - .enableInterfaceBinding(true) - .enableSocketTagging(true) - .enableProxying(true) - // TODO: uncomment once platform cert validation is fixed. - // .enablePlatformCertificatesValidation(true) - .addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}") - .addStringAccessor("demo-accessor", { "PlatformString" }) - .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") } - .setEventTracker({ - for (entry in it.entries) { - Log.d("MainActivity", "Event emitted: ${entry.key}, ${entry.value}") - } - }) - .setLogger { - Log.d("MainActivity", it) - } - .build() + engine = + AndroidEngineBuilder(application) + .addLogLevel(LogLevel.DEBUG) + .addPlatformFilter(::DemoFilter) + .addPlatformFilter(::BufferDemoFilter) + .addPlatformFilter(::AsyncDemoFilter) + .enableDNSCache(true) + // required by DNS cache + .addKeyValueStore("reserved.platform_store", SharedPreferencesStore(preferences)) + .enableInterfaceBinding(true) + .enableSocketTagging(true) + .enableProxying(true) + // TODO: uncomment once platform cert validation is fixed. + // .enablePlatformCertificatesValidation(true) + .addNativeFilter( + "envoy.filters.http.buffer", + "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":5242880}" + ) + .addStringAccessor("demo-accessor", { "PlatformString" }) + .setOnEngineRunning { Log.d("MainActivity", "Envoy async internal setup completed") } + .setEventTracker({ + for (entry in it.entries) { + Log.d("MainActivity", "Event emitted: ${entry.key}, ${entry.value}") + } + }) + .setLogger { Log.d("MainActivity", it) } + .build() recyclerView = findViewById(R.id.recycler_view) as RecyclerView recyclerView.layoutManager = LinearLayoutManager(this) viewAdapter = ResponseRecyclerViewAdapter() recyclerView.adapter = viewAdapter - val dividerItemDecoration = DividerItemDecoration( - recyclerView.context, DividerItemDecoration.VERTICAL - ) + val dividerItemDecoration = + DividerItemDecoration(recyclerView.context, DividerItemDecoration.VERTICAL) recyclerView.addItemDecoration(dividerItemDecoration) thread.start() val handler = Handler(thread.looper) @@ -117,12 +117,10 @@ class MainActivity : Activity() { // Note: this request will use an h2 stream for the upstream request. // The Java example uses http/1.1. This is done on purpose to test both paths in end-to-end // tests in CI. - val requestHeaders = RequestHeadersBuilder( - RequestMethod.GET, REQUEST_SCHEME, REQUEST_AUTHORITY, REQUEST_PATH - ) - .enableRequestCompression(CompressionAlgorithm.GZIP) - .addSocketTag(1,2) - .build() + val requestHeaders = + RequestHeadersBuilder(RequestMethod.GET, REQUEST_SCHEME, REQUEST_AUTHORITY, REQUEST_PATH) + .addSocketTag(1, 2) + .build() engine .streamClient() .newStreamPrototype() diff --git a/mobile/test/kotlin/integration/BUILD b/mobile/test/kotlin/integration/BUILD index 3b9b4f53621f..f26d908b847d 100644 --- a/mobile/test/kotlin/integration/BUILD +++ b/mobile/test/kotlin/integration/BUILD @@ -1,5 +1,10 @@ -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test", "envoy_mobile_jni_kt_test") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_lib.bzl", "envoy_mobile_kt_library") +load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test", "envoy_mobile_jni_kt_test") + +licenses(["notice"]) # Apache 2 + +envoy_mobile_package() envoy_mobile_jni_kt_test( name = "engine_start_test", @@ -8,8 +13,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -22,8 +32,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -36,8 +51,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -50,8 +70,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -64,37 +89,51 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], ) envoy_mobile_jni_kt_test( - name = "stat_flush_integration_test", + name = "cancel_stream_test", srcs = [ - "StatFlushIntegrationTest.kt", - "TestStatsdServer.kt", + "CancelStreamTest.kt", ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], ) envoy_mobile_jni_kt_test( - name = "cancel_stream_test", + name = "stream_idle_timeout_test", srcs = [ - "CancelStreamTest.kt", + "StreamIdleTimeoutTest.kt", ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -107,8 +146,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -121,8 +165,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -135,8 +184,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -153,26 +207,13 @@ envoy_mobile_jni_kt_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], - deps = [ - "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", - ], -) - -envoy_mobile_jni_kt_test( - name = "receive_data_delivery_limits_test", - srcs = [ - "ReceiveDataWithDeliveryLimitsTest.kt", - ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, - native_deps = [ - "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -185,14 +226,18 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], ) -#envoy_mobile_jni_kt_test( envoy_mobile_android_test( name = "send_data_test", srcs = [ @@ -200,8 +245,13 @@ envoy_mobile_android_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//test/java/io/envoyproxy/envoymobile/engine/testing", @@ -215,8 +265,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -229,8 +284,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -243,8 +303,13 @@ envoy_mobile_jni_kt_test( ], native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], @@ -262,10 +327,48 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", ], ) + +envoy_mobile_android_test( + name = "xds_test", + srcs = [ + "XdsTest.kt", + ], + native_deps = [ + "@envoy//test/config/integration/certs", + "//test/common/jni:libenvoy_jni_with_test_extensions.so", + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", + deps = [ + ":test_utilities", + "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", + ], +) + +envoy_mobile_kt_library( + name = "test_utilities", + srcs = [ + "TestUtilities.kt", + ], + deps = [ + "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", + "@maven//:junit_junit", + ], +) diff --git a/mobile/test/kotlin/integration/CancelGRPCStreamTest.kt b/mobile/test/kotlin/integration/CancelGRPCStreamTest.kt index d442a3ac722a..c9bba2c7495a 100644 --- a/mobile/test/kotlin/integration/CancelGRPCStreamTest.kt +++ b/mobile/test/kotlin/integration/CancelGRPCStreamTest.kt @@ -1,6 +1,5 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.EnvoyError import io.envoyproxy.envoymobile.FilterDataStatus @@ -9,10 +8,10 @@ import io.envoyproxy.envoymobile.FilterTrailersStatus import io.envoyproxy.envoymobile.FinalStreamIntel import io.envoyproxy.envoymobile.GRPCClient import io.envoyproxy.envoymobile.GRPCRequestHeadersBuilder -import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.ResponseFilter import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer @@ -22,9 +21,11 @@ import java.util.concurrent.TimeUnit import org.assertj.core.api.Assertions.assertThat import org.junit.Test -private val filterName = "cancel_validation_filter" -private const val pbfType = "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" -private const val localErrorFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError" +private const val FILTER_NAME = "cancel_validation_filter" +private const val PBF_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" +private const val LOCAL_ERROR_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError" class CancelGRPCStreamTest { @@ -35,9 +36,7 @@ class CancelGRPCStreamTest { private val filterExpectation = CountDownLatch(1) private val onCancelCallbackExpectation = CountDownLatch(1) - class CancelValidationFilter( - private val latch: CountDownLatch - ) : ResponseFilter { + class CancelValidationFilter(private val latch: CountDownLatch) : ResponseFilter { override fun onResponseHeaders( headers: ResponseHeaders, endStream: Boolean, @@ -62,6 +61,7 @@ class CancelGRPCStreamTest { } override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} override fun onCancel(finalStreamIntel: FinalStreamIntel) { @@ -71,29 +71,27 @@ class CancelGRPCStreamTest { @Test fun `cancel grpc stream calls onCancel callback`() { - val engine = EngineBuilder(Standard()) - .addPlatformFilter( - name = filterName, - factory = { CancelValidationFilter(filterExpectation) } - ) - .addNativeFilter("envoy.filters.http.platform_bridge", "{'@type': $pbfType, platform_filter_name: $filterName}") - .addNativeFilter("envoy.filters.http.local_error", "{'@type': $localErrorFilterType}") - .setOnEngineRunning {} - .build() + val engine = + EngineBuilder(Standard()) + .addPlatformFilter( + name = FILTER_NAME, + factory = { CancelValidationFilter(filterExpectation) } + ) + .addNativeFilter( + "envoy.filters.http.platform_bridge", + "{'@type': $PBF_TYPE, platform_filter_name: $FILTER_NAME}" + ) + .addNativeFilter("envoy.filters.http.local_error", "{'@type': $LOCAL_ERROR_FILTER_TYPE}") + .build() val client = GRPCClient(engine.streamClient()) - val requestHeaders = GRPCRequestHeadersBuilder( - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() - - client.newGRPCStreamPrototype() - .setOnCancel { _ -> - onCancelCallbackExpectation.countDown() - } + val requestHeaders = + GRPCRequestHeadersBuilder(scheme = "https", authority = "example.com", path = "/test").build() + + client + .newGRPCStreamPrototype() + .setOnCancel { _ -> onCancelCallbackExpectation.countDown() } .start(Executors.newSingleThreadExecutor()) .sendHeaders(requestHeaders, false) .cancel() diff --git a/mobile/test/kotlin/integration/CancelStreamTest.kt b/mobile/test/kotlin/integration/CancelStreamTest.kt index 5e4c13d967ff..6993fe62f26e 100644 --- a/mobile/test/kotlin/integration/CancelStreamTest.kt +++ b/mobile/test/kotlin/integration/CancelStreamTest.kt @@ -1,6 +1,5 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.EnvoyError import io.envoyproxy.envoymobile.FilterDataStatus @@ -12,6 +11,7 @@ import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.ResponseFilter import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer @@ -21,7 +21,8 @@ import java.util.concurrent.TimeUnit import org.assertj.core.api.Assertions.assertThat import org.junit.Test -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" class CancelStreamTest { @@ -32,9 +33,7 @@ class CancelStreamTest { private val filterExpectation = CountDownLatch(1) private val runExpectation = CountDownLatch(1) - class CancelValidationFilter( - private val latch: CountDownLatch - ) : ResponseFilter { + class CancelValidationFilter(private val latch: CountDownLatch) : ResponseFilter { override fun onResponseHeaders( headers: ResponseHeaders, endStream: Boolean, @@ -59,6 +58,7 @@ class CancelStreamTest { } override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} override fun onCancel(finalStreamIntel: FinalStreamIntel) { @@ -68,29 +68,29 @@ class CancelStreamTest { @Test fun `cancel stream calls onCancel callback`() { - val engine = EngineBuilder(Standard()) - .addPlatformFilter( - name = "cancel_validation_filter", - factory = { CancelValidationFilter(filterExpectation) } - ) - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .setOnEngineRunning {} - .build() + val engine = + EngineBuilder(Standard()) + .addPlatformFilter( + name = "cancel_validation_filter", + factory = { CancelValidationFilter(filterExpectation) } + ) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() - - client.newStreamPrototype() - .setOnCancel { _ -> - runExpectation.countDown() - } + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() + + client + .newStreamPrototype() + .setOnCancel { _ -> runExpectation.countDown() } .start(Executors.newSingleThreadExecutor()) .sendHeaders(requestHeaders, false) .cancel() diff --git a/mobile/test/kotlin/integration/EngineApiTest.kt b/mobile/test/kotlin/integration/EngineApiTest.kt index 540e10806878..61b2f16e5218 100644 --- a/mobile/test/kotlin/integration/EngineApiTest.kt +++ b/mobile/test/kotlin/integration/EngineApiTest.kt @@ -1,7 +1,9 @@ package test.kotlin.integration +import com.google.protobuf.NullValue +import com.google.protobuf.Struct +import com.google.protobuf.Value import io.envoyproxy.envoymobile.Element -import io.envoyproxy.envoymobile.Engine import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.engine.JniLibrary @@ -18,11 +20,34 @@ class EngineApiTest { @Test fun `verify engine APIs`() { val countDownLatch = CountDownLatch(1) - val engine = EngineBuilder() - .addLogLevel(LogLevel.INFO) - .addStatsFlushSeconds(1) - .setOnEngineRunning { countDownLatch.countDown() } - .build() + val engine = + EngineBuilder() + .addLogLevel(LogLevel.INFO) + .setNodeId("node-id") + .setNodeLocality("region", "zone", "subzone") + .setNodeMetadata( + Struct.newBuilder() + .putFields("string_value", Value.newBuilder().setStringValue("string").build()) + .putFields("number_value", Value.newBuilder().setNumberValue(123.0).build()) + .putFields("bool_value", Value.newBuilder().setBoolValue(true).build()) + .putFields("null_value", Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) + .putFields( + "struct_value", + Value.newBuilder() + .setStructValue( + Struct.newBuilder() + .putFields( + "nested_value", + Value.newBuilder().setStringValue("nested_string").build() + ) + .build() + ) + .build() + ) + .build() + ) + .setOnEngineRunning { countDownLatch.countDown() } + .build() assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue() diff --git a/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt b/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt index fda5f1997679..6b2ebb069fa0 100644 --- a/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt +++ b/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt @@ -2,11 +2,8 @@ package test.kotlin.integration import android.content.Context import androidx.test.core.app.ApplicationProvider - import io.envoyproxy.envoymobile.AndroidEngineBuilder import io.envoyproxy.envoymobile.EnvoyError -import io.envoyproxy.envoymobile.Engine -import io.envoyproxy.envoymobile.engine.JniLibrary import io.envoyproxy.envoymobile.FilterDataStatus import io.envoyproxy.envoymobile.FilterHeadersStatus import io.envoyproxy.envoymobile.FilterTrailersStatus @@ -21,19 +18,17 @@ import io.envoyproxy.envoymobile.ResponseFilter import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.ResponseTrailers import io.envoyproxy.envoymobile.StreamIntel - +import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat -import org.junit.Assert.assertNotNull import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner -class ThrowingFilter: RequestFilter, ResponseFilter { +class ThrowingFilter : RequestFilter, ResponseFilter { override fun onRequestHeaders( headers: RequestHeaders, endStream: Boolean, @@ -42,13 +37,18 @@ class ThrowingFilter: RequestFilter, ResponseFilter { throw Exception("Simulated onRequestHeaders exception") } - override fun onRequestData(body: ByteBuffer, endStream: Boolean, streamIntel: StreamIntel): - FilterDataStatus { - return FilterDataStatus.Continue(body) + override fun onRequestData( + body: ByteBuffer, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterDataStatus { + return FilterDataStatus.Continue(body) } - override fun onRequestTrailers(trailers: RequestTrailers, streamIntel: StreamIntel): - FilterTrailersStatus { + override fun onRequestTrailers( + trailers: RequestTrailers, + streamIntel: StreamIntel + ): FilterTrailersStatus { return FilterTrailersStatus.Continue(trailers) } @@ -75,10 +75,7 @@ class ThrowingFilter: RequestFilter, ResponseFilter { return FilterTrailersStatus.Continue(trailers) } - override fun onError( - error: EnvoyError, - finalStreamIntel: FinalStreamIntel - ) {} + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} override fun onCancel(finalStreamIntel: FinalStreamIntel) {} @@ -97,31 +94,37 @@ class FilterThrowingExceptionTest { val onRespondeHeadersLatch = CountDownLatch(1) val onJNIExceptionEventLatch = CountDownLatch(2) - var expectedMessages = mutableListOf( - "Simulated onRequestHeaders exception||onRequestHeaders||", - "Simulated onResponseHeaders exception||onResponseHeaders||") + var expectedMessages = + mutableListOf( + "Simulated onRequestHeaders exception||onRequestHeaders||", + "Simulated onResponseHeaders exception||onResponseHeaders||" + ) val context = ApplicationProvider.getApplicationContext() val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .setEventTracker { event -> - if (event["name"] == "event_log" && event["log_name"] == "jni_cleared_pending_exception") { - assertThat(event["message"]).contains(expectedMessages.removeFirst()) - onJNIExceptionEventLatch.countDown() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .setEventTracker { event -> + if ( + event["name"] == "event_log" && event["log_name"] == "jni_cleared_pending_exception" + ) { + assertThat(event["message"]).contains(expectedMessages.removeFirst()) + onJNIExceptionEventLatch.countDown() + } } - } - .addPlatformFilter(::ThrowingFilter) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() - - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + .addPlatformFilter(::ThrowingFilter) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() + + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() diff --git a/mobile/test/kotlin/integration/GRPCReceiveErrorTest.kt b/mobile/test/kotlin/integration/GRPCReceiveErrorTest.kt index c6c21abd7382..19efe62602e9 100644 --- a/mobile/test/kotlin/integration/GRPCReceiveErrorTest.kt +++ b/mobile/test/kotlin/integration/GRPCReceiveErrorTest.kt @@ -1,6 +1,5 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.EnvoyError import io.envoyproxy.envoymobile.FilterDataStatus @@ -12,6 +11,7 @@ import io.envoyproxy.envoymobile.GRPCRequestHeadersBuilder import io.envoyproxy.envoymobile.ResponseFilter import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer @@ -21,12 +21,9 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val hcmType = - "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.EnvoyMobileHttpConnectionManager" -private const val pbfType = "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" -private const val localErrorFilterType = - "type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError" -private const val filterName = "error_validation_filter" +private const val PBF_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" +private const val FILTER_NAME = "error_validation_filter" class GRPCReceiveErrorTest { init { @@ -67,6 +64,7 @@ class GRPCReceiveErrorTest { override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) { receivedError.countDown() } + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} override fun onCancel(finalStreamIntel: FinalStreamIntel) { @@ -76,31 +74,30 @@ class GRPCReceiveErrorTest { @Test fun `errors on stream call onError callback`() { - val requestHeader = GRPCRequestHeadersBuilder( - scheme = "https", - authority = "example.com", - path = "/pb.api.v1.Foo/GetBar" - ).build() - - val engine = EngineBuilder(Standard()) - .addPlatformFilter( - name = filterName, - factory = { ErrorValidationFilter(filterReceivedError, filterNotCancelled) } - ) - .addNativeFilter("envoy.filters.http.platform_bridge", "{'@type': $pbfType, platform_filter_name: $filterName}") - .setOnEngineRunning {} - .build() + val requestHeader = + GRPCRequestHeadersBuilder( + scheme = "https", + authority = "example.com", + path = "/pb.api.v1.Foo/GetBar" + ) + .build() + + val engine = + EngineBuilder(Standard()) + .addPlatformFilter( + name = FILTER_NAME, + factory = { ErrorValidationFilter(filterReceivedError, filterNotCancelled) } + ) + .addNativeFilter( + "envoy.filters.http.platform_bridge", + "{'@type': $PBF_TYPE, platform_filter_name: $FILTER_NAME}" + ) + .build() GRPCClient(engine.streamClient()) .newGRPCStreamPrototype() - .setOnResponseHeaders { _, _, _ -> } - .setOnResponseMessage { _, _ -> } - .setOnError { _, _ -> - callbackReceivedError.countDown() - } - .setOnCancel { _ -> - fail("Unexpected call to onCancel response callback") - } + .setOnError { _, _ -> callbackReceivedError.countDown() } + .setOnCancel { _ -> fail("Unexpected call to onCancel response callback") } .start() .sendHeaders(requestHeader, false) .sendMessage(ByteBuffer.wrap(ByteArray(5))) diff --git a/mobile/test/kotlin/integration/KeyValueStoreTest.kt b/mobile/test/kotlin/integration/KeyValueStoreTest.kt index d62edd181c2f..fdc46031951e 100644 --- a/mobile/test/kotlin/integration/KeyValueStoreTest.kt +++ b/mobile/test/kotlin/integration/KeyValueStoreTest.kt @@ -1,22 +1,21 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.KeyValueStore import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.engine.JniLibrary -import java.nio.ByteBuffer import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val assertionFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" -private const val testKey = "foo" -private const val testValue = "bar" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" +private const val TEST_KEY = "foo" +private const val TEST_VALUE = "bar" class KeyValueStoreTest { @@ -29,28 +28,42 @@ class KeyValueStoreTest { val readExpectation = CountDownLatch(3) val saveExpectation = CountDownLatch(1) - val testKeyValueStore = object : KeyValueStore { - override fun read(key: String): String? { readExpectation.countDown(); return null } - override fun remove(key: String) {} - override fun save(key: String, value: String) { saveExpectation.countDown() } - } + val testKeyValueStore = + object : KeyValueStore { + override fun read(key: String): String? { + readExpectation.countDown() + return null + } + + override fun remove(key: String) {} - val engine = EngineBuilder(Standard()) + override fun save(key: String, value: String) { + saveExpectation.countDown() + } + } + + val engine = + EngineBuilder(Standard()) .addKeyValueStore("envoy.key_value.platform_test", testKeyValueStore) - .addNativeFilter("envoy.filters.http.test_kv_store", "{'@type': type.googleapis.com/envoymobile.extensions.filters.http.test_kv_store.TestKeyValueStore, kv_store_name: envoy.key_value.platform_test, test_key: $testKey, test_value: $testValue}") - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") + .addNativeFilter( + "envoy.filters.http.test_kv_store", + "{'@type': type.googleapis.com/envoymobile.extensions.filters.http.test_kv_store.TestKeyValueStore, kv_store_name: envoy.key_value.platform_test, test_key: $TEST_KEY, test_value: $TEST_VALUE}" + ) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() - client.newStreamPrototype() + client + .newStreamPrototype() .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeaders, true) diff --git a/mobile/test/kotlin/integration/ReceiveDataTest.kt b/mobile/test/kotlin/integration/ReceiveDataTest.kt index ba326eb9a819..a20bf37e37f4 100644 --- a/mobile/test/kotlin/integration/ReceiveDataTest.kt +++ b/mobile/test/kotlin/integration/ReceiveDataTest.kt @@ -1,9 +1,9 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer import java.util.concurrent.CountDownLatch @@ -12,7 +12,8 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" class ReceiveDataTest { @@ -23,25 +24,28 @@ class ReceiveDataTest { @Test fun `response headers and response data call onResponseHeaders and onResponseData`() { - val engine = EngineBuilder(Standard()) - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .build() + val engine = + EngineBuilder(Standard()) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() val headersExpectation = CountDownLatch(1) val dataExpectation = CountDownLatch(1) var status: Int? = null var body: ByteBuffer? = null - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { responseHeaders, _, _ -> status = responseHeaders.httpStatus headersExpectation.countDown() diff --git a/mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt b/mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt deleted file mode 100644 index 454465bb44fe..000000000000 --- a/mobile/test/kotlin/integration/ReceiveDataWithDeliveryLimitsTest.kt +++ /dev/null @@ -1,68 +0,0 @@ -package test.kotlin.integration - -import io.envoyproxy.envoymobile.Standard -import io.envoyproxy.envoymobile.EngineBuilder -import io.envoyproxy.envoymobile.RequestHeadersBuilder -import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.engine.JniLibrary -import java.nio.ByteBuffer -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.fail -import org.junit.Test - -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" - -class ReceiveDataTest { - - init { - JniLibrary.loadTestLibrary() - } - - @Test - fun `data is received with min delivery size set`() { - - val engine = EngineBuilder(Standard()) - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .build() - val client = engine.streamClient() - - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() - - val headersExpectation = CountDownLatch(1) - val dataExpectation = CountDownLatch(1) - - var status: Int? = null - var body: ByteBuffer? = null - client.newStreamPrototype() - .setOnResponseHeaders { responseHeaders, _, _ -> - status = responseHeaders.httpStatus - headersExpectation.countDown() - } - .setOnResponseData { data, _, _ -> - body = data - dataExpectation.countDown() - } - .setOnError { _, _ -> fail("Unexpected error") } - .setMinDeliverySize(10) - .start() - .sendHeaders(requestHeaders, true) - - headersExpectation.await(10, TimeUnit.SECONDS) - dataExpectation.await(10, TimeUnit.SECONDS) - engine.terminate() - - assertThat(headersExpectation.count).isEqualTo(0) - assertThat(dataExpectation.count).isEqualTo(0) - - assertThat(status).isEqualTo(200) - assertThat(body!!.array().toString(Charsets.UTF_8)).isEqualTo("data") - } -} diff --git a/mobile/test/kotlin/integration/ReceiveErrorTest.kt b/mobile/test/kotlin/integration/ReceiveErrorTest.kt index 0f279c3b87d5..c1ae21de4310 100644 --- a/mobile/test/kotlin/integration/ReceiveErrorTest.kt +++ b/mobile/test/kotlin/integration/ReceiveErrorTest.kt @@ -1,6 +1,5 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.EnvoyError import io.envoyproxy.envoymobile.FilterDataStatus @@ -11,6 +10,7 @@ import io.envoyproxy.envoymobile.GRPCRequestHeadersBuilder import io.envoyproxy.envoymobile.ResponseFilter import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer @@ -20,9 +20,11 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val pbfType = "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" -private const val localErrorFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError" -private const val filterName = "error_validation_filter" +private const val PBF_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" +private const val LOCAL_ERROR_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError" +private const val FILTER_NAME = "error_validation_filter" class ReceiveErrorTest { init { @@ -63,6 +65,7 @@ class ReceiveErrorTest { override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) { receivedError.countDown() } + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} override fun onCancel(finalStreamIntel: FinalStreamIntel) { @@ -72,25 +75,31 @@ class ReceiveErrorTest { @Test fun `errors on stream call onError callback`() { - val requestHeader = GRPCRequestHeadersBuilder( - scheme = "https", - authority = "doesnotexist.example.com", - path = "/test" - ).build() - - val engine = EngineBuilder(Standard()) - .addPlatformFilter( - name = filterName, - factory = { ErrorValidationFilter(filterReceivedError, filterNotCancelled) } - ) - .setOnEngineRunning {} - .addNativeFilter("envoy.filters.http.platform_bridge", "{'@type': $pbfType, platform_filter_name: $filterName}") - .addNativeFilter("envoy.filters.http.local_error", "{'@type': $localErrorFilterType}") - .build() + val requestHeader = + GRPCRequestHeadersBuilder( + scheme = "https", + authority = "doesnotexist.example.com", + path = "/test" + ) + .build() + + val engine = + EngineBuilder(Standard()) + .addPlatformFilter( + name = FILTER_NAME, + factory = { ErrorValidationFilter(filterReceivedError, filterNotCancelled) } + ) + .addNativeFilter( + "envoy.filters.http.platform_bridge", + "{'@type': $PBF_TYPE, platform_filter_name: $FILTER_NAME}" + ) + .addNativeFilter("envoy.filters.http.local_error", "{'@type': $LOCAL_ERROR_FILTER_TYPE}") + .build() var errorCode: Int? = null - engine.streamClient() + engine + .streamClient() .newStreamPrototype() .setOnResponseHeaders { _, _, _ -> fail("Headers received instead of expected error") } .setOnResponseData { _, _, _ -> fail("Data received instead of expected error") } @@ -100,9 +109,7 @@ class ReceiveErrorTest { errorCode = error.errorCode callbackReceivedError.countDown() } - .setOnCancel { _ -> - fail("Unexpected call to onCancel response callback") - } + .setOnCancel { _ -> fail("Unexpected call to onCancel response callback") } .start() .sendHeaders(requestHeader, true) diff --git a/mobile/test/kotlin/integration/ReceiveTrailersTest.kt b/mobile/test/kotlin/integration/ReceiveTrailersTest.kt index efb6f33e1a80..9dd3515efe06 100644 --- a/mobile/test/kotlin/integration/ReceiveTrailersTest.kt +++ b/mobile/test/kotlin/integration/ReceiveTrailersTest.kt @@ -1,19 +1,19 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.EnvoyError -import io.envoyproxy.envoymobile.ResponseFilter -import io.envoyproxy.envoymobile.RequestHeadersBuilder -import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.StreamIntel -import io.envoyproxy.envoymobile.FinalStreamIntel -import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.FilterDataStatus import io.envoyproxy.envoymobile.FilterHeadersStatus import io.envoyproxy.envoymobile.FilterTrailersStatus -import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.FinalStreamIntel +import io.envoyproxy.envoymobile.RequestHeadersBuilder +import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.RequestTrailersBuilder +import io.envoyproxy.envoymobile.ResponseFilter +import io.envoyproxy.envoymobile.ResponseHeaders +import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.Standard +import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer import java.util.concurrent.CountDownLatch @@ -22,22 +22,18 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val pbfType = "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" -private const val assertionFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" -private const val matcherTrailerName = "test-trailer" -private const val matcherTrailerValue = "test.code" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" +private const val MATCHER_TRAILER_NAME = "test-trailer" +private const val MATCHER_TRAILER_VALUE = "test.code" -class SendTrailersTest { +class ReceiveTrailersTest { init { JniLibrary.loadTestLibrary() } - - class ErrorValidationFilter( - private val onTrailers: CountDownLatch - ) : ResponseFilter { + class ErrorValidationFilter(private val onTrailers: CountDownLatch) : ResponseFilter { override fun onResponseHeaders( headers: ResponseHeaders, endStream: Boolean, @@ -62,52 +58,50 @@ class SendTrailersTest { return FilterTrailersStatus.Continue(trailers) } - override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) { - } + override fun onError(error: EnvoyError, finalStreamIntel: FinalStreamIntel) {} + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} - override fun onCancel(finalStreamIntel: FinalStreamIntel) { - } + override fun onCancel(finalStreamIntel: FinalStreamIntel) {} } @Test fun `successful sending of trailers`() { val trailersReceived = CountDownLatch(2) val expectation = CountDownLatch(2) - val engine = EngineBuilder(Standard()) - .addPlatformFilter( - name = "test_platform_filter", - factory = { ErrorValidationFilter(trailersReceived) } - ) - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .setOnEngineRunning { } - .build() + val engine = + EngineBuilder(Standard()) + .addPlatformFilter( + name = "test_platform_filter", + factory = { ErrorValidationFilter(trailersReceived) } + ) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .build() val client = engine.streamClient() - val builder = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) + val builder = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) builder.add("send-trailers", "true") val requestHeadersDefault = builder.build() val body = ByteBuffer.wrap("match_me".toByteArray(Charsets.UTF_8)) - val requestTrailers = RequestTrailersBuilder() - .add(matcherTrailerName, matcherTrailerValue) - .build() + val requestTrailers = + RequestTrailersBuilder().add(MATCHER_TRAILER_NAME, MATCHER_TRAILER_VALUE).build() var responseStatus: Int? = null - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { headers, _, _ -> responseStatus = headers.httpStatus expectation.countDown() } - .setOnError { _, _ -> - fail("Unexpected error") - } + .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeadersDefault, false) .sendData(body) @@ -118,14 +112,13 @@ class SendTrailersTest { builder.remove("send-trailers") builder.add("send-trailers", "empty") val requestHeadersEmpty = builder.build() - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { headers, _, _ -> responseStatus = headers.httpStatus expectation.countDown() } - .setOnError { _, _ -> - fail("Unexpected error") - } + .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeadersEmpty, false) .sendData(body) @@ -135,14 +128,13 @@ class SendTrailersTest { builder.remove("send-trailers") builder.add("send-trailers", "empty-value") val requestHeadersEmptyValue = builder.build() - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { headers, _, _ -> responseStatus = headers.httpStatus expectation.countDown() } - .setOnError { _, _ -> - fail("Unexpected error") - } + .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeadersEmptyValue, false) .sendData(body) diff --git a/mobile/test/kotlin/integration/ResetConnectivityStateTest.kt b/mobile/test/kotlin/integration/ResetConnectivityStateTest.kt index c5c0d6e113a0..9e1e9971b477 100644 --- a/mobile/test/kotlin/integration/ResetConnectivityStateTest.kt +++ b/mobile/test/kotlin/integration/ResetConnectivityStateTest.kt @@ -1,10 +1,10 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.ResponseHeaders +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.engine.JniLibrary import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit @@ -12,9 +12,8 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" -private val apiListenerType = "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.EnvoyMobileHttpConnectionManager" -private val assertionFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" // This test doesn't do what it advertises (https://github.com/envoyproxy/envoy/issues/25180) class ResetConnectivityStateTest { @@ -27,30 +26,31 @@ class ResetConnectivityStateTest { fun `successful request after connection drain`() { val headersExpectation = CountDownLatch(2) - val engine = EngineBuilder(Standard()) - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .build() + val engine = + EngineBuilder(Standard()) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() var resultHeaders1: ResponseHeaders? = null var resultEndStream1: Boolean? = null - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { responseHeaders, endStream, _ -> resultHeaders1 = responseHeaders resultEndStream1 = endStream headersExpectation.countDown() } - .setOnResponseData { _, endStream, _ -> - resultEndStream1 = endStream - } + .setOnResponseData { _, endStream, _ -> resultEndStream1 = endStream } .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeaders, true) @@ -61,15 +61,14 @@ class ResetConnectivityStateTest { var resultHeaders2: ResponseHeaders? = null var resultEndStream2: Boolean? = null - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { responseHeaders, endStream, _ -> resultHeaders2 = responseHeaders resultEndStream2 = endStream headersExpectation.countDown() } - .setOnResponseData { _, endStream, _ -> - resultEndStream2 = endStream - } + .setOnResponseData { _, endStream, _ -> resultEndStream2 = endStream } .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeaders, true) diff --git a/mobile/test/kotlin/integration/SendDataTest.kt b/mobile/test/kotlin/integration/SendDataTest.kt index e2917c1fc882..fb63b82aae15 100644 --- a/mobile/test/kotlin/integration/SendDataTest.kt +++ b/mobile/test/kotlin/integration/SendDataTest.kt @@ -1,13 +1,13 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.engine.testing.TestJni -import io.envoyproxy.envoymobile.engine.JniLibrary -import io.envoyproxy.envoymobile.engine.AndroidJniLibrary; +import io.envoyproxy.envoymobile.Standard +import io.envoyproxy.envoymobile.engine.AndroidJniLibrary import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification +import io.envoyproxy.envoymobile.engine.JniLibrary +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.nio.ByteBuffer import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit @@ -15,54 +15,55 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" -private const val assertionFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" -private const val requestStringMatch = "match_me" +private const val ASSERTION_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" +private const val REQUEST_STRING_MATCH = "match_me" class SendDataTest { init { - AndroidJniLibrary.loadTestLibrary(); + AndroidJniLibrary.loadTestLibrary() JniLibrary.load() } @Test fun `successful sending data`() { - TestJni.startHttp2TestServer(); - val port = TestJni.getServerPort(); + TestJni.startHttp2TestServer() + val port = TestJni.getServerPort() val expectation = CountDownLatch(1) - val engine = EngineBuilder(Standard()) - .addNativeFilter("envoy.filters.http.assertion", "{'@type': $assertionFilterType, match_config: {http_request_generic_body_match: {patterns: [{string_match: $requestStringMatch}]}}}") - .setTrustChainVerification(TrustChainVerification.ACCEPT_UNTRUSTED) - .setOnEngineRunning { } - .build() + val engine = + EngineBuilder(Standard()) + .addNativeFilter( + "envoy.filters.http.assertion", + "{'@type': $ASSERTION_FILTER_TYPE, match_config: {http_request_generic_body_match: {patterns: [{string_match: $REQUEST_STRING_MATCH}]}}}" + ) + .setTrustChainVerification(TrustChainVerification.ACCEPT_UNTRUSTED) + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "localhost:" + port, - path = "/simple.txt" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "localhost:$port", + path = "/simple.txt" + ) + .build() - val body = ByteBuffer.wrap(requestStringMatch.toByteArray(Charsets.UTF_8)) + val body = ByteBuffer.wrap(REQUEST_STRING_MATCH.toByteArray(Charsets.UTF_8)) var responseStatus: Int? = null var responseEndStream = false - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { headers, endStream, _ -> responseStatus = headers.httpStatus responseEndStream = endStream expectation.countDown() } - .setOnResponseData { _, endStream, _ -> - responseEndStream = endStream - } - .setOnError { _, _ -> - fail("Unexpected error") - } + .setOnResponseData { _, endStream, _ -> responseEndStream = endStream } + .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeaders, false) .close(body) @@ -70,7 +71,7 @@ class SendDataTest { expectation.await(10, TimeUnit.SECONDS) engine.terminate() - TestJni.shutdownTestServer(); + TestJni.shutdownTestServer() assertThat(expectation.count).isEqualTo(0) assertThat(responseStatus).isEqualTo(200) diff --git a/mobile/test/kotlin/integration/SendHeadersTest.kt b/mobile/test/kotlin/integration/SendHeadersTest.kt index 45d388d6c219..1304a20bf38e 100644 --- a/mobile/test/kotlin/integration/SendHeadersTest.kt +++ b/mobile/test/kotlin/integration/SendHeadersTest.kt @@ -1,10 +1,10 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.ResponseHeaders +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.engine.JniLibrary import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit @@ -12,7 +12,8 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" class SendHeadersTest { @@ -24,30 +25,31 @@ class SendHeadersTest { fun `successful sending of request headers`() { val headersExpectation = CountDownLatch(1) - val engine = EngineBuilder(Standard()) - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .build() + val engine = + EngineBuilder(Standard()) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() var resultHeaders: ResponseHeaders? = null var resultEndStream: Boolean? = null - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { responseHeaders, endStream, _ -> resultHeaders = responseHeaders resultEndStream = endStream headersExpectation.countDown() } - .setOnResponseData { _, endStream, _ -> - resultEndStream = endStream - } + .setOnResponseData { _, endStream, _ -> resultEndStream = endStream } .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeaders, true) diff --git a/mobile/test/kotlin/integration/SendTrailersTest.kt b/mobile/test/kotlin/integration/SendTrailersTest.kt index 01b921b96d4f..ff5e906b5917 100644 --- a/mobile/test/kotlin/integration/SendTrailersTest.kt +++ b/mobile/test/kotlin/integration/SendTrailersTest.kt @@ -1,10 +1,10 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.RequestTrailersBuilder +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer import java.util.concurrent.CountDownLatch @@ -13,10 +13,12 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -private const val testResponseFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" -private const val assertionFilterType = "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" -private const val matcherTrailerName = "test-trailer" -private const val matcherTrailerValue = "test.code" +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" +private const val ASSERTION_FILTER_TYPE = + "type.googleapis.com/envoymobile.extensions.filters.http.assertion.Assertion" +private const val MATCHER_TRAILER_NAME = "test-trailer" +private const val MATCHER_TRAILER_VALUE = "test.code" class SendTrailersTest { @@ -28,37 +30,42 @@ class SendTrailersTest { fun `successful sending of trailers`() { val expectation = CountDownLatch(1) - val engine = EngineBuilder(Standard()) - .addNativeFilter("envoy.filters.http.assertion", "{'@type': $assertionFilterType, match_config: {http_request_trailers_match: {headers: [{name: 'test-trailer', exact_match: 'test.code'}]}}}") - .addNativeFilter("envoy.filters.http.buffer", "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":65000}") - .addNativeFilter("test_remote_response", "{'@type': $testResponseFilterType}") - .setOnEngineRunning { } - .build() + val engine = + EngineBuilder(Standard()) + .addNativeFilter( + "envoy.filters.http.assertion", + "{'@type': $ASSERTION_FILTER_TYPE, match_config: {http_request_trailers_match: {headers: [{name: 'test-trailer', exact_match: 'test.code'}]}}}" + ) + .addNativeFilter( + "envoy.filters.http.buffer", + "{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.buffer.v3.Buffer\",\"max_request_bytes\":65000}" + ) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() val body = ByteBuffer.wrap("match_me".toByteArray(Charsets.UTF_8)) - val requestTrailers = RequestTrailersBuilder() - .add(matcherTrailerName, matcherTrailerValue) - .build() + val requestTrailers = + RequestTrailersBuilder().add(MATCHER_TRAILER_NAME, MATCHER_TRAILER_VALUE).build() var responseStatus: Int? = null - client.newStreamPrototype() + client + .newStreamPrototype() .setOnResponseHeaders { headers, _, _ -> responseStatus = headers.httpStatus expectation.countDown() } - .setOnError { _, _ -> - fail("Unexpected error") - } + .setOnError { _, _ -> fail("Unexpected error") } .start() .sendHeaders(requestHeaders, false) .sendData(body) diff --git a/mobile/test/kotlin/integration/SetEventTrackerTest.kt b/mobile/test/kotlin/integration/SetEventTrackerTest.kt index 46dcf388e467..a6630eedb03f 100644 --- a/mobile/test/kotlin/integration/SetEventTrackerTest.kt +++ b/mobile/test/kotlin/integration/SetEventTrackerTest.kt @@ -19,34 +19,33 @@ class SetEventTrackerTest { @Test fun `set eventTracker`() { val countDownLatch = CountDownLatch(1) - val engine = EngineBuilder() - .setEventTracker { events -> - for (entry in events) { - assertThat(entry.key).isEqualTo("foo") - assertThat(entry.value).isEqualTo("bar") + val engine = + EngineBuilder() + .setEventTracker { events -> + for (entry in events) { + assertThat(entry.key).isEqualTo("foo") + assertThat(entry.value).isEqualTo("bar") + } + countDownLatch.countDown() } - countDownLatch.countDown() - } - .addNativeFilter( - "envoy.filters.http.test_event_tracker", - "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_event_tracker.TestEventTracker\",\"attributes\":{\"foo\":\"bar\"}}" - ) - .build() + .addNativeFilter( + "envoy.filters.http.test_event_tracker", + "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_event_tracker.TestEventTracker\",\"attributes\":{\"foo\":\"bar\"}}" + ) + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() - client - .newStreamPrototype() - .start() - .sendHeaders(requestHeaders, true) + client.newStreamPrototype().start().sendHeaders(requestHeaders, true) countDownLatch.await(30, TimeUnit.SECONDS) engine.terminate() @@ -56,20 +55,19 @@ class SetEventTrackerTest { @Test fun `engine should continue to run if no eventTracker is set and event is emitted`() { val countDownLatch = CountDownLatch(1) - val engine = EngineBuilder() - .addNativeFilter( - "envoy.filters.http.test_event_tracker", - "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_event_tracker.TestEventTracker\",\"attributes\":{\"foo\":\"bar\"}}" - ) - .build() + val engine = + EngineBuilder() + .addNativeFilter( + "envoy.filters.http.test_event_tracker", + "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_event_tracker.TestEventTracker\",\"attributes\":{\"foo\":\"bar\"}}" + ) + .build() val client = engine.streamClient() client .newStreamPrototype() - .setOnResponseData { _, _, _ -> - countDownLatch.countDown() - } + .setOnResponseData { _, _, _ -> countDownLatch.countDown() } .start() .close(ByteBuffer.allocate(1)) diff --git a/mobile/test/kotlin/integration/SetLoggerTest.kt b/mobile/test/kotlin/integration/SetLoggerTest.kt index fe4b1cbc28d5..0e3b9f980488 100644 --- a/mobile/test/kotlin/integration/SetLoggerTest.kt +++ b/mobile/test/kotlin/integration/SetLoggerTest.kt @@ -1,11 +1,11 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.Engine import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.engine.JniLibrary import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit @@ -22,21 +22,24 @@ class SetLoggerTest { fun `set logger`() { val countDownLatch = CountDownLatch(1) val logEventLatch = CountDownLatch(1) - val engine = EngineBuilder(Standard()) - .addLogLevel(LogLevel.DEBUG) - .addNativeFilter("test_logger", "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_logger.TestLogger\"}") - .setLogger { msg -> - if (msg.contains("starting main dispatch loop")) { - countDownLatch.countDown() + val engine = + EngineBuilder(Standard()) + .addLogLevel(LogLevel.DEBUG) + .addNativeFilter( + "test_logger", + "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_logger.TestLogger\"}" + ) + .setLogger { msg -> + if (msg.contains("starting main dispatch loop")) { + countDownLatch.countDown() + } } - } - .setEventTracker { event -> - if (event["log_name"] == "event_name") { - logEventLatch.countDown() + .setEventTracker { event -> + if (event["log_name"] == "event_name") { + logEventLatch.countDown() + } } - } - .setOnEngineRunning {} - .build() + .build() countDownLatch.await(30, TimeUnit.SECONDS) @@ -53,18 +56,20 @@ class SetLoggerTest { fun `engine should continue to run if no logger is set`() { val countDownLatch = CountDownLatch(1) val logEventLatch = CountDownLatch(1) - val engine = EngineBuilder(Standard()) - .setEventTracker { event -> - if (event["log_name"] == "event_name") { - logEventLatch.countDown() + val engine = + EngineBuilder(Standard()) + .setEventTracker { event -> + if (event["log_name"] == "event_name") { + logEventLatch.countDown() + } } - } - .addLogLevel(LogLevel.DEBUG) - .addNativeFilter("test_logger", "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_logger.TestLogger\"}") - .setOnEngineRunning { - countDownLatch.countDown() - } - .build() + .addLogLevel(LogLevel.DEBUG) + .addNativeFilter( + "test_logger", + "{\"@type\":\"type.googleapis.com/envoymobile.extensions.filters.http.test_logger.TestLogger\"}" + ) + .setOnEngineRunning { countDownLatch.countDown() } + .build() countDownLatch.await(30, TimeUnit.SECONDS) @@ -79,16 +84,15 @@ class SetLoggerTest { fun sendRequest(engine: Engine) { val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() - client.newStreamPrototype() - .start() - .sendHeaders(requestHeaders, true) + client.newStreamPrototype().start().sendHeaders(requestHeaders, true) } } diff --git a/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt b/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt deleted file mode 100644 index ad449a434d7c..000000000000 --- a/mobile/test/kotlin/integration/StatFlushIntegrationTest.kt +++ /dev/null @@ -1,78 +0,0 @@ -package test.kotlin.integration - -import io.envoyproxy.envoymobile.Element -import io.envoyproxy.envoymobile.Engine -import io.envoyproxy.envoymobile.EngineBuilder -import io.envoyproxy.envoymobile.LogLevel -import io.envoyproxy.envoymobile.engine.JniLibrary -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import org.assertj.core.api.Assertions.assertThat -import org.junit.After -import org.junit.Ignore -import org.junit.Test - -class StatFlushIntegrationTest { - private var engine: Engine? = null - - init { - JniLibrary.loadTestLibrary() - } - - @After - fun teardown() { - engine?.terminate() - engine = null - } - - @Test - fun `multiple stat sinks configured`() { - val countDownLatch = CountDownLatch(1) - engine = EngineBuilder() - .addLogLevel(LogLevel.DEBUG) - // Really high flush interval so it won't trigger during test execution. - .addStatsFlushSeconds(100) - .addStatsSinks(listOf(statsdSinkConfig(8125), statsdSinkConfig(5000))) - .addGrpcStatsDomain("example.com") - .setOnEngineRunning { countDownLatch.countDown() } - .build() - - assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue() - } - - @Test - fun `flush flushes to stats sink`() { - val countDownLatch = CountDownLatch(1) - engine = EngineBuilder() - .addLogLevel(LogLevel.DEBUG) - // Really high flush interval so it won't trigger during test execution. - .addStatsFlushSeconds(100) - .addStatsSinks(listOf(statsdSinkConfig(8125), statsdSinkConfig(5000))) - .setOnEngineRunning { countDownLatch.countDown() } - .build() - - assertThat(countDownLatch.await(30, TimeUnit.SECONDS)).isTrue() - - val statsdServer1 = TestStatsdServer() - statsdServer1.runAsync(8125) - - val statsdServer2 = TestStatsdServer() - statsdServer2.runAsync(5000) - - statsdServer1.setStatMatching { s -> s == "envoy.pulse.foo.bar:1|c" } - statsdServer2.setStatMatching { s -> s == "envoy.pulse.foo.bar:1|c" } - - engine!!.pulseClient().counter(Element("foo"), Element("bar")).increment(1) - engine!!.flushStats() - - statsdServer1.awaitStatMatching() - statsdServer2.awaitStatMatching() - } - - private fun statsdSinkConfig(port: Int): String { -return """{ name: envoy.stat_sinks.statsd, - typed_config: { - "@type": type.googleapis.com/envoy.config.metrics.v3.StatsdSink, - address: { socket_address: { address: 127.0.0.1, port_value: $port } } } }""" - } -} diff --git a/mobile/test/kotlin/integration/StreamIdleTimeoutTest.kt b/mobile/test/kotlin/integration/StreamIdleTimeoutTest.kt index 8aa185f5914d..f2d10e030593 100644 --- a/mobile/test/kotlin/integration/StreamIdleTimeoutTest.kt +++ b/mobile/test/kotlin/integration/StreamIdleTimeoutTest.kt @@ -1,6 +1,5 @@ package test.kotlin.integration -import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.EngineBuilder import io.envoyproxy.envoymobile.EnvoyError import io.envoyproxy.envoymobile.FilterDataStatus @@ -12,6 +11,7 @@ import io.envoyproxy.envoymobile.RequestMethod import io.envoyproxy.envoymobile.ResponseFilter import io.envoyproxy.envoymobile.ResponseHeaders import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.Standard import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary import java.nio.ByteBuffer @@ -22,7 +22,11 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.fail import org.junit.Test -class CancelStreamTest { +private const val TEST_RESPONSE_FILTER_TYPE = + "type.googleapis.com/" + + "envoymobile.extensions.filters.http.test_remote_response.TestRemoteResponse" + +class StreamIdleTimeoutTest { init { JniLibrary.loadTestLibrary() @@ -31,9 +35,7 @@ class CancelStreamTest { private val filterExpectation = CountDownLatch(1) private val callbackExpectation = CountDownLatch(1) - class IdleTimeoutValidationFilter( - private val latch: CountDownLatch - ) : ResponseFilter { + class IdleTimeoutValidationFilter(private val latch: CountDownLatch) : ResponseFilter { override fun onResponseHeaders( headers: ResponseHeaders, endStream: Boolean, @@ -61,41 +63,45 @@ class CancelStreamTest { assertThat(error.errorCode).isEqualTo(4) latch.countDown() } + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} override fun onCancel(finalStreamIntel: FinalStreamIntel) { - fail("Unexpected call to onCancel filter callback") + fail("Unexpected call to onCancel filter callback") } } @Test fun `stream idle timeout triggers onError callbacks`() { - val engine = EngineBuilder(Standard()).build() - .addPlatformFilter( - name = "idle_timeout_validation_filter", - factory = { IdleTimeoutValidationFilter(filterExpectation) } - ) - .addStreamIdleTimeoutSeconds(1) - .setOnEngineRunning {} - .build() + val engine = + EngineBuilder(Standard()) + .addPlatformFilter( + name = "idle_timeout_validation_filter", + factory = { IdleTimeoutValidationFilter(filterExpectation) } + ) + .addNativeFilter("test_remote_response", "{'@type': $TEST_RESPONSE_FILTER_TYPE}") + .addStreamIdleTimeoutSeconds(1) + .build() val client = engine.streamClient() - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "example.com", - path = "/test" - ) - .build() - - client.newStreamPrototype() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "example.com", + path = "/test" + ) + .build() + + client + .newStreamPrototype() .setOnError { error, _ -> assertThat(error.errorCode).isEqualTo(4) callbackExpectation.countDown() } .start(Executors.newSingleThreadExecutor()) - .sendHeaders(requestHeaders, true) + .sendHeaders(requestHeaders, false) filterExpectation.await(10, TimeUnit.SECONDS) callbackExpectation.await(10, TimeUnit.SECONDS) diff --git a/mobile/test/kotlin/integration/TestStatsdServer.kt b/mobile/test/kotlin/integration/TestStatsdServer.kt index 9e380e4e4c14..c8c21988ae50 100644 --- a/mobile/test/kotlin/integration/TestStatsdServer.kt +++ b/mobile/test/kotlin/integration/TestStatsdServer.kt @@ -20,32 +20,33 @@ class TestStatsdServer { val socket = DatagramSocket(port) socket.setSoTimeout(1000) // 1 second - thread = Thread( - fun() { - val buffer = ByteArray(256) - while (shutdownLatch.getCount() != 0L) { - val packet = DatagramPacket(buffer, buffer.size) - try { - socket.receive(packet) - } catch (e: SocketTimeoutException) { - // continue to next loop - continue - } catch (e: Exception) { - // TODO(snowp): Bubble up this error somehow. - return - } + thread = + Thread( + fun() { + val buffer = ByteArray(256) + while (shutdownLatch.getCount() != 0L) { + val packet = DatagramPacket(buffer, buffer.size) + try { + socket.receive(packet) + } catch (e: SocketTimeoutException) { + // continue to next loop + continue + } catch (e: Exception) { + // TODO(snowp): Bubble up this error somehow. + return + } - // TODO(snowp): Parse (or use a parser) so we can extract out individual metric names - // better. - val received = String(packet.getData(), packet.getOffset(), packet.getLength()) - val maybeMatch = matchCriteria.get() - if (maybeMatch != null && maybeMatch.invoke(received)) { - matchCriteria.set(null) - awaitNextStat.get().countDown() + // TODO(snowp): Parse (or use a parser) so we can extract out individual metric names + // better. + val received = String(packet.getData(), packet.getOffset(), packet.getLength()) + val maybeMatch = matchCriteria.get() + if (maybeMatch != null && maybeMatch.invoke(received)) { + matchCriteria.set(null) + awaitNextStat.get().countDown() + } } } - } - ) + ) thread!!.start() } diff --git a/mobile/test/kotlin/integration/TestUtilities.kt b/mobile/test/kotlin/integration/TestUtilities.kt new file mode 100644 index 000000000000..acb78ab2870e --- /dev/null +++ b/mobile/test/kotlin/integration/TestUtilities.kt @@ -0,0 +1,52 @@ +package test.kotlin.integration + +import io.envoyproxy.envoymobile.Engine +import java.time.Instant +import java.util.concurrent.TimeUnit +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds +import kotlin.time.DurationUnit +import org.junit.Assert.fail + +/** Gets the stats in the form of `Map`. */ +fun Engine.getStats(): Map { + // `dumpStats()` produces the following format: + // key1: value1 + // key2: value2 + // key3: value3 + // ... + val lines = dumpStats().split("\n") + return lines + .mapNotNull { + val keyValue = it.split(": ") + if (keyValue.size == 2) { + Pair(keyValue[0], keyValue[1]) + } else { + null + } + } + .toMap() +} + +/** + * Waits for 5 seconds (default) until the stat of the given [name] is greater than equal the + * specified [expectedValue]. + * + * @throws java.lang.AssertionError throw when the the operation timed out waiting for the stat + * value to be greater than the [expectedValue]. + */ +fun Engine.waitForStatGe( + name: String, + expectedValue: Long, + timeout: Duration = 5.seconds, +) { + val waitTime = Instant.now().plusMillis(timeout.toLong(DurationUnit.MILLISECONDS)) + var currentValue = getStats()[name] + while (currentValue == null || currentValue.toLong() < expectedValue) { + if (Instant.now() > waitTime) { + fail("Timed out waiting for $name to be greater than equal $expectedValue") + } + TimeUnit.MILLISECONDS.sleep(10) + currentValue = getStats()[name] + } +} diff --git a/mobile/test/kotlin/integration/XdsTest.kt b/mobile/test/kotlin/integration/XdsTest.kt new file mode 100644 index 000000000000..1551006ee8a2 --- /dev/null +++ b/mobile/test/kotlin/integration/XdsTest.kt @@ -0,0 +1,85 @@ +package test.kotlin.integration + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import io.envoyproxy.envoymobile.AndroidEngineBuilder +import io.envoyproxy.envoymobile.Engine +import io.envoyproxy.envoymobile.LogLevel +import io.envoyproxy.envoymobile.XdsBuilder +import io.envoyproxy.envoymobile.engine.AndroidJniLibrary +import io.envoyproxy.envoymobile.engine.JniLibrary +import io.envoyproxy.envoymobile.engine.testing.TestJni +import java.io.File +import java.util.concurrent.CountDownLatch +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class XdsTest { + private val appContext: Context = ApplicationProvider.getApplicationContext() + private lateinit var engine: Engine + + init { + AndroidJniLibrary.loadTestLibrary() + JniLibrary.load() + } + + @Before + fun setUp() { + val upstreamCert: String = + File("../envoy/test/config/integration/certs/upstreamcacert.pem").readText() + TestJni.initXdsTestServer() + val latch = CountDownLatch(1) + engine = + AndroidEngineBuilder(appContext) + .addLogLevel(LogLevel.DEBUG) + .setOnEngineRunning { latch.countDown() } + .setXds( + XdsBuilder( + TestJni.getXdsTestServerHost(), + TestJni.getXdsTestServerPort(), + ) + .setSslRootCerts(upstreamCert) + .addClusterDiscoveryService() + ) + .build() + latch.await() + TestJni.startXdsTestServer() + } + + @After + fun tearDown() { + engine.terminate() + TestJni.shutdownXdsTestServer() + } + + @Test + fun `test xDS with CDS`() { + // There are 2 initial clusters: base and base_clear. + engine.waitForStatGe("cluster_manager.cluster_added", 2) + val cdsResponse = + """ + version_info: v1 + resources: + - "@type": type.googleapis.com/envoy.config.cluster.v3.Cluster + name: my_cluster + type: STATIC + connect_timeout: 5s + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + {} + type_url: type.googleapis.com/envoy.config.cluster.v3.Cluster + nonce: nonce1 + """ + .trimIndent() + TestJni.sendDiscoveryResponse(cdsResponse) + // There are now 3 clusters: base, base_cluster, and my_cluster. + engine.waitForStatGe("cluster_manager.cluster_added", 3) + } +} diff --git a/mobile/test/kotlin/integration/proxying/BUILD b/mobile/test/kotlin/integration/proxying/BUILD index a19133bdbce3..9c14b6adfb18 100644 --- a/mobile/test/kotlin/integration/proxying/BUILD +++ b/mobile/test/kotlin/integration/proxying/BUILD @@ -1,15 +1,9 @@ -load("@envoy_mobile//bazel:kotlin_lib.bzl", "envoy_mobile_kt_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") -envoy_mobile_kt_library( - name = "proxy_lib", - srcs = ["Proxy.kt"], - visibility = ["//visibility:public"], - deps = [ - "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", - "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - ], -) +licenses(["notice"]) # Apache 2 + +envoy_mobile_package() envoy_mobile_android_test( name = "proxy_info_intent_perform_http_request_using_proxy_test", @@ -23,12 +17,17 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_and_listener_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - "//test/kotlin/integration/proxying:proxy_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", ], ) @@ -44,12 +43,17 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_and_listener_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - "//test/kotlin/integration/proxying:proxy_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", ], ) @@ -65,12 +69,17 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_and_listener_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - "//test/kotlin/integration/proxying:proxy_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", ], ) @@ -86,12 +95,17 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_and_listener_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - "//test/kotlin/integration/proxying:proxy_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", ], ) @@ -107,12 +121,17 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_and_listener_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - "//test/kotlin/integration/proxying:proxy_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", ], ) @@ -128,11 +147,16 @@ envoy_mobile_android_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_and_listener_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_and_listener_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", - "//test/kotlin/integration/proxying:proxy_lib", + "//test/java/io/envoyproxy/envoymobile/engine/testing", ], ) diff --git a/mobile/test/kotlin/integration/proxying/Proxy.kt b/mobile/test/kotlin/integration/proxying/Proxy.kt deleted file mode 100644 index 26f5a31edb63..000000000000 --- a/mobile/test/kotlin/integration/proxying/Proxy.kt +++ /dev/null @@ -1,192 +0,0 @@ -package test.kotlin.integration.proxying - -import android.content.Context - -import io.envoyproxy.envoymobile.AndroidEngineBuilder -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.EngineBuilder - -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -// A convenient wrapper for creating an builder for an engine that -// proxies network requests. -class Proxy constructor(val context: Context, val port: Int) { - fun http(): EngineBuilder { - val config = """ -static_resources: - listeners: - - name: base_api_listener - address: - socket_address: { protocol: TCP, address: 127.0.0.1, port_value: 10000 } - api_listener: - api_listener: - "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.EnvoyMobileHttpConnectionManager" - config: - stat_prefix: api_hcm - route_config: - name: api_router - virtual_hosts: - - name: api - domains: ["*"] - routes: - - match: { prefix: "/" } - direct_response: { status: 400, body: { inline_string: "not found" } } - - name: listener_proxy - address: - socket_address: { address: 127.0.0.1, port_value: $port } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: remote_hcm - route_config: - name: remote_route - virtual_hosts: - - name: remote_service - domains: ["*"] - routes: - - match: { prefix: "/" } - route: { cluster: cluster_proxy } - response_headers_to_add: - - append_action: OVERWRITE_IF_EXISTS_OR_ADD - header: - key: x-proxy-response - value: 'true' - http_filters: - - name: envoy.filters.http.local_error - typed_config: - "@type": type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError - - name: envoy.filters.http.dynamic_forward_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig - dns_cache_config: &dns_cache_config - name: base_dns_cache - dns_lookup_family: ALL - host_ttl: 86400s - dns_min_refresh_rate: 20s - dns_refresh_rate: 60s - dns_failure_refresh_rate: - base_interval: 2s - max_interval: 10s - dns_query_timeout: 25s - typed_dns_resolver_config: - name: envoy.network.dns_resolver.getaddrinfo - typed_config: {"@type":"type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig"} - - name: envoy.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: cluster_proxy - connect_timeout: 30s - lb_policy: CLUSTER_PROVIDED - dns_lookup_family: ALL - cluster_type: - name: envoy.clusters.dynamic_forward_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig - dns_cache_config: *dns_cache_config -layered_runtime: - layers: - - name: static_layer_0 - static_layer: - envoy: - # This disables envoy bug stats, which are filtered out of our stats inclusion list anyway - # Global stats do not play well with engines with limited lifetimes - disallow_global_stats: true -""" - return AndroidEngineBuilder(context, Custom(config)) - } - - fun https(): EngineBuilder { - val config = """ -static_resources: - listeners: - - name: base_api_listener - address: - socket_address: { protocol: TCP, address: 127.0.0.1, port_value: 10000 } - api_listener: - api_listener: - "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.EnvoyMobileHttpConnectionManager" - config: - stat_prefix: api_hcm - route_config: - name: api_router - virtual_hosts: - - name: api - domains: ["*"] - routes: - - match: { prefix: "/" } - direct_response: { status: 400, body: { inline_string: "not found" } } - - name: listener_proxy - address: - socket_address: { address: 127.0.0.1, port_value: $port } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: remote_hcm - route_config: - name: remote_route - virtual_hosts: - - name: remote_service - domains: ["*"] - routes: - - match: { connect_matcher: {} } - route: - cluster: cluster_proxy - upgrade_configs: - - upgrade_type: CONNECT - connect_config: - response_headers_to_add: - - append_action: OVERWRITE_IF_EXISTS_OR_ADD - header: - key: x-response-header-that-should-be-stripped - value: 'true' - http_filters: - - name: envoy.filters.http.local_error - typed_config: - "@type": type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError - - name: envoy.filters.http.dynamic_forward_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig - dns_cache_config: &dns_cache_config - name: base_dns_cache - dns_lookup_family: ALL - host_ttl: 86400s - dns_min_refresh_rate: 20s - dns_refresh_rate: 60s - dns_failure_refresh_rate: - base_interval: 2s - max_interval: 10s - dns_query_timeout: 25s - typed_dns_resolver_config: - name: envoy.network.dns_resolver.getaddrinfo - typed_config: {"@type":"type.googleapis.com/envoy.extensions.network.dns_resolver.getaddrinfo.v3.GetAddrInfoDnsResolverConfig"} - - name: envoy.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: cluster_proxy - connect_timeout: 30s - lb_policy: CLUSTER_PROVIDED - dns_lookup_family: ALL - cluster_type: - name: envoy.clusters.dynamic_forward_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig - dns_cache_config: *dns_cache_config -layered_runtime: - layers: - - name: static_layer_0 - static_layer: - envoy: - # This disables envoy bug stats, which are filtered out of our stats inclusion list anyway - # Global stats do not play well with engines with limited lifetimes - disallow_global_stats: true -""" - return AndroidEngineBuilder(context, Custom(config)) - } -} diff --git a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPRequestUsingProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPRequestUsingProxyTest.kt index b651c34f7411..9b47ba65f41c 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPRequestUsingProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPRequestUsingProxyTest.kt @@ -1,35 +1,28 @@ package test.kotlin.integration.proxying -import android.content.Intent import android.content.Context +import android.content.Intent import android.net.ConnectivityManager import android.net.Proxy import android.net.ProxyInfo import androidx.test.core.app.ApplicationProvider - import io.envoyproxy.envoymobile.AndroidEngineBuilder -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.Engine -import io.envoyproxy.envoymobile.engine.JniLibrary import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.ResponseHeaders -import io.envoyproxy.envoymobile.StreamIntel - +import io.envoyproxy.envoymobile.engine.JniLibrary +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.junit.runner.RunWith -import org.mockito.Mock import org.mockito.Mockito import org.robolectric.RobolectricTestRunner // ┌──────────────────┐ -// │ Proxy Engine │ +// │ Envoy Proxy │ // │ ┌──────────────┐ │ // ┌────────────────────────┐ ┌─┼─►listener_proxy│ │ // │http://api.lyft.com/ping│ ┌──────────────┬┘ │ └──────┬───────┘ │ ┌────────────┐ @@ -43,50 +36,46 @@ import org.robolectric.RobolectricTestRunner class PerformHTTPRequestUsingProxy { init { JniLibrary.loadTestLibrary() + JniLibrary.load() } @Test fun `performs an HTTP request through a proxy`() { - val port = (10001..11000).random() + TestJni.startHttpProxyTestServer() + val port = TestJni.getServerPort() val context = Mockito.spy(ApplicationProvider.getApplicationContext()) val connectivityManager: ConnectivityManager = Mockito.mock(ConnectivityManager::class.java) - Mockito.doReturn(connectivityManager).`when`(context).getSystemService(Context.CONNECTIVITY_SERVICE) - Mockito.`when`(connectivityManager.getDefaultProxy()).thenReturn(ProxyInfo.buildDirectProxy("127.0.0.1", port)) + Mockito.doReturn(connectivityManager) + .`when`(context) + .getSystemService(Context.CONNECTIVITY_SERVICE) + Mockito.`when`(connectivityManager.getDefaultProxy()) + .thenReturn(ProxyInfo.buildDirectProxy("127.0.0.1", port)) - val onProxyEngineRunningLatch = CountDownLatch(1) val onEngineRunningLatch = CountDownLatch(1) val onRespondeHeadersLatch = CountDownLatch(1) - val proxyEngineBuilder = Proxy(context, port) - .http() - val proxyEngine = proxyEngineBuilder - .addLogLevel(LogLevel.DEBUG) - .setOnEngineRunning { onProxyEngineRunningLatch.countDown() } - .build() - - onProxyEngineRunningLatch.await(10, TimeUnit.SECONDS) - assertThat(onProxyEngineRunningLatch.count).isEqualTo(0) - context.sendStickyBroadcast(Intent(Proxy.PROXY_CHANGE_ACTION)) val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() onEngineRunningLatch.await(10, TimeUnit.SECONDS) assertThat(onEngineRunningLatch.count).isEqualTo(0) - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "http", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "http", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() @@ -104,6 +93,6 @@ class PerformHTTPRequestUsingProxy { assertThat(onRespondeHeadersLatch.count).isEqualTo(0) engine.terminate() - proxyEngine.terminate() + TestJni.shutdownTestServer() } } diff --git a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestBadHostnameTest.kt b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestBadHostnameTest.kt index a2f6ce0c3b24..fd3641a1d043 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestBadHostnameTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestBadHostnameTest.kt @@ -1,35 +1,28 @@ package test.kotlin.integration.proxying -import android.content.Intent import android.content.Context +import android.content.Intent import android.net.ConnectivityManager import android.net.Proxy import android.net.ProxyInfo import androidx.test.core.app.ApplicationProvider - -import io.envoyproxy.envoymobile.LogLevel -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.Engine import io.envoyproxy.envoymobile.AndroidEngineBuilder +import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.ResponseHeaders -import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary - +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.junit.runner.RunWith -import org.mockito.Mock import org.mockito.Mockito import org.robolectric.RobolectricTestRunner // ┌──────────────────┐ -// │ Proxy Engine │ +// │ Envoy Proxy │ // │ ┌──────────────┐ │ // ┌─────────────────────────┐ ┌─┼─►listener_proxy│ │ // │https://api.lyft.com/ping│ ┌──────────────┬┘ │ └──────┬───────┘ │ ┌────────────┐ @@ -43,57 +36,51 @@ import org.robolectric.RobolectricTestRunner class PerformHTTPSRequestBadHostname { init { JniLibrary.loadTestLibrary() + JniLibrary.load() } @Test fun `attempts an HTTPs request through a proxy using an async DNS resolution that fails`() { - val port = (10001..11000).random() + TestJni.startHttpsProxyTestServer() + val port = TestJni.getServerPort() val context = Mockito.spy(ApplicationProvider.getApplicationContext()) val connectivityManager: ConnectivityManager = Mockito.mock(ConnectivityManager::class.java) - Mockito.doReturn(connectivityManager).`when`(context).getSystemService(Context.CONNECTIVITY_SERVICE) - Mockito.`when`(connectivityManager.getDefaultProxy()).thenReturn(ProxyInfo.buildDirectProxy("loopback", port)) + Mockito.doReturn(connectivityManager) + .`when`(context) + .getSystemService(Context.CONNECTIVITY_SERVICE) + Mockito.`when`(connectivityManager.getDefaultProxy()) + .thenReturn(ProxyInfo.buildDirectProxy("loopback", port)) val onEngineRunningLatch = CountDownLatch(1) - val onProxyEngineRunningLatch = CountDownLatch(1) val onErrorLatch = CountDownLatch(1) - val proxyEngineBuilder = Proxy(context, port).https() - val proxyEngine = proxyEngineBuilder - .addLogLevel(LogLevel.DEBUG) - .addDNSQueryTimeoutSeconds(2) - .setOnEngineRunning { onProxyEngineRunningLatch.countDown() } - .build() - - onProxyEngineRunningLatch.await(10, TimeUnit.SECONDS) - assertThat(onProxyEngineRunningLatch.count).isEqualTo(0) - context.sendStickyBroadcast(Intent(Proxy.PROXY_CHANGE_ACTION)) val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() onEngineRunningLatch.await(10, TimeUnit.SECONDS) assertThat(onEngineRunningLatch.count).isEqualTo(0) - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() .newStreamPrototype() - .setOnError { _, _ -> - onErrorLatch.countDown() - } + .setOnError { _, _ -> onErrorLatch.countDown() } .start(Executors.newSingleThreadExecutor()) .sendHeaders(requestHeaders, true) @@ -101,6 +88,6 @@ class PerformHTTPSRequestBadHostname { assertThat(onErrorLatch.count).isEqualTo(0) engine.terminate() - proxyEngine.terminate() + TestJni.shutdownTestServer() } } diff --git a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingAsyncProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingAsyncProxyTest.kt index c69cdfa55eea..a8883b7e48a1 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingAsyncProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingAsyncProxyTest.kt @@ -1,35 +1,28 @@ package test.kotlin.integration.proxying -import android.content.Intent import android.content.Context +import android.content.Intent import android.net.ConnectivityManager import android.net.Proxy import android.net.ProxyInfo import androidx.test.core.app.ApplicationProvider - -import io.envoyproxy.envoymobile.LogLevel -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.Engine import io.envoyproxy.envoymobile.AndroidEngineBuilder +import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.ResponseHeaders -import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary - +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.junit.runner.RunWith -import org.mockito.Mock import org.mockito.Mockito import org.robolectric.RobolectricTestRunner // ┌──────────────────┐ -// │ Proxy Engine │ +// │ Envoy Proxy │ // │ ┌──────────────┐ │ // ┌─────────────────────────┐ ┌─┼─►listener_proxy│ │ // │https://api.lyft.com/ping│ ┌──────────────┬┘ │ └──────┬───────┘ │ ┌────────────┐ @@ -43,49 +36,46 @@ import org.robolectric.RobolectricTestRunner class PerformHTTPSRequestUsingAsyncProxyTest { init { JniLibrary.loadTestLibrary() + JniLibrary.load() } @Test fun `performs an HTTPs request through a proxy using async DNS resolution`() { - val port = (10001..11000).random() + TestJni.startHttpsProxyTestServer() + val port = TestJni.getServerPort() val context = Mockito.spy(ApplicationProvider.getApplicationContext()) val connectivityManager: ConnectivityManager = Mockito.mock(ConnectivityManager::class.java) - Mockito.doReturn(connectivityManager).`when`(context).getSystemService(Context.CONNECTIVITY_SERVICE) - Mockito.`when`(connectivityManager.getDefaultProxy()).thenReturn(ProxyInfo.buildDirectProxy("localhost", port)) + Mockito.doReturn(connectivityManager) + .`when`(context) + .getSystemService(Context.CONNECTIVITY_SERVICE) + Mockito.`when`(connectivityManager.getDefaultProxy()) + .thenReturn(ProxyInfo.buildDirectProxy("localhost", port)) val onEngineRunningLatch = CountDownLatch(1) - val onProxyEngineRunningLatch = CountDownLatch(1) val onRespondeHeadersLatch = CountDownLatch(1) - val proxyEngineBuilder = Proxy(ApplicationProvider.getApplicationContext(), port).https() - val proxyEngine = proxyEngineBuilder - .addLogLevel(LogLevel.DEBUG) - .setOnEngineRunning { onProxyEngineRunningLatch.countDown() } - .build() - - onProxyEngineRunningLatch.await(10, TimeUnit.SECONDS) - assertThat(onProxyEngineRunningLatch.count).isEqualTo(0) - context.sendStickyBroadcast(Intent(Proxy.PROXY_CHANGE_ACTION)) val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() onEngineRunningLatch.await(10, TimeUnit.SECONDS) assertThat(onEngineRunningLatch.count).isEqualTo(0) - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() @@ -103,6 +93,6 @@ class PerformHTTPSRequestUsingAsyncProxyTest { assertThat(onRespondeHeadersLatch.count).isEqualTo(0) engine.terminate() - proxyEngine.terminate() + TestJni.shutdownTestServer() } } diff --git a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingProxyTest.kt index 2eea4d7f8c11..46c3837c0195 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyInfoIntentPerformHTTPSRequestUsingProxyTest.kt @@ -1,35 +1,28 @@ package test.kotlin.integration.proxying -import android.content.Intent import android.content.Context +import android.content.Intent import android.net.ConnectivityManager import android.net.Proxy import android.net.ProxyInfo import androidx.test.core.app.ApplicationProvider - -import io.envoyproxy.envoymobile.LogLevel -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.Engine import io.envoyproxy.envoymobile.AndroidEngineBuilder +import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.ResponseHeaders -import io.envoyproxy.envoymobile.StreamIntel import io.envoyproxy.envoymobile.engine.JniLibrary - +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.junit.runner.RunWith -import org.mockito.Mock import org.mockito.Mockito import org.robolectric.RobolectricTestRunner // ┌──────────────────┐ -// │ Proxy Engine │ +// │ Envoy Proxy │ // │ ┌──────────────┐ │ // ┌─────────────────────────┐ ┌─┼─►listener_proxy│ │ // │https://api.lyft.com/ping│ ┌──────────────┬┘ │ └──────┬───────┘ │ ┌────────────┐ @@ -43,49 +36,46 @@ import org.robolectric.RobolectricTestRunner class PerformHTTPSRequestUsingProxy { init { JniLibrary.loadTestLibrary() + JniLibrary.load() } @Test fun `performs an HTTPs request through a proxy`() { - val port = (10001..11000).random() + TestJni.startHttpsProxyTestServer() + val port = TestJni.getServerPort() val context = Mockito.spy(ApplicationProvider.getApplicationContext()) val connectivityManager: ConnectivityManager = Mockito.mock(ConnectivityManager::class.java) - Mockito.doReturn(connectivityManager).`when`(context).getSystemService(Context.CONNECTIVITY_SERVICE) - Mockito.`when`(connectivityManager.getDefaultProxy()).thenReturn(ProxyInfo.buildDirectProxy("127.0.0.1", port)) + Mockito.doReturn(connectivityManager) + .`when`(context) + .getSystemService(Context.CONNECTIVITY_SERVICE) + Mockito.`when`(connectivityManager.getDefaultProxy()) + .thenReturn(ProxyInfo.buildDirectProxy("127.0.0.1", port)) val onEngineRunningLatch = CountDownLatch(1) - val onProxyEngineRunningLatch = CountDownLatch(1) val onRespondeHeadersLatch = CountDownLatch(1) - val proxyEngineBuilder = Proxy(context, port).https() - val proxyEngine = proxyEngineBuilder - .addLogLevel(LogLevel.DEBUG) - .setOnEngineRunning { onProxyEngineRunningLatch.countDown() } - .build() - - onProxyEngineRunningLatch.await(10, TimeUnit.SECONDS) - assertThat(onProxyEngineRunningLatch.count).isEqualTo(0) - context.sendStickyBroadcast(Intent(Proxy.PROXY_CHANGE_ACTION)) val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() onEngineRunningLatch.await(10, TimeUnit.SECONDS) assertThat(onEngineRunningLatch.count).isEqualTo(0) - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "https", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() @@ -103,6 +93,6 @@ class PerformHTTPSRequestUsingProxy { assertThat(onRespondeHeadersLatch.count).isEqualTo(0) engine.terminate() - proxyEngine.terminate() + TestJni.shutdownTestServer() } } diff --git a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestUsingProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestUsingProxyTest.kt index 31030df2bab4..384ce579988f 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestUsingProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestUsingProxyTest.kt @@ -1,35 +1,27 @@ package test.kotlin.integration.proxying -import android.content.Intent import android.content.Context import android.net.ConnectivityManager -import android.net.Proxy import android.net.ProxyInfo import androidx.test.core.app.ApplicationProvider - import io.envoyproxy.envoymobile.AndroidEngineBuilder -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.Engine -import io.envoyproxy.envoymobile.engine.JniLibrary import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.ResponseHeaders -import io.envoyproxy.envoymobile.StreamIntel - +import io.envoyproxy.envoymobile.engine.AndroidJniLibrary +import io.envoyproxy.envoymobile.engine.JniLibrary +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.junit.runner.RunWith -import org.mockito.Mock import org.mockito.Mockito import org.robolectric.RobolectricTestRunner // ┌──────────────────┐ -// │ Proxy Engine │ +// │ Envoy Proxy │ // │ ┌──────────────┐ │ // ┌────────────────────────┐ ┌─┼─►listener_proxy│ │ // │http://api.lyft.com/ping│ ┌──────────────┬┘ │ └──────┬───────┘ │ ┌────────────┐ @@ -42,49 +34,46 @@ import org.robolectric.RobolectricTestRunner @RunWith(RobolectricTestRunner::class) class PerformHTTPRequestUsingProxy { init { + AndroidJniLibrary.loadTestLibrary() JniLibrary.loadTestLibrary() + JniLibrary.load() } @Test fun `performs an HTTP request through a proxy`() { - val port = (10001..11000).random() + TestJni.startHttpProxyTestServer() + val port = TestJni.getServerPort() val context = Mockito.spy(ApplicationProvider.getApplicationContext()) val connectivityManager: ConnectivityManager = Mockito.mock(ConnectivityManager::class.java) - Mockito.doReturn(connectivityManager).`when`(context).getSystemService(Context.CONNECTIVITY_SERVICE) - Mockito.`when`(connectivityManager.getDefaultProxy()).thenReturn(ProxyInfo.buildDirectProxy("127.0.0.1", port)) + Mockito.doReturn(connectivityManager) + .`when`(context) + .getSystemService(Context.CONNECTIVITY_SERVICE) + Mockito.`when`(connectivityManager.getDefaultProxy()) + .thenReturn(ProxyInfo.buildDirectProxy("127.0.0.1", port)) - val onProxyEngineRunningLatch = CountDownLatch(1) val onEngineRunningLatch = CountDownLatch(1) val onRespondeHeadersLatch = CountDownLatch(1) - val proxyEngineBuilder = Proxy(context, port) - .http() - val proxyEngine = proxyEngineBuilder - .addLogLevel(LogLevel.DEBUG) - .setOnEngineRunning { onProxyEngineRunningLatch.countDown() } - .build() - - onProxyEngineRunningLatch.await(10, TimeUnit.SECONDS) - assertThat(onProxyEngineRunningLatch.count).isEqualTo(0) - val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() onEngineRunningLatch.await(10, TimeUnit.SECONDS) assertThat(onEngineRunningLatch.count).isEqualTo(0) - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "http", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "http", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() @@ -102,6 +91,6 @@ class PerformHTTPRequestUsingProxy { assertThat(onRespondeHeadersLatch.count).isEqualTo(0) engine.terminate() - proxyEngine.terminate() + TestJni.shutdownTestServer() } } diff --git a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt index 82eb5f982b75..baffa3c9abad 100644 --- a/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt +++ b/mobile/test/kotlin/integration/proxying/ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt @@ -1,36 +1,27 @@ package test.kotlin.integration.proxying -import android.content.Intent import android.content.Context import android.net.ConnectivityManager -import android.net.Proxy import android.net.ProxyInfo import android.net.Uri import androidx.test.core.app.ApplicationProvider - import io.envoyproxy.envoymobile.AndroidEngineBuilder -import io.envoyproxy.envoymobile.Custom -import io.envoyproxy.envoymobile.Engine -import io.envoyproxy.envoymobile.engine.JniLibrary import io.envoyproxy.envoymobile.LogLevel import io.envoyproxy.envoymobile.RequestHeadersBuilder import io.envoyproxy.envoymobile.RequestMethod -import io.envoyproxy.envoymobile.ResponseHeaders -import io.envoyproxy.envoymobile.StreamIntel - +import io.envoyproxy.envoymobile.engine.JniLibrary +import io.envoyproxy.envoymobile.engine.testing.TestJni import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit - import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.junit.runner.RunWith -import org.mockito.Mock import org.mockito.Mockito import org.robolectric.RobolectricTestRunner // ┌──────────────────┐ -// │ Proxy Engine │ +// │ Envoy Proxy │ // │ ┌──────────────┐ │ // ┌─────────────────────────┐ │ │listener_proxy│ │ // │https://api.lyft.com/ping│ ┌──────────────┐ │ └──────┬───────┘ │ ┌────────────┐ @@ -46,48 +37,43 @@ import org.robolectric.RobolectricTestRunner class PerformHTTPRequestUsingProxy { init { JniLibrary.loadTestLibrary() + JniLibrary.load() } @Test fun `performs an HTTP request through a proxy`() { - val port = (10001..11000).random() + TestJni.startHttpProxyTestServer() val context = Mockito.spy(ApplicationProvider.getApplicationContext()) val connectivityManager: ConnectivityManager = Mockito.mock(ConnectivityManager::class.java) - Mockito.doReturn(connectivityManager).`when`(context).getSystemService(Context.CONNECTIVITY_SERVICE) - Mockito.`when`(connectivityManager.getDefaultProxy()).thenReturn(ProxyInfo.buildPacProxy(Uri.parse("https://example.com"))) + Mockito.doReturn(connectivityManager) + .`when`(context) + .getSystemService(Context.CONNECTIVITY_SERVICE) + Mockito.`when`(connectivityManager.getDefaultProxy()) + .thenReturn(ProxyInfo.buildPacProxy(Uri.parse("https://example.com"))) - val onProxyEngineRunningLatch = CountDownLatch(1) val onEngineRunningLatch = CountDownLatch(1) val onRespondeHeadersLatch = CountDownLatch(1) - val proxyEngineBuilder = Proxy(context, port) - .http() - val proxyEngine = proxyEngineBuilder - .addLogLevel(LogLevel.DEBUG) - .setOnEngineRunning { onProxyEngineRunningLatch.countDown() } - .build() - - onProxyEngineRunningLatch.await(10, TimeUnit.SECONDS) - assertThat(onProxyEngineRunningLatch.count).isEqualTo(0) - val builder = AndroidEngineBuilder(context) - val engine = builder - .addLogLevel(LogLevel.DEBUG) - .enableProxying(true) - .setOnEngineRunning { onEngineRunningLatch.countDown() } - .build() + val engine = + builder + .addLogLevel(LogLevel.DEBUG) + .enableProxying(true) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() onEngineRunningLatch.await(10, TimeUnit.SECONDS) assertThat(onEngineRunningLatch.count).isEqualTo(0) - val requestHeaders = RequestHeadersBuilder( - method = RequestMethod.GET, - scheme = "http", - authority = "api.lyft.com", - path = "/ping" - ) - .build() + val requestHeaders = + RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "http", + authority = "api.lyft.com", + path = "/ping" + ) + .build() engine .streamClient() @@ -95,7 +81,7 @@ class PerformHTTPRequestUsingProxy { .setOnResponseHeaders { responseHeaders, _, _ -> val status = responseHeaders.httpStatus ?: 0L assertThat(status).isEqualTo(301) - assertThat(responseHeaders.value("x-proxy-response")).isNull(); + assertThat(responseHeaders.value("x-proxy-response")).isNull() onRespondeHeadersLatch.countDown() } .start(Executors.newSingleThreadExecutor()) @@ -105,6 +91,6 @@ class PerformHTTPRequestUsingProxy { assertThat(onRespondeHeadersLatch.count).isEqualTo(0) engine.terminate() - proxyEngine.terminate() + TestJni.shutdownTestServer() } } diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD b/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD index 4f052d6aceef..39aa8ecb6256 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_jni_kt_test", "envoy_mobile_kt_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_jni_kt_test( name = "engine_builder_test", srcs = [ @@ -13,8 +16,13 @@ envoy_mobile_jni_kt_test( }, native_deps = [ "//test/common/jni:libenvoy_jni_with_test_extensions.so", - "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", - ], + ] + select({ + "@platforms//os:macos": [ + "//test/common/jni:libenvoy_jni_with_test_extensions_jnilib", + ], + "//conditions:default": [], + }), + native_lib_name = "envoy_jni_with_test_extensions", deps = [ "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt index 1703b08690d7..31dbbc3481a5 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/EngineBuilderTest.kt @@ -1,10 +1,10 @@ package io.envoyproxy.envoymobile import io.envoyproxy.envoymobile.engine.EnvoyEngine +import io.envoyproxy.envoymobile.engine.JniLibrary import org.assertj.core.api.Assertions.assertThat import org.junit.Test import org.mockito.Mockito.mock -import io.envoyproxy.envoymobile.engine.JniLibrary class EngineBuilderTest { private lateinit var engineBuilder: EngineBuilder @@ -24,16 +24,6 @@ class EngineBuilderTest { assertThat(engine.logLevel).isEqualTo(LogLevel.DEBUG) } - @Test - fun `specifying stats domain overrides default`() { - engineBuilder = EngineBuilder(Standard()) - engineBuilder.addEngineType { envoyEngine } - engineBuilder.addGrpcStatsDomain("stats.envoyproxy.io") - - val engine = engineBuilder.build() as EngineImpl - assertThat(engine.envoyConfiguration.grpcStatsDomain).isEqualTo("stats.envoyproxy.io") - } - @Test fun `enabling interface binding overrides default`() { engineBuilder = EngineBuilder(Standard()) @@ -121,7 +111,8 @@ class EngineBuilderTest { engineBuilder.addH2ConnectionKeepaliveIdleIntervalMilliseconds(1234) val engine = engineBuilder.build() as EngineImpl - assertThat(engine.envoyConfiguration.h2ConnectionKeepaliveIdleIntervalMilliseconds).isEqualTo(1234) + assertThat(engine.envoyConfiguration.h2ConnectionKeepaliveIdleIntervalMilliseconds) + .isEqualTo(1234) } @Test @@ -144,16 +135,6 @@ class EngineBuilderTest { assertThat(engine.envoyConfiguration.maxConnectionsPerHost).isEqualTo(1234) } - @Test - fun `specifying stats flush overrides default`() { - engineBuilder = EngineBuilder(Standard()) - engineBuilder.addEngineType { envoyEngine } - engineBuilder.addStatsFlushSeconds(1234) - - val engine = engineBuilder.build() as EngineImpl - assertThat(engine.envoyConfiguration.statsFlushSeconds).isEqualTo(1234) - } - @Test fun `specifying stream idle timeout overrides default`() { engineBuilder = EngineBuilder(Standard()) @@ -207,25 +188,26 @@ class EngineBuilderTest { @Test fun `specifying xDS works`() { var xdsBuilder = XdsBuilder("fake_test_address", 0) - xdsBuilder.setAuthenticationToken("x-goog-api-key", "A1B2C3") - xdsBuilder.setJwtAuthenticationToken("my_jwt_token") + xdsBuilder + .addInitialStreamHeader("x-goog-api-key", "A1B2C3") + .addInitialStreamHeader("x-android-package", "com.google.myapp") xdsBuilder.setSslRootCerts("my_root_certs") - xdsBuilder.setSni("fake_test_address"); xdsBuilder.addRuntimeDiscoveryService("some_rtds_resource") - xdsBuilder.addClusterDiscoveryService("xdstp://fake_test_address/envoy.config.cluster.v3.Cluster/xyz") + xdsBuilder.addClusterDiscoveryService( + "xdstp://fake_test_address/envoy.config.cluster.v3.Cluster/xyz" + ) engineBuilder = EngineBuilder(Standard()) engineBuilder.addEngineType { envoyEngine } engineBuilder.setXds(xdsBuilder) val engine = engineBuilder.build() as EngineImpl assertThat(engine.envoyConfiguration.xdsAddress).isEqualTo("fake_test_address") - assertThat(engine.envoyConfiguration.xdsAuthHeader).isEqualTo("x-goog-api-key") - assertThat(engine.envoyConfiguration.xdsAuthToken).isEqualTo("A1B2C3") - assertThat(engine.envoyConfiguration.xdsJwtToken).isEqualTo("my_jwt_token") + assertThat(engine.envoyConfiguration.xdsGrpcInitialMetadata) + .isEqualTo(mapOf("x-goog-api-key" to "A1B2C3", "x-android-package" to "com.google.myapp")) assertThat(engine.envoyConfiguration.xdsRootCerts).isEqualTo("my_root_certs") - assertThat(engine.envoyConfiguration.xdsSni).isEqualTo("fake_test_address") assertThat(engine.envoyConfiguration.rtdsResourceName).isEqualTo("some_rtds_resource") - assertThat(engine.envoyConfiguration.cdsResourcesLocator).isEqualTo("xdstp://fake_test_address/envoy.config.cluster.v3.Cluster/xyz") + assertThat(engine.envoyConfiguration.cdsResourcesLocator) + .isEqualTo("xdstp://fake_test_address/envoy.config.cluster.v3.Cluster/xyz") } @Test diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCRequestHeadersBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCRequestHeadersBuilderTest.kt index 63122ff60f4b..07bf88dd2e13 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCRequestHeadersBuilderTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCRequestHeadersBuilderTest.kt @@ -6,57 +6,59 @@ import org.junit.Test class GRPCRequestHeadersBuilderTest { @Test fun `adds scheme to header`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar").build() assertThat(headers.value(":scheme")).containsExactly("https") assertThat(headers.scheme).isEqualTo("https") } @Test fun `adds authority to header`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar").build() assertThat(headers.value(":authority")).containsExactly("envoyproxy.io") assertThat(headers.authority).isEqualTo("envoyproxy.io") } @Test fun `adds path to header`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar").build() assertThat(headers.value(":path")).containsExactly("/pb.api.v1.Foo/GetBar") assertThat(headers.path).isEqualTo("/pb.api.v1.Foo/GetBar") } @Test fun `adds grpc content type header`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar").build() assertThat(headers.value("content-type")).containsExactly("application/grpc") } @Test fun `uses http post`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar").build() assertThat(headers.method).isEqualTo(RequestMethod.POST) assertThat(headers.value(":method")).containsExactly("POST") } @Test fun `adds timeout header when set to value`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .addtimeoutMs(200) - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") + .addtimeoutMs(200) + .build() assertThat(headers.value("grpc-timeout")).containsExactly("200m") } @Test fun `removes timeout header when set to null`() { - val headers = GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") - .addtimeoutMs(200) - .addtimeoutMs(null) - .build() + val headers = + GRPCRequestHeadersBuilder("https", "envoyproxy.io", "/pb.api.v1.Foo/GetBar") + .addtimeoutMs(200) + .addtimeoutMs(null) + .build() assertThat(headers.value("grpc-timeout")).isNull() } } diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCStreamTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCStreamTest.kt index 3cb4a2738ded..37a6c2ffc0ce 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCStreamTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/GRPCStreamTest.kt @@ -21,10 +21,7 @@ class GRPCStreamTest { stream.onRequestData = { data, _ -> sentData.write(data.array()) } } - GRPCClient(streamClient) - .newGRPCStreamPrototype() - .start(Executor {}) - .sendMessage(message1) + GRPCClient(streamClient).newGRPCStreamPrototype().start(Executor {}).sendMessage(message1) assertThat(sentData.size()).isEqualTo(5 + message1.array().count()) } @@ -36,10 +33,7 @@ class GRPCStreamTest { stream.onRequestData = { data, _ -> sentData.write(data.array()) } } - GRPCClient(streamClient) - .newGRPCStreamPrototype() - .start(Executor {}) - .sendMessage(message1) + GRPCClient(streamClient).newGRPCStreamPrototype().start(Executor {}).sendMessage(message1) assertThat(sentData.toByteArray()[0]).isEqualTo(0) } @@ -51,12 +45,10 @@ class GRPCStreamTest { stream.onRequestData = { data, _ -> sentData.write(data.array()) } } - GRPCClient(streamClient) - .newGRPCStreamPrototype() - .start(Executor {}) - .sendMessage(message1) + GRPCClient(streamClient).newGRPCStreamPrototype().start(Executor {}).sendMessage(message1) - val size = ByteBuffer.wrap(sentData.toByteArray().sliceArray(1 until 5)).order(ByteOrder.BIG_ENDIAN).int + val size = + ByteBuffer.wrap(sentData.toByteArray().sliceArray(1 until 5)).order(ByteOrder.BIG_ENDIAN).int assertThat(size).isEqualTo(message1.array().count()) } @@ -67,27 +59,20 @@ class GRPCStreamTest { stream.onRequestData = { data, _ -> sentData.write(data.array()) } } - GRPCClient(streamClient) - .newGRPCStreamPrototype() - .start(Executor {}) - .sendMessage(message1) + GRPCClient(streamClient).newGRPCStreamPrototype().start(Executor {}).sendMessage(message1) - assertThat(sentData.toByteArray().sliceArray(5 until sentData.size())).isEqualTo(message1.array()) + assertThat(sentData.toByteArray().sliceArray(5 until sentData.size())) + .isEqualTo(message1.array()) } @Test fun `cancel calls a stream callback`() { val countDownLatch = CountDownLatch(1) val streamClient = MockStreamClient { stream -> - stream.onCancel = { - countDownLatch.countDown() - } + stream.onCancel = { countDownLatch.countDown() } } - GRPCClient(streamClient) - .newGRPCStreamPrototype() - .start(Executor {}) - .cancel() + GRPCClient(streamClient).newGRPCStreamPrototype().start(Executor {}).cancel() assertThat(countDownLatch.await(2000, TimeUnit.MILLISECONDS)).isTrue() } @@ -97,7 +82,8 @@ class GRPCStreamTest { @Test(timeout = 1000L) fun `headers callback passes headers`() { val countDownLatch = CountDownLatch(1) - val expectedHeaders = ResponseHeaders(mapOf("grpc-status" to listOf("1"), "x-other" to listOf("foo", "bar"))) + val expectedHeaders = + ResponseHeaders(mapOf("grpc-status" to listOf("1"), "x-other" to listOf("foo", "bar"))) var stream: MockStream? = null val streamClient = MockStreamClient { stream = it } @@ -117,14 +103,16 @@ class GRPCStreamTest { @Test(timeout = 1000L) fun `trailers callback passes trailers`() { val countDownLatch = CountDownLatch(1) - val expectedTrailers = ResponseTrailers(mapOf("x-foo" to listOf("bar"), "x-baz" to listOf("1", "2"))) + val expectedTrailers = + ResponseTrailers(mapOf("x-foo" to listOf("bar"), "x-baz" to listOf("1", "2"))) var stream: MockStream? = null val streamClient = MockStreamClient { stream = it } GRPCClient(streamClient) .newGRPCStreamPrototype() .setOnResponseTrailers { trailers, _ -> - assertThat(trailers.caseSensitiveHeaders()).isEqualTo(expectedTrailers.caseSensitiveHeaders()) + assertThat(trailers.caseSensitiveHeaders()) + .isEqualTo(expectedTrailers.caseSensitiveHeaders()) countDownLatch.countDown() } .start(Executor {}) @@ -164,21 +152,30 @@ class GRPCStreamTest { val streamClient = MockStreamClient { stream = it } val firstMessage = byteArrayOf(0x1, 0x2, 0x3, 0x4, 0x5) - val firstMessageBuffer = ByteBuffer.wrap( - byteArrayOf( - 0x0, // Compression flag - 0x0, 0x0, 0x0, 0x5 // Length bytes - ) + firstMessage - ) + val firstMessageBuffer = + ByteBuffer.wrap( + byteArrayOf( + 0x0, // Compression flag + 0x0, + 0x0, + 0x0, + 0x5 // Length bytes + ) + firstMessage + ) val secondMessage = byteArrayOf(0x6, 0x7, 0x8, 0x9, 0x0, 0x1) - val secondMessageBufferPart1 = ByteBuffer.wrap( - byteArrayOf( - 0x0, // Compression flag - 0x0, 0x0, 0x0, 0x6 // Length bytes - ) + secondMessage.sliceArray(0 until 2) - ) - val secondMessageBufferPart2 = ByteBuffer.wrap(secondMessage.sliceArray(2 until secondMessage.count())) + val secondMessageBufferPart1 = + ByteBuffer.wrap( + byteArrayOf( + 0x0, // Compression flag + 0x0, + 0x0, + 0x0, + 0x6 // Length bytes + ) + secondMessage.sliceArray(0 until 2) + ) + val secondMessageBufferPart2 = + ByteBuffer.wrap(secondMessage.sliceArray(2 until secondMessage.count())) GRPCClient(streamClient) .newGRPCStreamPrototype() @@ -212,12 +209,16 @@ class GRPCStreamTest { } .start(Executor {}) - val emptyMessage = ByteBuffer.wrap( - byteArrayOf( - 0x0, // Compression flag - 0x0, 0x0, 0x0, 0x0 // Length bytes + val emptyMessage = + ByteBuffer.wrap( + byteArrayOf( + 0x0, // Compression flag + 0x0, + 0x0, + 0x0, + 0x0 // Length bytes + ) ) - ) stream?.receiveData(emptyMessage, false) countDownLatch.await() @@ -229,20 +230,28 @@ class GRPCStreamTest { var stream: MockStream? = null val streamClient = MockStreamClient { stream = it } - val emptyMessageBuffer = ByteBuffer.wrap( - byteArrayOf( - 0x0, // Compression flag - 0x0, 0x0, 0x0, 0x0 // Length bytes + val emptyMessageBuffer = + ByteBuffer.wrap( + byteArrayOf( + 0x0, // Compression flag + 0x0, + 0x0, + 0x0, + 0x0 // Length bytes + ) ) - ) val secondMessage = byteArrayOf(0x6, 0x7, 0x8, 0x9, 0x0, 0x1) - val secondMessageBuffer = ByteBuffer.wrap( - byteArrayOf( - 0x0, // Compression flag - 0x0, 0x0, 0x0, 0x6 // Length bytes - ) + secondMessage - ) + val secondMessageBuffer = + ByteBuffer.wrap( + byteArrayOf( + 0x0, // Compression flag + 0x0, + 0x0, + 0x0, + 0x6 // Length bytes + ) + secondMessage + ) GRPCClient(streamClient) .newGRPCStreamPrototype() diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersBuilderTest.kt index f67a90dfc362..0cbdcc09d1be 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersBuilderTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersBuilderTest.kt @@ -6,92 +6,89 @@ import org.junit.Test class HeadersBuilderTest { @Test fun `adding new header adds to list of header keys`() { - val headers = RequestHeadersBuilder(mutableMapOf()) - .add("x-foo", "1") - .add("x-foo", "2") - .build() + val headers = RequestHeadersBuilder(mutableMapOf()).add("x-foo", "1").add("x-foo", "2").build() assertThat(headers.value("x-foo")).containsExactly("1", "2") } @Test fun `adding header performs a case-insensitive header name lookup`() { - val headers = RequestHeadersBuilder(mutableMapOf()) - .add("fOo", "abc") - .add("foo", "123") - .build() + val headers = RequestHeadersBuilder(mutableMapOf()).add("fOo", "abc").add("foo", "123").build() assertThat(headers.caseSensitiveHeaders()).isEqualTo(mapOf("fOo" to listOf("abc", "123"))) } @Test fun `removing specific header key removes all of its values`() { - val headers = RequestHeadersBuilder(mutableMapOf()) - .add("x-foo", "1") - .add("x-foo", "2") - .remove("x-foo") - .build() + val headers = + RequestHeadersBuilder(mutableMapOf()) + .add("x-foo", "1") + .add("x-foo", "2") + .remove("x-foo") + .build() assertThat(headers.caseSensitiveHeaders()).doesNotContainKey("x-foo") } @Test fun `removing specific header key does not remove other keys`() { - val headers = RequestHeadersBuilder(mutableMapOf()) - .add("x-foo", "123") - .add("x-bar", "abc") - .build() + val headers = + RequestHeadersBuilder(mutableMapOf()).add("x-foo", "123").add("x-bar", "abc").build() assertThat(headers.value("x-foo")).containsExactly("123") assertThat(headers.value("x-bar")).containsExactly("abc") } @Test fun `removing specific header key performs a case-insensitive header name lookup`() { - val headers = RequestHeadersBuilder(mutableMapOf()) - .set("foo", mutableListOf("123")) - .remove("fOo") - .build() + val headers = + RequestHeadersBuilder(mutableMapOf()).set("foo", mutableListOf("123")).remove("fOo").build() assertThat(headers.caseSensitiveHeaders()).isEmpty() } @Test fun `setting header replaces existing headers with matching name`() { - val headers = RequestHeadersBuilder(mutableMapOf()) - .add("x-foo", "123") - .set("x-foo", mutableListOf("abc")) - .build() + val headers = + RequestHeadersBuilder(mutableMapOf()) + .add("x-foo", "123") + .set("x-foo", mutableListOf("abc")) + .build() assertThat(headers.value("x-foo")).containsExactly("abc") } @Test fun `setting header replaces performs a case-insensitive header name lookup`() { - val headers = RequestHeadersBuilder(mapOf()) - .set("foo", mutableListOf("123")) - .set("fOo", mutableListOf("abc")) - .build() + val headers = + RequestHeadersBuilder(mapOf()) + .set("foo", mutableListOf("123")) + .set("fOo", mutableListOf("abc")) + .build() assertThat(headers.caseSensitiveHeaders()).isEqualTo(mapOf("fOo" to listOf("abc"))) } @Test fun `test initialization is case-insensitive, preserves casing and processes headers in alphabetical order`() { - val headers = RequestHeadersBuilder(mutableMapOf("a" to mutableListOf("456"), "A" to mutableListOf("123"))).build() + val headers = + RequestHeadersBuilder(mutableMapOf("a" to mutableListOf("456"), "A" to mutableListOf("123"))) + .build() assertThat(headers.caseSensitiveHeaders()).isEqualTo(mapOf("A" to listOf("123", "456"))) } @Test fun `test restricted headers are not settable`() { - val headers = RequestHeadersBuilder(method = RequestMethod.GET, authority = "example.com", path = "/") - .add("host", "example.com") - .add("Host", "example.com") - .add("hostWithSuffix", "foo.bar") - .set(":scheme", mutableListOf("http")) - .set(":path", mutableListOf("/nope")) - .build() - .caseSensitiveHeaders() - val expected = mapOf( - ":authority" to listOf("example.com"), - ":path" to listOf("/"), - ":method" to listOf("GET"), - ":scheme" to listOf("https"), - "hostWithSuffix" to listOf("foo.bar"), - ) + val headers = + RequestHeadersBuilder(method = RequestMethod.GET, authority = "example.com", path = "/") + .add("host", "example.com") + .add("Host", "example.com") + .add("hostWithSuffix", "foo.bar") + .set(":scheme", mutableListOf("http")) + .set(":path", mutableListOf("/nope")) + .build() + .caseSensitiveHeaders() + val expected = + mapOf( + ":authority" to listOf("example.com"), + ":path" to listOf("/"), + ":method" to listOf("GET"), + ":scheme" to listOf("https"), + "hostWithSuffix" to listOf("foo.bar"), + ) assertThat(headers).isEqualTo(expected) } } diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersContainerTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersContainerTest.kt index 483c4591810a..e0740e0d20ed 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersContainerTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/HeadersContainerTest.kt @@ -13,13 +13,17 @@ class HeadersContainerest { @Test fun `instantiation with mutable list of values is case-insensitive, preserves casing and processes in alphabetical order`() { - val container = HeadersContainer(mapOf("a" to mutableListOf("456"), "A" to mutableListOf("123"))) + val container = + HeadersContainer( + mapOf("a" to mutableListOf("456"), "A" to mutableListOf("123")) + ) assertThat(container.caseSensitiveHeaders()).isEqualTo(mapOf("A" to listOf("123", "456"))) } @Test fun `creation with immutable list of values is case-insensitive, preserves casing and processes in alphabetical order`() { - val container = HeadersContainer.create(mapOf("a" to listOf("456"), "A" to listOf("123"))) + val container = + HeadersContainer.create(mapOf("a" to listOf("456"), "A" to listOf("123"))) assertThat(container.caseSensitiveHeaders()).isEqualTo(mapOf("A" to listOf("123", "456"))) } @@ -46,7 +50,7 @@ class HeadersContainerest { val container = HeadersContainer(mapOf()) container.set("x-foo", mutableListOf("abc")) - assertThat( container.value("x-foo")).isEqualTo(listOf("abc")) + assertThat(container.value("x-foo")).isEqualTo(listOf("abc")) } @Test diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/PulseClientImplTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/PulseClientImplTest.kt index 2f2960480092..99a2403cbe64 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/PulseClientImplTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/PulseClientImplTest.kt @@ -14,11 +14,9 @@ import org.mockito.MockitoAnnotations class PulseClientImplTest { private var envoyEngine: EnvoyEngine = mock(EnvoyEngine::class.java) - @Captor - private lateinit var elementsCaptor: ArgumentCaptor + @Captor private lateinit var elementsCaptor: ArgumentCaptor - @Captor - private lateinit var tagsCaptor: ArgumentCaptor> + @Captor private lateinit var tagsCaptor: ArgumentCaptor> @Before fun setup() { @@ -31,9 +29,8 @@ class PulseClientImplTest { val counter = pulseClient.counter(Element("test"), Element("stat")) counter.increment() val countCaptor = ArgumentCaptor.forClass(Int::class.java) - verify(envoyEngine).recordCounterInc( - elementsCaptor.capture(), tagsCaptor.capture(), countCaptor.capture() - ) + verify(envoyEngine) + .recordCounterInc(elementsCaptor.capture(), tagsCaptor.capture(), countCaptor.capture()) assertThat(elementsCaptor.getValue()).isEqualTo("test.stat") assertThat(countCaptor.getValue()).isEqualTo(1) assertThat(tagsCaptor.getValue().size).isEqualTo(0) @@ -42,15 +39,16 @@ class PulseClientImplTest { @Test fun `counter delegates to engine with tags and count`() { val pulseClient = PulseClientImpl(envoyEngine) - val counter = pulseClient.counter( - Element("test"), Element("stat"), - tags = TagsBuilder().add("testKey1", "testValue1").add("testKey2", "testValue2").build() - ) + val counter = + pulseClient.counter( + Element("test"), + Element("stat"), + tags = TagsBuilder().add("testKey1", "testValue1").add("testKey2", "testValue2").build() + ) counter.increment(5) val countCaptor = ArgumentCaptor.forClass(Int::class.java) - verify(envoyEngine).recordCounterInc( - elementsCaptor.capture(), tagsCaptor.capture(), countCaptor.capture() - ) + verify(envoyEngine) + .recordCounterInc(elementsCaptor.capture(), tagsCaptor.capture(), countCaptor.capture()) assertThat(elementsCaptor.getValue()).isEqualTo("test.stat") assertThat(countCaptor.getValue()).isEqualTo(5) diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderTest.kt index cc71d0c22ff2..fbf33a0176dd 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/RequestHeadersBuilderTest.kt @@ -6,11 +6,14 @@ import org.junit.Test class RequestHeadersBuilderTest { @Test fun `adds method to headers`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .build() assertThat(headers.value(":method")).containsExactly("POST") assertThat(headers.method).isEqualTo(RequestMethod.POST) @@ -18,11 +21,14 @@ class RequestHeadersBuilderTest { @Test fun `adds scheme to headers`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .build() assertThat(headers.value(":scheme")).containsExactly("https") assertThat(headers.scheme).isEqualTo("https") @@ -30,11 +36,14 @@ class RequestHeadersBuilderTest { @Test fun `adds authority to headers`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .build() assertThat(headers.value(":authority")).containsExactly("envoyproxy.io") assertThat(headers.authority).isEqualTo("envoyproxy.io") @@ -42,11 +51,14 @@ class RequestHeadersBuilderTest { @Test fun `adds path to headers`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .build() assertThat(headers.value(":path")).containsExactly("/mock") assertThat(headers.path).isEqualTo("/mock") @@ -54,28 +66,34 @@ class RequestHeadersBuilderTest { @Test fun `joins header values with the same key`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .add("x-foo", "1") - .add("x-foo", "2") - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .add("x-foo", "1") + .add("x-foo", "2") + .build() assertThat(headers.value("x-foo")).containsExactly("1", "2") } @Test fun `cannot publicly add headers with restricted prefix`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .add(":x-foo", "123") - .add("x-envoy-mobile-foo", "abc") - .add("host", "example.com") - .add("hostWithSuffix", "foo.bar") - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .add(":x-foo", "123") + .add("x-envoy-mobile-foo", "abc") + .add("host", "example.com") + .add("hostWithSuffix", "foo.bar") + .build() assertThat(headers.caseSensitiveHeaders()).doesNotContainKey(":x-foo") assertThat(headers.caseSensitiveHeaders()).doesNotContainKey("x-envoy-mobile-foo") @@ -85,13 +103,16 @@ class RequestHeadersBuilderTest { @Test fun `cannot publicly set headers with restricted prefix`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .set(":x-foo", mutableListOf("123")) - .set("x-envoy-mobile-foo", mutableListOf("abc")) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .set(":x-foo", mutableListOf("123")) + .set("x-envoy-mobile-foo", mutableListOf("abc")) + .build() assertThat(headers.caseSensitiveHeaders()).doesNotContainKey(":x-foo") assertThat(headers.caseSensitiveHeaders()).doesNotContainKey("x-envoy-mobile-foo") @@ -99,25 +120,31 @@ class RequestHeadersBuilderTest { @Test fun `cannot publicly remove headers with restricted prefix`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .remove(":path") - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .remove(":path") + .build() assertThat(headers.value(":path")).contains("/mock") } @Test fun `can internally set headers with restricted prefix`() { - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .internalSet(":x-foo", mutableListOf("123")) - .internalSet("x-envoy-mobile-foo", mutableListOf("abc")) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .internalSet(":x-foo", mutableListOf("123")) + .internalSet("x-envoy-mobile-foo", mutableListOf("abc")) + .build() assertThat(headers.value(":x-foo")).containsExactly("123") assertThat(headers.value("x-envoy-mobile-foo")).containsExactly("abc") @@ -125,49 +152,60 @@ class RequestHeadersBuilderTest { @Test fun `includes retry policy headers`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 123, - retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), - perRetryTimeoutMS = 9001 - ) + val retryPolicy = + RetryPolicy( + maxRetryCount = 123, + retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), + perRetryTimeoutMS = 9001 + ) val retryPolicyHeaders = retryPolicy.outboundHeaders() - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .addRetryPolicy(retryPolicy) - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .addRetryPolicy(retryPolicy) + .build() assertThat(headers.caseSensitiveHeaders()).containsAllEntriesOf(retryPolicyHeaders) } @Test fun `retry policy takes precedence over manually set retry headers`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 123, - retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), - perRetryTimeoutMS = 9001 - ) - - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .add("x-envoy-max-retries", "override") - .addRetryPolicy(retryPolicy) - .build() + val retryPolicy = + RetryPolicy( + maxRetryCount = 123, + retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), + perRetryTimeoutMS = 9001 + ) + + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .add("x-envoy-max-retries", "override") + .addRetryPolicy(retryPolicy) + .build() assertThat(headers.value("x-envoy-max-retries")).containsExactly("123") } @Test fun `converting to request headers and back maintains equality`() { - val headers1 = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .build() + val headers1 = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .build() val headers2 = headers1.toRequestHeadersBuilder().build() assertThat(headers1.caseSensitiveHeaders()).isEqualTo(headers2.caseSensitiveHeaders()) @@ -175,28 +213,32 @@ class RequestHeadersBuilderTest { @Test fun `converting retry policy to headers and back creates the same retry policy`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 123, - retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), - perRetryTimeoutMS = 9001 - ) - - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .addRetryPolicy(retryPolicy) - .build() - - assertThat(retryPolicy.outboundHeaders()).isEqualTo(RetryPolicy.from(headers)!!.outboundHeaders()) + val retryPolicy = + RetryPolicy( + maxRetryCount = 123, + retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), + perRetryTimeoutMS = 9001 + ) + + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .addRetryPolicy(retryPolicy) + .build() + + assertThat(retryPolicy.outboundHeaders()) + .isEqualTo(RetryPolicy.from(headers)!!.outboundHeaders()) } @Test fun `converting request method to string and back creates the same request method`() { assertThat(RequestMethod.enumValue(RequestMethod.DELETE.stringValue)) .isEqualTo(RequestMethod.DELETE) - assertThat(RequestMethod.enumValue(RequestMethod.GET.stringValue)) - .isEqualTo(RequestMethod.GET) + assertThat(RequestMethod.enumValue(RequestMethod.GET.stringValue)).isEqualTo(RequestMethod.GET) assertThat(RequestMethod.enumValue(RequestMethod.HEAD.stringValue)) .isEqualTo(RequestMethod.HEAD) assertThat(RequestMethod.enumValue(RequestMethod.OPTIONS.stringValue)) @@ -205,8 +247,7 @@ class RequestHeadersBuilderTest { .isEqualTo(RequestMethod.PATCH) assertThat(RequestMethod.enumValue(RequestMethod.POST.stringValue)) .isEqualTo(RequestMethod.POST) - assertThat(RequestMethod.enumValue(RequestMethod.PUT.stringValue)) - .isEqualTo(RequestMethod.PUT) + assertThat(RequestMethod.enumValue(RequestMethod.PUT.stringValue)).isEqualTo(RequestMethod.PUT) assertThat(RequestMethod.enumValue(RequestMethod.TRACE.stringValue)) .isEqualTo(RequestMethod.TRACE) } diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/ResponseHeadersTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/ResponseHeadersTest.kt index 1aaf2e7bc7ff..261f9d8cc08e 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/ResponseHeadersTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/ResponseHeadersTest.kt @@ -30,17 +30,13 @@ class ResponseHeadersTest { @Test fun `adding HTTP status code sets the appropriate header`() { - val headers = ResponseHeadersBuilder() - .addHttpStatus(200) - .build() + val headers = ResponseHeadersBuilder().addHttpStatus(200).build() assertThat(headers.value(":status")).containsExactly("200") } @Test fun `adding negative HTTP status code no-ops`() { - val headers = ResponseHeadersBuilder() - .addHttpStatus(-123) - .build() + val headers = ResponseHeadersBuilder().addHttpStatus(-123).build() assertThat(headers.value(":status")).isNull() } diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapperTest.kt b/mobile/test/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapperTest.kt index 5f41594724ab..52989f9df44c 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapperTest.kt +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/RetryPolicyMapperTest.kt @@ -6,59 +6,70 @@ import org.junit.Test class RetryPolicyMapperTest { @Test fun `converting to headers with per retry timeout includes all headers`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 3, - retryOn = listOf( - RetryRule.STATUS_5XX, - RetryRule.GATEWAY_ERROR, - RetryRule.CONNECT_FAILURE, - RetryRule.REFUSED_STREAM, - RetryRule.RETRIABLE_4XX, - RetryRule.RETRIABLE_HEADERS, - RetryRule.RESET - ), - retryStatusCodes = listOf(400, 422, 500), - perRetryTimeoutMS = 15000, - totalUpstreamTimeoutMS = 60000 - ) + val retryPolicy = + RetryPolicy( + maxRetryCount = 3, + retryOn = + listOf( + RetryRule.STATUS_5XX, + RetryRule.GATEWAY_ERROR, + RetryRule.CONNECT_FAILURE, + RetryRule.REFUSED_STREAM, + RetryRule.RETRIABLE_4XX, + RetryRule.RETRIABLE_HEADERS, + RetryRule.RESET + ), + retryStatusCodes = listOf(400, 422, 500), + perRetryTimeoutMS = 15000, + totalUpstreamTimeoutMS = 60000 + ) - assertThat(retryPolicy.outboundHeaders()).isEqualTo( - mapOf( - "x-envoy-max-retries" to listOf("3"), - "x-envoy-retriable-status-codes" to listOf("400", "422", "500"), - "x-envoy-retry-on" to listOf( - "5xx", "gateway-error", "connect-failure", "refused-stream", "retriable-4xx", - "retriable-headers", "reset", "retriable-status-codes" - ), - "x-envoy-upstream-rq-per-try-timeout-ms" to listOf("15000"), - "x-envoy-upstream-rq-timeout-ms" to listOf("60000") + assertThat(retryPolicy.outboundHeaders()) + .isEqualTo( + mapOf( + "x-envoy-max-retries" to listOf("3"), + "x-envoy-retriable-status-codes" to listOf("400", "422", "500"), + "x-envoy-retry-on" to + listOf( + "5xx", + "gateway-error", + "connect-failure", + "refused-stream", + "retriable-4xx", + "retriable-headers", + "reset", + "retriable-status-codes" + ), + "x-envoy-upstream-rq-per-try-timeout-ms" to listOf("15000"), + "x-envoy-upstream-rq-timeout-ms" to listOf("60000") + ) ) - ) } @Test fun `converting from header values delimited with comma yields individual enum values`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 3, - retryOn = listOf( - RetryRule.STATUS_5XX, - RetryRule.GATEWAY_ERROR - ), - retryStatusCodes = listOf(400, 422, 500), - perRetryTimeoutMS = 15000, - totalUpstreamTimeoutMS = 60000 - ) + val retryPolicy = + RetryPolicy( + maxRetryCount = 3, + retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR), + retryStatusCodes = listOf(400, 422, 500), + perRetryTimeoutMS = 15000, + totalUpstreamTimeoutMS = 60000 + ) - val headers = RequestHeadersBuilder( - method = RequestMethod.POST, scheme = "https", - authority = "envoyproxy.io", path = "/mock" - ) - .add("x-envoy-max-retries", "3") - .add("x-envoy-retriable-status-codes", "400,422,500") - .add("x-envoy-retry-on", "5xx,gateway-error") - .add("x-envoy-upstream-rq-per-try-timeout-ms", "15000") - .add("x-envoy-upstream-rq-timeout-ms", "60000") - .build() + val headers = + RequestHeadersBuilder( + method = RequestMethod.POST, + scheme = "https", + authority = "envoyproxy.io", + path = "/mock" + ) + .add("x-envoy-max-retries", "3") + .add("x-envoy-retriable-status-codes", "400,422,500") + .add("x-envoy-retry-on", "5xx,gateway-error") + .add("x-envoy-upstream-rq-per-try-timeout-ms", "15000") + .add("x-envoy-upstream-rq-timeout-ms", "60000") + .build() val retryPolicyFromHeaders = RetryPolicy.from(headers)!! @@ -67,10 +78,11 @@ class RetryPolicyMapperTest { @Test fun `converting to headers without retry timeout excludes per retry timeout header`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 123, - retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR) - ) + val retryPolicy = + RetryPolicy( + maxRetryCount = 123, + retryOn = listOf(RetryRule.STATUS_5XX, RetryRule.GATEWAY_ERROR) + ) assertThat(retryPolicy.outboundHeaders()) .doesNotContainKey("x-envoy-upstream-rq-per-try-timeout-ms") @@ -78,19 +90,21 @@ class RetryPolicyMapperTest { @Test fun `converting to headers without upstream timeout includes zero for timeout header`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 123, - retryOn = listOf(RetryRule.STATUS_5XX), - totalUpstreamTimeoutMS = null - ) + val retryPolicy = + RetryPolicy( + maxRetryCount = 123, + retryOn = listOf(RetryRule.STATUS_5XX), + totalUpstreamTimeoutMS = null + ) - assertThat(retryPolicy.outboundHeaders()).isEqualTo( - mapOf( - "x-envoy-max-retries" to listOf("123"), - "x-envoy-retry-on" to listOf("5xx"), - "x-envoy-upstream-rq-timeout-ms" to listOf("0") + assertThat(retryPolicy.outboundHeaders()) + .isEqualTo( + mapOf( + "x-envoy-max-retries" to listOf("123"), + "x-envoy-retry-on" to listOf("5xx"), + "x-envoy-upstream-rq-timeout-ms" to listOf("0") + ) ) - ) } @Test(expected = IllegalArgumentException::class) @@ -105,20 +119,22 @@ class RetryPolicyMapperTest { @Test fun `converting headers without retry status code does not set retriable status code headers`() { - val retryPolicy = RetryPolicy( - maxRetryCount = 123, - retryOn = listOf( - RetryRule.STATUS_5XX, - RetryRule.GATEWAY_ERROR, - RetryRule.CONNECT_FAILURE, - RetryRule.REFUSED_STREAM, - RetryRule.RETRIABLE_4XX, - RetryRule.RETRIABLE_HEADERS, - RetryRule.RESET - ), - retryStatusCodes = emptyList(), - totalUpstreamTimeoutMS = null - ) + val retryPolicy = + RetryPolicy( + maxRetryCount = 123, + retryOn = + listOf( + RetryRule.STATUS_5XX, + RetryRule.GATEWAY_ERROR, + RetryRule.CONNECT_FAILURE, + RetryRule.REFUSED_STREAM, + RetryRule.RETRIABLE_4XX, + RetryRule.RETRIABLE_HEADERS, + RetryRule.RESET + ), + retryStatusCodes = emptyList(), + totalUpstreamTimeoutMS = null + ) val headers = retryPolicy.outboundHeaders() assertThat(headers["x-envoy-retriable-status-codes"]).isNull() diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD b/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD index bb6919774be8..faea616ea3a9 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_kt_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_kt_test( name = "element_test", srcs = [ diff --git a/mobile/test/non_hermetic/BUILD b/mobile/test/non_hermetic/BUILD index 37af76e1edf8..98c0db304e5a 100644 --- a/mobile/test/non_hermetic/BUILD +++ b/mobile/test/non_hermetic/BUILD @@ -1,20 +1,19 @@ load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test_binary", - "envoy_package", - "envoy_select_google_grpc", + "envoy_mobile_package", + "envoy_select_envoy_mobile_xds", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() # This is a envoy_cc_test_binary instead of an envoy_cc_test because we don't want it to be run # when `bazel test //test/...` is called. envoy_cc_test_binary( name = "gcp_traffic_director_integration_test", - # The test relies on the Google gRPC library. - srcs = envoy_select_google_grpc( + srcs = envoy_select_envoy_mobile_xds( ["gcp_traffic_director_integration_test.cc"], "@envoy", ), @@ -31,13 +30,10 @@ envoy_cc_test_binary( "//library/common/types:c_types_lib", "//test/common/integration:base_client_integration_test_lib", "@envoy//source/common/config:api_version_lib", - "@envoy//source/common/grpc:google_grpc_creds_lib", "@envoy//source/common/protobuf:utility_lib_header", "@envoy//source/extensions/clusters/strict_dns:strict_dns_cluster_lib", - "@envoy//source/extensions/config_subscription/grpc:grpc_mux_lib", - "@envoy//source/extensions/config_subscription/grpc:grpc_subscription_lib", - "@envoy//source/extensions/config_subscription/grpc:new_grpc_mux_lib", "@envoy//source/extensions/health_checkers/http:health_checker_lib", + "@envoy//source/extensions/load_balancing_policies/round_robin:config", "@envoy//test/common/grpc:grpc_client_integration_lib", "@envoy//test/integration:http_integration_lib", "@envoy//test/test_common:environment_lib", @@ -45,6 +41,7 @@ envoy_cc_test_binary( "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_build_config//:extension_registry", "@envoy_build_config//:test_extensions", ], ) diff --git a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc index c055f5a6217f..dffb30925195 100644 --- a/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc +++ b/mobile/test/non_hermetic/gcp_traffic_director_integration_test.cc @@ -10,13 +10,10 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/config_source.pb.h" -#include "source/common/grpc/google_grpc_creds_impl.h" #include "source/common/protobuf/utility.h" #include "source/extensions/clusters/strict_dns/strict_dns_cluster.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_factory.h" -#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "source/extensions/health_checkers/http/health_checker_impl.h" +#include "source/extensions/load_balancing_policies/round_robin/config.h" #include "test/common/grpc/grpc_client_integration.h" #include "test/common/integration/base_client_integration_test.h" @@ -24,9 +21,9 @@ #include "absl/strings/substitute.h" #include "absl/synchronization/notification.h" +#include "extension_registry.h" #include "gtest/gtest.h" #include "library/common/data/utility.h" -#include "library/common/engine_handle.h" #include "library/common/types/c_types.h" #include "tools/cpp/runfiles/runfiles.h" @@ -37,10 +34,10 @@ using ::Envoy::Grpc::SotwOrDelta; using ::Envoy::Network::Address::IpVersion; // The One-Platform API endpoint for Traffic Director. -constexpr char TD_API_ENDPOINT[] = "staging-trafficdirectorconsumermesh.sandbox.googleapis.com"; +constexpr char TD_API_ENDPOINT[] = "trafficdirectorconsumermesh.googleapis.com"; // The project number of the project, found on the main page of the project in // Google Cloud Console. -constexpr char PROJECT_ID[] = "947171374466"; +constexpr char PROJECT_ID[] = "33303528656"; // Tests that Envoy Mobile can connect to Traffic Director (an xDS management server offered by GCP) // via a test GCP project, and can pull down xDS config for the given project. @@ -52,28 +49,27 @@ class GcpTrafficDirectorIntegrationTest // TODO(https://github.com/envoyproxy/envoy/issues/27848): remove these force registrations // once the EngineBuilder APIs support conditional force registration. - // Force register the Google gRPC library. - Grpc::forceRegisterDefaultGoogleGrpcCredentialsFactory(); - // Force register the gRPC mux implementations. - Config::forceRegisterGrpcMuxFactory(); - Config::forceRegisterNewGrpcMuxFactory(); - Config::forceRegisterAdsConfigSubscriptionFactory(); + // Register the extensions required for Envoy Mobile. + ExtensionRegistry::registerFactories(); + // Force register the cluster factories used by the test. Upstream::forceRegisterStrictDnsClusterFactory(); Upstream::forceRegisterHttpHealthCheckerFactory(); + Extensions::LoadBalancingPolices::RoundRobin::forceRegisterFactory(); std::string root_certs(TestEnvironment::readFileToStringForTest( TestEnvironment::runfilesPath("test/config/integration/certs/google_root_certs.pem"))); - // API key for the `bct-staging-td-consumer-mesh` GCP test project. - const char* api_key = std::getenv("GCP_TEST_PROJECT_API_KEY"); - RELEASE_ASSERT(api_key != nullptr, "GCP_TEST_PROJECT_API_KEY environment variable not set."); + // API key for the `bct-prod-td-consumer-mesh` GCP test project. + const char* api_key = std::getenv("GCP_TEST_PROJECT_PROD_API_KEY"); + RELEASE_ASSERT(api_key != nullptr, + "GCP_TEST_PROJECT_PROD_API_KEY environment variable not set."); Platform::XdsBuilder xds_builder(/*xds_server_address=*/std::string(TD_API_ENDPOINT), /*xds_server_port=*/443); - xds_builder.setAuthenticationToken("x-goog-api-key", std::string(api_key)); - xds_builder.setSslRootCerts(std::move(root_certs)); - xds_builder.addClusterDiscoveryService(); + xds_builder.addInitialStreamHeader("x-goog-api-key", std::string(api_key)) + .setSslRootCerts(std::move(root_certs)) + .addClusterDiscoveryService(); builder_.addLogLevel(Platform::LogLevel::trace) .setNodeId(absl::Substitute("projects/$0/networks/default/nodes/111222333444", PROJECT_ID)) .setXds(std::move(xds_builder)); diff --git a/mobile/test/objective-c/BUILD b/mobile/test/objective-c/BUILD index a67acb0ae0ef..1d977756c4ca 100644 --- a/mobile/test/objective-c/BUILD +++ b/mobile/test/objective-c/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_objc_test", "envoy_objc_library") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_objc_test( name = "envoy_bridge_utility_test", srcs = [ diff --git a/mobile/test/performance/BUILD b/mobile/test/performance/BUILD index 3b5b3c9b5689..b77bc54c243d 100644 --- a/mobile/test/performance/BUILD +++ b/mobile/test/performance/BUILD @@ -1,8 +1,8 @@ -load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_binary", "envoy_package") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_cc_binary", "envoy_mobile_package") licenses(["notice"]) # Apache 2 -envoy_package() +envoy_mobile_package() envoy_cc_binary( name = "test_binary_size", diff --git a/mobile/test/swift/BUILD b/mobile/test/swift/BUILD index e19d6d04652a..83b7b81a1a75 100644 --- a/mobile/test/swift/BUILD +++ b/mobile/test/swift/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_swift_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_swift_test( name = "test", size = "large", diff --git a/mobile/test/swift/EngineBuilderTests.swift b/mobile/test/swift/EngineBuilderTests.swift index c2713374592b..c0019ccb2e30 100644 --- a/mobile/test/swift/EngineBuilderTests.swift +++ b/mobile/test/swift/EngineBuilderTests.swift @@ -109,20 +109,6 @@ final class EngineBuilderTests: XCTestCase { self.waitForExpectations(timeout: 0.01) } - func testAddinggrpcStatsDomainAddsToConfigurationWhenRunningEnvoy() { - let expectation = self.expectation(description: "Run called with expected data") - MockEnvoyEngine.onRunWithConfig = { config, _ in - XCTAssertEqual("stats.envoyproxy.io", config.grpcStatsDomain) - expectation.fulfill() - } - - _ = EngineBuilder() - .addEngineType(MockEnvoyEngine.self) - .addGrpcStatsDomain("stats.envoyproxy.io") - .build() - self.waitForExpectations(timeout: 0.01) - } - func testAddingConnectTimeoutSecondsAddsToConfigurationWhenRunningEnvoy() { let expectation = self.expectation(description: "Run called with expected data") MockEnvoyEngine.onRunWithConfig = { config, _ in @@ -250,20 +236,6 @@ final class EngineBuilderTests: XCTestCase { self.waitForExpectations(timeout: 0.01) } - func testAddingStatsFlushSecondsAddsToConfigurationWhenRunningEnvoy() { - let expectation = self.expectation(description: "Run called with expected data") - MockEnvoyEngine.onRunWithConfig = { config, _ in - XCTAssertEqual(42, config.statsFlushSeconds) - expectation.fulfill() - } - - _ = EngineBuilder() - .addEngineType(MockEnvoyEngine.self) - .addStatsFlushSeconds(42) - .build() - self.waitForExpectations(timeout: 0.01) - } - func testAddingStreamIdleTimeoutSecondsAddsToConfigurationWhenRunningEnvoy() { let expectation = self.expectation(description: "Run called with expected data") MockEnvoyEngine.onRunWithConfig = { config, _ in @@ -356,7 +328,7 @@ final class EngineBuilderTests: XCTestCase { self.waitForExpectations(timeout: 0.01) } -#if ENVOY_GOOGLE_GRPC +#if ENVOY_MOBILE_XDS func testAddingRtdsConfigurationWhenRunningEnvoy() { let xdsBuilder = XdsBuilder(xdsServerAddress: "FAKE_SWIFT_ADDRESS", xdsServerPort: 0) .addRuntimeDiscoveryService(resourceName: "some_rtds_resource", timeoutInSeconds: 14325) @@ -394,9 +366,9 @@ final class EngineBuilderTests: XCTestCase { func testAddingXdsSecurityConfigurationWhenRunningEnvoy() { let xdsBuilder = XdsBuilder(xdsServerAddress: "FAKE_SWIFT_ADDRESS", xdsServerPort: 0) - .setAuthenticationToken(header: "x-goog-api-key", token: "A1B2C3") + .addInitialStreamHeader(header: "x-goog-api-key", value: "A1B2C3") + .addInitialStreamHeader(header: "x-android-package", value: "com.google.myapp") .setSslRootCerts(rootCerts: "fake_ssl_root_certs") - .setSni(sni: "fake_sni_address") .addRuntimeDiscoveryService(resourceName: "some_rtds_resource", timeoutInSeconds: 14325) let bootstrapDebugDescription = EngineBuilder() .addEngineType(MockEnvoyEngine.self) @@ -404,23 +376,9 @@ final class EngineBuilderTests: XCTestCase { .bootstrapDebugDescription() XCTAssertTrue(bootstrapDebugDescription.contains("x-goog-api-key")) XCTAssertTrue(bootstrapDebugDescription.contains("A1B2C3")) + XCTAssertTrue(bootstrapDebugDescription.contains("x-android-package")) + XCTAssertTrue(bootstrapDebugDescription.contains("com.google.myapp")) XCTAssertTrue(bootstrapDebugDescription.contains("fake_ssl_root_certs")) - XCTAssertTrue(bootstrapDebugDescription.contains("fake_sni_address")) - } - - func testAddingXdsJwtSecurityConfigurationWhenRunningEnvoy() { - let xdsBuilder = XdsBuilder(xdsServerAddress: "FAKE_SWIFT_ADDRESS", xdsServerPort: 0) - .setJwtAuthenticationToken(token: "fake_jwt_token", tokenLifetimeInSeconds: 12345) - .setSslRootCerts(rootCerts: "fake_ssl_root_certs") - .setSni(sni: "fake_sni_address") - .addRuntimeDiscoveryService(resourceName: "some_rtds_resource", timeoutInSeconds: 14325) - let bootstrapDebugDescription = EngineBuilder() - .addEngineType(MockEnvoyEngine.self) - .setXds(xdsBuilder) - .bootstrapDebugDescription() - XCTAssertTrue(bootstrapDebugDescription.contains("fake_jwt_token")) - XCTAssertTrue(bootstrapDebugDescription.contains("fake_ssl_root_certs")) - XCTAssertTrue(bootstrapDebugDescription.contains("fake_sni_address")) } #endif diff --git a/mobile/test/swift/apps/baseline/BUILD b/mobile/test/swift/apps/baseline/BUILD index 8801f241871a..0eccd319f4e3 100644 --- a/mobile/test/swift/apps/baseline/BUILD +++ b/mobile/test/swift/apps/baseline/BUILD @@ -1,9 +1,12 @@ load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application") load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + swift_library( name = "appmain", srcs = glob(["*.swift"]), diff --git a/mobile/test/swift/apps/experimental/BUILD b/mobile/test/swift/apps/experimental/BUILD index 3ac85595124a..0eccd319f4e3 100644 --- a/mobile/test/swift/apps/experimental/BUILD +++ b/mobile/test/swift/apps/experimental/BUILD @@ -1,9 +1,12 @@ -load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application") load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") +load("//bazel:config.bzl", "MINIMUM_IOS_VERSION") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + swift_library( name = "appmain", srcs = glob(["*.swift"]), diff --git a/mobile/test/swift/cxx/BUILD b/mobile/test/swift/cxx/BUILD index 179c2d58ca50..3599e37554e6 100644 --- a/mobile/test/swift/cxx/BUILD +++ b/mobile/test/swift/cxx/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_swift_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_swift_test( name = "test", srcs = [ diff --git a/mobile/test/swift/integration/BUILD b/mobile/test/swift/integration/BUILD index 86be3c4319e3..4830157ab739 100644 --- a/mobile/test/swift/integration/BUILD +++ b/mobile/test/swift/integration/BUILD @@ -1,23 +1,11 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_swift_test") licenses(["notice"]) # Apache 2 -# TODO(jpsim): Fix remote execution for all the tests in this file. +envoy_mobile_package() -envoy_mobile_swift_test( - name = "test", - srcs = [ - "StatFlushIntegrationTest.swift", - ], - tags = [ - "no-remote-exec", - ], - visibility = ["//visibility:public"], - deps = [ - ":test_extensions", - "//library/objective-c:envoy_engine_objc_lib", - ], -) +# TODO(jpsim): Fix remote execution for all the tests in this file. envoy_mobile_swift_test( name = "end_to_end_networking_test", @@ -107,6 +95,7 @@ envoy_mobile_swift_test( deps = [ ":test_extensions", "//library/objective-c:envoy_engine_objc_lib", + "//test/objective-c:envoy_test_server", ], ) diff --git a/mobile/test/swift/integration/EngineApiTest.swift b/mobile/test/swift/integration/EngineApiTest.swift index 0892eeafd37b..c0194661b3ec 100644 --- a/mobile/test/swift/integration/EngineApiTest.swift +++ b/mobile/test/swift/integration/EngineApiTest.swift @@ -14,7 +14,6 @@ final class EngineApiTest: XCTestCase { let engine = EngineBuilder() .addLogLevel(.debug) - .addStatsFlushSeconds(1) .setOnEngineRunning { engineExpectation.fulfill() } diff --git a/mobile/test/swift/integration/IdleTimeoutTest.swift b/mobile/test/swift/integration/IdleTimeoutTest.swift index 3726b243fa57..c35a1a4fe928 100644 --- a/mobile/test/swift/integration/IdleTimeoutTest.swift +++ b/mobile/test/swift/integration/IdleTimeoutTest.swift @@ -1,5 +1,6 @@ import Envoy import EnvoyEngine +import EnvoyTestServer import Foundation import TestExtensions import XCTest @@ -11,83 +12,7 @@ final class IdleTimeoutTests: XCTestCase { } func testIdleTimeout() { - let idleTimeout = "0.5s" - let remotePort = Int.random(in: 10001...11000) - // swiftlint:disable:next line_length - let hcmType = "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" - // swiftlint:disable:next line_length - let emhcmType = "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.EnvoyMobileHttpConnectionManager" - let pbfType = - "type.googleapis.com/envoymobile.extensions.filters.http.platform_bridge.PlatformBridge" - let localErrorFilterType = - "type.googleapis.com/envoymobile.extensions.filters.http.local_error.LocalError" let filterName = "reset_idle_test_filter" - let config = -""" -static_resources: - listeners: - - name: fake_remote_listener - address: - socket_address: { protocol: TCP, address: 127.0.0.1, port_value: \(remotePort) } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": \(hcmType) - stat_prefix: remote_hcm - route_config: - name: remote_route - virtual_hosts: - - name: remote_service - domains: ["*"] - routes: - - match: { prefix: "/" } - direct_response: { status: 200 } - http_filters: - - name: envoy.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - - name: base_api_listener - address: - socket_address: { protocol: TCP, address: 0.0.0.0, port_value: 10000 } - api_listener: - api_listener: - "@type": \(emhcmType) - config: - stat_prefix: api_hcm - stream_idle_timeout: \(idleTimeout) - route_config: - name: api_router - virtual_hosts: - - name: api - domains: ["*"] - routes: - - match: { prefix: "/" } - route: { cluster: fake_remote } - http_filters: - - name: envoy.filters.http.platform_bridge - typed_config: - "@type": \(pbfType) - platform_filter_name: \(filterName) - - name: envoy.filters.http.local_error - typed_config: - "@type": \(localErrorFilterType) - - name: envoy.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: fake_remote - connect_timeout: 0.25s - type: STATIC - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: fake_remote - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: { address: 127.0.0.1, port_value: \(remotePort) } -""" class IdleTimeoutValidationFilter: AsyncResponseFilter, ResponseFilter { let timeoutExpectation: XCTestExpectation @@ -132,7 +57,7 @@ static_resources: } func onError(_ error: EnvoyError, streamIntel: FinalStreamIntel) { - XCTAssertEqual(error.errorCode, 0) + XCTAssertEqual(error.errorCode, 4) timeoutExpectation.fulfill() } @@ -147,8 +72,11 @@ static_resources: let callbackExpectation = self.expectation(description: "Stream idle timeout received by callbacks") - let engine = EngineBuilder(yaml: config) + EnvoyTestServer.startHttp1PlaintextServer() + + let engine = EngineBuilder() .addLogLevel(.trace) + .addStreamIdleTimeoutSeconds(1) .addPlatformFilter( name: filterName, factory: { IdleTimeoutValidationFilter(timeoutExpectation: filterExpectation) } @@ -157,21 +85,23 @@ static_resources: let client = engine.streamClient() - let requestHeaders = RequestHeadersBuilder(method: .get, scheme: "https", - authority: "example.com", path: "/test") - .build() + let port = String(EnvoyTestServer.getEnvoyPort()) + let requestHeaders = RequestHeadersBuilder( + method: .get, scheme: "http", authority: "localhost:" + port, path: "/" + ) + .build() client .newStreamPrototype() .setOnError { error, _ in - XCTAssertEqual(error.errorCode, 0) + XCTAssertEqual(error.errorCode, 4) callbackExpectation.fulfill() } .setOnCancel { _ in XCTFail("Unexpected call to onCancel filter callback") } .start() - .sendHeaders(requestHeaders, endStream: true) + .sendHeaders(requestHeaders, endStream: false) XCTAssertEqual( XCTWaiter.wait(for: [filterExpectation, callbackExpectation], timeout: 10), @@ -179,5 +109,6 @@ static_resources: ) engine.terminate() + EnvoyTestServer.shutdownTestServer() } } diff --git a/mobile/test/swift/integration/StatFlushIntegrationTest.swift b/mobile/test/swift/integration/StatFlushIntegrationTest.swift deleted file mode 100644 index 2e084dc58718..000000000000 --- a/mobile/test/swift/integration/StatFlushIntegrationTest.swift +++ /dev/null @@ -1,69 +0,0 @@ -import Envoy -import Foundation -import TestExtensions -import XCTest - -final class StatFlushIntegrationTest: XCTestCase { - override static func setUp() { - super.setUp() - register_test_extensions() - } - - func testLotsOfFlushesWithHistograms() throws { - let engineExpectation = self.expectation(description: "Engine Running") - - let engine = EngineBuilder() - .addLogLevel(.debug) - .addStatsFlushSeconds(1) - .setOnEngineRunning { - engineExpectation.fulfill() - } - .build() - - XCTAssertEqual(XCTWaiter.wait(for: [engineExpectation], timeout: 10), .completed) - - let pulseClient = engine.pulseClient() - let counter = pulseClient.counter(elements: ["foo", "bar", "distribution"]) - - counter.increment(count: 100) - - // Hit flushStats() many times in a row to make sure that there are no issues with - // concurrent flushing. - for _ in 0...100 { - engine.flushStats() - } - - engine.terminate() - } - - func testMultipleStatSinks() throws { - let engineExpectation = self.expectation(description: "Engine Running") - - let engine = EngineBuilder() - .addLogLevel(.debug) - .addStatsFlushSeconds(1) - .addGrpcStatsDomain("example.com") - .addStatsSinks( - [statsdSinkConfig(port: 1234), statsdSinkConfig(port: 5555)] - ) - .setOnEngineRunning { - engineExpectation.fulfill() - } - .build() - - XCTAssertEqual(XCTWaiter.wait(for: [engineExpectation], timeout: 10), .completed) - - engine.terminate() - } - - func statsdSinkConfig(port: Int) -> String { - return """ - { name: envoy.stat_sinks.statsd, - typed_config: { - "@type": type.googleapis.com/envoy.config.metrics.v3.StatsdSink, - address: { socket_address: { address: 127.0.0.1, port_value: \(port) } } - } - } -""" - } -} diff --git a/mobile/test/swift/stats/BUILD b/mobile/test/swift/stats/BUILD index 079158d2cdae..dbc34bd9c78a 100644 --- a/mobile/test/swift/stats/BUILD +++ b/mobile/test/swift/stats/BUILD @@ -1,7 +1,10 @@ +load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") load("@envoy_mobile//bazel:apple.bzl", "envoy_mobile_swift_test") licenses(["notice"]) # Apache 2 +envoy_mobile_package() + envoy_mobile_swift_test( name = "test", srcs = [ diff --git a/mobile/third_party/rbe_configs/config/BUILD b/mobile/third_party/rbe_configs/config/BUILD index 35069fba0019..be86c7f0724e 100644 --- a/mobile/third_party/rbe_configs/config/BUILD +++ b/mobile/third_party/rbe_configs/config/BUILD @@ -42,7 +42,7 @@ platform( "@bazel_tools//tools/cpp:clang", ], exec_properties = { - "container-image": "docker://envoyproxy/envoy-build-ubuntu:mobile-56f235b141079013e64912d676fe7da981368402@sha256:b3cfc59c2fd1a86a2b12d303324f33d7f7248458233f3be2959fab119b11fa6f", + "container-image": "docker://envoyproxy/envoy-build-ubuntu:mobile-fd9ec000fdd72d5c5e4e4ef16db4f9103058779e@sha256:1db9bac6578115179fe686cd0e58ee340e2cb3737d19d9136a264d5d94351961", "OSFamily": "Linux", "Pool": "linux", }, @@ -57,7 +57,7 @@ platform( "@bazel_tools//tools/cpp:clang", ], exec_properties = { - "container-image": "docker://envoyproxy/envoy-build-ubuntu:mobile-56f235b141079013e64912d676fe7da981368402@sha256:b3cfc59c2fd1a86a2b12d303324f33d7f7248458233f3be2959fab119b11fa6f", + "container-image": "docker://envoyproxy/envoy-build-ubuntu:mobile-fd9ec000fdd72d5c5e4e4ef16db4f9103058779e@sha256:1db9bac6578115179fe686cd0e58ee340e2cb3737d19d9136a264d5d94351961", "OSFamily": "Linux", "Pool": "linux", # Necessary to workaround https://github.com/google/sanitizers/issues/916, otherwise, dangling threads in the diff --git a/mobile/tools/check_format.sh b/mobile/tools/check_format.sh index 0a350b8f5e63..257a8e97506b 100755 --- a/mobile/tools/check_format.sh +++ b/mobile/tools/check_format.sh @@ -30,7 +30,7 @@ FORMAT_ARGS=( ./Envoy.xcodeproj/ ./dist/ ./bazel/envoy_mobile_swift_bazel_support.bzl ./bazel/envoy_mobile_repositories.bzl - ./examples/swift/swiftpm/Packages/Envoy.xcframework + ./examples/swift/swiftpm/Packages/Envoy.xcframework ./tmp --skip_envoy_build_rule_check) if [[ -n "$TARGET_PATH" ]]; then FORMAT_ARGS+=("$TARGET_PATH") @@ -41,9 +41,23 @@ FORMAT_ARGS+=( ./library/objective-c ./test/java ./test/java ./test/objective-c ./test/swift ./experimental/swift --build_fixer_check_excluded_paths - ./envoy ./BUILD ./dist ./examples ./library/java - ./library/kotlin ./library/objective-c ./library/swift - ./library/common/extensions ./test/java ./test/kotlin ./test/objective-c - ./test/swift ./experimental/swift) + ./envoy ./BUILD ./dist) export ENVOY_BAZEL_PREFIX="@envoy" && ./bazelw run @envoy//tools/code_format:check_format -- "${ENVOY_FORMAT_ACTION}" --path "$PWD" "${FORMAT_ARGS[@]}" + +KTFMT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"/ktfmt.sh +KOTLIN_DIRS=( + "library/kotlin" + "test/kotlin" + "examples/kotlin" +) +if [[ "${ENVOY_FORMAT_ACTION}" == "fix" ]]; then + "${KTFMT}" "${KOTLIN_DIRS[@]}" +else + NEEDS_FORMAT=$("${KTFMT}" --dry-run "${KOTLIN_DIRS[@]}") + if [[ -n "${NEEDS_FORMAT}" ]]; then + echo "ERROR: Run 'tools/check_format.sh fix' to fix" + echo "${NEEDS_FORMAT}" + exit 1 + fi +fi diff --git a/mobile/tools/ktfmt.sh b/mobile/tools/ktfmt.sh new file mode 100755 index 000000000000..2f585038a07c --- /dev/null +++ b/mobile/tools/ktfmt.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -euo pipefail + +ktfmt_version="0.46" +readonly ktfmt_version + +ktfmt_url="https://repo1.maven.org/maven2/com/facebook/ktfmt/${ktfmt_version}/ktfmt-${ktfmt_version}-jar-with-dependencies.jar" +readonly ktfmt_url + +ktfmt_sha256="97fc7fbd194d01a9fa45d8147c0552403003d55bac4ab89d84d7bb4d5e3f48de" +readonly ktfmt_sha256 + +jdk_url="https://cdn.azul.com/zulu/bin/zulu11.68.17-ca-jdk11.0.21-linux_x64.tar.gz" +readonly jdk_url + +jdk_sha256="725aba257da4bca14959060fea3faf59005eafdc2d5ccc3cb745403c5b60fb27" +readonly jdk_sha256 + +script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly script_root + +ktfmt_jar="$script_root/tmp/ktfmt/versions/ktfmt-0.46.jar" +readonly ktfmt_jar + +jdk="$script_root/tmp/jdk/versions/jdk11" +readonly jdk + +java="${jdk}"/bin/java +readonly java + +check_sha256sum() { + sha256="$1" + binary="$2" + sha_check=$(echo "${sha256}" "${binary}" | sha256sum --quiet --check || true) + echo "${sha_check}" + if [[ -n "${sha_check}" ]]; then + echo "Deleting ${binary}" >&2 + rm -f "$binary" + exit 1 + fi +} + +download_jdk() { + mkdir -p "${jdk}" + + download_temp_dir=$(mktemp -d) + jdk_tar_gz="${download_temp_dir}/jdk.tar.gz" + curl --fail -L --retry 5 --retry-connrefused --silent --progress-bar \ + --output "${jdk_tar_gz}" "$jdk_url" + + check_sha256sum "${jdk_sha256}" "${jdk_tar_gz}" + + tar -C "${jdk}" -xf "${jdk_tar_gz}" --strip-components=1 +} + +download_ktfmt() { + mkdir -p "$(dirname "${ktfmt_jar}")" + + curl --fail -L --retry 5 --retry-connrefused --silent --progress-bar \ + --output "$ktfmt_jar" "$ktfmt_url" + + check_sha256sum "${ktfmt_sha256}" "${ktfmt_jar}" +} + +# TODO(fredyw): Use CI's JDK when available. +if [[ ! -f "${java}" ]]; then + download_jdk +fi + +if [[ ! -f "${ktfmt_jar}" ]]; then + download_ktfmt +fi + +"${java}" -jar "${ktfmt_jar}" --google-style "$@" diff --git a/mobile/tools/what_to_run.sh b/mobile/tools/what_to_run.sh deleted file mode 100755 index 8daacd10892e..000000000000 --- a/mobile/tools/what_to_run.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -BRANCH_NAME="$GITHUB_REF_NAME" -BASE_COMMIT="$(git merge-base origin/main HEAD)" -CHANGED_FILES="$(git diff "${BASE_COMMIT}" --name-only)" -CHANGE_MATCH='^mobile/|^bazel/repository_locations\.bzl|^\.bazelrc|^\.bazelversion|^\.github/workflows/mobile-*|^\.github/workflows/_env.yml^tools/code_format/check_format.py' - -# The logic in this file is roughly: -# -# pull_request + changed files = run all mobile CI -# -# main = run some mobile CI -# -# all other commits = run minimal mobile CI (these jobs have no conditions) -# -# Branches are not currently tested, altho that should be restricted by the workflow - - -run_default_ci () { - { - echo "mobile_android_build=true" - echo "mobile_android_tests=true" - echo "mobile_asan=true" - echo "mobile_cc_tests=true" - echo "mobile_compile_time_options=true" - echo "mobile_coverage=true" - echo "mobile_formatting=true" - echo "mobile_ios_build=true" - echo "mobile_ios_tests=true" - echo "mobile_release_validation=true" - echo "mobile_tsan=true" - } >> "$GITHUB_OUTPUT" -} - -run_ci_for_changed_files () { - run_default_ci - { - echo "mobile_android_build_all=true" - echo "mobile_ios_build_all=true" - } >> "$GITHUB_OUTPUT" -} - -if [[ "$BRANCH_NAME" == "main" ]]; then - run_default_ci - exit 0 -fi - -if grep -qE "$CHANGE_MATCH" <<< "$CHANGED_FILES"; then - run_ci_for_changed_files -fi diff --git a/releases.asc b/releases.asc new file mode 100644 index 000000000000..8910f7738297 --- /dev/null +++ b/releases.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGNidjsBEADJeNLHg1Jj6r+988j6IGpg55Gbw1QlNt0iDrJGHYuDONtti70E +F3tJZmdreuue89jQZgDc9cBMdAixHpxenuBwfuKCT99txYocQGwZKVCu9U3sbRA6 +eDaD4adj+o05pznCzde+Evz4g0T/BIquj/EWuk6DH5BbOwI80XyOPaumef7hBqyU +goUPBIMkbh/JsnVAPtFsPDbHZsNO/lHDt5d7bbfWdtv58LIoGP/yNvufV7+vwymr +rWyuTBHQzanwDZwGGhJdaz6ubaiJ6eupheLS0d3xXJr4r2ZB3nGeCzW0g257Rd1Z +3FHhFmMNGROtSfja7Mk5Z2R/dp+dLbMHzFlo+GFkOUGNjXI3JbUCYlm+6+4nrwU6 +o45LJv9TusMHi6ncfCziV0WCDUGlL7dzNqjT5Fux/P/InjvnH/Iz7e/awdUnim1P +nJsOrT4pwfPJWYBtkf2aRmu+NICs5sqy6HIZgs09yOTOVyb5eKy2SV0APrGWq0OH +MkWd2WIwWuBeowByzGUymsrq7G2aQwqyJe4JYxqq8KFbjRisqhZqZ+xHxGuIvwAI +cxs/EmQLKsk/b06j5uojucv6SPKWSGGf7cO7PDfDqZxkQTNmcqkYHkGJA5D+hLsW +kt1RKIRRFineRUANIvwaFP5Ce06CX3urvOi8ZY0OL0jOx6dp2ldVoYOdSwARAQAB +tDZFbnZveSBtYWludGFpbmVycyA8ZW52b3ktbWFpbnRhaW5lcnNAZ29vZ2xlZ3Jv +dXBzLmNvbT6JAk4EEwEKADgWIQQK/Og2uk0dNXY8hSPYzcN1AYHzHwUCY2J2OwIb +AwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRDYzcN1AYHzHwDsD/kBLu+5BDx9 +2Cr0cHn6dENBVS3wL53RCKSG0Uzlh+SY90BN7WTI5KyACt3SZYZHbbZbAYvvXbr5 +9ooh0U+eGi3HsIqrP7GdrDC+8gtJB+Pud9+ltMCgU92IW/YAUSI/IgCbCOru6czc +H2Lah8S272YOpds/wuaiRSiMCU8TROcFXMfriGvcmNKwrJsRhJVARDZC3DB6m/ve +jWHAIX2eY7WhXyS6DJhs2iVrwz6fDxAiji5zrEZo+4CEUIfBojb+KeaC7PpusHu1 +Sleq+Ck6zekAoDX1MmbPOeBT8qCoTNQ1w30jj6CFW8T5ncVottPRIzQGe0S350HR +o8JLWdEAAds4QIHsvnaBAKfRHPRPo4Uw5n4/xklLJIXzbafhN/goLA/AC9tznQzq +A95SoMn7fMXMd/sLVOs4V1XHQsCaXt9gR+Qwd1tqBNC/mZBrCPkJiStnRrTufhRb +R97z1lCynJMJhdqeRX7VnwncNS62U30pfV0mtYpS3URKqc62Jm3+fJalnMPaeXfM +xgjlerxVNkYXFNb+5NEMQSKuGI4Uq+kLTmajk0aS4cJGM+JvjRx9UtW2rbfz9Svh +8dSKV89wVQ/Zu9R3dAontooo9eC5WTHYGoiPDjFqBGsaLePQyJzMwWJ/XYG8ClF1 +zQnAJL32VY1ECG7GbXw4tWtYdIErBnIckrkCDQRjYnY7ARAAto3DSLlcFkzKLBZ8 +9hmjgonSZlHyMMAP8Ys+js8jKic2aDp7IPorrQSD8autoahbnPBe/TMlkGf+pD7z +iqsIImClNAJrqArRifNEwHO96uMAFsa7UiyyUz+P0n7zR3oznasZo7r64oCE5f60 ++xbaoruyGiffiBnJ8uVqxEKvijilj2vCdc7ON+KOv6E6/iiBeQTEvTuSTs5Gm8IF +hRG2Ia4SHOCjdpglPU1mgoTF9PzFBbsAILLJxGhM8LldIeELELrofu760IvRapC+ +4baSW9eu4WuHraiD26ZpEZKAQgtO8YFPQyltBSbLB2CU7i6kBlXTUL34h8BOfzwM +lFdwz7aTQlY3ZxLtjh4FXdLlM0YqolmJjShIOyd865mlx2BqFnqQigNHXZdqQiEc +06+YN6CxichTiLCCN9+UwTjO7w/QTYOyb4vBs5ubMB54KE+peUor56i9/UdpYxmT +eaugXX9M74mNDpWcdfBwJZJHCjTYbX+NZnxNysyf5QWr3roSBPLHFMzL/fbBsbWe +kZ59p4LcBNLD6wIGtqbrX7fF5mH28B5ZFGV8q8yI3mpt13adIsuuw+S8dLfQrQwP +ipitYHsBQ0F6JtGFs8DzD7f2KoLlVbXgP40adoG1dPs3NmiGLVZkjmqDaHzL6Fwk +G41YcBDHyqSOpQMVS6pJiWJIvsMAEQEAAYkCNgQYAQoAIBYhBAr86Da6TR01djyF +I9jNw3UBgfMfBQJjYnY7AhsMAAoJENjNw3UBgfMffWwQALhSp2fedt/oRIEP8rdN +yFex6FSXlZQOMuGyeo6PeDob23O+WcTIcS5POM7UckERen+30ZPHsOET3jIPH4h5 +wIFuhUe/ap2JT6+CmsW+2kPz7nHGFQwCdjRScFOmRpEiu8+7X2l5LqUaO0PiC88m +n4fCWH1mIDqdGLxjd2EoNBt8vlQUt58wQbRPTs7P79209o44zYz04TTNL8iXzr3B +ngK13Z8QwaR1H56Ba3DdyU2xG016W1pN4B4C7DgTNWZwzRJkd3vg5AMzV6ZnP5+D +U38oQElfpGBy6YQCtC/ZVPDFNL0JaevraIG1LdFBqpUCu2bz4FURd1KoUITFdxeY +6sqL018q8wtTb7nRKMqPXiGelAlLvSaML8cq8CefUAlhjihhpJs14B1M+KOuCOt9 +r0VHdH1Zp5FOTutA+QgTwVDngNWacoxjMY/zaZxLPrB9SppPpR4UNUDvcXCBJzgr +34tnUdDvcb7fuysexVUiUCfs9dMuv8TxCYmaVjY2Hq7oBEN8pM1o05FfLqCXwdrB +lkCRorH68WyLJv/FKZTyfVyHn32WOjr5DKGI0rT/nwkRRaI0SQbKEejvUxlq1WdK +cn7oQEM2sZdH3v85GQU0fA/0V4UV12pha1b1p1rbRhdJh4h/iY+rUdRz/YyOMlz5 +xn73SlvaQy7ttkC8vuPiVI6g +=YtjB +-----END PGP PUBLIC KEY BLOCK----- diff --git a/source/common/access_log/BUILD b/source/common/access_log/BUILD index 92959e853067..869662afa0f4 100644 --- a/source/common/access_log/BUILD +++ b/source/common/access_log/BUILD @@ -22,11 +22,11 @@ envoy_cc_library( "//envoy/filesystem:filesystem_interface", "//envoy/http:header_map_interface", "//envoy/runtime:runtime_interface", - "//envoy/server:access_log_config_interface", "//envoy/upstream:upstream_interface", "//source/common/common:assert_lib", "//source/common/common:utility_lib", "//source/common/config:utility_lib", + "//source/common/formatter:substitution_formatter_lib", "//source/common/http:header_map_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index 7df906595462..24972a34d325 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -53,9 +53,9 @@ bool ComparisonFilter::compareAgainstValue(uint64_t lhs) const { } FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLogFilter& config, - Server::Configuration::CommonFactoryContext& context) { - Runtime::Loader& runtime = context.runtime(); - Random::RandomGenerator& random = context.api().randomGenerator(); + Server::Configuration::FactoryContext& context) { + Runtime::Loader& runtime = context.serverFactoryContext().runtime(); + Random::RandomGenerator& random = context.serverFactoryContext().api().randomGenerator(); ProtobufMessage::ValidationVisitor& validation_visitor = context.messageValidationVisitor(); switch (config.filter_specifier_case()) { case envoy::config::accesslog::v3::AccessLogFilter::FilterSpecifierCase::kStatusCodeFilter: @@ -98,16 +98,14 @@ FilterPtr FilterFactory::fromProto(const envoy::config::accesslog::v3::AccessLog return nullptr; } -bool TraceableRequestFilter::evaluate(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, AccessLogType) const { +bool TraceableRequestFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& info) const { const Tracing::Decision decision = Tracing::TracerUtility::shouldTraceRequest(info); return decision.traced && decision.reason == Tracing::Reason::ServiceForced; } -bool StatusCodeFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType) const { +bool StatusCodeFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& info) const { if (!info.responseCode()) { return compareAgainstValue(0ULL); } @@ -115,9 +113,8 @@ bool StatusCodeFilter::evaluate(const StreamInfo::StreamInfo& info, const Http:: return compareAgainstValue(info.responseCode().value()); } -bool DurationFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType) const { +bool DurationFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& info) const { absl::optional duration = info.currentDuration(); if (!duration.has_value()) { return false; @@ -133,9 +130,8 @@ RuntimeFilter::RuntimeFilter(const envoy::config::accesslog::v3::RuntimeFilter& percent_(config.percent_sampled()), use_independent_randomness_(config.use_independent_randomness()) {} -bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo& stream_info, - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, AccessLogType) const { +bool RuntimeFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) const { // This code is verbose to avoid preallocating a random number that is not needed. uint64_t random_value; if (use_independent_randomness_) { @@ -160,7 +156,7 @@ bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo& stream_info, OperatorFilter::OperatorFilter( const Protobuf::RepeatedPtrField& configs, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { for (const auto& config : configs) { auto filter = FilterFactory::fromProto(config, context); if (filter != nullptr) { @@ -170,22 +166,18 @@ OperatorFilter::OperatorFilter( } OrFilter::OrFilter(const envoy::config::accesslog::v3::OrFilter& config, - Server::Configuration::CommonFactoryContext& context) + Server::Configuration::FactoryContext& context) : OperatorFilter(config.filters(), context) {} AndFilter::AndFilter(const envoy::config::accesslog::v3::AndFilter& config, - Server::Configuration::CommonFactoryContext& context) + Server::Configuration::FactoryContext& context) : OperatorFilter(config.filters(), context) {} -bool OrFilter::evaluate(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - AccessLogType access_log_type) const { +bool OrFilter::evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const { bool result = false; for (auto& filter : filters_) { - result |= filter->evaluate(info, request_headers, response_headers, response_trailers, - access_log_type); + result |= filter->evaluate(context, info); if (result) { break; @@ -195,15 +187,11 @@ bool OrFilter::evaluate(const StreamInfo::StreamInfo& info, return result; } -bool AndFilter::evaluate(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - AccessLogType access_log_type) const { +bool AndFilter::evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const { bool result = true; for (auto& filter : filters_) { - result &= filter->evaluate(info, request_headers, response_headers, response_trailers, - access_log_type); + result &= filter->evaluate(context, info); if (!result) { break; @@ -213,20 +201,17 @@ bool AndFilter::evaluate(const StreamInfo::StreamInfo& info, return result; } -bool NotHealthCheckFilter::evaluate(const StreamInfo::StreamInfo& info, - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, AccessLogType) const { +bool NotHealthCheckFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& info) const { return !info.healthCheck(); } HeaderFilter::HeaderFilter(const envoy::config::accesslog::v3::HeaderFilter& config) : header_data_(std::make_unique(config.header())) {} -bool HeaderFilter::evaluate(const StreamInfo::StreamInfo&, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType) const { - return Http::HeaderUtility::matchHeaders(request_headers, *header_data_); +bool HeaderFilter::evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo&) const { + return Http::HeaderUtility::matchHeaders(context.requestHeaders(), *header_data_); } ResponseFlagFilter::ResponseFlagFilter( @@ -240,9 +225,8 @@ ResponseFlagFilter::ResponseFlagFilter( } } -bool ResponseFlagFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType) const { +bool ResponseFlagFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& info) const { if (configured_flags_ != 0) { return info.intersectResponseFlags(configured_flags_); } @@ -257,14 +241,12 @@ GrpcStatusFilter::GrpcStatusFilter(const envoy::config::accesslog::v3::GrpcStatu exclude_ = config.exclude(); } -bool GrpcStatusFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - AccessLogType) const { +bool GrpcStatusFilter::evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const { Grpc::Status::GrpcStatus status = Grpc::Status::WellKnownGrpcStatus::Unknown; const auto& optional_status = - Grpc::Common::getGrpcStatus(response_trailers, response_headers, info); + Grpc::Common::getGrpcStatus(context.responseTrailers(), context.responseHeaders(), info); if (optional_status.has_value()) { status = optional_status.value(); } @@ -286,10 +268,9 @@ LogTypeFilter::LogTypeFilter(const envoy::config::accesslog::v3::LogTypeFilter& exclude_ = config.exclude(); } -bool LogTypeFilter::evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType access_log_type) const { - const bool found = types_.contains(access_log_type); +bool LogTypeFilter::evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo&) const { + const bool found = types_.contains(context.accessLogType()); return exclude_ ? !found : found; } @@ -315,9 +296,8 @@ MetadataFilter::MetadataFilter(const envoy::config::accesslog::v3::MetadataFilte present_matcher_ = Matchers::ValueMatcher::create(present_val); } -bool MetadataFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType) const { +bool MetadataFilter::evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& info) const { const auto& value = Envoy::Config::Metadata::metadataValue(&info.dynamicMetadata(), filter_, path_); // If the key corresponds to a set value in dynamic metadata, return true if the value matches the @@ -331,37 +311,19 @@ bool MetadataFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::Re return default_match_; } -namespace { - -template -InstanceSharedPtr makeAccessLogInstance(const envoy::config::accesslog::v3::AccessLog& config, - FactoryContext& context) { +InstanceSharedPtr AccessLogFactory::fromProto(const envoy::config::accesslog::v3::AccessLog& config, + Server::Configuration::FactoryContext& context) { FilterPtr filter; if (config.has_filter()) { filter = FilterFactory::fromProto(config.filter(), context); } - auto& factory = - Config::Utility::getAndCheckFactory(config); + auto& factory = Config::Utility::getAndCheckFactory(config); ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( config, context.messageValidationVisitor(), factory); return factory.createAccessLogInstance(*message, std::move(filter), context); } -} // namespace - -InstanceSharedPtr -AccessLogFactory::fromProto(const envoy::config::accesslog::v3::AccessLog& config, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return makeAccessLogInstance(config, context); -} - -InstanceSharedPtr -AccessLogFactory::fromProto(const envoy::config::accesslog::v3::AccessLog& config, - Server::Configuration::CommonFactoryContext& context) { - return makeAccessLogInstance(config, context); -} - } // namespace AccessLog } // namespace Envoy diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index a60a09bbcb56..298b481cbd2e 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -10,12 +10,12 @@ #include "envoy/config/accesslog/v3/accesslog.pb.h" #include "envoy/config/typed_config.h" #include "envoy/runtime/runtime.h" -#include "envoy/server/access_log_config.h" #include "envoy/type/v3/percent.pb.h" #include "source/common/common/matchers.h" #include "source/common/common/utility.h" #include "source/common/config/utility.h" +#include "source/common/formatter/http_specific_formatter.h" #include "source/common/grpc/status.h" #include "source/common/http/header_utility.h" #include "source/common/protobuf/protobuf.h" @@ -35,7 +35,7 @@ class FilterFactory { * Read a filter definition from proto and instantiate a concrete filter class. */ static FilterPtr fromProto(const envoy::config::accesslog::v3::AccessLogFilter& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::FactoryContext& context); }; /** @@ -62,9 +62,8 @@ class StatusCodeFilter : public ComparisonFilter { : ComparisonFilter(config.comparison(), runtime) {} // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; }; /** @@ -77,9 +76,8 @@ class DurationFilter : public ComparisonFilter { : ComparisonFilter(config.comparison(), runtime) {} // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; }; /** @@ -89,7 +87,7 @@ class OperatorFilter : public Filter { public: OperatorFilter( const Protobuf::RepeatedPtrField& configs, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::FactoryContext& context); protected: std::vector filters_; @@ -101,12 +99,11 @@ class OperatorFilter : public Filter { class AndFilter : public OperatorFilter { public: AndFilter(const envoy::config::accesslog::v3::AndFilter& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::FactoryContext& context); // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; }; /** @@ -115,12 +112,11 @@ class AndFilter : public OperatorFilter { class OrFilter : public OperatorFilter { public: OrFilter(const envoy::config::accesslog::v3::OrFilter& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::FactoryContext& context); // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; }; /** @@ -131,9 +127,8 @@ class NotHealthCheckFilter : public Filter { NotHealthCheckFilter() = default; // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; }; /** @@ -142,9 +137,8 @@ class NotHealthCheckFilter : public Filter { class TraceableRequestFilter : public Filter { public: // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; }; /** @@ -156,9 +150,8 @@ class RuntimeFilter : public Filter { Random::RandomGenerator& random); // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; private: Runtime::Loader& runtime_; @@ -176,9 +169,8 @@ class HeaderFilter : public Filter { HeaderFilter(const envoy::config::accesslog::v3::HeaderFilter& config); // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; private: const Http::HeaderUtility::HeaderDataPtr header_data_; @@ -192,9 +184,8 @@ class ResponseFlagFilter : public Filter { ResponseFlagFilter(const envoy::config::accesslog::v3::ResponseFlagFilter& config); // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; private: uint64_t configured_flags_{}; @@ -213,9 +204,8 @@ class GrpcStatusFilter : public Filter { GrpcStatusFilter(const envoy::config::accesslog::v3::GrpcStatusFilter& config); // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; private: GrpcStatusHashSet statuses_; @@ -238,9 +228,8 @@ class LogTypeFilter : public Filter { LogTypeFilter(const envoy::config::accesslog::v3::LogTypeFilter& filter_config); - bool evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType access_log_type) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; private: LogTypeHashSet types_; @@ -254,9 +243,8 @@ class MetadataFilter : public Filter { public: MetadataFilter(const envoy::config::accesslog::v3::MetadataFilter& filter_config); - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType) const override; + bool evaluate(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const override; private: Matchers::ValueMatcherConstSharedPtr present_matcher_; @@ -268,27 +256,6 @@ class MetadataFilter : public Filter { const std::string filter_; }; -/** - * Extension filter factory that reads from ExtensionFilter proto. - */ -class ExtensionFilterFactory : public Config::TypedFactory { -public: - ~ExtensionFilterFactory() override = default; - - /** - * Create a particular extension filter implementation from a config proto. If the - * implementation is unable to produce a filter with the provided parameters, it should throw an - * EnvoyException. The returned pointer should never be nullptr. - * @param config supplies the custom configuration for this filter type. - * @param context supplies the server factory context. - * @return an instance of extension filter implementation from a config proto. - */ - virtual FilterPtr createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, - Server::Configuration::CommonFactoryContext& context) PURE; - - std::string category() const override { return "envoy.access_loggers.extension_filters"; } -}; - /** * Access log factory that reads the configuration from proto. */ @@ -298,16 +265,8 @@ class AccessLogFactory { * Read a filter definition from proto and instantiate an Instance. This method is used * to create access log instances that need access to listener properties. */ - static InstanceSharedPtr - fromProto(const envoy::config::accesslog::v3::AccessLog& config, - Server::Configuration::ListenerAccessLogFactoryContext& context); - - /** - * Read a filter definition from proto and instantiate an Instance. This method does not - * have access to listener properties, for example for access loggers of admin interface. - */ static InstanceSharedPtr fromProto(const envoy::config::accesslog::v3::AccessLog& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::FactoryContext& context); /** * Template method to create an access log filter from proto configuration for non-HTTP access @@ -316,7 +275,7 @@ class AccessLogFactory { template static FilterBasePtr accessLogFilterFromProto(const envoy::config::accesslog::v3::AccessLogFilter& config, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { if (!config.has_extension_filter()) { ExceptionUtil::throwEnvoyException( "Access log filter: only extension filter is supported by non-HTTP access loggers."); @@ -324,10 +283,7 @@ class AccessLogFactory { auto& factory = Config::Utility::getAndCheckFactory>( config.extension_filter()); - auto typed_filter_config = Config::Utility::translateToFactoryConfig( - config.extension_filter(), context.messageValidationVisitor(), factory); - - return factory.createFilter(*typed_filter_config, context); + return factory.createFilter(config.extension_filter(), context); } /** @@ -337,7 +293,7 @@ class AccessLogFactory { template static InstanceBaseSharedPtr accessLoggerFromProto(const envoy::config::accesslog::v3::AccessLog& config, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { FilterBasePtr filter; if (config.has_filter()) { filter = accessLogFilterFromProto(config.filter(), context); diff --git a/source/common/access_log/access_log_manager_impl.cc b/source/common/access_log/access_log_manager_impl.cc index ba3486ca545f..27e289359d73 100644 --- a/source/common/access_log/access_log_manager_impl.cc +++ b/source/common/access_log/access_log_manager_impl.cc @@ -54,8 +54,8 @@ AccessLogFileImpl::AccessLogFileImpl(Filesystem::FilePtr&& file, Event::Dispatch flush_timer_->enableTimer(flush_interval_msec_); auto open_result = open(); if (!open_result.return_value_) { - throw EnvoyException(fmt::format("unable to open file '{}': {}", file_->path(), - open_result.err_->getErrorDetails())); + throwEnvoyExceptionOrPanic(fmt::format("unable to open file '{}': {}", file_->path(), + open_result.err_->getErrorDetails())); } } diff --git a/source/common/buffer/BUILD b/source/common/buffer/BUILD index b1cd4bb68dfc..0545b3b1558d 100644 --- a/source/common/buffer/BUILD +++ b/source/common/buffer/BUILD @@ -30,6 +30,7 @@ envoy_cc_library( "//source/common/common:non_copyable", "//source/common/common:utility_lib", "//source/common/event:libevent_lib", + "@com_google_absl//absl/functional:any_invocable", ], ) diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 058657dc5abd..317f4b4a06c1 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -14,6 +14,8 @@ #include "source/common/common/utility.h" #include "source/common/event/libevent.h" +#include "absl/functional/any_invocable.h" + namespace Envoy { namespace Buffer { @@ -30,7 +32,7 @@ namespace Buffer { * +-----------------+----------------+----------------------+ * ^ ^ ^ ^ * | | | | - * base_ data() base_ + reservable_ base_ + capacity_ + * base_ base_ + data_ base_ + reservable_ base_ + capacity_ */ class Slice { public: @@ -90,7 +92,7 @@ class Slice { : capacity_(fragment.size()), storage_(nullptr), base_(static_cast(const_cast(fragment.data()))), reservable_(fragment.size()) { - addDrainTracker([&fragment]() { fragment.done(); }); + releasor_ = [&fragment]() { fragment.done(); }; } Slice(Slice&& rhs) noexcept { @@ -101,6 +103,7 @@ class Slice { reservable_ = rhs.reservable_; drain_trackers_ = std::move(rhs.drain_trackers_); account_ = std::move(rhs.account_); + releasor_.swap(rhs.releasor_); rhs.capacity_ = 0; rhs.base_ = nullptr; @@ -119,6 +122,11 @@ class Slice { reservable_ = rhs.reservable_; drain_trackers_ = std::move(rhs.drain_trackers_); account_ = std::move(rhs.account_); + if (releasor_) { + releasor_(); + } + releasor_ = rhs.releasor_; + rhs.releasor_ = nullptr; rhs.capacity_ = 0; rhs.base_ = nullptr; @@ -129,7 +137,12 @@ class Slice { return *this; } - ~Slice() { callAndClearDrainTrackersAndCharges(); } + ~Slice() { + callAndClearDrainTrackersAndCharges(); + if (releasor_) { + releasor_(); + } + } /** * @return true if the data in the slice is mutable @@ -307,6 +320,9 @@ class Slice { void transferDrainTrackersTo(Slice& destination) { destination.drain_trackers_.splice(destination.drain_trackers_.end(), drain_trackers_); ASSERT(drain_trackers_.empty()); + // The releasor needn't to be transferred, and actually if there is releasor, this + // slice can't coalesce. Then there won't be a chance to calling this method. + ASSERT(releasor_ == nullptr); } /** @@ -397,6 +413,9 @@ class Slice { /** Account associated with this slice. This may be null. When * coalescing with another slice, we do not transfer over their account. */ BufferMemoryAccountSharedPtr account_; + + /** The releasor for the BufferFragment */ + std::function releasor_; }; class OwnedImpl; @@ -587,8 +606,8 @@ class BufferFragmentImpl : NonCopyable, public BufferFragment { */ BufferFragmentImpl( const void* data, size_t size, - const std::function& releasor) - : data_(data), size_(size), releasor_(releasor) {} + absl::AnyInvocable releasor) + : data_(data), size_(size), releasor_(std::move(releasor)) {} // Buffer::BufferFragment const void* data() const override { return data_; } @@ -602,7 +621,7 @@ class BufferFragmentImpl : NonCopyable, public BufferFragment { private: const void* const data_; const size_t size_; - const std::function releasor_; + absl::AnyInvocable releasor_; }; class LibEventInstance : public Instance { diff --git a/source/common/buffer/watermark_buffer.cc b/source/common/buffer/watermark_buffer.cc index f7e3297baebc..f2b6208a663a 100644 --- a/source/common/buffer/watermark_buffer.cc +++ b/source/common/buffer/watermark_buffer.cc @@ -209,22 +209,22 @@ uint64_t WatermarkBufferFactory::resetAccountsGivenPressure(float pressure) { const uint32_t buckets_to_clear = std::min( std::floor(pressure * BufferMemoryAccountImpl::NUM_MEMORY_CLASSES_) + 1, 8); - uint32_t last_bucket_to_clear = BufferMemoryAccountImpl::NUM_MEMORY_CLASSES_ - buckets_to_clear; - ENVOY_LOG_MISC(warn, "resetting streams in buckets >= {}", last_bucket_to_clear); - // Clear buckets, prioritizing the buckets with larger streams. uint32_t num_streams_reset = 0; + uint32_t num_buckets_reset = 0; for (uint32_t buckets_cleared = 0; buckets_cleared < buckets_to_clear; ++buckets_cleared) { const uint32_t bucket_to_clear = BufferMemoryAccountImpl::NUM_MEMORY_CLASSES_ - buckets_cleared - 1; - ENVOY_LOG_MISC(warn, "resetting {} streams in bucket {}.", - size_class_account_sets_[bucket_to_clear].size(), bucket_to_clear); - - auto it = size_class_account_sets_[bucket_to_clear].begin(); - while (it != size_class_account_sets_[bucket_to_clear].end()) { - if (num_streams_reset >= kMaxNumberOfStreamsToResetPerInvocation) { - return num_streams_reset; - } + absl::flat_hash_set& bucket = + size_class_account_sets_[bucket_to_clear]; + + if (bucket.empty()) { + continue; + } + ++num_buckets_reset; + + auto it = bucket.begin(); + while (it != bucket.end() && num_streams_reset < kMaxNumberOfStreamsToResetPerInvocation) { auto next = std::next(it); // This will trigger an erase, which avoids rehashing and invalidates the // iterator *it*. *next* is still valid. @@ -233,7 +233,10 @@ uint64_t WatermarkBufferFactory::resetAccountsGivenPressure(float pressure) { ++num_streams_reset; } } - + if (num_buckets_reset > 0) { + ENVOY_LOG_MISC(warn, "resetting {} streams in {} buckets, {} empty buckets", num_streams_reset, + num_buckets_reset, buckets_to_clear - num_buckets_reset); + } return num_streams_reset; } diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 14bcf9917df7..f1fb01081e22 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -207,7 +207,10 @@ envoy_cc_library( "json_escape_string.h", "logger.h", ], - external_deps = ["abseil_synchronization"], + external_deps = [ + "abseil_synchronization", + "abseil_flat_hash_map", + ], deps = [ ":base_logger_lib", ":fmt_lib", @@ -348,7 +351,7 @@ envoy_cc_library( "//envoy/registry", "//source/common/protobuf:utility_lib", "//source/common/stats:symbol_table_lib", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg_cc_proto", "@com_googlesource_code_re2//:re2", "@envoy_api//envoy/extensions/regex_engines/v3:pkg_cc_proto", "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", diff --git a/source/common/common/cleanup.h b/source/common/common/cleanup.h index ae1d3cb0d55a..3d34bf51b805 100644 --- a/source/common/common/cleanup.h +++ b/source/common/common/cleanup.h @@ -13,6 +13,10 @@ class Cleanup { Cleanup(std::function f) : f_(std::move(f)) {} ~Cleanup() { f_(); } + // Copying leads to cleanup multiple times, so only allow move. + Cleanup(const Cleanup&) = delete; + Cleanup(Cleanup&&) = default; + void cancel() { cancelled_ = true; f_ = []() {}; diff --git a/source/common/common/fine_grain_logger.cc b/source/common/common/fine_grain_logger.cc index 3d9f53eb62d0..5fd65e845740 100644 --- a/source/common/common/fine_grain_logger.cc +++ b/source/common/common/fine_grain_logger.cc @@ -93,14 +93,9 @@ std::string FineGrainLogContext::listFineGrainLoggers() ABSL_LOCKS_EXCLUDED(fine void FineGrainLogContext::setAllFineGrainLoggers(spdlog::level::level_enum level) ABSL_LOCKS_EXCLUDED(fine_grain_log_lock_) { absl::ReaderMutexLock l(&fine_grain_log_lock_); - if (verbosity_update_info_.empty()) { - for (const auto& it : *fine_grain_log_map_) { - it.second->set_level(level); - } - } else { - for (const auto& [key, logger] : *fine_grain_log_map_) { - logger->set_level(getLogLevel(key)); - } + verbosity_update_info_.clear(); + for (const auto& it : *fine_grain_log_map_) { + it.second->set_level(level); } } @@ -174,9 +169,6 @@ spdlog::logger* FineGrainLogContext::createLogger(const std::string& key) void FineGrainLogContext::updateVerbosityDefaultLevel(level_enum level) { { absl::WriterMutexLock wl(&fine_grain_log_lock_); - if (level == verbosity_default_level_) { - return; - } verbosity_default_level_ = level; } @@ -229,18 +221,17 @@ level_enum FineGrainLogContext::getLogLevel(absl::string_view file) const { } } - absl::string_view stem = file, stem_basename = basename; + absl::string_view stem_basename = basename; { const size_t sep = stem_basename.find('.'); if (sep != stem_basename.npos) { - stem.remove_suffix(stem_basename.size() - sep); stem_basename.remove_suffix(stem_basename.size() - sep); } } for (const auto& info : verbosity_update_info_) { if (info.update_is_path) { // If there are any slashes in the pattern, try to match the full path name. - if (safeFileNameMatch(info.update_pattern, stem)) { + if (safeFileNameMatch(info.update_pattern, file)) { return info.log_level; } } else if (safeFileNameMatch(info.update_pattern, stem_basename)) { diff --git a/source/common/common/hash.h b/source/common/common/hash.h index 01b1af1f0240..b1af0f03660b 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "source/common/common/macros.h" #include "source/common/common/safe_memcpy.h" @@ -26,6 +27,58 @@ class HashUtil { return XXH64(input.data(), input.size(), seed); } + /** + * Return 64-bit hash from deterministically serializing a value in + * an endian-independent way and hashing it with the xxHash algorithm. + * + * enums are excluded because they're not needed; if they were to be + * supported later they should have a separate function because enum + * sizes are implementation-specific. + * + * bools have an inlined specialization below because they too have + * implementation-specific sizes. + * + * floating point values have a specialization because just endian- + * ordering the binary leaves many possible binary representations of + * NaN, for example. + * + * @param input supplies the value to hash. + * @param seed supplies the hash seed which defaults to 0. + * See https://github.com/Cyan4973/xxHash for details. + */ + template < + typename ValueType, + std::enable_if_t && !std::is_enum_v, bool> = true> + static uint64_t xxHash64Value(ValueType input, uint64_t seed = 0) { +#if defined(ABSL_IS_LITTLE_ENDIAN) + return XXH64(reinterpret_cast(&input), sizeof(input), seed); +#else + char buf[sizeof(input)]; + const char* p = reinterpret_cast(&input); + for (size_t i = 0; i < sizeof(input); i++) { + buf[i] = p[sizeof(input) - i - 1]; + } + return XXH64(buf, sizeof(input), seed); +#endif + } + template , bool> = true> + static uint64_t xxHash64FloatingPoint(FloatingPoint input, uint64_t seed = 0) { + if (std::isnan(input)) { + return XXH64("NaN", 3, seed); + } + if (std::isinf(input)) { + return XXH64("Inf", 3, seed); + } + int exp; + FloatingPoint frac = std::frexp(input, &exp); + seed = xxHash64Value(exp, seed); + // Turn the fraction between -1 and 1 we have into an integer we can + // hash endian-independently, using the largest possible range. + int64_t mantissa = frac * 9223372036854775808.0; // 2^63 + return xxHash64Value(mantissa, seed); + } + /** * Return 64-bit hash from the xxHash algorithm for a collection of strings. * @param input supplies the absl::Span to hash. @@ -51,6 +104,20 @@ class HashUtil { } }; +// Explicit specialization for bool because its size may be implementation dependent. +template <> inline uint64_t HashUtil::xxHash64Value(bool input, uint64_t seed) { + char b = input ? 1 : 0; + return XXH64(&b, sizeof(b), seed); +} +// Explicit specialization for float and double because IEEE binary representation of NaN +// can be inconsistent. +template <> inline uint64_t HashUtil::xxHash64Value(double input, uint64_t seed) { + return xxHash64FloatingPoint(input, seed); +} +template <> inline uint64_t HashUtil::xxHash64Value(float input, uint64_t seed) { + return xxHash64FloatingPoint(input, seed); +} + /** * From * (https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libstdc%2b%2b-v3/libsupc%2b%2b/hash_bytes.cc). diff --git a/source/common/common/logger.cc b/source/common/common/logger.cc index 77e499c9e702..57e65b316d24 100644 --- a/source/common/common/logger.cc +++ b/source/common/common/logger.cc @@ -158,7 +158,8 @@ void DelegatingLogSink::setTlsDelegate(SinkDelegate* sink) { *tlsSink() = sink; SinkDelegate* DelegatingLogSink::tlsDelegate() { return *tlsSink(); } -static Context* current_context = nullptr; +static std::atomic current_context = nullptr; +static_assert(std::atomic::is_always_lock_free); Context::Context(spdlog::level::level_enum log_level, const std::string& log_format, Thread::BasicLockable& lock, bool should_escape, bool enable_fine_grain_logging) @@ -171,7 +172,7 @@ Context::Context(spdlog::level::level_enum log_level, const std::string& log_for Context::~Context() { current_context = save_context_; if (current_context != nullptr) { - current_context->activate(); + current_context.load()->activate(); } else { Registry::getSink()->clearLock(); } @@ -197,7 +198,7 @@ void Context::activate() { bool Context::useFineGrainLogger() { if (current_context) { - return current_context->enable_fine_grain_logging_; + return current_context.load()->enable_fine_grain_logging_; } return false; } @@ -209,28 +210,31 @@ void Context::changeAllLogLevels(spdlog::level::level_enum level) { Registry::setLogLevel(level); } else { // Level setting with Fine-Grain Logger. - FINE_GRAIN_LOG(info, "change all log levels: level='{}'", - spdlog::level::level_string_views[level]); - getFineGrainLogContext().setAllFineGrainLoggers(level); + FINE_GRAIN_LOG( + info, + "change all log levels and default verbosity level for fine grain loggers: level='{}'", + spdlog::level::level_string_views[level]); + getFineGrainLogContext().updateVerbosityDefaultLevel(level); } } void Context::enableFineGrainLogger() { if (current_context) { - current_context->enable_fine_grain_logging_ = true; - current_context->fine_grain_default_level_ = current_context->log_level_; - current_context->fine_grain_log_format_ = current_context->log_format_; - if (current_context->log_format_ == Logger::Logger::DEFAULT_LOG_FORMAT) { - current_context->fine_grain_log_format_ = kDefaultFineGrainLogFormat; + current_context.load()->enable_fine_grain_logging_ = true; + current_context.load()->fine_grain_default_level_ = current_context.load()->log_level_; + current_context.load()->fine_grain_log_format_ = current_context.load()->log_format_; + if (current_context.load()->log_format_ == Logger::Logger::DEFAULT_LOG_FORMAT) { + current_context.load()->fine_grain_log_format_ = kDefaultFineGrainLogFormat; } getFineGrainLogContext().setDefaultFineGrainLogLevelFormat( - current_context->fine_grain_default_level_, current_context->fine_grain_log_format_); + current_context.load()->fine_grain_default_level_, + current_context.load()->fine_grain_log_format_); } } void Context::disableFineGrainLogger() { if (current_context) { - current_context->enable_fine_grain_logging_ = false; + current_context.load()->enable_fine_grain_logging_ = false; } } @@ -238,14 +242,14 @@ std::string Context::getFineGrainLogFormat() { if (!current_context) { // Context is not instantiated in benchmark test return kDefaultFineGrainLogFormat; } - return current_context->fine_grain_log_format_; + return current_context.load()->fine_grain_log_format_; } spdlog::level::level_enum Context::getFineGrainDefaultLevel() { if (!current_context) { return spdlog::level::info; } - return current_context->fine_grain_default_level_; + return current_context.load()->fine_grain_default_level_; } std::vector& Registry::allLoggers() { diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 756c3f942a5a..a90e43628e1e 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -39,6 +39,7 @@ const static bool should_log = true; FUNCTION(aws) \ FUNCTION(assert) \ FUNCTION(backtrace) \ + FUNCTION(basic_auth) \ FUNCTION(cache_filter) \ FUNCTION(client) \ FUNCTION(config) \ @@ -54,6 +55,7 @@ const static bool should_log = true; FUNCTION(file) \ FUNCTION(filter) \ FUNCTION(forward_proxy) \ + FUNCTION(geolocation) \ FUNCTION(grpc) \ FUNCTION(happy_eyeballs) \ FUNCTION(hc) \ diff --git a/source/common/common/matchers.cc b/source/common/common/matchers.cc index 1026016fdd51..ac623b85ad8d 100644 --- a/source/common/common/matchers.cc +++ b/source/common/common/matchers.cc @@ -31,6 +31,8 @@ ValueMatcherConstSharedPtr ValueMatcher::create(const envoy::type::matcher::v3:: return std::make_shared(v.present_match()); case envoy::type::matcher::v3::ValueMatcher::MatchPatternCase::kListMatch: return std::make_shared(v.list_match()); + case envoy::type::matcher::v3::ValueMatcher::MatchPatternCase::kOrMatch: + return std::make_shared(v.or_match()); case envoy::type::matcher::v3::ValueMatcher::MatchPatternCase::MATCH_PATTERN_NOT_SET: break; // Fall through to PANIC. } @@ -88,6 +90,22 @@ bool ListMatcher::match(const ProtobufWkt::Value& value) const { return false; } +OrMatcher::OrMatcher(const envoy::type::matcher::v3::OrMatcher& matcher) { + or_matchers_.reserve(matcher.value_matchers().size()); + for (const auto& or_matcher : matcher.value_matchers()) { + or_matchers_.push_back(ValueMatcher::create(or_matcher)); + } +} + +bool OrMatcher::match(const ProtobufWkt::Value& value) const { + for (const auto& or_matcher : or_matchers_) { + if (or_matcher->match(value)) { + return true; + } + } + return false; +} + MetadataMatcher::MetadataMatcher(const envoy::type::matcher::v3::MetadataMatcher& matcher) : matcher_(matcher) { for (const auto& seg : matcher.path()) { diff --git a/source/common/common/matchers.h b/source/common/common/matchers.h index 5bdd03bb84a1..e96f3689322f 100644 --- a/source/common/common/matchers.h +++ b/source/common/common/matchers.h @@ -172,6 +172,16 @@ class ListMatcher : public ValueMatcher { ValueMatcherConstSharedPtr oneof_value_matcher_; }; +class OrMatcher : public ValueMatcher { +public: + OrMatcher(const envoy::type::matcher::v3::OrMatcher& matcher); + + bool match(const ProtobufWkt::Value& value) const override; + +private: + std::vector or_matchers_; +}; + class MetadataMatcher { public: MetadataMatcher(const envoy::type::matcher::v3::MetadataMatcher& matcher); diff --git a/source/common/common/regex.cc b/source/common/common/regex.cc index 0281d658988d..086de371b7b3 100644 --- a/source/common/common/regex.cc +++ b/source/common/common/regex.cc @@ -17,7 +17,7 @@ CompiledGoogleReMatcher::CompiledGoogleReMatcher(const std::string& regex, bool do_program_size_check) : regex_(regex, re2::RE2::Quiet) { if (!regex_.ok()) { - throw EnvoyException(regex_.error()); + throwEnvoyExceptionOrPanic(regex_.error()); } if (do_program_size_check && Runtime::isRuntimeInitialized()) { @@ -25,10 +25,11 @@ CompiledGoogleReMatcher::CompiledGoogleReMatcher(const std::string& regex, const uint32_t max_program_size_error_level = Runtime::getInteger("re2.max_program_size.error_level", 100); if (regex_program_size > max_program_size_error_level) { - throw EnvoyException(fmt::format("regex '{}' RE2 program size of {} > max program size of " - "{} set for the error level threshold. Increase " - "configured max program size if necessary.", - regex, regex_program_size, max_program_size_error_level)); + throwEnvoyExceptionOrPanic( + fmt::format("regex '{}' RE2 program size of {} > max program size of " + "{} set for the error level threshold. Increase " + "configured max program size if necessary.", + regex, regex_program_size, max_program_size_error_level)); } const uint32_t max_program_size_warn_level = @@ -52,9 +53,10 @@ CompiledGoogleReMatcher::CompiledGoogleReMatcher( const uint32_t max_program_size = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.google_re2(), max_program_size, 100); if (regex_program_size > max_program_size) { - throw EnvoyException(fmt::format("regex '{}' RE2 program size of {} > max program size of " - "{}. Increase configured max program size if necessary.", - config.regex(), regex_program_size, max_program_size)); + throwEnvoyExceptionOrPanic( + fmt::format("regex '{}' RE2 program size of {} > max program size of " + "{}. Increase configured max program size if necessary.", + config.regex(), regex_program_size, max_program_size)); } } } diff --git a/source/common/common/shared_token_bucket_impl.cc b/source/common/common/shared_token_bucket_impl.cc index f0942226afcf..63807a683595 100644 --- a/source/common/common/shared_token_bucket_impl.cc +++ b/source/common/common/shared_token_bucket_impl.cc @@ -11,7 +11,7 @@ const char SharedTokenBucketImpl::ResetCheckSyncPoint[] = "post_reset_check"; SharedTokenBucketImpl::SharedTokenBucketImpl(uint64_t max_tokens, TimeSource& time_source, double fill_rate) - : impl_(max_tokens, time_source, fill_rate), reset_once_(false) {} + : impl_(max_tokens, time_source, fill_rate) {} uint64_t SharedTokenBucketImpl::consume(uint64_t tokens, bool allow_partial) { Thread::LockGuard lock(mutex_); diff --git a/source/common/common/shared_token_bucket_impl.h b/source/common/common/shared_token_bucket_impl.h index 51594c3da57f..09cdda5cfcb6 100644 --- a/source/common/common/shared_token_bucket_impl.h +++ b/source/common/common/shared_token_bucket_impl.h @@ -43,7 +43,7 @@ class SharedTokenBucketImpl : public TokenBucket { private: Thread::MutexBasicLockable mutex_; TokenBucketImpl impl_ ABSL_GUARDED_BY(mutex_); - bool reset_once_ ABSL_GUARDED_BY(mutex_); + bool reset_once_ ABSL_GUARDED_BY(mutex_){false}; mutable Thread::ThreadSynchronizer synchronizer_; // Used only for testing. friend class SharedTokenBucketImplTest; }; diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 3c425812fa4e..144d57f69a12 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -408,7 +408,7 @@ std::string StringUtil::removeTokens(absl::string_view source, absl::string_view uint32_t StringUtil::itoa(char* out, size_t buffer_size, uint64_t i) { // The maximum size required for an unsigned 64-bit integer is 21 chars (including null). if (buffer_size < 21) { - throw std::invalid_argument("itoa buffer too small"); + throwExceptionOrPanic(std::invalid_argument, "itoa buffer too small"); } char* current = out; @@ -651,7 +651,7 @@ InlineString::InlineString(const char* str, size_t size) : size_(size) { } void ExceptionUtil::throwEnvoyException(const std::string& message) { - throw EnvoyException(message); + throwEnvoyExceptionOrPanic(message); } } // namespace Envoy diff --git a/source/common/common/utility.h b/source/common/common/utility.h index a616c3aebeff..a689c90397e0 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -803,7 +803,7 @@ class InlineString : public InlineStorage { /** * @return a std::string copy of the InlineString. */ - std::string toString() const { return std::string(data_, size_); } + std::string toString() const { return {data_, size_}; } /** * @return a string_view into the InlineString. diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 0813e2bb694d..dffb0c2b7f3f 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -68,7 +68,7 @@ envoy_cc_library( deps = [ "//envoy/config:subscription_interface", "//source/common/protobuf:utility_lib", - "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/core/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) @@ -206,7 +206,7 @@ envoy_cc_library( hdrs = ["xds_resource.h"], deps = [ "//source/common/http:utility_lib", - "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/core/v3:pkg_cc_proto", ], ) @@ -236,8 +236,8 @@ envoy_cc_library( "//source/common/stats:stats_matcher_lib", "//source/common/stats:tag_producer_lib", "//source/common/version:api_version_lib", - "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", - "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_github_cncf_xds//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/v3:pkg_cc_proto", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", @@ -287,7 +287,7 @@ envoy_cc_library( deps = [ "//source/common/common:macros", "//source/common/protobuf:utility_lib", - "@com_github_cncf_udpa//xds/core/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/common/config/datasource.cc b/source/common/config/datasource.cc index 3ee0110e7149..cf7af0e0f71c 100644 --- a/source/common/config/datasource.cc +++ b/source/common/config/datasource.cc @@ -15,12 +15,29 @@ static constexpr uint32_t RetryInitialDelayMilliseconds = 1000; static constexpr uint32_t RetryMaxDelayMilliseconds = 10 * 1000; static constexpr uint32_t RetryCount = 1; -std::string read(const envoy::config::core::v3::DataSource& source, bool allow_empty, - Api::Api& api) { +std::string read(const envoy::config::core::v3::DataSource& source, bool allow_empty, Api::Api& api, + uint64_t max_size) { std::string data; + absl::StatusOr file_or_error; switch (source.specifier_case()) { case envoy::config::core::v3::DataSource::SpecifierCase::kFilename: - data = api.fileSystem().fileReadToEnd(source.filename()); + if (max_size > 0) { + if (!api.fileSystem().fileExists(source.filename())) { + throwEnvoyExceptionOrPanic(fmt::format("file {} does not exist", source.filename())); + } + const ssize_t size = api.fileSystem().fileSize(source.filename()); + if (size < 0) { + throwEnvoyExceptionOrPanic( + absl::StrCat("cannot determine size of file ", source.filename())); + } + if (static_cast(size) > max_size) { + throwEnvoyExceptionOrPanic(fmt::format("file {} size is {} bytes; maximum is {}", + source.filename(), size, max_size)); + } + } + file_or_error = api.fileSystem().fileReadToEnd(source.filename()); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + data = file_or_error.value(); break; case envoy::config::core::v3::DataSource::SpecifierCase::kInlineBytes: data = source.inline_bytes(); @@ -31,7 +48,7 @@ std::string read(const envoy::config::core::v3::DataSource& source, bool allow_e case envoy::config::core::v3::DataSource::SpecifierCase::kEnvironmentVariable: { const char* environment_variable = std::getenv(source.environment_variable().c_str()); if (environment_variable == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Environment variable doesn't exist: {}", source.environment_variable())); } data = environment_variable; @@ -39,12 +56,12 @@ std::string read(const envoy::config::core::v3::DataSource& source, bool allow_e } default: if (!allow_empty) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Unexpected DataSource::specifier_case(): {}", source.specifier_case())); } } if (!allow_empty && data.empty()) { - throw EnvoyException("DataSource cannot be empty"); + throwEnvoyExceptionOrPanic("DataSource cannot be empty"); } return data; } diff --git a/source/common/config/datasource.h b/source/common/config/datasource.h index 9f884bd40968..326c45a3d149 100644 --- a/source/common/config/datasource.h +++ b/source/common/config/datasource.h @@ -24,11 +24,13 @@ namespace DataSource { * @param source data source. * @param allow_empty return an empty string if no DataSource case is specified. * @param api reference to the Api object + * @param max_size max size limit of file to read, default 0 means no limit, and if the file data + * would exceed the limit, it will throw a EnvoyException. * @return std::string with DataSource contents. * @throw EnvoyException if no DataSource case is specified and !allow_empty. */ -std::string read(const envoy::config::core::v3::DataSource& source, bool allow_empty, - Api::Api& api); +std::string read(const envoy::config::core::v3::DataSource& source, bool allow_empty, Api::Api& api, + uint64_t max_size = 0); /** * @param source data source. diff --git a/source/common/config/metadata.h b/source/common/config/metadata.h index 25978f51e188..ffa24cb19711 100644 --- a/source/common/config/metadata.h +++ b/source/common/config/metadata.h @@ -141,5 +141,19 @@ template class TypedMetadataImpl : public TypedMetadata absl::node_hash_map> data_; }; +// MetadataPack is struct that contains both the proto and typed metadata. +template struct MetadataPack { + MetadataPack(const envoy::config::core::v3::Metadata& metadata) + : proto_metadata_(metadata), typed_metadata_(proto_metadata_) {} + MetadataPack() : proto_metadata_(), typed_metadata_(proto_metadata_) {} + + const envoy::config::core::v3::Metadata proto_metadata_; + const TypedMetadataImpl typed_metadata_; +}; + +template using MetadataPackPtr = std::unique_ptr>; +template +using MetadataPackSharedPtr = std::shared_ptr>; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 21ff380a9745..c06588a1300e 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -67,11 +67,11 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( switch (api_config_source.api_type()) { PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; case envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC: - throw EnvoyException("Unsupported config source AGGREGATED_GRPC"); + throwEnvoyExceptionOrPanic("Unsupported config source AGGREGATED_GRPC"); case envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC: - throw EnvoyException("Unsupported config source AGGREGATED_DELTA_GRPC"); + throwEnvoyExceptionOrPanic("Unsupported config source AGGREGATED_DELTA_GRPC"); case envoy::config::core::v3::ApiConfigSource::DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE: - throw EnvoyException( + throwEnvoyExceptionOrPanic( "REST_LEGACY no longer a supported ApiConfigSource. " "Please specify an explicit supported api_type in the following config:\n" + config.DebugString()); @@ -86,7 +86,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( break; } if (subscription_type.empty()) { - throw EnvoyException("Invalid API config source API type"); + throwEnvoyExceptionOrPanic("Invalid API config source API type"); } break; } @@ -95,13 +95,13 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( break; } default: - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Missing config source specifier in envoy::config::core::v3::ConfigSource"); } ConfigSubscriptionFactory* factory = Registry::FactoryRegistry::getFactory(subscription_type); if (factory == nullptr) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "Didn't find a registered config subscription factory implementation for name: '{}'", subscription_type)); } @@ -113,7 +113,7 @@ SubscriptionPtr createFromFactoryOrThrow(ConfigSubscriptionFactory::Subscription ConfigSubscriptionFactory* factory = Registry::FactoryRegistry::getFactory(subscription_type); if (factory == nullptr) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "Didn't find a registered config subscription factory implementation for name: '{}'", subscription_type)); } @@ -153,7 +153,7 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( } case xds::core::v3::ResourceLocator::XDSTP: { if (resource_type != collection_locator.resource_type()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("xdstp:// type does not match {} in {}", resource_type, Config::XdsResourceIdentifier::encodeUrl(collection_locator))); } @@ -179,8 +179,8 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( "envoy.config_subscription.aggregated_grpc_collection"); } default: - throw EnvoyException(fmt::format("Unknown xdstp:// transport API type in {}", - api_config_source.DebugString())); + throwEnvoyExceptionOrPanic(fmt::format("Unknown xdstp:// transport API type in {}", + api_config_source.DebugString())); } } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kAds: { @@ -191,14 +191,15 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( return createFromFactoryOrThrow(data, "envoy.config_subscription.ads_collection"); } default: - throw EnvoyException("Missing or not supported config source specifier in " - "envoy::config::core::v3::ConfigSource for a collection. Only ADS and " - "gRPC in delta-xDS mode are supported."); + throwEnvoyExceptionOrPanic( + "Missing or not supported config source specifier in " + "envoy::config::core::v3::ConfigSource for a collection. Only ADS and " + "gRPC in delta-xDS mode are supported."); } } default: // TODO(htuch): Implement HTTP semantics for collection ResourceLocators. - throw EnvoyException("Unsupported code path"); + throwEnvoyExceptionOrPanic("Unsupported code path"); } } diff --git a/source/common/config/ttl.h b/source/common/config/ttl.h index 880fe42a60f8..3c679a350dc1 100644 --- a/source/common/config/ttl.h +++ b/source/common/config/ttl.h @@ -44,7 +44,7 @@ class TtlManager { TtlManager& parent_; }; - ScopedTtlUpdate scopedTtlUpdate() { return ScopedTtlUpdate(*this); } + ScopedTtlUpdate scopedTtlUpdate() { return {*this}; } void add(std::chrono::milliseconds ttl, const std::string& name); diff --git a/source/common/config/utility.cc b/source/common/config/utility.cc index ac770ddf66d4..c1db95c678e7 100644 --- a/source/common/config/utility.cc +++ b/source/common/config/utility.cc @@ -33,11 +33,11 @@ Upstream::ClusterConstOptRef Utility::checkCluster(absl::string_view error_prefi bool allow_added_via_api) { const auto cluster = cm.clusters().getCluster(cluster_name); if (!cluster.has_value()) { - throw EnvoyException(fmt::format("{}: unknown cluster '{}'", error_prefix, cluster_name)); + throwEnvoyExceptionOrPanic(fmt::format("{}: unknown cluster '{}'", error_prefix, cluster_name)); } if (!allow_added_via_api && cluster->get().info()->addedViaApi()) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "{}: invalid cluster '{}': currently only static (non-CDS) clusters are supported", error_prefix, cluster_name)); } @@ -55,7 +55,7 @@ Utility::checkClusterAndLocalInfo(absl::string_view error_prefix, absl::string_v void Utility::checkLocalInfo(absl::string_view error_prefix, const LocalInfo::LocalInfo& local_info) { if (local_info.clusterName().empty() || local_info.nodeName().empty()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("{}: node 'id' and 'cluster' are required. Set it either in 'node' config or " "via --service-node and --service-cluster options.", error_prefix, local_info.node().DebugString())); @@ -66,7 +66,7 @@ void Utility::checkFilesystemSubscriptionBackingPath(const std::string& path, Ap // TODO(junr03): the file might be deleted between this check and the // watch addition. if (!api.fileSystem().fileExists(path)) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "paths must refer to an existing path in the system: '{}' does not exist", path)); } } @@ -84,32 +84,32 @@ void checkApiConfigSourceNames(const envoy::config::core::v3::ApiConfigSource& a api_config_source.api_type() == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); if (api_config_source.cluster_names().empty() && api_config_source.grpc_services().empty()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("API configs must have either a gRPC service or a cluster name defined: {}", api_config_source.DebugString())); } if (is_grpc) { if (!api_config_source.cluster_names().empty()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("{}::(DELTA_)GRPC must not have a cluster name specified: {}", api_config_source.GetTypeName(), api_config_source.DebugString())); } if (api_config_source.grpc_services().size() > 1) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("{}::(DELTA_)GRPC must have a single gRPC service specified: {}", api_config_source.GetTypeName(), api_config_source.DebugString())); } } else { if (!api_config_source.grpc_services().empty()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("{}, if not a gRPC type, must not have a gRPC service specified: {}", api_config_source.GetTypeName(), api_config_source.DebugString())); } if (api_config_source.cluster_names().size() != 1) { - throw EnvoyException(fmt::format("{} must have a singleton cluster name specified: {}", - api_config_source.GetTypeName(), - api_config_source.DebugString())); + throwEnvoyExceptionOrPanic(fmt::format("{} must have a singleton cluster name specified: {}", + api_config_source.GetTypeName(), + api_config_source.DebugString())); } } } @@ -120,9 +120,10 @@ void Utility::validateClusterName(const Upstream::ClusterManager::ClusterSet& pr const std::string& config_source) { const auto& it = primary_clusters.find(cluster_name); if (it == primary_clusters.end()) { - throw EnvoyException(fmt::format("{} must have a statically defined non-EDS cluster: '{}' does " - "not exist, was added via api, or is an EDS cluster", - config_source, cluster_name)); + throwEnvoyExceptionOrPanic( + fmt::format("{} must have a statically defined non-EDS cluster: '{}' does " + "not exist, was added via api, or is an EDS cluster", + config_source, cluster_name)); } } @@ -179,7 +180,7 @@ Utility::getGrpcControlPlane(const envoy::config::core::v3::ApiConfigSource& api std::chrono::milliseconds Utility::apiConfigSourceRefreshDelay( const envoy::config::core::v3::ApiConfigSource& api_config_source) { if (!api_config_source.has_refresh_delay()) { - throw EnvoyException("refresh_delay is required for REST API configuration sources"); + throwEnvoyExceptionOrPanic("refresh_delay is required for REST API configuration sources"); } return std::chrono::milliseconds( @@ -238,8 +239,9 @@ Grpc::AsyncClientFactoryPtr Utility::factoryForGrpcApiConfigSource( if (api_config_source.api_type() != envoy::config::core::v3::ApiConfigSource::GRPC && api_config_source.api_type() != envoy::config::core::v3::ApiConfigSource::DELTA_GRPC) { - throw EnvoyException(fmt::format("{} type must be gRPC: {}", api_config_source.GetTypeName(), - api_config_source.DebugString())); + throwEnvoyExceptionOrPanic(fmt::format("{} type must be gRPC: {}", + api_config_source.GetTypeName(), + api_config_source.DebugString())); } envoy::config::core::v3::GrpcService grpc_service; @@ -273,7 +275,7 @@ void Utility::translateOpaqueConfig(const ProtobufWkt::Any& typed_config, #ifdef ENVOY_ENABLE_YAML MessageUtil::jsonConvert(typed_struct.value(), validation_visitor, out_proto); #else - throw EnvoyException("Attempting to use JSON typed structs with JSON compiled out"); + IS_ENVOY_BUG("Attempting to use JSON typed structs with JSON compiled out"); #endif } } else if (type == legacy_typed_struct_type) { @@ -289,7 +291,7 @@ void Utility::translateOpaqueConfig(const ProtobufWkt::Any& typed_config, MessageUtil::jsonConvert(typed_struct.value(), validation_visitor, out_proto); #else UNREFERENCED_PARAMETER(validation_visitor); - throw EnvoyException("Attempting to use legacy JSON structs with JSON compiled out"); + IS_ENVOY_BUG("Attempting to use legacy JSON structs with JSON compiled out"); #endif } } // out_proto is expecting Struct, unpack directly @@ -301,7 +303,7 @@ void Utility::translateOpaqueConfig(const ProtobufWkt::Any& typed_config, MessageUtil::unpackTo(typed_config, struct_config); MessageUtil::jsonConvert(struct_config, validation_visitor, out_proto); #else - throw EnvoyException("Attempting to use JSON structs with JSON compiled out"); + IS_ENVOY_BUG("Attempting to use JSON structs with JSON compiled out"); #endif } } @@ -318,7 +320,7 @@ JitteredExponentialBackOffStrategyPtr Utility::buildJitteredExponentialBackOffSt PROTOBUF_GET_MS_OR_DEFAULT(backoff.value(), max_interval, base_interval_ms * 10); if (max_interval_ms < base_interval_ms) { - throw EnvoyException("max_interval must be greater than or equal to the base_interval"); + throwEnvoyExceptionOrPanic("max_interval must be greater than or equal to the base_interval"); } return std::make_unique(base_interval_ms, max_interval_ms, random); @@ -326,13 +328,13 @@ JitteredExponentialBackOffStrategyPtr Utility::buildJitteredExponentialBackOffSt // default_base_interval_ms must be greater than zero if (default_base_interval_ms == 0) { - throw EnvoyException("default_base_interval_ms must be greater than zero"); + throwEnvoyExceptionOrPanic("default_base_interval_ms must be greater than zero"); } // default maximum interval is specified if (default_max_interval_ms != absl::nullopt) { if (default_max_interval_ms.value() < default_base_interval_ms) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "default_max_interval_ms must be greater than or equal to the default_base_interval_ms"); } return std::make_unique( diff --git a/source/common/config/well_known_names.cc b/source/common/config/well_known_names.cc index 5c220c93defe..7fa39a3f4358 100644 --- a/source/common/config/well_known_names.cc +++ b/source/common/config/well_known_names.cc @@ -198,6 +198,12 @@ TagNameValues::TagNameValues() { // listener_local_rate_limit.(.) addTokenized(LOCAL_LISTENER_RATELIMIT_PREFIX, "listener_local_ratelimit.$.**"); + + // dns_filter.(.).** + addTokenized(DNS_FILTER_PREFIX, "dns_filter.$.**"); + + // connection_limit.(.)* + addTokenized(CONNECTION_LIMIT_PREFIX, "connection_limit.$.**"); } void TagNameValues::addRe2(const std::string& name, const std::string& regex, diff --git a/source/common/config/well_known_names.h b/source/common/config/well_known_names.h index bfe14fd9c284..1a9da7161f8f 100644 --- a/source/common/config/well_known_names.h +++ b/source/common/config/well_known_names.h @@ -117,6 +117,10 @@ class TagNameValues { const std::string LOCAL_NETWORK_RATELIMIT_PREFIX = "envoy.local_network_ratelimit_prefix"; // Stats prefix for the Local Ratelimit listener filter const std::string LOCAL_LISTENER_RATELIMIT_PREFIX = "envoy.local_listener_ratelimit_prefix"; + // Stats prefix for the dns filter + const std::string DNS_FILTER_PREFIX = "envoy.dns_filter_prefix"; + // Stats prefix for the Connection limit filter + const std::string CONNECTION_LIMIT_PREFIX = "envoy.connection_limit_prefix"; // Stats prefix for the TCP Proxy network filter const std::string TCP_PREFIX = "envoy.tcp_prefix"; // Stats prefix for the UDP Proxy network filter diff --git a/source/common/config/xds_resource.cc b/source/common/config/xds_resource.cc index 31f3c9cc27ca..7334f1a5602a 100644 --- a/source/common/config/xds_resource.cc +++ b/source/common/config/xds_resource.cc @@ -104,7 +104,7 @@ std::string XdsResourceIdentifier::encodeUrl(const xds::core::v3::ResourceLocato namespace { -void decodePath(absl::string_view path, std::string* resource_type, std::string& id) { +absl::Status decodePath(absl::string_view path, std::string* resource_type, std::string& id) { // This is guaranteed by Http::Utility::extractHostPathFromUrn. ASSERT(absl::StartsWith(path, "/")); const std::vector path_components = absl::StrSplit(path.substr(1), '/'); @@ -112,27 +112,26 @@ void decodePath(absl::string_view path, std::string* resource_type, std::string& if (resource_type != nullptr) { *resource_type = std::string(path_components[0]); if (resource_type->empty()) { - throw XdsResourceIdentifier::DecodeException( - fmt::format("Resource type missing from {}", path)); + return absl::InvalidArgumentError(fmt::format("Resource type missing from {}", path)); } id_it = std::next(id_it); } id = PercentEncoding::decode(absl::StrJoin(id_it, path_components.cend(), "/")); + return absl::OkStatus(); } void decodeQueryParams(absl::string_view query_params, xds::core::v3::ContextParams& context_params) { - Http::Utility::QueryParams query_params_components = - Http::Utility::parseQueryString(query_params); - for (const auto& it : query_params_components) { + auto query_params_components = Http::Utility::QueryParamsMulti::parseQueryString(query_params); + for (const auto& it : query_params_components.data()) { (*context_params.mutable_params())[PercentEncoding::decode(it.first)] = - PercentEncoding::decode(it.second); + PercentEncoding::decode(it.second[0]); } } -void decodeFragment( - absl::string_view fragment, - Protobuf::RepeatedPtrField& directives) { +absl::Status +decodeFragment(absl::string_view fragment, + Protobuf::RepeatedPtrField& directives) { const std::vector fragment_components = absl::StrSplit(fragment, ','); for (const absl::string_view& fragment_component : fragment_components) { if (absl::StartsWith(fragment_component, "alt=")) { @@ -141,18 +140,19 @@ void decodeFragment( } else if (absl::StartsWith(fragment_component, "entry=")) { directives.Add()->set_entry(PercentEncoding::decode(fragment_component.substr(6))); } else { - throw XdsResourceIdentifier::DecodeException( + return absl::InvalidArgumentError( fmt::format("Unknown fragment component {}", fragment_component)); - ; } } + return absl::OkStatus(); } } // namespace -xds::core::v3::ResourceName XdsResourceIdentifier::decodeUrn(absl::string_view resource_urn) { +absl::StatusOr +XdsResourceIdentifier::decodeUrn(absl::string_view resource_urn) { if (!hasXdsTpScheme(resource_urn)) { - throw XdsResourceIdentifier::DecodeException( + return absl::InvalidArgumentError( fmt::format("{} does not have an xdstp: scheme", resource_urn)); } absl::string_view host, path; @@ -164,8 +164,11 @@ xds::core::v3::ResourceName XdsResourceIdentifier::decodeUrn(absl::string_view r decodeQueryParams(path.substr(query_params_start), *decoded_resource_name.mutable_context()); path = path.substr(0, query_params_start); } - decodePath(path, decoded_resource_name.mutable_resource_type(), - *decoded_resource_name.mutable_id()); + auto status = decodePath(path, decoded_resource_name.mutable_resource_type(), + *decoded_resource_name.mutable_id()); + if (!status.ok()) { + return status; + } return decoded_resource_name; } @@ -175,7 +178,8 @@ xds::core::v3::ResourceLocator XdsResourceIdentifier::decodeUrl(absl::string_vie xds::core::v3::ResourceLocator decoded_resource_locator; const size_t fragment_start = path.find('#'); if (fragment_start != absl::string_view::npos) { - decodeFragment(path.substr(fragment_start + 1), *decoded_resource_locator.mutable_directives()); + THROW_IF_NOT_OK(decodeFragment(path.substr(fragment_start + 1), + *decoded_resource_locator.mutable_directives())); path = path.substr(0, fragment_start); } if (hasXdsTpScheme(resource_url)) { @@ -185,10 +189,10 @@ xds::core::v3::ResourceLocator XdsResourceIdentifier::decodeUrl(absl::string_vie } else if (absl::StartsWith(resource_url, "file:")) { decoded_resource_locator.set_scheme(xds::core::v3::ResourceLocator::FILE); // File URLs only have a path and fragment. - decodePath(path, nullptr, *decoded_resource_locator.mutable_id()); + THROW_IF_NOT_OK(decodePath(path, nullptr, *decoded_resource_locator.mutable_id())); return decoded_resource_locator; } else { - throw XdsResourceIdentifier::DecodeException( + throwEnvoyExceptionOrPanic( fmt::format("{} does not have a xdstp:, http: or file: scheme", resource_url)); } decoded_resource_locator.set_authority(PercentEncoding::decode(host)); @@ -198,8 +202,8 @@ xds::core::v3::ResourceLocator XdsResourceIdentifier::decodeUrl(absl::string_vie *decoded_resource_locator.mutable_exact_context()); path = path.substr(0, query_params_start); } - decodePath(path, decoded_resource_locator.mutable_resource_type(), - *decoded_resource_locator.mutable_id()); + THROW_IF_NOT_OK(decodePath(path, decoded_resource_locator.mutable_resource_type(), + *decoded_resource_locator.mutable_id())); return decoded_resource_locator; } diff --git a/source/common/config/xds_resource.h b/source/common/config/xds_resource.h index 9011fd96cd4c..6f362e589c1f 100644 --- a/source/common/config/xds_resource.h +++ b/source/common/config/xds_resource.h @@ -44,27 +44,21 @@ class XdsResourceIdentifier { return encodeUrl(resource_locator, {}); } - // Thrown when an exception occurs during URI decoding. - class DecodeException : public EnvoyException { - public: - DecodeException(const std::string& what) : EnvoyException(what) {} - }; - /** * Decode a xdstp:// URN string to a xds::core::v3::ResourceName. * * @param resource_urn xdstp:// resource URN. * @return xds::core::v3::ResourceName resource name message for resource_urn. - * @throws DecodeException when parsing fails. + * @throws EnvoyException when parsing fails. */ - static xds::core::v3::ResourceName decodeUrn(absl::string_view resource_urn); + static absl::StatusOr decodeUrn(absl::string_view resource_urn); /** * Decode a xdstp:// URL string to a xds::core::v3::ResourceLocator. * * @param resource_url xdstp:// resource URL. * @return xds::core::v3::ResourceLocator resource name message for resource_url. - * @throws DecodeException when parsing fails. + * @throws EnvoyException when parsing fails. */ static xds::core::v3::ResourceLocator decodeUrl(absl::string_view resource_url); diff --git a/source/common/conn_pool/conn_pool_base.cc b/source/common/conn_pool/conn_pool_base.cc index ae614b9a78ac..9ff006726811 100644 --- a/source/common/conn_pool/conn_pool_base.cc +++ b/source/common/conn_pool/conn_pool_base.cc @@ -448,13 +448,24 @@ bool ConnPoolImplBase::isIdleImpl() const { connecting_clients_.empty() && early_data_clients_.empty(); } +/* + This method may be invoked once or twice. + It is called first time in ConnPoolImplBase::onConnectionEvent for Local/RemoteClose events. + The second time it is called from Envoy::Tcp::ActiveTcpClient::~ActiveTcpClient via + ConnPoolImplBase::checkForIdleAndCloseIdleConnsIfDraining. + + The logic must be constructed in such way that the method is called once or twice. + See PR 30807 description for explanation why idle callbacks are deleted after being called. +*/ void ConnPoolImplBase::checkForIdleAndNotify() { if (isIdleImpl()) { - ENVOY_LOG(debug, "invoking idle callbacks - is_draining_for_deletion_={}", - is_draining_for_deletion_); + ENVOY_LOG(debug, "invoking {} idle callback(s) - is_draining_for_deletion_={}", + idle_callbacks_.size(), is_draining_for_deletion_); for (const Instance::IdleCb& cb : idle_callbacks_) { cb(); } + // Clear callbacks, so they are not executed if checkForIdleAndNotify is called again. + idle_callbacks_.clear(); } } diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 608cd4dcce57..41d65b47e3cc 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -41,15 +41,12 @@ envoy_cc_library( "//envoy/common:time_interface", "//envoy/event:signal_interface", "//envoy/network:client_connection_factory", - "//envoy/network:listen_socket_interface", - "//envoy/network:listener_interface", "//source/common/common:assert_lib", "//source/common/common:thread_lib", "//source/common/config:utility_lib", "//source/common/filesystem:watcher_lib", "//source/common/network:address_lib", "//source/common/network:default_client_connection_factory", - "//source/common/network:listener_lib", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", ], ) diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 1723e02f41a6..778c8c779287 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -26,8 +26,6 @@ #include "source/common/filesystem/watcher_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/connection_impl.h" -#include "source/common/network/tcp_listener_impl.h" -#include "source/common/network/udp_listener_impl.h" #include "source/common/runtime/runtime_features.h" #include "event2/event.h" @@ -65,13 +63,12 @@ DispatcherImpl::DispatcherImpl(const std::string& name, Api::Api& api, api.bootstrap().overload_manager().buffer_factory_config())) {} DispatcherImpl::DispatcherImpl(const std::string& name, Thread::ThreadFactory& thread_factory, - TimeSource& time_source, Random::RandomGenerator& random_generator, + TimeSource& time_source, Random::RandomGenerator&, Filesystem::Instance& file_system, Event::TimeSystem& time_system, const ScaledRangeTimerManagerFactory& scaled_timer_factory, const Buffer::WatermarkFactorySharedPtr& watermark_factory) : name_(name), thread_factory_(thread_factory), time_source_(time_source), - random_generator_(random_generator), file_system_(file_system), - buffer_factory_(watermark_factory), + file_system_(file_system), buffer_factory_(watermark_factory), scheduler_(time_system.createScheduler(base_scheduler_, base_scheduler_)), thread_local_delete_cb_( base_scheduler_.createSchedulableCallback([this]() -> void { runThreadLocalDelete(); })), @@ -191,26 +188,6 @@ Filesystem::WatcherPtr DispatcherImpl::createFilesystemWatcher() { return Filesystem::WatcherPtr{new Filesystem::WatcherImpl(*this, file_system_)}; } -Network::ListenerPtr -DispatcherImpl::createListener(Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb, - Runtime::Loader& runtime, - const Network::ListenerConfig& listener_config) { - ASSERT(isThreadSafe()); - return std::make_unique( - *this, random_generator_, runtime, std::move(socket), cb, listener_config.bindToPort(), - listener_config.ignoreGlobalConnLimit(), - listener_config.maxConnectionsToAcceptPerSocketEvent()); -} - -Network::UdpListenerPtr -DispatcherImpl::createUdpListener(Network::SocketSharedPtr socket, - Network::UdpListenerCallbacks& cb, - const envoy::config::core::v3::UdpSocketConfig& config) { - ASSERT(isThreadSafe()); - return std::make_unique(*this, std::move(socket), cb, timeSource(), - config); -} - TimerPtr DispatcherImpl::createTimer(TimerCb cb) { ASSERT(isThreadSafe()); return createTimerInternal(cb); diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index c051c4c088f7..6b133fe10984 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -43,6 +43,7 @@ class DispatcherImpl : Logger::Loggable, DispatcherImpl(const std::string& name, Api::Api& api, Event::TimeSystem& time_system, const ScaledRangeTimerManagerFactory& scaled_timer_factory, const Buffer::WatermarkFactorySharedPtr& watermark_factory); + // TODO(alyssawilk) remove random_generator. DispatcherImpl(const std::string& name, Thread::ThreadFactory& thread_factory, TimeSource& time_source, Random::RandomGenerator& random_generator, Filesystem::Instance& file_system, Event::TimeSystem& time_system, @@ -75,12 +76,6 @@ class DispatcherImpl : Logger::Loggable, FileEventPtr createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events) override; Filesystem::WatcherPtr createFilesystemWatcher() override; - Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime, - const Network::ListenerConfig& listener_config) override; - Network::UdpListenerPtr - createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, - const envoy::config::core::v3::UdpSocketConfig& config) override; TimerPtr createTimer(TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerType timer_type, TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerMinimum minimum, TimerCb cb) override; @@ -149,7 +144,6 @@ class DispatcherImpl : Logger::Loggable, const std::string name_; Thread::ThreadFactory& thread_factory_; TimeSource& time_source_; - Random::RandomGenerator& random_generator_; Filesystem::Instance& file_system_; std::string stats_prefix_; DispatcherStatsPtr stats_; diff --git a/source/common/filesystem/directory.h b/source/common/filesystem/directory.h index 46389873a40b..925326c14e32 100644 --- a/source/common/filesystem/directory.h +++ b/source/common/filesystem/directory.h @@ -10,11 +10,16 @@ namespace Envoy { namespace Filesystem { // This class does not do any validation of input data. Do not use with untrusted inputs. +// +// DirectoryIteratorImpl will fail silently and act like an empty iterator in case of +// error opening the directory, and will silently skip files that can't be stat'ed. Check +// DirectoryIteratorImpl::status() after initialization and after each increment if +// silent failure is not the desired behavior. class Directory { public: Directory(const std::string& directory_path) : directory_path_(directory_path) {} - DirectoryIteratorImpl begin() { return DirectoryIteratorImpl(directory_path_); } + DirectoryIteratorImpl begin() { return {directory_path_}; } DirectoryIteratorImpl end() { return {}; } diff --git a/source/common/filesystem/inotify/watcher_impl.cc b/source/common/filesystem/inotify/watcher_impl.cc index 61f8d2b69d93..864da79f5b59 100644 --- a/source/common/filesystem/inotify/watcher_impl.cc +++ b/source/common/filesystem/inotify/watcher_impl.cc @@ -35,12 +35,14 @@ WatcherImpl::~WatcherImpl() { close(inotify_fd_); } void WatcherImpl::addWatch(absl::string_view path, uint32_t events, OnChangedCb callback) { // Because of general inotify pain, we always watch the directory that the file lives in, // and then synthetically raise per file events. - const PathSplitResult result = file_system_.splitPathFromFilename(path); + auto result_or_error = file_system_.splitPathFromFilename(path); + THROW_IF_STATUS_NOT_OK(result_or_error, throw); + const PathSplitResult result = result_or_error.value(); const uint32_t watch_mask = IN_MODIFY | IN_MOVED_TO; int watch_fd = inotify_add_watch(inotify_fd_, std::string(result.directory_).c_str(), watch_mask); if (watch_fd == -1) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("unable to add filesystem watch for file {}: {}", path, errorDetails(errno))); } diff --git a/source/common/filesystem/kqueue/watcher_impl.cc b/source/common/filesystem/kqueue/watcher_impl.cc index 1cf9e0865bf5..200c08dc8104 100644 --- a/source/common/filesystem/kqueue/watcher_impl.cc +++ b/source/common/filesystem/kqueue/watcher_impl.cc @@ -35,7 +35,7 @@ WatcherImpl::~WatcherImpl() { void WatcherImpl::addWatch(absl::string_view path, uint32_t events, Watcher::OnChangedCb cb) { FileWatchPtr watch = addWatch(path, events, cb, false); if (watch == nullptr) { - throw EnvoyException(absl::StrCat("invalid watch path ", path)); + throwEnvoyExceptionOrPanic(absl::StrCat("invalid watch path ", path)); } } @@ -49,7 +49,9 @@ WatcherImpl::FileWatchPtr WatcherImpl::addWatch(absl::string_view path, uint32_t return nullptr; } - watch_fd = open(std::string(file_system_.splitPathFromFilename(path).directory_).c_str(), 0); + const auto result_or_error = file_system_.splitPathFromFilename(path); + THROW_IF_STATUS_NOT_OK(result_or_error, throw); + watch_fd = open(std::string(result_or_error.value().directory_).c_str(), 0); if (watch_fd == -1) { return nullptr; } @@ -71,7 +73,7 @@ WatcherImpl::FileWatchPtr WatcherImpl::addWatch(absl::string_view path, uint32_t reinterpret_cast(watch_fd)); if (kevent(queue_, &event, 1, nullptr, 0, nullptr) == -1 || event.flags & EV_ERROR) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("unable to add filesystem watch for file {}: {}", path, errorDetails(errno))); } @@ -106,7 +108,10 @@ void WatcherImpl::onKqueueEvent() { ASSERT(file != nullptr); ASSERT(watch_fd == file->fd_); - auto pathname = file_system_.splitPathFromFilename(file->file_); + absl::StatusOr pathname_or_error = + file_system_.splitPathFromFilename(file->file_); + THROW_IF_STATUS_NOT_OK(pathname_or_error, throw); + PathSplitResult& pathname = pathname_or_error.value(); if (file->watching_dir_) { if (event.fflags & NOTE_DELETE) { diff --git a/source/common/filesystem/posix/directory_iterator_impl.cc b/source/common/filesystem/posix/directory_iterator_impl.cc index a50514d991dc..a2a87766ee9d 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.cc +++ b/source/common/filesystem/posix/directory_iterator_impl.cc @@ -4,13 +4,20 @@ #include "source/common/common/utility.h" #include "source/common/filesystem/directory_iterator_impl.h" +#include "absl/strings/strip.h" + namespace Envoy { namespace Filesystem { DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& directory_path) - : directory_path_(directory_path), os_sys_calls_(Api::OsSysCallsSingleton::get()) { + : directory_path_(absl::StripSuffix(directory_path, "/")), + os_sys_calls_(Api::OsSysCallsSingleton::get()) { openDirectory(); - nextEntry(); + if (status_.ok()) { + nextEntry(); + } else { + entry_ = {"", FileType::Other, absl::nullopt}; + } } DirectoryIteratorImpl::~DirectoryIteratorImpl() { @@ -28,27 +35,35 @@ void DirectoryIteratorImpl::openDirectory() { DIR* temp_dir = ::opendir(directory_path_.c_str()); dir_ = temp_dir; if (!dir_) { - throw EnvoyException( - fmt::format("unable to open directory {}: {}", directory_path_, errorDetails(errno))); + status_ = absl::ErrnoToStatus(errno, fmt::format("unable to open directory {}: {}", + directory_path_, errorDetails(errno))); } } void DirectoryIteratorImpl::nextEntry() { errno = 0; dirent* entry = ::readdir(dir_); - if (entry == nullptr && errno != 0) { - throw EnvoyException( - fmt::format("unable to iterate directory {}: {}", directory_path_, errorDetails(errno))); - } if (entry == nullptr) { entry_ = {"", FileType::Other, absl::nullopt}; + if (errno != 0) { + status_ = absl::ErrnoToStatus(errno, fmt::format("unable to iterate directory {}: {}", + directory_path_, errorDetails(errno))); + } } else { - entry_ = makeEntry(entry->d_name); + auto result = makeEntry(entry->d_name); + status_ = result.status(); + if (!status_.ok()) { + // If we failed to stat one file it might have just been deleted, + // keep iterating over the rest of the dir. + return nextEntry(); + } else { + entry_ = result.value(); + } } } -DirectoryEntry DirectoryIteratorImpl::makeEntry(absl::string_view filename) const { +absl::StatusOr DirectoryIteratorImpl::makeEntry(absl::string_view filename) const { const std::string full_path = absl::StrCat(directory_path_, "/", filename); struct stat stat_buf; const Api::SysCallIntResult result = os_sys_calls_.stat(full_path.c_str(), &stat_buf); @@ -62,10 +77,8 @@ DirectoryEntry DirectoryIteratorImpl::makeEntry(absl::string_view filename) cons return DirectoryEntry{std::string{filename}, FileType::Regular, absl::nullopt}; } } - // TODO: throwing an exception here makes this dangerous to use in worker threads, - // and in general since it's not clear to the user of Directory that an exception - // may be thrown. Perhaps make this return StatusOr and handle failures gracefully. - throw EnvoyException(fmt::format("unable to stat file: '{}' ({})", full_path, errno)); + return absl::ErrnoToStatus( + result.errno_, fmt::format("unable to stat file: '{}' ({})", full_path, result.errno_)); } else if (S_ISDIR(stat_buf.st_mode)) { return DirectoryEntry{std::string{filename}, FileType::Directory, absl::nullopt}; } else if (S_ISREG(stat_buf.st_mode)) { diff --git a/source/common/filesystem/posix/directory_iterator_impl.h b/source/common/filesystem/posix/directory_iterator_impl.h index 03d45cb7512f..4b6aeef5d0c0 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.h +++ b/source/common/filesystem/posix/directory_iterator_impl.h @@ -29,7 +29,7 @@ class DirectoryIteratorImpl : public DirectoryIterator { void nextEntry(); void openDirectory(); - DirectoryEntry makeEntry(absl::string_view filename) const; + absl::StatusOr makeEntry(absl::string_view filename) const; std::string directory_path_; DIR* dir_{nullptr}; diff --git a/source/common/filesystem/posix/filesystem_impl.cc b/source/common/filesystem/posix/filesystem_impl.cc index 24c0bf1c05f3..567eb43e15ff 100644 --- a/source/common/filesystem/posix/filesystem_impl.cc +++ b/source/common/filesystem/posix/filesystem_impl.cc @@ -137,9 +137,11 @@ static constexpr absl::optional systemTimeFromTimespec(const struct return timespecToChrono(t); } -static FileInfo infoFromStat(absl::string_view path, const struct stat& s, FileType type) { - return { - std::string{InstanceImplPosix().splitPathFromFilename(path).file_}, +static Api::IoCallResult infoFromStat(absl::string_view path, const struct stat& s, + FileType type) { + auto result_or_error = InstanceImplPosix().splitPathFromFilename(path); + FileInfo ret = { + result_or_error.status().ok() ? std::string{result_or_error.value().file_} : "", s.st_size, type, #ifdef _DARWIN_FEATURE_64_BIT_INODE @@ -152,9 +154,10 @@ static FileInfo infoFromStat(absl::string_view path, const struct stat& s, FileT systemTimeFromTimespec(s.st_mtim), #endif }; + return result_or_error.status().ok() ? resultSuccess(ret) : resultFailure(ret, EINVAL); } -static FileInfo infoFromStat(absl::string_view path, const struct stat& s) { +static Api::IoCallResult infoFromStat(absl::string_view path, const struct stat& s) { return infoFromStat(path, s, typeFromStat(s)); } @@ -164,7 +167,7 @@ Api::IoCallResult FileImplPosix::info() { if (::fstat(fd_, &s) != 0) { return resultFailure({}, errno); } - return resultSuccess(infoFromStat(path(), s)); + return infoFromStat(path(), s); } Api::IoCallResult InstanceImplPosix::stat(absl::string_view path) { @@ -177,12 +180,12 @@ Api::IoCallResult InstanceImplPosix::stat(absl::string_view path) { // but the reference is broken as the target could not be stat()'ed. // After confirming this with an lstat, treat this file entity as // a regular file, which may be unlink()'ed. - return resultSuccess(infoFromStat(path, s, FileType::Regular)); + return infoFromStat(path, s, FileType::Regular); } } return resultFailure({}, errno); } - return resultSuccess(infoFromStat(path, s)); + return infoFromStat(path, s); } Api::IoCallBoolResult InstanceImplPosix::createPath(absl::string_view path) { @@ -282,14 +285,14 @@ ssize_t InstanceImplPosix::fileSize(const std::string& path) { return info.st_size; } -std::string InstanceImplPosix::fileReadToEnd(const std::string& path) { +absl::StatusOr InstanceImplPosix::fileReadToEnd(const std::string& path) { if (illegalPath(path)) { - throw EnvoyException(absl::StrCat("Invalid path: ", path)); + return absl::InvalidArgumentError(absl::StrCat("Invalid path: ", path)); } std::ifstream file(path); if (file.fail()) { - throw EnvoyException(absl::StrCat("unable to read file: ", path)); + return absl::InvalidArgumentError(absl::StrCat("unable to read file: ", path)); } std::stringstream file_string; @@ -298,17 +301,17 @@ std::string InstanceImplPosix::fileReadToEnd(const std::string& path) { return file_string.str(); } -PathSplitResult InstanceImplPosix::splitPathFromFilename(absl::string_view path) { +absl::StatusOr InstanceImplPosix::splitPathFromFilename(absl::string_view path) { size_t last_slash = path.rfind('/'); if (last_slash == std::string::npos) { - throw EnvoyException(fmt::format("invalid file path {}", path)); + return absl::InvalidArgumentError(fmt::format("invalid file path {}", path)); } absl::string_view name = path.substr(last_slash + 1); // truncate all trailing slashes, except root slash if (last_slash == 0) { ++last_slash; } - return {path.substr(0, last_slash), name}; + return PathSplitResult{path.substr(0, last_slash), name}; } bool InstanceImplPosix::illegalPath(const std::string& path) { diff --git a/source/common/filesystem/posix/filesystem_impl.h b/source/common/filesystem/posix/filesystem_impl.h index e952a129f93e..a17ab3973e5e 100644 --- a/source/common/filesystem/posix/filesystem_impl.h +++ b/source/common/filesystem/posix/filesystem_impl.h @@ -58,8 +58,8 @@ class InstanceImplPosix : public Instance { bool fileExists(const std::string& path) override; bool directoryExists(const std::string& path) override; ssize_t fileSize(const std::string& path) override; - std::string fileReadToEnd(const std::string& path) override; - PathSplitResult splitPathFromFilename(absl::string_view path) override; + absl::StatusOr fileReadToEnd(const std::string& path) override; + absl::StatusOr splitPathFromFilename(absl::string_view path) override; bool illegalPath(const std::string& path) override; Api::IoCallResult stat(absl::string_view path) override; Api::IoCallBoolResult createPath(absl::string_view path) override; diff --git a/source/common/filesystem/win32/directory_iterator_impl.cc b/source/common/filesystem/win32/directory_iterator_impl.cc index 1dbf6ea73b4d..381be7899251 100644 --- a/source/common/filesystem/win32/directory_iterator_impl.cc +++ b/source/common/filesystem/win32/directory_iterator_impl.cc @@ -3,20 +3,27 @@ #include "source/common/common/fmt.h" #include "source/common/filesystem/directory_iterator_impl.h" +#include "absl/strings/strip.h" + namespace Envoy { namespace Filesystem { DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& directory_path) : DirectoryIterator(), find_handle_(INVALID_HANDLE_VALUE) { + absl::string_view path = absl::StripSuffix(directory_path, "/"); + path = absl::StripSuffix(path, "\\"); WIN32_FIND_DATA find_data; - const std::string glob = directory_path + "\\*"; + const std::string glob = absl::StrCat(path, "\\*"); find_handle_ = ::FindFirstFile(glob.c_str(), &find_data); if (find_handle_ == INVALID_HANDLE_VALUE) { - throw EnvoyException( - fmt::format("unable to open directory {}: {}", directory_path, ::GetLastError())); + status_ = + absl::UnknownError(fmt::format("unable to open directory {}: {}", path, ::GetLastError())); + } + if (status_.ok()) { + entry_ = makeEntry(find_data); + } else { + entry_ = {"", FileType::Other, absl::nullopt}; } - - entry_ = makeEntry(find_data); } DirectoryIteratorImpl::~DirectoryIteratorImpl() { @@ -29,12 +36,12 @@ DirectoryIteratorImpl& DirectoryIteratorImpl::operator++() { WIN32_FIND_DATA find_data; const BOOL ret = ::FindNextFile(find_handle_, &find_data); const DWORD err = ::GetLastError(); - if (ret == 0 && err != ERROR_NO_MORE_FILES) { - throw EnvoyException(fmt::format("unable to iterate directory: {}", err)); - } if (ret == 0) { entry_ = {"", FileType::Other, absl::nullopt}; + if (err != ERROR_NO_MORE_FILES) { + status_ = absl::UnknownError(fmt::format("unable to iterate directory: {}", err)); + } } else { entry_ = makeEntry(find_data); } diff --git a/source/common/filesystem/win32/filesystem_impl.cc b/source/common/filesystem/win32/filesystem_impl.cc index d400cea68b13..476ba49101fd 100644 --- a/source/common/filesystem/win32/filesystem_impl.cc +++ b/source/common/filesystem/win32/filesystem_impl.cc @@ -150,21 +150,26 @@ static FileType fileTypeFromAttributeData(const WIN32_FILE_ATTRIBUTE_DATA& data) return FileType::Regular; } -static FileInfo fileInfoFromAttributeData(absl::string_view path, - const WIN32_FILE_ATTRIBUTE_DATA& data) { +static Api::IoCallResult +fileInfoFromAttributeData(absl::string_view path, const WIN32_FILE_ATTRIBUTE_DATA& data) { absl::optional sz; FileType type = fileTypeFromAttributeData(data); if (type == FileType::Regular) { sz = fileSizeFromAttributeData(data); } - return { - std::string{InstanceImplWin32().splitPathFromFilename(path).file_}, + absl::StatusOr result_or_error = InstanceImplWin32().splitPathFromFilename(path); + FileInfo ret{ + result_or_error.ok() ? std::string{result_or_error.value().file_} : "", sz, type, systemTimeFromFileTime(data.ftCreationTime), systemTimeFromFileTime(data.ftLastAccessTime), systemTimeFromFileTime(data.ftLastWriteTime), }; + if (result_or_error.status().ok()) { + return resultSuccess(ret); + } + return resultFailure(ret, ERROR_INVALID_DATA); } Api::IoCallResult FileImplWin32::info() { @@ -174,7 +179,7 @@ Api::IoCallResult FileImplWin32::info() { if (!result) { return resultFailure({}, ::GetLastError()); } - return resultSuccess(fileInfoFromAttributeData(path(), data)); + return fileInfoFromAttributeData(path(), data); } Api::IoCallResult InstanceImplWin32::stat(absl::string_view path) { @@ -184,7 +189,7 @@ Api::IoCallResult InstanceImplWin32::stat(absl::string_view path) { if (!result) { return resultFailure({}, ::GetLastError()); } - return resultSuccess(fileInfoFromAttributeData(full_path, data)); + return fileInfoFromAttributeData(full_path, data); } Api::IoCallBoolResult InstanceImplWin32::createPath(absl::string_view path) { @@ -276,9 +281,9 @@ ssize_t InstanceImplWin32::fileSize(const std::string& path) { return result; } -std::string InstanceImplWin32::fileReadToEnd(const std::string& path) { +absl::StatusOr InstanceImplWin32::fileReadToEnd(const std::string& path) { if (illegalPath(path)) { - throw EnvoyException(absl::StrCat("Invalid path: ", path)); + return absl::InvalidArgumentError(absl::StrCat("Invalid path: ", path)); } // In integration tests (and potentially in production) we rename config files and this creates @@ -290,9 +295,9 @@ std::string InstanceImplWin32::fileReadToEnd(const std::string& path) { if (fd == INVALID_HANDLE) { auto last_error = ::GetLastError(); if (last_error == ERROR_FILE_NOT_FOUND) { - throw EnvoyException(absl::StrCat("Invalid path: ", path)); + return absl::InvalidArgumentError(absl::StrCat("Invalid path: ", path)); } - throw EnvoyException(absl::StrCat("unable to read file: ", path)); + return absl::InvalidArgumentError(absl::StrCat("unable to read file: ", path)); } DWORD buffer_size = 100; DWORD bytes_read = 0; @@ -303,10 +308,10 @@ std::string InstanceImplWin32::fileReadToEnd(const std::string& path) { auto last_error = ::GetLastError(); if (last_error == ERROR_FILE_NOT_FOUND) { CloseHandle(fd); - throw EnvoyException(absl::StrCat("Invalid path: ", path)); + return absl::InvalidArgumentError(absl::StrCat("Invalid path: ", path)); } CloseHandle(fd); - throw EnvoyException(absl::StrCat("unable to read file: ", path)); + return absl::InvalidArgumentError(absl::StrCat("unable to read file: ", path)); } complete_buffer.insert(complete_buffer.end(), buffer.begin(), buffer.begin() + bytes_read); } while (bytes_read == buffer_size); @@ -314,10 +319,10 @@ std::string InstanceImplWin32::fileReadToEnd(const std::string& path) { return std::string(complete_buffer.begin(), complete_buffer.end()); } -PathSplitResult InstanceImplWin32::splitPathFromFilename(absl::string_view path) { +absl::StatusOr InstanceImplWin32::splitPathFromFilename(absl::string_view path) { size_t last_slash = path.find_last_of(":/\\"); if (last_slash == std::string::npos) { - throw EnvoyException(fmt::format("invalid file path {}", path)); + return absl::InvalidArgumentError(fmt::format("invalid file path {}", path)); } absl::string_view name = path.substr(last_slash + 1); // Truncate all trailing slashes, but retain the entire @@ -325,7 +330,7 @@ PathSplitResult InstanceImplWin32::splitPathFromFilename(absl::string_view path) if (last_slash == 0 || path[last_slash] == ':' || path[last_slash - 1] == ':') { ++last_slash; } - return {path.substr(0, last_slash), name}; + return PathSplitResult{path.substr(0, last_slash), name}; } // clang-format off diff --git a/source/common/filesystem/win32/filesystem_impl.h b/source/common/filesystem/win32/filesystem_impl.h index 6a4ccf5521f3..f7a5139c8af8 100644 --- a/source/common/filesystem/win32/filesystem_impl.h +++ b/source/common/filesystem/win32/filesystem_impl.h @@ -92,8 +92,8 @@ class InstanceImplWin32 : public Instance { bool fileExists(const std::string& path) override; bool directoryExists(const std::string& path) override; ssize_t fileSize(const std::string& path) override; - std::string fileReadToEnd(const std::string& path) override; - PathSplitResult splitPathFromFilename(absl::string_view path) override; + absl::StatusOr fileReadToEnd(const std::string& path) override; + absl::StatusOr splitPathFromFilename(absl::string_view path) override; bool illegalPath(const std::string& path) override; Api::IoCallResult stat(absl::string_view path) override; Api::IoCallBoolResult createPath(absl::string_view path) override; diff --git a/source/common/filesystem/win32/watcher_impl.cc b/source/common/filesystem/win32/watcher_impl.cc index eaacb61cfd2f..601c787461bf 100644 --- a/source/common/filesystem/win32/watcher_impl.cc +++ b/source/common/filesystem/win32/watcher_impl.cc @@ -55,7 +55,9 @@ void WatcherImpl::addWatch(absl::string_view path, uint32_t events, OnChangedCb return; } - const PathSplitResult result = file_system_.splitPathFromFilename(path); + const absl::StatusOr result_or_error = file_system_.splitPathFromFilename(path); + THROW_IF_STATUS_NOT_OK(result_or_error, throw); + const PathSplitResult& result = result_or_error.value(); // ReadDirectoryChangesW only has a Unicode version, so we need // to use wide strings here const std::wstring directory = wstring_converter_.from_bytes(std::string(result.directory_)); diff --git a/source/common/filter/config_discovery_impl.cc b/source/common/filter/config_discovery_impl.cc index e9ce25cd6e86..0aafca8f05b6 100644 --- a/source/common/filter/config_discovery_impl.cc +++ b/source/common/filter/config_discovery_impl.cc @@ -225,7 +225,7 @@ void FilterConfigProviderManagerImplBase::applyLastOrDefaultConfig( bool last_config_valid = false; if (subscription->lastConfig()) { TRY_ASSERT_MAIN_THREAD { - THROW_IF_NOT_OK(provider.validateTypeUrl(subscription->lastTypeUrl())) + THROW_IF_NOT_OK(provider.validateTypeUrl(subscription->lastTypeUrl())); provider.validateMessage(filter_config_name, *subscription->lastConfig(), subscription->lastFactoryName()); last_config_valid = true; @@ -252,9 +252,10 @@ void FilterConfigProviderManagerImplBase::validateProtoConfigDefaultFactory( const bool null_default_factory, const std::string& filter_config_name, absl::string_view type_url) const { if (null_default_factory) { - throw EnvoyException(fmt::format("Error: cannot find filter factory {} for default filter " - "configuration with type URL {}.", - filter_config_name, type_url)); + throwEnvoyExceptionOrPanic( + fmt::format("Error: cannot find filter factory {} for default filter " + "configuration with type URL {}.", + filter_config_name, type_url)); } } diff --git a/source/common/filter/config_discovery_impl.h b/source/common/filter/config_discovery_impl.h index 60f53060fb40..16889d5a4e46 100644 --- a/source/common/filter/config_discovery_impl.h +++ b/source/common/filter/config_discovery_impl.h @@ -127,7 +127,7 @@ class DynamicFilterConfigProviderImpl : public DynamicFilterConfigProviderImplBa if (default_configuration_) { auto status = onConfigUpdate(*default_configuration_, "", nullptr); if (!status.ok()) { - throw EnvoyException(std::string(status.message())); + throwEnvoyExceptionOrPanic(std::string(status.message())); } } } @@ -219,8 +219,13 @@ class HttpDynamicFilterConfigProviderImpl instantiateFilterFactory(const Protobuf::Message& message) const override { auto* factory = Registry::FactoryRegistry::getFactoryByType( message.GetTypeName()); - return {factory->name(), - factory->createFilterFactoryFromProto(message, getStatPrefix(), factory_context_)}; + absl::StatusOr error_or_factory = + factory->createFilterFactoryFromProto(message, getStatPrefix(), factory_context_); + if (!error_or_factory.status().ok()) { + throwEnvoyExceptionOrPanic(std::string(error_or_factory.status().message())); + } + + return {factory->name(), error_or_factory.value()}; } Server::Configuration::ServerFactoryContext& server_context_; @@ -304,7 +309,7 @@ class ListenerDynamicFilterConfigProviderImpl : public DynamicFilterConfigProvid const std::string& filter_chain_type, absl::string_view stat_prefix, const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher) : DynamicFilterConfigProviderImpl( - subscription, require_type_urls, factory_context.threadLocal(), + subscription, require_type_urls, factory_context.serverFactoryContext().threadLocal(), std::move(default_config), last_filter_in_filter_chain, filter_chain_type, stat_prefix, listener_filter_matcher), factory_context_(factory_context) {} diff --git a/source/common/formatter/BUILD b/source/common/formatter/BUILD index dd70e30c6625..5047a90f877e 100644 --- a/source/common/formatter/BUILD +++ b/source/common/formatter/BUILD @@ -13,6 +13,7 @@ envoy_cc_library( srcs = [ "http_specific_formatter.cc", "stream_info_formatter.cc", + "substitution_formatter.cc", ], hdrs = [ "http_specific_formatter.h", @@ -48,6 +49,7 @@ envoy_cc_library( ":substitution_formatter_lib", "//source/common/config:utility_lib", "//source/common/protobuf", + "//source/server:generic_factory_context_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/common/formatter/http_specific_formatter.cc b/source/common/formatter/http_specific_formatter.cc index c86194ccd9cf..c68638d8eb6a 100644 --- a/source/common/formatter/http_specific_formatter.cc +++ b/source/common/formatter/http_specific_formatter.cc @@ -201,7 +201,8 @@ GrpcStatusFormatter::Format GrpcStatusFormatter::parseFormat(absl::string_view f return GrpcStatusFormatter::Number; } - throw EnvoyException("GrpcStatusFormatter only supports CAMEL_STRING, SNAKE_STRING or NUMBER."); + throwEnvoyExceptionOrPanic( + "GrpcStatusFormatter only supports CAMEL_STRING, SNAKE_STRING or NUMBER."); } GrpcStatusFormatter::GrpcStatusFormatter(const std::string& main_header, diff --git a/source/common/formatter/stream_info_formatter.cc b/source/common/formatter/stream_info_formatter.cc index ac305351c6c2..7d72800f3aaa 100644 --- a/source/common/formatter/stream_info_formatter.cc +++ b/source/common/formatter/stream_info_formatter.cc @@ -140,7 +140,7 @@ FilterStateFormatter::create(const std::string& format, const absl::optional) { + return std::make_unique( + format, + std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + return stream_info.timeSource().systemTime(); + })); + }}}, {"DYNAMIC_METADATA", {CommandSyntaxChecker::PARAMS_REQUIRED, [](const std::string& format, absl::optional max_length) { @@ -1429,6 +1438,26 @@ const StreamInfoFormatterProviderLookupTable& getKnownStreamInfoFormatterProvide [](const std::string& key, absl::optional max_length) { return std::make_unique(key, max_length); }}}, + {"UPSTREAM_CONNECTION_POOL_READY_DURATION", + {CommandSyntaxChecker::COMMAND_ONLY, + [](const std::string&, const absl::optional&) { + return std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) + -> absl::optional { + if (auto upstream_info = stream_info.upstreamInfo(); + upstream_info.has_value()) { + if (auto connection_pool_callback_latency = + upstream_info.value() + .get() + .upstreamTiming() + .connectionPoolCallbackLatency(); + connection_pool_callback_latency.has_value()) { + return connection_pool_callback_latency; + } + } + return absl::nullopt; + }); + }}}, }); } diff --git a/source/common/formatter/stream_info_formatter.h b/source/common/formatter/stream_info_formatter.h index 2aa683c03a90..5ad34ef617e2 100644 --- a/source/common/formatter/stream_info_formatter.h +++ b/source/common/formatter/stream_info_formatter.h @@ -274,7 +274,7 @@ class CommonStreamInfoFormatterBase : public FormatterProviderBase static FormatterBasePtr fromProtoConfig(const envoy::config::core::v3::SubstitutionFormatString& config, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::GenericFactoryContext& context) { // Instantiate formatter extensions. std::vector> commands; for (const auto& formatter : config.formatters()) { auto* factory = Envoy::Config::Utility::getFactory>(formatter); if (!factory) { - throw EnvoyException(absl::StrCat("Formatter not found: ", formatter.name())); + throwEnvoyExceptionOrPanic(absl::StrCat("Formatter not found: ", formatter.name())); } auto typed_config = Envoy::Config::Utility::translateAnyToFactoryConfig( formatter.typed_config(), context.messageValidationVisitor(), *factory); auto parser = factory->createCommandParserFromProto(*typed_config, context); if (!parser) { - throw EnvoyException(absl::StrCat("Failed to create command parser: ", formatter.name())); + throwEnvoyExceptionOrPanic( + absl::StrCat("Failed to create command parser: ", formatter.name())); } commands.push_back(std::move(parser)); } @@ -56,8 +58,9 @@ class SubstitutionFormatStringUtils { commands); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormatSource: return std::make_unique>( - Config::DataSource::read(config.text_format_source(), true, context.api()), false, - commands); + Config::DataSource::read(config.text_format_source(), true, + context.serverFactoryContext().api()), + config.omit_empty_values(), commands); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::FORMAT_NOT_SET: PANIC_DUE_TO_PROTO_UNSET; } diff --git a/source/common/formatter/substitution_format_utility.cc b/source/common/formatter/substitution_format_utility.cc index baaf382896bf..82a906d3ed3d 100644 --- a/source/common/formatter/substitution_format_utility.cc +++ b/source/common/formatter/substitution_format_utility.cc @@ -16,15 +16,15 @@ void CommandSyntaxChecker::verifySyntax(CommandSyntaxFlags flags, const std::str const std::string& subcommand, const absl::optional& length) { if ((flags == COMMAND_ONLY) && ((subcommand.length() != 0) || length.has_value())) { - throw EnvoyException(fmt::format("{} does not take any parameters or length", command)); + throwEnvoyExceptionOrPanic(fmt::format("{} does not take any parameters or length", command)); } if ((flags & PARAMS_REQUIRED).any() && (subcommand.length() == 0)) { - throw EnvoyException(fmt::format("{} requires parameters", command)); + throwEnvoyExceptionOrPanic(fmt::format("{} requires parameters", command)); } if ((flags & LENGTH_ALLOWED).none() && length.has_value()) { - throw EnvoyException(fmt::format("{} does not allow length to be specified.", command)); + throwEnvoyExceptionOrPanic(fmt::format("{} does not allow length to be specified.", command)); } } @@ -93,7 +93,7 @@ void SubstitutionFormatUtils::parseSubcommandHeaders(const std::string& subcomma alternative_header = ""; parseSubcommand(subcommand, '?', main_header, alternative_header, subs); if (!subs.empty()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( // Header format rules support only one alternative header. // docs/root/configuration/observability/access_log/access_log.rst#format-rules absl::StrCat("More than 1 alternative header specified in token: ", subcommand)); @@ -102,7 +102,8 @@ void SubstitutionFormatUtils::parseSubcommandHeaders(const std::string& subcomma // The main and alternative header should not contain invalid characters {NUL, LR, CF}. if (!Envoy::Http::validHeaderString(main_header) || !Envoy::Http::validHeaderString(alternative_header)) { - throw EnvoyException("Invalid header configuration. Format string contains null or newline."); + throwEnvoyExceptionOrPanic( + "Invalid header configuration. Format string contains null or newline."); } } diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc new file mode 100644 index 000000000000..34f397b0ef26 --- /dev/null +++ b/source/common/formatter/substitution_formatter.cc @@ -0,0 +1,58 @@ +#include "source/common/formatter/substitution_formatter.h" + +namespace Envoy { +namespace Formatter { + +const std::regex& SubstitutionFormatParser::commandWithArgsRegex() { + // The following regex is used to check validity of the formatter command and to + // extract groups. + // The formatter command has the following format: + // % COMMAND(SUBCOMMAND):LENGTH% + // % signs at the beginning and end are used by parser to find next COMMAND. + // COMMAND must always be present and must consist of characters: "A-Z", "0-9" or "_". + // SUBCOMMAND presence depends on the COMMAND. Format is flexible but cannot contain ")".: + // - for some commands SUBCOMMAND is not allowed (for example %PROTOCOL%) + // - for some commands SUBCOMMAND is required (for example %REQ(:AUTHORITY)%, just %REQ% will + // cause error) + // - for some commands SUBCOMMAND is optional (for example %START_TIME% and + // %START_TIME(%f.%1f.%2f.%3f)% are both correct). + // LENGTH presence depends on the command. Some + // commands allow LENGTH to be specified, so not. Regex is used to validate the syntax and also + // to extract values for COMMAND, SUBCOMMAND and LENGTH. + // + // Below is explanation of capturing and non-capturing groups. Non-capturing groups are used + // to specify that certain part of the formatter command is optional and should contain specific + // characters. Capturing groups are used to extract the values when regex is matched against + // formatter command string. + // + // clang-format off + // Non-capturing group specifying optional :LENGTH ---------------------- + // | + // Non-capturing group specifying optional (SUBCOMMAND)--- | + // | | + // Non-capturing group specifying mandatory COMMAND | | + // which uses only A-Z, 0-9 and _ characters | | + // Group is used only to specify allowed characters. | | + // | | | + // | | | + // _________________ _______________ _____________ + // | | | | | | + CONSTRUCT_ON_FIRST_USE(std::regex, + R"EOF(^%((?:[A-Z]|[0-9]|_)+)(?:\(([^\)]*)\))?(?::([0-9]+))?%)EOF"); + // |__________________| |______| |______| + // | | | + // Capturing group specifying COMMAND -- | | + // The index of this group is 1. | | + // | | + // Capturing group for SUBCOMMAND. If present, it will ----- | + // contain SUBCOMMAND without "(" and ")". The index | + // of SUBCOMMAND group is 2. | + // | + // Capturing group for LENGTH. If present, it will ------------------------- + // contain just number without ":". The index of + // LENGTH group is 3. + // clang-format on +} + +} // namespace Formatter +} // namespace Envoy diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 02dce31b9668..bc450ca95173 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -34,54 +34,6 @@ class SubstitutionFormatParser { std::string current_token; std::vector> formatters; - // The following regex is used to check validity of the formatter command and to - // extract groups. - // The formatter command has the following format: - // % COMMAND(SUBCOMMAND):LENGTH% - // % signs at the beginning and end are used by parser to find next COMMAND. - // COMMAND must always be present and must consist of characters: "A-Z", "0-9" or "_". - // SUBCOMMAND presence depends on the COMMAND. Format is flexible but cannot contain ")".: - // - for some commands SUBCOMMAND is not allowed (for example %PROTOCOL%) - // - for some commands SUBCOMMAND is required (for example %REQ(:AUTHORITY)%, just %REQ% will - // cause error) - // - for some commands SUBCOMMAND is optional (for example %START_TIME% and - // %START_TIME(%f.%1f.%2f.%3f)% are both correct). - // LENGTH presence depends on the command. Some - // commands allow LENGTH to be specified, so not. Regex is used to validate the syntax and also - // to extract values for COMMAND, SUBCOMMAND and LENGTH. - // - // Below is explanation of capturing and non-capturing groups. Non-capturing groups are used - // to specify that certain part of the formatter command is optional and should contain specific - // characters. Capturing groups are used to extract the values when regex is matched against - // formatter command string. - // - // clang-format off - // Non-capturing group specifying optional :LENGTH ------------------------------------- - // | - // Non-capturing group specifying optional (SUBCOMMAND)------------------ | - // | | - // Non-capturing group specifying mandatory COMMAND | | - // which uses only A-Z, 0-9 and _ characters ----- | | - // Group is used only to specify allowed characters. | | | - // | | | - // | | | - // _________________ _______________ _____________ - // | | | | | | - const std::regex command_w_args_regex(R"EOF(^%((?:[A-Z]|[0-9]|_)+)(?:\(([^\)]*)\))?(?::([0-9]+))?%)EOF"); - // |__________________| |______| |______| - // | | | - // Capturing group specifying COMMAND ----------------- | | - // The index of this group is 1. | | - // | | - // Capturing group for SUBCOMMAND. If present, it will -------------------- | - // contain SUBCOMMAND without "(" and ")". The index | - // of SUBCOMMAND group is 2. | - // | - // Capturing group for LENGTH. If present, it will ---------------------------------------- - // contain just number without ":". The index of - // LENGTH group is 3. - // clang-format on - for (size_t pos = 0; pos < format.size(); ++pos) { if (format[pos] != '%') { current_token += format[pos]; @@ -105,8 +57,8 @@ class SubstitutionFormatParser { std::smatch m; const std::string search_space = std::string(format.substr(pos)); - if (!std::regex_search(search_space, m, command_w_args_regex)) { - throw EnvoyException( + if (!std::regex_search(search_space, m, commandWithArgsRegex())) { + throwEnvoyExceptionOrPanic( fmt::format("Incorrect configuration: {}. Couldn't find valid command at position {}", format, pos)); } @@ -121,7 +73,7 @@ class SubstitutionFormatParser { if (m.str(3).length() != 0) { size_t length_value; if (!absl::SimpleAtoi(m.str(3), &length_value)) { - throw EnvoyException(absl::StrCat("Length must be an integer, given: ", m.str(3))); + throwEnvoyExceptionOrPanic(absl::StrCat("Length must be an integer, given: ", m.str(3))); } max_length = length_value; } @@ -171,6 +123,9 @@ class SubstitutionFormatParser { return formatters; } + +private: + static const std::regex& commandWithArgsRegex(); }; inline constexpr absl::string_view DefaultUnspecifiedValueStringView = "-"; @@ -317,7 +272,7 @@ template class StructFormatterBase { output->emplace(pair.first, toFormatNumberValue(pair.second.number_value())); break; default: - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Only string values, nested structs, list values and number values are " "supported in structured access log format."); } @@ -346,7 +301,7 @@ template class StructFormatterBase { break; default: - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Only string values, nested structs, list values and number values are " "supported in structured access log format."); } diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 96d1ed06353d..9ab2790a7c1c 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -55,6 +55,7 @@ envoy_cc_library( "//envoy/singleton:manager_interface", "//envoy/thread_local:thread_local_interface", "//envoy/upstream:cluster_manager_interface", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ] + envoy_select_google_grpc([":google_async_client_lib"]), ) @@ -100,6 +101,7 @@ envoy_cc_library( "//source/common/common:macros", "//source/common/common:safe_memcpy_lib", "//source/common/common:utility_lib", + "//source/common/grpc:codec_lib", "//source/common/grpc:status_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", diff --git a/source/common/grpc/async_client_impl.cc b/source/common/grpc/async_client_impl.cc index 7e03eaab7b28..962d68ae3705 100644 --- a/source/common/grpc/async_client_impl.cc +++ b/source/common/grpc/async_client_impl.cc @@ -123,6 +123,7 @@ void AsyncStreamImpl::onHeaders(Http::ResponseHeaderMapPtr&& headers, bool end_s // TODO(mattklein123): clang-tidy is showing a use after move when passing to // onReceiveInitialMetadata() above. This looks like an actual bug that I will fix in a // follow up. + // NOLINTNEXTLINE(bugprone-use-after-move) onTrailers(Http::createHeaderMap(*headers)); return; } @@ -254,7 +255,8 @@ void AsyncRequestImpl::cancel() { } void AsyncRequestImpl::onCreateInitialMetadata(Http::RequestHeaderMap& metadata) { - current_span_->injectContext(metadata, nullptr); + Tracing::HttpTraceContext trace_context(metadata); + current_span_->injectContext(trace_context, nullptr); callbacks_.onCreateInitialMetadata(metadata); } diff --git a/source/common/grpc/async_client_manager_impl.cc b/source/common/grpc/async_client_manager_impl.cc index e741ec56383d..73595f9ea0c1 100644 --- a/source/common/grpc/async_client_manager_impl.cc +++ b/source/common/grpc/async_client_manager_impl.cc @@ -1,5 +1,7 @@ #include "source/common/grpc/async_client_manager_impl.h" +#include + #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/stats/scope.h" @@ -17,6 +19,8 @@ namespace Envoy { namespace Grpc { namespace { +constexpr uint64_t DefaultEntryIdleDuration{50000}; + // Validates a string for gRPC header key compliance. This is a subset of legal HTTP characters. // See https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md bool validateGrpcHeaderChars(absl::string_view key) { @@ -46,16 +50,21 @@ AsyncClientFactoryImpl::AsyncClientFactoryImpl(Upstream::ClusterManager& cm, if (skip_cluster_check) { return; } - cm_.checkActiveStaticCluster(config.envoy_grpc().cluster_name()); + THROW_IF_NOT_OK(cm_.checkActiveStaticCluster(config.envoy_grpc().cluster_name())); } -AsyncClientManagerImpl::AsyncClientManagerImpl(Upstream::ClusterManager& cm, - ThreadLocal::Instance& tls, TimeSource& time_source, - Api::Api& api, const StatNames& stat_names) +AsyncClientManagerImpl::AsyncClientManagerImpl( + Upstream::ClusterManager& cm, ThreadLocal::Instance& tls, TimeSource& time_source, + Api::Api& api, const StatNames& stat_names, + const envoy::config::bootstrap::v3::Bootstrap::GrpcAsyncClientManagerConfig& config) : cm_(cm), tls_(tls), time_source_(time_source), api_(api), stat_names_(stat_names), raw_async_client_cache_(tls_) { - raw_async_client_cache_.set([](Event::Dispatcher& dispatcher) { - return std::make_shared(dispatcher); + + const auto max_cached_entry_idle_duration = std::chrono::milliseconds( + PROTOBUF_GET_MS_OR_DEFAULT(config, max_cached_entry_idle_duration, DefaultEntryIdleDuration)); + + raw_async_client_cache_.set([max_cached_entry_idle_duration](Event::Dispatcher& dispatcher) { + return std::make_shared(dispatcher, max_cached_entry_idle_duration); }); #ifdef ENVOY_GOOGLE_GRPC google_tls_slot_ = tls.allocateSlot(); @@ -84,7 +93,7 @@ GoogleAsyncClientFactoryImpl::GoogleAsyncClientFactoryImpl( UNREFERENCED_PARAMETER(config_); UNREFERENCED_PARAMETER(api_); UNREFERENCED_PARAMETER(stat_names_); - throw EnvoyException("Google C++ gRPC client is not linked"); + throwEnvoyExceptionOrPanic("Google C++ gRPC client is not linked"); #else ASSERT(google_tls_slot_ != nullptr); #endif @@ -94,7 +103,7 @@ GoogleAsyncClientFactoryImpl::GoogleAsyncClientFactoryImpl( for (const auto& header : config.initial_metadata()) { // Validate key if (!validateGrpcHeaderChars(header.key())) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Illegal characters in gRPC initial metadata header key: {}.", header.key())); } @@ -102,7 +111,7 @@ GoogleAsyncClientFactoryImpl::GoogleAsyncClientFactoryImpl( // Binary base64 encoded - handled by the GRPC library if (!::absl::EndsWith(header.key(), "-bin") && !validateGrpcCompatibleAsciiHeaderValue(header.value())) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "Illegal ASCII value for gRPC initial metadata header key: {}.", header.key())); } } @@ -161,8 +170,9 @@ RawAsyncClientSharedPtr AsyncClientManagerImpl::getOrCreateRawAsyncClientWithHas return client; } -AsyncClientManagerImpl::RawAsyncClientCache::RawAsyncClientCache(Event::Dispatcher& dispatcher) - : dispatcher_(dispatcher) { +AsyncClientManagerImpl::RawAsyncClientCache::RawAsyncClientCache( + Event::Dispatcher& dispatcher, std::chrono::milliseconds max_cached_entry_idle_duration) + : dispatcher_(dispatcher), max_cached_entry_idle_duration_(max_cached_entry_idle_duration) { cache_eviction_timer_ = dispatcher.createTimer([this] { evictEntriesAndResetEvictionTimer(); }); } @@ -203,14 +213,20 @@ void AsyncClientManagerImpl::RawAsyncClientCache::evictEntriesAndResetEvictionTi MonotonicTime now = dispatcher_.timeSource().monotonicTime(); // Evict all the entries that have expired. while (!lru_list_.empty()) { - MonotonicTime next_expire = lru_list_.back().accessed_time_ + EntryTimeoutInterval; - if (now >= next_expire) { + const MonotonicTime next_expire = + lru_list_.back().accessed_time_ + max_cached_entry_idle_duration_; + std::chrono::seconds time_to_next_expire_sec = + std::chrono::duration_cast(next_expire - now); + // since 'now' and 'next_expire' are in nanoseconds, the following condition is to + // check if the difference between them is less than 1 second. If we don't do this, the + // timer will be enabled with 0 seconds, which will cause the timer to fire immediately. + // This will cause cpu spike. + if (time_to_next_expire_sec.count() <= 0) { // Erase the expired entry. lru_map_.erase(lru_list_.back().config_with_hash_key_); lru_list_.pop_back(); } else { - cache_eviction_timer_->enableTimer( - std::chrono::duration_cast(next_expire - now)); + cache_eviction_timer_->enableTimer(time_to_next_expire_sec); return; } } diff --git a/source/common/grpc/async_client_manager_impl.h b/source/common/grpc/async_client_manager_impl.h index eadf3009490c..a01f02af41a8 100644 --- a/source/common/grpc/async_client_manager_impl.h +++ b/source/common/grpc/async_client_manager_impl.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/api/api.h" +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/grpc/async_client_manager.h" #include "envoy/singleton/manager.h" @@ -46,8 +47,10 @@ class GoogleAsyncClientFactoryImpl : public AsyncClientFactory { class AsyncClientManagerImpl : public AsyncClientManager { public: - AsyncClientManagerImpl(Upstream::ClusterManager& cm, ThreadLocal::Instance& tls, - TimeSource& time_source, Api::Api& api, const StatNames& stat_names); + AsyncClientManagerImpl( + Upstream::ClusterManager& cm, ThreadLocal::Instance& tls, TimeSource& time_source, + Api::Api& api, const StatNames& stat_names, + const envoy::config::bootstrap::v3::Bootstrap::GrpcAsyncClientManagerConfig& config); RawAsyncClientSharedPtr getOrCreateRawAsyncClient(const envoy::config::core::v3::GrpcService& config, Stats::Scope& scope, bool skip_cluster_check) override; @@ -61,7 +64,8 @@ class AsyncClientManagerImpl : public AsyncClientManager { bool skip_cluster_check) override; class RawAsyncClientCache : public ThreadLocal::ThreadLocalObject { public: - explicit RawAsyncClientCache(Event::Dispatcher& dispatcher); + explicit RawAsyncClientCache(Event::Dispatcher& dispatcher, + std::chrono::milliseconds max_cached_entry_idle_duration); void setCache(const GrpcServiceConfigWithHashKey& config_with_hash_key, const RawAsyncClientSharedPtr& client); @@ -83,7 +87,7 @@ class AsyncClientManagerImpl : public AsyncClientManager { absl::flat_hash_map lru_map_; Event::Dispatcher& dispatcher_; Envoy::Event::TimerPtr cache_eviction_timer_; - static constexpr std::chrono::seconds EntryTimeoutInterval{50}; + const std::chrono::milliseconds max_cached_entry_idle_duration_; }; private: diff --git a/source/common/grpc/codec.h b/source/common/grpc/codec.h index 6eee1ac49680..6140442fb84c 100644 --- a/source/common/grpc/codec.h +++ b/source/common/grpc/codec.h @@ -12,7 +12,7 @@ namespace Grpc { const uint8_t GRPC_FH_DEFAULT = 0b0u; // Last bit for a compressed message. const uint8_t GRPC_FH_COMPRESSED = 0b1u; -// Bit specifies end-of-stream response in Buf Connect. +// Bit specifies end-of-stream response in Connect. const uint8_t CONNECT_FH_EOS = 0b10u; constexpr uint64_t GRPC_FRAME_HEADER_SIZE = sizeof(uint8_t) + sizeof(uint32_t); diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 3cbe02aabffb..aa8058783bb8 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -15,6 +15,7 @@ #include "source/common/common/macros.h" #include "source/common/common/safe_memcpy.h" #include "source/common/common/utility.h" +#include "source/common/grpc/codec.h" #include "source/common/http/header_utility.h" #include "source/common/http/headers.h" #include "source/common/http/message_impl.h" @@ -294,7 +295,7 @@ std::string Common::typeUrl(const std::string& qualified_name) { void Common::prependGrpcFrameHeader(Buffer::Instance& buffer) { std::array header; - header[0] = 0; // flags + header[0] = GRPC_FH_DEFAULT; // flags const uint32_t nsize = htonl(buffer.length()); safeMemcpyUnsafeDst(&header[1], &nsize); buffer.prepend(absl::string_view(&header[0], 5)); diff --git a/source/common/grpc/common.h b/source/common/grpc/common.h index e9f7117eec2e..6d3059c27ff4 100644 --- a/source/common/grpc/common.h +++ b/source/common/grpc/common.h @@ -64,14 +64,14 @@ class Common { /** * @param headers the headers to parse. - * @return bool indicating whether the header is a Buf Connect request header. + * @return bool indicating whether the header is a Connect request header. * This is determined by checking for the connect protocol version header and a path header. */ static bool isConnectRequestHeaders(const Http::RequestHeaderMap& headers); /** * @param headers the headers to parse. - * @return bool indicating whether the header is a Buf Connect streaming request header. + * @return bool indicating whether the header is a Connect streaming request header. * This is determined by checking for the connect streaming content type and a path header. */ static bool isConnectStreamingRequestHeaders(const Http::RequestHeaderMap& headers); @@ -93,7 +93,7 @@ class Common { /** * @param headers the headers to parse. - * @return bool indicating whether the header is a Buf Connect streaming response header. + * @return bool indicating whether the header is a Connect streaming response header. */ static bool isConnectStreamingResponseHeaders(const Http::ResponseHeaderMap& headers); diff --git a/source/common/grpc/google_async_client_impl.cc b/source/common/grpc/google_async_client_impl.cc index 326319585a73..eea658811b45 100644 --- a/source/common/grpc/google_async_client_impl.cc +++ b/source/common/grpc/google_async_client_impl.cc @@ -463,7 +463,8 @@ void GoogleAsyncRequestImpl::cancel() { } void GoogleAsyncRequestImpl::onCreateInitialMetadata(Http::RequestHeaderMap& metadata) { - current_span_->injectContext(metadata, nullptr); + Tracing::HttpTraceContext trace_context(metadata); + current_span_->injectContext(trace_context, nullptr); callbacks_.onCreateInitialMetadata(metadata); } diff --git a/source/common/grpc/google_grpc_utils.cc b/source/common/grpc/google_grpc_utils.cc index a3510685233c..fbe891306692 100644 --- a/source/common/grpc/google_grpc_utils.cc +++ b/source/common/grpc/google_grpc_utils.cc @@ -79,7 +79,7 @@ grpc::ByteBuffer GoogleGrpcUtils::makeByteBuffer(Buffer::InstancePtr&& buffer_in slices.emplace_back(raw_slice.mem_, raw_slice.len_, &BufferInstanceContainer::derefBufferInstanceContainer, container); } - return {&slices[0], slices.size()}; + return {&slices[0], slices.size()}; // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) } class GrpcSliceBufferFragmentImpl : public Buffer::BufferFragment { diff --git a/source/common/http/BUILD b/source/common/http/BUILD index acf2e80fdf70..d1ee13a01bf4 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -150,6 +150,7 @@ envoy_cc_library( srcs = ["dependency_manager.cc"], hdrs = ["dependency_manager.h"], external_deps = [ + "abseil_flat_hash_set", "abseil_status", ], deps = [ diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index 18167336065a..0ec47e45e26c 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -140,6 +140,8 @@ void AsyncStreamImpl::encodeTrailers(ResponseTrailerMapPtr&& trailers) { } void AsyncStreamImpl::sendHeaders(RequestHeaderMap& headers, bool end_stream) { + request_headers_ = &headers; + if (Http::Headers::get().MethodValues.Head == headers.getMethodValue()) { is_head_request_ = true; } @@ -182,6 +184,8 @@ void AsyncStreamImpl::sendData(Buffer::Instance& data, bool end_stream) { } void AsyncStreamImpl::sendTrailers(RequestTrailerMap& trailers) { + request_trailers_ = &trailers; + ASSERT(dispatcher().isThreadSafe()); // See explanation in sendData. if (local_closed_) { @@ -276,7 +280,8 @@ AsyncRequestSharedImpl::AsyncRequestSharedImpl(AsyncClientImpl& parent, } void AsyncRequestImpl::initialize() { - child_span_->injectContext(request_->headers(), nullptr); + Tracing::HttpTraceContext trace_context(request_->headers()); + child_span_->injectContext(trace_context, nullptr); sendHeaders(request_->headers(), request_->body().length() == 0); if (request_->body().length() != 0) { // It's possible this will be a no-op due to a local response synchronously generated in @@ -287,7 +292,8 @@ void AsyncRequestImpl::initialize() { } void AsyncOngoingRequestImpl::initialize() { - child_span_->injectContext(*request_headers_, nullptr); + Tracing::HttpTraceContext trace_context(*request_headers_); + child_span_->injectContext(trace_context, nullptr); sendHeaders(*request_headers_, false); } diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 4d5c80527a33..1631e8383a61 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -208,13 +208,10 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, } // The async client won't pause if sending 1xx headers so simply swallow any. void encode1xxHeaders(ResponseHeaderMapPtr&&) override {} - ResponseHeaderMapOptRef informationalHeaders() const override { return {}; } void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) override; - ResponseHeaderMapOptRef responseHeaders() const override { return {}; } void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(ResponseTrailerMapPtr&& trailers) override; - ResponseTrailerMapOptRef responseTrailers() const override { return {}; } void encodeMetadata(MetadataMapPtr&&) override {} void onDecoderFilterAboveWriteBufferHighWatermark() override { ++high_watermark_calls_; @@ -251,9 +248,19 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, OptRef downstreamCallbacks() override { return {}; } OptRef upstreamCallbacks() override { return {}; } void resetIdleTimer() override {} - void setUpstreamOverrideHost(absl::string_view) override {} - absl::optional upstreamOverrideHost() const override { return {}; } + void setUpstreamOverrideHost(Upstream::LoadBalancerContext::OverrideHost) override {} + absl::optional + upstreamOverrideHost() const override { + return absl::nullopt; + } absl::string_view filterConfigName() const override { return ""; } + RequestHeaderMapOptRef requestHeaders() override { return makeOptRefFromPtr(request_headers_); } + RequestTrailerMapOptRef requestTrailers() override { + return makeOptRefFromPtr(request_trailers_); + } + ResponseHeaderMapOptRef informationalHeaders() override { return {}; } + ResponseHeaderMapOptRef responseHeaders() override { return {}; } + ResponseTrailerMapOptRef responseTrailers() override { return {}; } // ScopeTrackedObject void dumpState(std::ostream& os, int indent_level) const override { @@ -275,6 +282,8 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, Buffer::InstancePtr buffered_body_; Buffer::BufferMemoryAccountSharedPtr account_{nullptr}; absl::optional buffer_limit_{absl::nullopt}; + RequestHeaderMap* request_headers_{}; + RequestTrailerMap* request_trailers_{}; bool encoded_response_headers_{}; bool is_grpc_request_{}; bool is_head_request_{false}; diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 52f8188c205c..a07eb825f789 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -66,6 +66,7 @@ namespace Http { COUNTER(downstream_rq_rejected_via_ip_detection) \ COUNTER(downstream_rq_response_before_rq_complete) \ COUNTER(downstream_rq_rx_reset) \ + COUNTER(downstream_rq_too_many_premature_resets) \ COUNTER(downstream_rq_timeout) \ COUNTER(downstream_rq_header_timeout) \ COUNTER(downstream_rq_too_large) \ diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 3071e7809cf6..c219d9699c47 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1,7 +1,9 @@ #include "source/common/http/conn_manager_impl.h" +#include #include #include +#include #include #include #include @@ -55,6 +57,18 @@ namespace Envoy { namespace Http { +const absl::string_view ConnectionManagerImpl::PrematureResetTotalStreamCountKey = + "overload.premature_reset_total_stream_count"; +const absl::string_view ConnectionManagerImpl::PrematureResetMinStreamLifetimeSecondsKey = + "overload.premature_reset_min_stream_lifetime_seconds"; +// Runtime key for maximum number of requests that can be processed from a single connection per +// I/O cycle. Requests over this limit are deferred until the next I/O cycle. +const absl::string_view ConnectionManagerImpl::MaxRequestsPerIoCycle = + "http.max_requests_per_io_cycle"; +// Don't attempt to intelligently delay close: https://github.com/envoyproxy/envoy/issues/30010 +const absl::string_view ConnectionManagerImpl::OptionallyDelayClose = + "http1.optionally_delay_close"; + bool requestWasConnect(const RequestHeaderMapSharedPtr& headers, Protocol protocol) { if (!headers) { return false; @@ -110,6 +124,8 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config, /*node_id=*/local_info_.node().id(), /*server_name=*/config_.serverName(), /*proxy_status_config=*/config_.proxyStatusConfig())), + max_requests_during_dispatch_( + runtime_.snapshot().getInteger(ConnectionManagerImpl::MaxRequestsPerIoCycle, UINT32_MAX)), refresh_rtt_after_request_( Runtime::runtimeFeatureEnabled("envoy.reloadable_features.refresh_rtt_after_request")) { ENVOY_LOG_ONCE_IF( @@ -127,6 +143,10 @@ const ResponseHeaderMap& ConnectionManagerImpl::continueHeader() { void ConnectionManagerImpl::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) { read_callbacks_ = &callbacks; dispatcher_ = &callbacks.connection().dispatcher(); + if (max_requests_during_dispatch_ != UINT32_MAX) { + deferred_request_processing_callback_ = + dispatcher_->createSchedulableCallback([this]() -> void { onDeferredRequestProcessing(); }); + } stats_.named_.downstream_cx_total_.inc(); stats_.named_.downstream_cx_active_.inc(); @@ -196,7 +216,8 @@ ConnectionManagerImpl::~ConnectionManagerImpl() { void ConnectionManagerImpl::checkForDeferredClose(bool skip_delay_close) { Network::ConnectionCloseType close = Network::ConnectionCloseType::FlushWriteAndDelay; - if (skip_delay_close) { + if (runtime_.snapshot().getBoolean(ConnectionManagerImpl::OptionallyDelayClose, true) && + skip_delay_close) { close = Network::ConnectionCloseType::FlushWrite; } if (drain_state_ == DrainState::Closing && streams_.empty() && !codec_->wantsToWrite()) { @@ -273,6 +294,12 @@ void ConnectionManagerImpl::doEndStream(ActiveStream& stream, bool check_for_def } void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { + if (!stream.state_.is_internally_destroyed_) { + ++closed_non_internally_destroyed_requests_; + if (isPrematureRstStream(stream)) { + ++number_premature_stream_resets_; + } + } if (stream.max_stream_duration_timer_ != nullptr) { stream.max_stream_duration_timer_->disableTimer(); stream.max_stream_duration_timer_ = nullptr; @@ -291,19 +318,17 @@ void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { stream.access_log_flush_timer_ = nullptr; } - if (stream.expand_agnostic_stream_lifetime_) { - // Only destroy the active stream if the underlying codec has notified us of - // completion or we've internal redirect the stream. - if (!stream.canDestroyStream()) { - // Track that this stream is not expecting any additional calls apart from - // codec notification. - stream.state_.is_zombie_stream_ = true; - return; - } + // Only destroy the active stream if the underlying codec has notified us of + // completion or we've internal redirect the stream. + if (!stream.canDestroyStream()) { + // Track that this stream is not expecting any additional calls apart from + // codec notification. + stream.state_.is_zombie_stream_ = true; + return; + } - if (stream.response_encoder_ != nullptr) { - stream.response_encoder_->getStream().registerCodecEventCallbacks(nullptr); - } + if (stream.response_encoder_ != nullptr) { + stream.response_encoder_->getStream().registerCodecEventCallbacks(nullptr); } stream.completeRequest(); @@ -349,6 +374,7 @@ void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { if (connection_idle_timer_ && streams_.empty()) { connection_idle_timer_->enableTimer(config_.idleTimeout().value()); } + maybeDrainDueToPrematureResets(); } RequestDecoderHandlePtr ConnectionManagerImpl::newStreamHandle(ResponseEncoder& response_encoder, @@ -397,9 +423,7 @@ RequestDecoder& ConnectionManagerImpl::newStream(ResponseEncoder& response_encod new_stream->state_.is_internally_created_ = is_internally_created; new_stream->response_encoder_ = &response_encoder; new_stream->response_encoder_->getStream().addCallbacks(*new_stream); - if (new_stream->expand_agnostic_stream_lifetime_) { - new_stream->response_encoder_->getStream().registerCodecEventCallbacks(new_stream.get()); - } + new_stream->response_encoder_->getStream().registerCodecEventCallbacks(new_stream.get()); new_stream->response_encoder_->getStream().setFlushTimeout(new_stream->idle_timeout_ms_); new_stream->streamInfo().setDownstreamBytesMeter(response_encoder.getStream().bytesMeter()); // If the network connection is backed up, the stream should be made aware of it on creation. @@ -453,6 +477,7 @@ void ConnectionManagerImpl::createCodec(Buffer::Instance& data) { } Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { + requests_during_dispatch_count_ = 0; if (!codec_) { // Http3 codec should have been instantiated by now. createCodec(data); @@ -619,6 +644,61 @@ void ConnectionManagerImpl::doConnectionClose( } } +bool ConnectionManagerImpl::isPrematureRstStream(const ActiveStream& stream) const { + // Check if the request was prematurely reset, by comparing its lifetime to the configured + // threshold. + ASSERT(!stream.state_.is_internally_destroyed_); + absl::optional duration = + stream.filter_manager_.streamInfo().currentDuration(); + + // Check if request lifetime is longer than the premature reset threshold. + if (duration) { + const uint64_t lifetime = std::chrono::duration_cast(*duration).count(); + const uint64_t min_lifetime = runtime_.snapshot().getInteger( + ConnectionManagerImpl::PrematureResetMinStreamLifetimeSecondsKey, 1); + if (lifetime > min_lifetime) { + return false; + } + } + + // If request has completed before configured threshold, also check if the Envoy proxied the + // response from the upstream. Requests without the response status were reset. + // TODO(RyanTheOptimist): Possibly support half_closed_local instead. + return !stream.filter_manager_.streamInfo().responseCode(); +} + +// Sends a GOAWAY if too many streams have been reset prematurely on this +// connection. +void ConnectionManagerImpl::maybeDrainDueToPrematureResets() { + if (!Runtime::runtimeFeatureEnabled( + "envoy.restart_features.send_goaway_for_premature_rst_streams") || + closed_non_internally_destroyed_requests_ == 0) { + return; + } + + const uint64_t limit = + runtime_.snapshot().getInteger(ConnectionManagerImpl::PrematureResetTotalStreamCountKey, 500); + + if (closed_non_internally_destroyed_requests_ < limit) { + // Even though the total number of streams have not reached `limit`, check if the number of bad + // streams is high enough that even if every subsequent stream is good, the connection + // would be closed once the limit is reached, and if so close the connection now. + if (number_premature_stream_resets_ * 2 < limit) { + return; + } + } else { + if (number_premature_stream_resets_ * 2 < closed_non_internally_destroyed_requests_) { + return; + } + } + + if (read_callbacks_->connection().state() == Network::Connection::State::Open) { + stats_.named_.downstream_rq_too_many_premature_resets_.inc(); + doConnectionClose(Network::ConnectionCloseType::Abort, absl::nullopt, + "too_many_premature_resets"); + } +} + void ConnectionManagerImpl::onGoAway(GoAwayErrorCode) { // Currently we do nothing with remote go away frames. In the future we can decide to no longer // push resources if applicable. @@ -765,8 +845,6 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect StreamInfo::FilterState::LifeSpan::Connection), request_response_timespan_(new Stats::HistogramCompletableTimespanImpl( connection_manager_.stats_.named_.downstream_rq_time_, connection_manager_.timeSource())), - expand_agnostic_stream_lifetime_( - Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)), header_validator_( connection_manager.config_.makeHeaderValidator(connection_manager.codec_->protocol())) { ASSERT(!connection_manager.config_.isRoutable() || @@ -1112,6 +1190,12 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt bool end_stream) { ENVOY_STREAM_LOG(debug, "request headers complete (end_stream={}):\n{}", *this, end_stream, *headers); + // We only want to record this when reading the headers the first time, not when recreating + // a stream. + if (!filter_manager_.remoteDecodeComplete()) { + filter_manager_.streamInfo().downstreamTiming().onLastDownstreamHeaderRxByteReceived( + connection_manager_.dispatcher_->timeSource()); + } ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); request_headers_ = std::move(headers); @@ -1226,7 +1310,7 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt // Rewrites the host of CONNECT-UDP requests. if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") && - HeaderUtility::isConnectUdp(*request_headers_) && + HeaderUtility::isConnectUdpRequest(*request_headers_) && !HeaderUtility::rewriteAuthorityForConnectUdp(*request_headers_)) { sendLocalReply(Code::NotFound, "The path is incorrect for CONNECT-UDP", nullptr, absl::nullopt, StreamInfo::ResponseCodeDetails::get().InvalidPath); @@ -1341,7 +1425,12 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt traceRequest(); } - filter_manager_.decodeHeaders(*request_headers_, end_stream); + if (!connection_manager_.shouldDeferRequestProxyingToNextIoCycle()) { + filter_manager_.decodeHeaders(*request_headers_, end_stream); + } else { + state_.deferred_to_next_io_iteration_ = true; + state_.deferred_end_stream_ = end_stream; + } // Reset it here for both global and overridden cases. resetIdleTimer(); @@ -1353,8 +1442,9 @@ void ConnectionManagerImpl::ActiveStream::traceRequest() { ConnectionManagerImpl::chargeTracingStats(tracing_decision.reason, connection_manager_.config_.tracingStats()); + Tracing::HttpTraceContext trace_context(*request_headers_); active_span_ = connection_manager_.tracer().startSpan( - *this, *request_headers_, filter_manager_.streamInfo(), tracing_decision); + *this, trace_context, filter_manager_.streamInfo(), tracing_decision); if (!active_span_) { return; @@ -1408,8 +1498,15 @@ void ConnectionManagerImpl::ActiveStream::decodeData(Buffer::Instance& data, boo connection_manager_.read_callbacks_->connection().dispatcher()); maybeEndDecode(end_stream); filter_manager_.streamInfo().addBytesReceived(data.length()); - - filter_manager_.decodeData(data, end_stream); + if (!state_.deferred_to_next_io_iteration_) { + filter_manager_.decodeData(data, end_stream); + } else { + if (!deferred_data_) { + deferred_data_ = std::make_unique(); + } + deferred_data_->move(data); + state_.deferred_end_stream_ = end_stream; + } } void ConnectionManagerImpl::ActiveStream::decodeTrailers(RequestTrailerMapPtr&& trailers) { @@ -1425,15 +1522,21 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(RequestTrailerMapPtr&& return; } maybeEndDecode(true); - filter_manager_.decodeTrailers(*request_trailers_); + if (!state_.deferred_to_next_io_iteration_) { + filter_manager_.decodeTrailers(*request_trailers_); + } } void ConnectionManagerImpl::ActiveStream::decodeMetadata(MetadataMapPtr&& metadata_map) { resetIdleTimer(); - // After going through filters, the ownership of metadata_map will be passed to terminal filter. - // The terminal filter may encode metadata_map to the next hop immediately or store metadata_map - // and encode later when connection pool is ready. - filter_manager_.decodeMetadata(*metadata_map); + if (!state_.deferred_to_next_io_iteration_) { + // After going through filters, the ownership of metadata_map will be passed to terminal filter. + // The terminal filter may encode metadata_map to the next hop immediately or store metadata_map + // and encode later when connection pool is ready. + filter_manager_.decodeMetadata(*metadata_map); + } else { + deferred_metadata_.push(std::move(metadata_map)); + } } void ConnectionManagerImpl::ActiveStream::disarmRequestTimeout() { @@ -1717,12 +1820,9 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade state_.is_tunneling_ = true; } - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.prohibit_route_refresh_after_response_headers_sent")) { - // Block route cache if the response headers is received and processed. Because after this - // point, the cached route should never be updated or refreshed. - blockRouteCache(); - } + // Block route cache if the response headers is received and processed. Because after this + // point, the cached route should never be updated or refreshed. + blockRouteCache(); if (connection_manager_.drain_state_ != DrainState::NotDraining && connection_manager_.codec_->protocol() < Protocol::Http2) { @@ -1917,6 +2017,11 @@ uint32_t ConnectionManagerImpl::ActiveStream::maxPathTagLength() const { return connection_manager_tracing_config_->max_path_tag_length_; } +bool ConnectionManagerImpl::ActiveStream::spawnUpstreamSpan() const { + ASSERT(connection_manager_tracing_config_.has_value()); + return connection_manager_tracing_config_->spawn_upstream_span_; +} + const Router::RouteEntry::UpgradeMap* ConnectionManagerImpl::ActiveStream::upgradeMap() { // We must check if the 'cached_route_' optional is populated since this function can be called // early via sendLocalReply(), before the cached route is populated. @@ -2133,5 +2238,76 @@ void ConnectionManagerImpl::ActiveStream::resetStream(Http::StreamResetReason, a connection_manager_.doEndStream(*this); } +bool ConnectionManagerImpl::ActiveStream::onDeferredRequestProcessing() { + // TODO(yanavlasov): Merge this with the filter manager continueIteration() method + if (!state_.deferred_to_next_io_iteration_) { + return false; + } + state_.deferred_to_next_io_iteration_ = false; + bool end_stream = state_.deferred_end_stream_ && deferred_data_ == nullptr && + request_trailers_ == nullptr && deferred_metadata_.empty(); + filter_manager_.decodeHeaders(*request_headers_, end_stream); + if (end_stream) { + return true; + } + // Send metadata before data, as data may have an associated end_stream. + while (!deferred_metadata_.empty()) { + MetadataMapPtr& metadata = deferred_metadata_.front(); + filter_manager_.decodeMetadata(*metadata); + deferred_metadata_.pop(); + } + // Filter manager will return early from decodeData and decodeTrailers if + // request has completed. + if (deferred_data_ != nullptr) { + end_stream = state_.deferred_end_stream_ && request_trailers_ == nullptr; + filter_manager_.decodeData(*deferred_data_, end_stream); + } + if (request_trailers_ != nullptr) { + filter_manager_.decodeTrailers(*request_trailers_); + } + return true; +} + +bool ConnectionManagerImpl::shouldDeferRequestProxyingToNextIoCycle() { + // Do not defer this stream if stream deferral is disabled + if (deferred_request_processing_callback_ == nullptr) { + return false; + } + // Defer this stream if there are already deferred streams, so they are not + // processed out of order + if (deferred_request_processing_callback_->enabled()) { + return true; + } + ++requests_during_dispatch_count_; + bool defer = requests_during_dispatch_count_ > max_requests_during_dispatch_; + if (defer) { + deferred_request_processing_callback_->scheduleCallbackNextIteration(); + } + return defer; +} + +void ConnectionManagerImpl::onDeferredRequestProcessing() { + if (streams_.empty()) { + return; + } + requests_during_dispatch_count_ = 1; // 1 stream is always let through + // Streams are inserted at the head of the list. As such process deferred + // streams in the reverse order. + auto reverse_iter = std::prev(streams_.end()); + bool at_first_element = false; + do { + at_first_element = reverse_iter == streams_.begin(); + // Move the iterator to the previous item in case the `onDeferredRequestProcessing` call removes + // the stream from the list. + auto previous_element = std::prev(reverse_iter); + bool was_deferred = (*reverse_iter)->onDeferredRequestProcessing(); + if (was_deferred && shouldDeferRequestProxyingToNextIoCycle()) { + break; + } + reverse_iter = previous_element; + // TODO(yanavlasov): see if `rend` can be used. + } while (!at_first_element); +} + } // namespace Http } // namespace Envoy diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 47749a9b77bf..9c97f9eb6350 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -118,6 +118,16 @@ class ConnectionManagerImpl : Logger::Loggable, void setClearHopByHopResponseHeaders(bool value) { clear_hop_by_hop_response_headers_ = value; } bool clearHopByHopResponseHeaders() const { return clear_hop_by_hop_response_headers_; } + // This runtime key configures the number of streams which must be closed on a connection before + // envoy will potentially drain a connection due to excessive prematurely reset streams. + static const absl::string_view PrematureResetTotalStreamCountKey; + + // The minimum lifetime of a stream, in seconds, in order not to be considered + // prematurely closed. + static const absl::string_view PrematureResetMinStreamLifetimeSecondsKey; + static const absl::string_view MaxRequestsPerIoCycle; + static const absl::string_view OptionallyDelayClose; + private: struct ActiveStream; class MobileConnectionManagerImpl; @@ -310,12 +320,8 @@ class ConnectionManagerImpl : Logger::Loggable, void blockRouteCache(); // Return true if the cached route is blocked. bool routeCacheBlocked() const { - ENVOY_BUG(!route_cache_blocked_, - "Should never try to refresh or clear the route cache when " - "it is blocked! To temporarily ignore this new constraint, " - "set runtime flag " - "`envoy.reloadable_features.prohibit_route_refresh_after_response_headers_sent` " - "to `false`"); + ENVOY_BUG(!route_cache_blocked_, "Should never try to refresh or clear the route cache when " + "it is blocked!"); return route_cache_blocked_; } @@ -340,7 +346,7 @@ class ConnectionManagerImpl : Logger::Loggable, : codec_saw_local_complete_(false), codec_encode_complete_(false), on_reset_stream_called_(false), is_zombie_stream_(false), successful_upgrade_(false), is_internally_destroyed_(false), is_internally_created_(false), is_tunneling_(false), - decorated_propagate_(true) {} + decorated_propagate_(true), deferred_to_next_io_iteration_(false) {} // It's possibly for the codec to see the completed response but not fully // encode it. @@ -365,6 +371,14 @@ class ConnectionManagerImpl : Logger::Loggable, bool is_tunneling_ : 1; bool decorated_propagate_ : 1; + + // Indicates that sending headers to the filter manager is deferred to the + // next I/O cycle. If data or trailers are received when this flag is set + // they are deferred too. + // TODO(yanavlasov): encapsulate the entire state of deferred streams into a separate + // structure, so it can be atomically created and cleared. + bool deferred_to_next_io_iteration_ : 1; + bool deferred_end_stream_ : 1; }; bool canDestroyStream() const { @@ -412,7 +426,12 @@ class ConnectionManagerImpl : Logger::Loggable, // HTTP connection manager configuration, then the entire connection is closed. bool validateTrailers(); - std::weak_ptr stillAlive() { return std::weak_ptr(still_alive_); } + std::weak_ptr stillAlive() { return {still_alive_}; } + + // Dispatch deferred headers, body and trailers to the filter manager. + // Return true if this stream was deferred and dispatched pending headers, body and trailers (if + // present). Return false if this stream was not deferred. + bool onDeferredRequestProcessing(); ConnectionManagerImpl& connection_manager_; OptRef connection_manager_tracing_config_; @@ -454,8 +473,6 @@ class ConnectionManagerImpl : Logger::Loggable, std::chrono::milliseconds idle_timeout_ms_{}; State state_; - const bool expand_agnostic_stream_lifetime_; - // Snapshot of the route configuration at the time of request is started. This is used to ensure // that the same route configuration is used throughout the lifetime of the request. This // snapshot will be cleared when the cached route is blocked. Because after that we will not @@ -500,8 +517,11 @@ class ConnectionManagerImpl : Logger::Loggable, const Tracing::CustomTagMap* customTags() const override; bool verbose() const override; uint32_t maxPathTagLength() const override; + bool spawnUpstreamSpan() const override; std::shared_ptr still_alive_ = std::make_shared(true); + std::unique_ptr deferred_data_; + std::queue deferred_metadata_; }; using ActiveStreamPtr = std::unique_ptr; @@ -569,6 +589,18 @@ class ConnectionManagerImpl : Logger::Loggable, void doConnectionClose(absl::optional close_type, absl::optional response_flag, absl::string_view details); + // Returns true if a RST_STREAM for the given stream is premature. Premature + // means the RST_STREAM arrived before response headers were sent and than + // the stream was alive for short period of time. This period is specified + // by the optional runtime value PrematureResetMinStreamLifetimeSecondsKey, + // or one second if that is not present. + bool isPrematureRstStream(const ActiveStream& stream) const; + // Sends a GOAWAY if both sufficient streams have been closed on a connection + // and at least half have been prematurely reset? + void maybeDrainDueToPrematureResets(); + + bool shouldDeferRequestProxyingToNextIoCycle(); + void onDeferredRequestProcessing(); enum class DrainState { NotDraining, Draining, Closing }; @@ -609,7 +641,16 @@ class ConnectionManagerImpl : Logger::Loggable, bool clear_hop_by_hop_response_headers_{true}; // The number of requests accumulated on the current connection. uint64_t accumulated_requests_{}; + // The number of requests closed on the current connection which were + // not internally destroyed + uint64_t closed_non_internally_destroyed_requests_{}; + // The number of requests that received a premature RST_STREAM, according to + // the definition given in `isPrematureRstStream()`. + uint64_t number_premature_stream_resets_{0}; const std::string proxy_name_; // for Proxy-Status. + uint32_t requests_during_dispatch_count_{0}; + const uint32_t max_requests_during_dispatch_{UINT32_MAX}; + Event::SchedulableCallbackPtr deferred_request_processing_callback_; const bool refresh_rtt_after_request_{}; }; diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 107cdd9fe270..b9fb637485db 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -92,6 +92,9 @@ ConnectionManagerUtility::MutateRequestHeadersResult ConnectionManagerUtility::m if (!Utility::isUpgrade(request_headers)) { request_headers.removeConnection(); request_headers.removeUpgrade(); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.sanitize_te")) { + request_headers.removeTE(); + } } // Clean proxy headers. @@ -298,9 +301,7 @@ void ConnectionManagerUtility::cleanInternalHeaders( request_headers.removeEnvoyDecoratorOperation(); request_headers.removeEnvoyDownstreamServiceCluster(); request_headers.removeEnvoyDownstreamServiceNode(); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.sanitize_original_path")) { - request_headers.removeEnvoyOriginalPath(); - } + request_headers.removeEnvoyOriginalPath(); } // Headers to be stripped from edge *and* intermediate-hop external requests. diff --git a/source/common/http/conn_pool_grid.cc b/source/common/http/conn_pool_grid.cc index 900818fa3f29..4a3c09dc418a 100644 --- a/source/common/http/conn_pool_grid.cc +++ b/source/common/http/conn_pool_grid.cc @@ -46,7 +46,7 @@ ConnectivityGrid::WrapperCallbacks::WrapperCallbacks(ConnectivityGrid& grid, ConnectivityGrid::WrapperCallbacks::ConnectionAttemptCallbacks::ConnectionAttemptCallbacks( WrapperCallbacks& parent, PoolIterator it) - : parent_(parent), pool_it_(it), cancellable_(nullptr) {} + : parent_(parent), pool_it_(it) {} ConnectivityGrid::WrapperCallbacks::ConnectionAttemptCallbacks::~ConnectionAttemptCallbacks() { if (cancellable_ != nullptr) { diff --git a/source/common/http/conn_pool_grid.h b/source/common/http/conn_pool_grid.h index 969d2e6ae5aa..feaa9ac8963c 100644 --- a/source/common/http/conn_pool_grid.h +++ b/source/common/http/conn_pool_grid.h @@ -69,7 +69,7 @@ class ConnectivityGrid : public ConnectionPool::Instance, const PoolIterator pool_it_; // The handle to cancel this connection attempt. // This is owned by the pool which created it. - Cancellable* cancellable_; + Cancellable* cancellable_{nullptr}; }; using ConnectionAttemptCallbacksPtr = std::unique_ptr; diff --git a/source/common/http/filter_chain_helper.cc b/source/common/http/filter_chain_helper.cc index be7634cca46f..4963dd6e3ae0 100644 --- a/source/common/http/filter_chain_helper.cc +++ b/source/common/http/filter_chain_helper.cc @@ -10,52 +10,38 @@ #include "source/common/config/utility.h" #include "source/common/http/utility.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/filters/http/common/pass_through_filter.h" namespace Envoy { namespace Http { -// Allows graceful handling of missing configuration for ECDS. -class MissingConfigFilter : public Http::PassThroughDecoderFilter { -public: - Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap&, bool) override { - decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::NoFilterConfigFound); - decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, EMPTY_STRING, nullptr, - absl::nullopt, EMPTY_STRING); - return Http::FilterHeadersStatus::StopIteration; - } -}; - -static Http::FilterFactoryCb MissingConfigFilterFactory = - [](Http::FilterChainFactoryCallbacks& cb) { - cb.addStreamDecoderFilter(std::make_shared()); - }; - void FilterChainUtility::createFilterChainForFactories( Http::FilterChainManager& manager, const FilterChainOptions& options, const FilterFactoriesList& filter_factories) { bool added_missing_config_filter = false; for (const auto& filter_config_provider : filter_factories) { // If this filter is disabled explicitly, skip trying to create it. - if (options.filterDisabled(filter_config_provider->name())) { + if (options.filterDisabled(filter_config_provider.provider->name()) + .value_or(filter_config_provider.disabled)) { continue; } - auto config = filter_config_provider->config(); + auto config = filter_config_provider.provider->config(); if (config.has_value()) { Filter::NamedHttpFilterFactoryCb& factory_cb = config.value().get(); - manager.applyFilterFactoryCb({filter_config_provider->name(), factory_cb.name}, + manager.applyFilterFactoryCb({filter_config_provider.provider->name(), factory_cb.name}, factory_cb.factory_cb); continue; } // If a filter config is missing after warming, inject a local reply with status 500. if (!added_missing_config_filter) { - ENVOY_LOG(trace, "Missing filter config for a provider {}", filter_config_provider->name()); + ENVOY_LOG(trace, "Missing filter config for a provider {}", + filter_config_provider.provider->name()); manager.applyFilterFactoryCb({}, MissingConfigFilterFactory); added_missing_config_filter = true; } else { - ENVOY_LOG(trace, "Provider {} missing a filter config", filter_config_provider->name()); + ENVOY_LOG(trace, "Provider {} missing a filter config", + filter_config_provider.provider->name()); } } } @@ -80,7 +66,7 @@ FilterChainUtility::createSingletonDownstreamFilterConfigProviderManager( downstream_filter_config_provider_manager = context.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(downstream_filter_config_provider_manager), - [] { return std::make_shared(); }); + [] { return std::make_shared(); }, true); return downstream_filter_config_provider_manager; } diff --git a/source/common/http/filter_chain_helper.h b/source/common/http/filter_chain_helper.h index d8c9c725c064..39d6ce6dd6af 100644 --- a/source/common/http/filter_chain_helper.h +++ b/source/common/http/filter_chain_helper.h @@ -6,9 +6,11 @@ #include "envoy/filter/config_provider_manager.h" #include "envoy/http/filter.h" +#include "source/common/common/empty_string.h" #include "source/common/common/logger.h" #include "source/common/filter/config_discovery_impl.h" #include "source/common/http/dependency_manager.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" namespace Envoy { namespace Http { @@ -20,10 +22,32 @@ using UpstreamFilterConfigProviderManager = Filter::FilterConfigProviderManager; +// Allows graceful handling of missing configuration for ECDS. +class MissingConfigFilter : public Http::PassThroughDecoderFilter { +public: + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap&, bool) override { + decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::NoFilterConfigFound); + decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, EMPTY_STRING, nullptr, + absl::nullopt, EMPTY_STRING); + return Http::FilterHeadersStatus::StopIteration; + } +}; + +static Http::FilterFactoryCb MissingConfigFilterFactory = + [](Http::FilterChainFactoryCallbacks& cb) { + cb.addStreamDecoderFilter(std::make_shared()); + }; + class FilterChainUtility : Logger::Loggable { public: - using FilterFactoriesList = - std::list>; + struct FilterFactoryProvider { + Filter::FilterConfigProviderPtr provider; + // If true, this filter is disabled by default and must be explicitly enabled by + // route configuration. + bool disabled{}; + }; + + using FilterFactoriesList = std::list; using FiltersList = Protobuf::RepeatedPtrField< envoy::extensions::filters::network::http_connection_manager::v3::HttpFilter>; @@ -43,8 +67,7 @@ class FilterChainUtility : Logger::Loggable { template class FilterChainHelper : Logger::Loggable { public: - using FilterFactoriesList = - std::list>; + using FilterFactoriesList = FilterChainUtility::FilterFactoriesList; using FilterConfigProviderManager = Filter::FilterConfigProviderManager; @@ -85,12 +108,25 @@ class FilterChainHelper : Logger::Loggable { bool last_filter_in_current_config, FilterFactoriesList& filter_factories, DependencyManager& dependency_manager) { ENVOY_LOG(debug, " {} filter #{}", prefix, i); + + const bool disabled_by_default = proto_config.disabled(); + + // Ensure the terminal filter will not be disabled by default. Because the terminal filter must + // be the last filter in the chain (ensured by 'validateTerminalFilters') and terminal flag of + // dynamic filter could not determined and may changed at runtime, so we check the last filter + // flag here as alternative. + if (last_filter_in_current_config && disabled_by_default) { + return absl::InvalidArgumentError(fmt::format( + "Error: the last (terminal) filter ({}) in the chain cannot be disabled by default.", + proto_config.name())); + } + if (proto_config.config_type_case() == envoy::extensions::filters::network::http_connection_manager::v3::HttpFilter:: ConfigTypeCase::kConfigDiscovery) { return processDynamicFilterConfig(proto_config.name(), proto_config.config_discovery(), filter_factories, filter_chain_type, - last_filter_in_current_config); + last_filter_in_current_config, disabled_by_default); } // Now see if there is a factory that will accept the config. @@ -104,8 +140,12 @@ class FilterChainHelper : Logger::Loggable { } ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( proto_config, server_context_.messageValidationVisitor(), *factory); - Http::FilterFactoryCb callback = + absl::StatusOr callback_or_error = factory->createFilterFactoryFromProto(*message, stats_prefix_, factory_context_); + if (!callback_or_error.status().ok()) { + return callback_or_error.status(); + } + Http::FilterFactoryCb callback = callback_or_error.value(); dependency_manager.registerFilter(factory->name(), *factory->dependencies()); const bool is_terminal = factory->isTerminalFilterByProto(*message, server_context_); Config::Utility::validateTerminalFilters(proto_config.name(), factory->name(), @@ -119,7 +159,7 @@ class FilterChainHelper : Logger::Loggable { MessageUtil::getJsonStringFromMessageOrError( static_cast(proto_config.typed_config()))); #endif - filter_factories.push_back(std::move(filter_config_provider)); + filter_factories.push_back({std::move(filter_config_provider), disabled_by_default}); return absl::OkStatus(); } @@ -128,7 +168,7 @@ class FilterChainHelper : Logger::Loggable { const envoy::config::core::v3::ExtensionConfigSource& config_discovery, FilterFactoriesList& filter_factories, const std::string& filter_chain_type, - bool last_filter_in_current_config) { + bool last_filter_in_current_config, bool disabled_by_default) { ENVOY_LOG(debug, " dynamic filter name: {}", name); if (config_discovery.apply_default_config_without_warming() && !config_discovery.has_default_config()) { @@ -148,7 +188,7 @@ class FilterChainHelper : Logger::Loggable { auto filter_config_provider = filter_config_provider_manager_.createDynamicFilterConfigProvider( config_discovery, name, server_context_, factory_context_, cluster_manager_, last_filter_in_current_config, filter_chain_type, nullptr); - filter_factories.push_back(std::move(filter_config_provider)); + filter_factories.push_back({std::move(filter_config_provider), disabled_by_default}); return absl::OkStatus(); } diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index 324e619edb1a..c7cd775321a2 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -297,11 +297,13 @@ ActiveStreamFilterBase::mostSpecificPerFilterConfig() const { result = current_route->mostSpecificPerFilterConfig(filter_context_.filter_name); if (result != nullptr) { - ENVOY_LOG_FIRST_N(warn, 10, - "No per filter config is found by filter config name and fallback to use " - "filter canonical name. This is deprecated and will be forbidden very " - "soon. Please use the filter config name to index per filter config. See " - "https://github.com/envoyproxy/envoy/issues/29461 for more detail."); + ENVOY_LOG_FIRST_N( + warn, 10, + "No per filter config is found by filter config name \"{}\" and fallback to use " + "filter canonical name \"{}\". This is deprecated and will be forbidden very " + "soon. Please use the filter config name to index per filter config. See " + "https://github.com/envoyproxy/envoy/issues/29461 for more detail.", + filter_context_.config_name, filter_context_.filter_name); } } return result; @@ -328,12 +330,14 @@ void ActiveStreamFilterBase::traversePerFilterConfig( } current_route->traversePerFilterConfig( - filter_context_.filter_name, [&cb](const Router::RouteSpecificFilterConfig& config) { - ENVOY_LOG_FIRST_N(warn, 10, - "No per filter config is found by filter config name and fallback to use " - "filter canonical name. This is deprecated and will be forbidden very " - "soon. Please use the filter config name to index per filter config. See " - "https://github.com/envoyproxy/envoy/issues/29461 for more detail."); + filter_context_.filter_name, [&cb, this](const Router::RouteSpecificFilterConfig& config) { + ENVOY_LOG_FIRST_N( + warn, 10, + "No per filter config is found by filter config name \"{}\" and fallback to use " + "filter canonical name \"{}\". This is deprecated and will be forbidden very " + "soon. Please use the filter config name to index per filter config. See " + "https://github.com/envoyproxy/envoy/issues/29461 for more detail.", + filter_context_.config_name, filter_context_.filter_name); cb(config); }); } @@ -352,6 +356,37 @@ OptRef ActiveStreamFilterBase::upstreamCallbacks( return parent_.filter_manager_callbacks_.upstreamCallbacks(); } +RequestHeaderMapOptRef ActiveStreamFilterBase::requestHeaders() { + return parent_.filter_manager_callbacks_.requestHeaders(); +} +RequestTrailerMapOptRef ActiveStreamFilterBase::requestTrailers() { + return parent_.filter_manager_callbacks_.requestTrailers(); +} +ResponseHeaderMapOptRef ActiveStreamFilterBase::informationalHeaders() { + return parent_.filter_manager_callbacks_.informationalHeaders(); +} +ResponseHeaderMapOptRef ActiveStreamFilterBase::responseHeaders() { + return parent_.filter_manager_callbacks_.responseHeaders(); +} +ResponseTrailerMapOptRef ActiveStreamFilterBase::responseTrailers() { + return parent_.filter_manager_callbacks_.responseTrailers(); +} + +void ActiveStreamFilterBase::sendLocalReply( + Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, absl::string_view details) { + if (!streamInfo().filterState()->hasData(LocalReplyFilterStateKey)) { + streamInfo().filterState()->setData( + LocalReplyFilterStateKey, + std::make_shared(filter_context_.config_name), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::FilterChain); + } + + parent_.sendLocalReply(code, body, modify_headers, grpc_status, details); +} + bool ActiveStreamDecoderFilter::canContinue() { // It is possible for the connection manager to respond directly to a request even while // a filter is trying to continue. If a response has already happened, we should not @@ -407,6 +442,14 @@ void ActiveStreamDecoderFilter::drainSavedRequestMetadata() { } void ActiveStreamDecoderFilter::handleMetadataAfterHeadersCallback() { + if (parent_.state_.decoder_filter_chain_aborted_ && + Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.stop_decode_metadata_on_local_reply")) { + // The decoder filter chain has been aborted, possibly due to a local reply. In this case, + // there's no reason to decode saved metadata. + getSavedRequestMetadata()->clear(); + return; + } // If we drain accumulated metadata, the iteration must start with the current filter. const bool saved_state = iterate_from_current_filter_; iterate_from_current_filter_ = true; @@ -456,7 +499,7 @@ void ActiveStreamDecoderFilter::sendLocalReply( Code code, absl::string_view body, std::function modify_headers, const absl::optional grpc_status, absl::string_view details) { - parent_.sendLocalReply(code, body, modify_headers, grpc_status, details); + ActiveStreamFilterBase::sendLocalReply(code, body, modify_headers, grpc_status, details); } void ActiveStreamDecoderFilter::encode1xxHeaders(ResponseHeaderMapPtr&& headers) { @@ -469,10 +512,6 @@ void ActiveStreamDecoderFilter::encode1xxHeaders(ResponseHeaderMapPtr&& headers) } } -ResponseHeaderMapOptRef ActiveStreamDecoderFilter::informationalHeaders() const { - return parent_.filter_manager_callbacks_.informationalHeaders(); -} - void ActiveStreamDecoderFilter::encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) { parent_.streamInfo().setResponseCodeDetails(details); @@ -480,10 +519,6 @@ void ActiveStreamDecoderFilter::encodeHeaders(ResponseHeaderMapPtr&& headers, bo parent_.encodeHeaders(nullptr, *parent_.filter_manager_callbacks_.responseHeaders(), end_stream); } -ResponseHeaderMapOptRef ActiveStreamDecoderFilter::responseHeaders() const { - return parent_.filter_manager_callbacks_.responseHeaders(); -} - void ActiveStreamDecoderFilter::encodeData(Buffer::Instance& data, bool end_stream) { parent_.encodeData(nullptr, data, end_stream, FilterManager::FilterIterationStartState::CanStartFromCurrent); @@ -494,10 +529,6 @@ void ActiveStreamDecoderFilter::encodeTrailers(ResponseTrailerMapPtr&& trailers) parent_.encodeTrailers(nullptr, *parent_.filter_manager_callbacks_.responseTrailers()); } -ResponseTrailerMapOptRef ActiveStreamDecoderFilter::responseTrailers() const { - return parent_.filter_manager_callbacks_.responseTrailers(); -} - void ActiveStreamDecoderFilter::encodeMetadata(MetadataMapPtr&& metadata_map_ptr) { parent_.encodeMetadata(nullptr, std::move(metadata_map_ptr)); } @@ -546,8 +577,14 @@ void FilterManager::decodeHeaders(ActiveStreamDecoderFilter* filter, RequestHead ASSERT(!(state_.filter_call_state_ & FilterCallState::DecodeHeaders)); state_.filter_call_state_ |= FilterCallState::DecodeHeaders; (*entry)->end_stream_ = (end_stream && continue_data_entry == decoder_filters_.end()); + if ((*entry)->end_stream_) { + state_.filter_call_state_ |= FilterCallState::EndOfStream; + } FilterHeadersStatus status = (*entry)->decodeHeaders(headers, (*entry)->end_stream_); state_.filter_call_state_ &= ~FilterCallState::DecodeHeaders; + if ((*entry)->end_stream_) { + state_.filter_call_state_ &= ~FilterCallState::EndOfStream; + } if (state_.decoder_filter_chain_aborted_) { executeLocalReplyIfPrepared(); ENVOY_STREAM_LOG(trace, @@ -677,7 +714,7 @@ void FilterManager::decodeData(ActiveStreamDecoderFilter* filter, Buffer::Instan // is called in decodeData during a previous filter invocation, at which point we communicate to // the current and future filters that the stream has not yet ended. if (end_stream) { - state_.filter_call_state_ |= FilterCallState::LastDataFrame; + state_.filter_call_state_ |= FilterCallState::EndOfStream; } recordLatestDataFilter(entry, state_.latest_data_decoding_filter_, decoder_filters_); @@ -690,7 +727,7 @@ void FilterManager::decodeData(ActiveStreamDecoderFilter* filter, Buffer::Instan } state_.filter_call_state_ &= ~FilterCallState::DecodeData; if (end_stream) { - state_.filter_call_state_ &= ~FilterCallState::LastDataFrame; + state_.filter_call_state_ &= ~FilterCallState::EndOfStream; } ENVOY_STREAM_LOG(trace, "decode data called: filter={} status={}", *this, (*entry)->filter_context_.config_name, static_cast(status)); @@ -731,7 +768,7 @@ void FilterManager::decodeData(ActiveStreamDecoderFilter* filter, Buffer::Instan RequestTrailerMap& FilterManager::addDecodedTrailers() { // Trailers can only be added during the last data frame (i.e. end_stream = true). - ASSERT(state_.filter_call_state_ & FilterCallState::LastDataFrame); + ASSERT(state_.filter_call_state_ & FilterCallState::EndOfStream); filter_manager_callbacks_.setRequestTrailers(RequestTrailerMapImpl::create()); return *filter_manager_callbacks_.requestTrailers(); @@ -831,7 +868,8 @@ void FilterManager::decodeMetadata(ActiveStreamDecoderFilter* filter, MetadataMa if (state_.decoder_filter_chain_aborted_) { // If the decoder filter chain has been aborted, then either: // 1. This filter has sent a local reply from decode metadata. - // 2. This filter is the terminal http filter, and an upstream filter has sent a local reply. + // 2. This filter is the terminal http filter, and an upstream HTTP filter has sent a local + // reply. ASSERT((status == FilterMetadataStatus::StopIterationForLocalReply) || (std::next(entry) == decoder_filters_.end())); executeLocalReplyIfPrepared(); @@ -1157,7 +1195,7 @@ void FilterManager::maybeContinueEncoding( void FilterManager::encodeHeaders(ActiveStreamEncoderFilter* filter, ResponseHeaderMap& headers, bool end_stream) { - // See encodeHeaders() comments in include/envoy/http/filter.h for why the 1xx precondition holds. + // See encodeHeaders() comments in envoy/http/filter.h for why the 1xx precondition holds. ASSERT(!CodeUtility::is1xx(Utility::getResponseStatus(headers)) || Utility::getResponseStatus(headers) == enumToInt(Http::Code::SwitchingProtocols)); filter_manager_callbacks_.resetIdleTimer(); @@ -1172,6 +1210,9 @@ void FilterManager::encodeHeaders(ActiveStreamEncoderFilter* filter, ResponseHea ASSERT(!(state_.filter_call_state_ & FilterCallState::EncodeHeaders)); state_.filter_call_state_ |= FilterCallState::EncodeHeaders; (*entry)->end_stream_ = (end_stream && continue_data_entry == encoder_filters_.end()); + if ((*entry)->end_stream_) { + state_.filter_call_state_ |= FilterCallState::EndOfStream; + } FilterHeadersStatus status = (*entry)->handle_->encodeHeaders(headers, (*entry)->end_stream_); if (state_.encoder_filter_chain_aborted_) { ENVOY_STREAM_LOG(trace, @@ -1185,6 +1226,9 @@ void FilterManager::encodeHeaders(ActiveStreamEncoderFilter* filter, ResponseHea "encodeHeaders when end_stream is already false"); state_.filter_call_state_ &= ~FilterCallState::EncodeHeaders; + if ((*entry)->end_stream_) { + state_.filter_call_state_ &= ~FilterCallState::EndOfStream; + } ENVOY_STREAM_LOG(trace, "encode headers called: filter={} status={}", *this, (*entry)->filter_context_.config_name, static_cast(status)); @@ -1292,7 +1336,7 @@ void FilterManager::encodeMetadata(ActiveStreamEncoderFilter* filter, ResponseTrailerMap& FilterManager::addEncodedTrailers() { // Trailers can only be added during the last data frame (i.e. end_stream = true). - ASSERT(state_.filter_call_state_ & FilterCallState::LastDataFrame); + ASSERT(state_.filter_call_state_ & FilterCallState::EndOfStream); // Trailers can only be added once. ASSERT(!filter_manager_callbacks_.responseTrailers()); @@ -1351,7 +1395,7 @@ void FilterManager::encodeData(ActiveStreamEncoderFilter* filter, Buffer::Instan // the current and future filters that the stream has not yet ended. state_.filter_call_state_ |= FilterCallState::EncodeData; if (end_stream) { - state_.filter_call_state_ |= FilterCallState::LastDataFrame; + state_.filter_call_state_ |= FilterCallState::EndOfStream; } recordLatestDataFilter(entry, state_.latest_data_encoding_filter_, encoder_filters_); @@ -1368,7 +1412,7 @@ void FilterManager::encodeData(ActiveStreamEncoderFilter* filter, Buffer::Instan } state_.filter_call_state_ &= ~FilterCallState::EncodeData; if (end_stream) { - state_.filter_call_state_ &= ~FilterCallState::LastDataFrame; + state_.filter_call_state_ &= ~FilterCallState::EndOfStream; } ENVOY_STREAM_LOG(trace, "encode data called: filter={} status={}", *this, (*entry)->filter_context_.config_name, static_cast(status)); @@ -1523,8 +1567,8 @@ bool FilterManager::createFilterChain() { } } - // This filter chain options is only used for the downstream filter chains for now. So, try to - // set valid initial route only when the downstream callbacks is available. + // This filter chain options is only used for the downstream HTTP filter chains for now. So, try + // to set valid initial route only when the downstream callbacks is available. FilterChainOptionsImpl options( filter_manager_callbacks_.downstreamCallbacks().has_value() ? streamInfo().route() : nullptr); filter_chain_factory_.createFilterChain(*this, false, options); @@ -1710,7 +1754,8 @@ void ActiveStreamEncoderFilter::sendLocalReply( Code code, absl::string_view body, std::function modify_headers, const absl::optional grpc_status, absl::string_view details) { - parent_.sendLocalReply(code, body, modify_headers, grpc_status, details); + + ActiveStreamFilterBase::sendLocalReply(code, body, modify_headers, grpc_status, details); } void ActiveStreamEncoderFilter::responseDataTooLarge() { @@ -1743,11 +1788,13 @@ Buffer::BufferMemoryAccountSharedPtr ActiveStreamDecoderFilter::account() const return parent_.account(); } -void ActiveStreamDecoderFilter::setUpstreamOverrideHost(absl::string_view host) { - parent_.upstream_override_host_.emplace(std::move(host)); +void ActiveStreamDecoderFilter::setUpstreamOverrideHost( + Upstream::LoadBalancerContext::OverrideHost upstream_override_host) { + parent_.upstream_override_host_ = upstream_override_host; } -absl::optional ActiveStreamDecoderFilter::upstreamOverrideHost() const { +absl::optional +ActiveStreamDecoderFilter::upstreamOverrideHost() const { return parent_.upstream_override_host_; } diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 5efc602878bc..42c02e594d23 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -36,6 +36,25 @@ class DownstreamFilterManager; struct ActiveStreamFilterBase; +constexpr absl::string_view LocalReplyFilterStateKey = + "envoy.filters.network.http_connection_manager.local_reply_owner"; +class LocalReplyOwnerObject : public StreamInfo::FilterState::Object { +public: + LocalReplyOwnerObject(const std::string& filter_config_name) + : filter_config_name_(filter_config_name) {} + + ProtobufTypes::MessagePtr serializeAsProto() const override { + auto message = std::make_unique(); + message->set_value(filter_config_name_); + return message; + } + + absl::optional serializeAsString() const override { return filter_config_name_; } + +private: + const std::string filter_config_name_; +}; + /** * Base class wrapper for both stream encoder and decoder filters. * @@ -106,6 +125,11 @@ struct ActiveStreamFilterBase : public virtual StreamFilterCallbacks, OptRef downstreamCallbacks() override; OptRef upstreamCallbacks() override; absl::string_view filterConfigName() const override { return filter_context_.config_name; } + RequestHeaderMapOptRef requestHeaders() override; + RequestTrailerMapOptRef requestTrailers() override; + ResponseHeaderMapOptRef informationalHeaders() override; + ResponseHeaderMapOptRef responseHeaders() override; + ResponseTrailerMapOptRef responseTrailers() override; // Functions to set or get iteration state. bool canIterate() { return iteration_state_ == IterationState::Continue; } @@ -132,6 +156,11 @@ struct ActiveStreamFilterBase : public virtual StreamFilterCallbacks, Router::RouteConstSharedPtr getRoute() const; + void sendLocalReply(Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details); + // A vector to save metadata when the current filter's [de|en]codeMetadata() can not be called, // either because [de|en]codeHeaders() of the current filter returns StopAllIteration or because // [de|en]codeHeaders() adds new metadata to [de|en]code, but we don't know @@ -218,13 +247,10 @@ struct ActiveStreamDecoderFilter : public ActiveStreamFilterBase, const absl::optional grpc_status, absl::string_view details) override; void encode1xxHeaders(ResponseHeaderMapPtr&& headers) override; - ResponseHeaderMapOptRef informationalHeaders() const override; void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) override; - ResponseHeaderMapOptRef responseHeaders() const override; void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(ResponseTrailerMapPtr&& trailers) override; - ResponseTrailerMapOptRef responseTrailers() const override; void encodeMetadata(MetadataMapPtr&& metadata_map_ptr) override; void onDecoderFilterAboveWriteBufferHighWatermark() override; void onDecoderFilterBelowWriteBufferLowWatermark() override; @@ -239,8 +265,8 @@ struct ActiveStreamDecoderFilter : public ActiveStreamFilterBase, Network::Socket::OptionsSharedPtr getUpstreamSocketOptions() const override; Buffer::BufferMemoryAccountSharedPtr account() const override; - void setUpstreamOverrideHost(absl::string_view host) override; - absl::optional upstreamOverrideHost() const override; + void setUpstreamOverrideHost(Upstream::LoadBalancerContext::OverrideHost) override; + absl::optional upstreamOverrideHost() const override; // Each decoder filter instance checks if the request passed to the filter is gRPC // so that we can issue gRPC local responses to gRPC requests. Filter's decodeHeaders() @@ -495,7 +521,7 @@ class FilterManagerCallbacks { virtual Http1StreamEncoderOptionsOptRef http1StreamEncoderOptions() PURE; /** - * Returns the UpstreamStreamFilterCallbacks for upstream filters. + * Returns the UpstreamStreamFilterCallbacks for upstream HTTP filters. */ virtual OptRef upstreamCallbacks() { return {}; } @@ -516,7 +542,7 @@ class FilterManagerCallbacks { virtual const ScopeTrackedObject& scope() PURE; /** - * Returns a handle to the downstream callbacks, if available. + * Returns the DownstreamStreamFilterCallbacks for downstream HTTP filters. */ virtual OptRef downstreamCallbacks() { return {}; } }; @@ -591,6 +617,9 @@ class OverridableRemoteConnectionInfoSetterStreamInfo : public StreamInfo::Strea OptRef filterChainInfo() const override { return StreamInfoImpl::downstreamAddressProvider().filterChainInfo(); } + OptRef listenerInfo() const override { + return StreamInfoImpl::downstreamAddressProvider().listenerInfo(); + } private: Network::Address::InstanceConstSharedPtr overridden_downstream_remote_address_; @@ -662,22 +691,15 @@ class FilterManager : public ScopeTrackedObject, void applyFilterFactoryCb(FilterContext context, FilterFactoryCb& factory) override; void log(AccessLog::AccessLogType access_log_type) { - RequestHeaderMap* request_headers = nullptr; - if (filter_manager_callbacks_.requestHeaders()) { - request_headers = filter_manager_callbacks_.requestHeaders().ptr(); - } - ResponseHeaderMap* response_headers = nullptr; - if (filter_manager_callbacks_.responseHeaders()) { - response_headers = filter_manager_callbacks_.responseHeaders().ptr(); - } - ResponseTrailerMap* response_trailers = nullptr; - if (filter_manager_callbacks_.responseTrailers()) { - response_trailers = filter_manager_callbacks_.responseTrailers().ptr(); - } + const Formatter::HttpFormatterContext log_context{ + filter_manager_callbacks_.requestHeaders().ptr(), + filter_manager_callbacks_.responseHeaders().ptr(), + filter_manager_callbacks_.responseTrailers().ptr(), + {}, + access_log_type}; for (const auto& log_handler : access_log_handlers_) { - log_handler->log(request_headers, response_headers, response_trailers, streamInfo(), - access_log_type); + log_handler->log(log_context, streamInfo()); } } @@ -918,8 +940,8 @@ class FilterManager : public ScopeTrackedObject, public: FilterChainOptionsImpl(Router::RouteConstSharedPtr route) : route_(std::move(route)) {} - bool filterDisabled(absl::string_view config_name) const override { - return route_ != nullptr && route_->filterDisabled(config_name); + absl::optional filterDisabled(absl::string_view config_name) const override { + return route_ != nullptr ? route_->filterDisabled(config_name) : absl::nullopt; } private: @@ -1010,7 +1032,7 @@ class FilterManager : public ScopeTrackedObject, std::list watermark_callbacks_; Network::Socket::OptionsSharedPtr upstream_options_ = std::make_shared(); - absl::optional upstream_override_host_; + absl::optional upstream_override_host_; const FilterChainFactory& filter_chain_factory_; // TODO(snowp): Once FM has been moved to its own file we'll make these private classes of FM, @@ -1037,9 +1059,11 @@ class FilterManager : public ScopeTrackedObject, // to verify we do not encode1xx headers more than once per // filter. static constexpr uint32_t Encode1xxHeaders = 0x100; - // Used to indicate that we're processing the final [En|De]codeData frame, - // i.e. end_stream = true - static constexpr uint32_t LastDataFrame = 0x200; + // Used to indicate that one of the following conditions is true: + // 1. The filter manager is processing the final [En|De]codeData frame. + // 2. The filter manager is in [En|De]codeHeaders and is processing a + // header-only request or response. + static constexpr uint32_t EndOfStream = 0x200; // Masks for filter call state. static constexpr uint32_t IsDecodingMask = DecodeHeaders | DecodeData | DecodeMetadata | DecodeTrailers; diff --git a/source/common/http/hash_policy.cc b/source/common/http/hash_policy.cc index ca97775f318e..d107a33e6172 100644 --- a/source/common/http/hash_policy.cc +++ b/source/common/http/hash_policy.cc @@ -142,11 +142,11 @@ class QueryParameterHashMethod : public HashMethodImplBase { const HeaderEntry* header = headers.Path(); if (header) { - Http::Utility::QueryParams query_parameters = - Http::Utility::parseQueryString(header->value().getStringView()); - const auto& iter = query_parameters.find(parameter_name_); - if (iter != query_parameters.end()) { - hash = HashUtil::xxHash64(iter->second); + Http::Utility::QueryParamsMulti query_parameters = + Http::Utility::QueryParamsMulti::parseQueryString(header->value().getStringView()); + const auto val = query_parameters.getFirstValue(parameter_name_); + if (val.has_value()) { + hash = HashUtil::xxHash64(val.value()); } } return hash; @@ -217,9 +217,9 @@ HashPolicyImpl::HashPolicyImpl( hash_impls_.emplace_back( new FilterStateHashMethod(hash_policy->filter_state().key(), hash_policy->terminal())); break; - default: - throw EnvoyException( - absl::StrCat("Unsupported hash policy ", hash_policy->policy_specifier_case())); + case envoy::config::route::v3::RouteAction::HashPolicy::PolicySpecifierCase:: + POLICY_SPECIFIER_NOT_SET: + PANIC("hash policy not set"); } } } diff --git a/source/common/http/header_map_impl.cc b/source/common/http/header_map_impl.cc index d296a22c2eb7..1431e66ffe6b 100644 --- a/source/common/http/header_map_impl.cc +++ b/source/common/http/header_map_impl.cc @@ -26,11 +26,6 @@ constexpr absl::string_view DelimiterForInlineHeaders{","}; constexpr absl::string_view DelimiterForInlineCookies{"; "}; const static int kMinHeadersForLazyMap = 3; // Optimal hard-coded value based on benchmarks. -bool validatedLowerCaseString(absl::string_view str) { - auto lower_case_str = LowerCaseString(str); - return lower_case_str == str; -} - absl::string_view delimiterByHeader(const LowerCaseString& key) { if (key == Http::Headers::get().Cookie) { return DelimiterForInlineCookies; @@ -531,61 +526,5 @@ HeaderMapImplUtility::getAllHeaderMapImplInfo() { return ret; } -absl::string_view RequestHeaderMapImpl::protocol() const { return getProtocolValue(); } - -absl::string_view RequestHeaderMapImpl::host() const { return getHostValue(); } - -absl::string_view RequestHeaderMapImpl::path() const { return getPathValue(); } - -absl::string_view RequestHeaderMapImpl::method() const { return getMethodValue(); } - -void RequestHeaderMapImpl::forEach(Tracing::TraceContext::IterateCallback callback) const { - HeaderMapImpl::iterate([cb = std::move(callback)](const HeaderEntry& entry) { - if (cb(entry.key().getStringView(), entry.value().getStringView())) { - return HeaderMap::Iterate::Continue; - } - return HeaderMap::Iterate::Break; - }); -} - -absl::optional RequestHeaderMapImpl::getByKey(absl::string_view key) const { - ASSERT(validatedLowerCaseString(key)); - auto result = const_cast(this)->getExisting(key); - - if (result.empty()) { - return absl::nullopt; - } - return result[0]->value().getStringView(); -} - -void RequestHeaderMapImpl::setByKey(absl::string_view key, absl::string_view val) { - ASSERT(validatedLowerCaseString(key)); - HeaderMapImpl::removeExisting(key); - - HeaderString new_key; - new_key.setCopy(key); - HeaderString new_val; - new_val.setCopy(val); - - HeaderMapImpl::insertByKey(std::move(new_key), std::move(new_val)); -} - -void RequestHeaderMapImpl::setByReferenceKey(absl::string_view key, absl::string_view val) { - ASSERT(validatedLowerCaseString(key)); - HeaderMapImpl::removeExisting(key); - - HeaderString new_val; - new_val.setCopy(val); - - HeaderMapImpl::insertByKey(HeaderString(key), std::move(new_val)); -} - -void RequestHeaderMapImpl::setByReference(absl::string_view key, absl::string_view val) { - ASSERT(validatedLowerCaseString(key)); - HeaderMapImpl::removeExisting(key); - - HeaderMapImpl::insertByKey(HeaderString(key), HeaderString(val)); -} - } // namespace Http } // namespace Envoy diff --git a/source/common/http/header_map_impl.h b/source/common/http/header_map_impl.h index df1d76431bb7..df7c911e93de 100644 --- a/source/common/http/header_map_impl.h +++ b/source/common/http/header_map_impl.h @@ -491,17 +491,6 @@ class RequestHeaderMapImpl final : public TypedHeaderMapImpl, INLINE_REQ_RESP_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS) INLINE_REQ_RESP_NUMERIC_HEADERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS) - // Tracing::TraceContext - absl::string_view protocol() const override; - absl::string_view host() const override; - absl::string_view path() const override; - absl::string_view method() const override; - void forEach(Tracing::TraceContext::IterateCallback callback) const override; - absl::optional getByKey(absl::string_view key) const override; - void setByKey(absl::string_view key, absl::string_view val) override; - void setByReferenceKey(absl::string_view key, absl::string_view val) override; - void setByReference(absl::string_view key, absl::string_view val) override; - protected: // NOTE: Because inline_headers_ is a variable size member, it must be the last member in the // most derived class. This forces the definition of the following three functions to also be diff --git a/source/common/http/header_mutation.cc b/source/common/http/header_mutation.cc index 7c88a3bf8be0..1f31ae858143 100644 --- a/source/common/http/header_mutation.cc +++ b/source/common/http/header_mutation.cc @@ -18,10 +18,9 @@ class AppendMutation : public HeaderEvaluator, public Envoy::Router::HeadersToAd AppendMutation(const HeaderValueOption& header_value_option) : HeadersToAddEntry(header_value_option), header_name_(header_value_option.header().key()) {} - void evaluateHeaders(Http::HeaderMap& headers, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + void evaluateHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const override { - std::string value = formatter_->format(request_headers, response_headers, stream_info); + const std::string value = formatter_->formatWithContext(context, stream_info); if (!value.empty() || add_if_empty_) { switch (append_action_) { @@ -57,8 +56,7 @@ class RemoveMutation : public HeaderEvaluator { public: RemoveMutation(const std::string& header_name) : header_name_(header_name) {} - void evaluateHeaders(Http::HeaderMap& headers, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, + void evaluateHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&) const override { headers.remove(header_name_); } @@ -84,11 +82,10 @@ HeaderMutations::HeaderMutations(const ProtoHeaderMutatons& header_mutations) { } void HeaderMutations::evaluateHeaders(Http::HeaderMap& headers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const { for (const auto& mutation : header_mutations_) { - mutation->evaluateHeaders(headers, request_headers, response_headers, stream_info); + mutation->evaluateHeaders(headers, context, stream_info); } } diff --git a/source/common/http/header_mutation.h b/source/common/http/header_mutation.h index f1cad1e08c47..961fa468e177 100644 --- a/source/common/http/header_mutation.h +++ b/source/common/http/header_mutation.h @@ -17,8 +17,7 @@ class HeaderMutations : public HeaderEvaluator { HeaderMutations(const ProtoHeaderMutatons& header_mutations); // Http::HeaderEvaluator - void evaluateHeaders(Http::HeaderMap& headers, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + void evaluateHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const override; private: diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index 398cdab44ac9..f83084ac9ece 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -14,7 +14,10 @@ #include "absl/strings/match.h" #include "nghttp2/nghttp2.h" + +#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS #include "quiche/common/structured_headers.h" +#endif #include "quiche/http2/adapter/header_validator.h" namespace Envoy { @@ -239,11 +242,19 @@ bool HeaderUtility::isConnect(const RequestHeaderMap& headers) { return headers.Method() && headers.Method()->value() == Http::Headers::get().MethodValues.Connect; } -bool HeaderUtility::isConnectUdp(const RequestHeaderMap& headers) { +bool HeaderUtility::isConnectUdpRequest(const RequestHeaderMap& headers) { return headers.Upgrade() && absl::EqualsIgnoreCase(headers.getUpgradeValue(), Http::Headers::get().UpgradeValues.ConnectUdp); } +bool HeaderUtility::isConnectUdpResponse(const ResponseHeaderMap& headers) { + // In connect-udp case, Envoy will transform the H2 headers to H1 upgrade headers. + // A valid response should have SwitchingProtocol status and connect-udp upgrade. + return headers.Upgrade() && Utility::getResponseStatus(headers) == 101 && + absl::EqualsIgnoreCase(headers.getUpgradeValue(), + Http::Headers::get().UpgradeValues.ConnectUdp); +} + bool HeaderUtility::isConnectResponse(const RequestHeaderMap* request_headers, const ResponseHeaderMap& response_headers) { return request_headers && isConnect(*request_headers) && @@ -639,12 +650,12 @@ std::string HeaderUtility::addEncodingToAcceptEncoding(absl::string_view accept_ } bool HeaderUtility::isStandardConnectRequest(const Http::RequestHeaderMap& headers) { - return headers.method() == Http::Headers::get().MethodValues.Connect && + return headers.getMethodValue() == Http::Headers::get().MethodValues.Connect && headers.getProtocolValue().empty(); } bool HeaderUtility::isExtendedH2ConnectRequest(const Http::RequestHeaderMap& headers) { - return headers.method() == Http::Headers::get().MethodValues.Connect && + return headers.getMethodValue() == Http::Headers::get().MethodValues.Connect && !headers.getProtocolValue().empty(); } diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index fb0e863a69b3..58c765a8a27a 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -167,7 +167,12 @@ class HeaderUtility { /** * @brief a helper function to determine if the headers represent a CONNECT-UDP request. */ - static bool isConnectUdp(const RequestHeaderMap& headers); + static bool isConnectUdpRequest(const RequestHeaderMap& headers); + + /** + * @brief a helper function to determine if the headers represent a CONNECT-UDP response. + */ + static bool isConnectUdpResponse(const ResponseHeaderMap& headers); /** * @brief a helper function to determine if the headers represent an accepted CONNECT response. diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 5c1252a8e2c6..24ee070ae6fb 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -176,6 +176,7 @@ class HeaderValues { const LowerCaseString EnvoyOriginalMethod{absl::StrCat(prefix(), "-original-method")}; const LowerCaseString EnvoyOriginalPath{absl::StrCat(prefix(), "-original-path")}; const LowerCaseString EnvoyOverloaded{absl::StrCat(prefix(), "-overloaded")}; + const LowerCaseString EnvoyDropOverload{absl::StrCat(prefix(), "-drop-overload")}; const LowerCaseString EnvoyRateLimited{absl::StrCat(prefix(), "-ratelimited")}; const LowerCaseString EnvoyRetryOn{absl::StrCat(prefix(), "-retry-on")}; const LowerCaseString EnvoyRetryGrpcOn{absl::StrCat(prefix(), "-retry-grpc-on")}; @@ -279,6 +280,10 @@ class HeaderValues { const std::string True{"true"}; } EnvoyOverloadedValues; + struct { + const std::string True{"true"}; + } EnvoyDropOverloadValues; + struct { const std::string True{"true"}; } EnvoyRateLimitedValues; diff --git a/source/common/http/http1/balsa_parser.cc b/source/common/http/http1/balsa_parser.cc index 4221aae94795..bd77276d8ce5 100644 --- a/source/common/http/http1/balsa_parser.cc +++ b/source/common/http/http1/balsa_parser.cc @@ -5,9 +5,9 @@ #include #include "source/common/common/assert.h" -#include "source/common/common/regex.h" #include "source/common/http/headers.h" +#include "absl/strings/ascii.h" #include "absl/strings/match.h" namespace Envoy { @@ -22,6 +22,7 @@ using ::quiche::BalsaHeaders; constexpr absl::string_view kColonSlashSlash = "://"; // Response must start with "HTTP". constexpr char kResponseFirstByte = 'H'; +constexpr absl::string_view kHttpVersionPrefix = "HTTP/"; // Allowed characters for field names according to Section 5.1 // and for methods according to Section 9.1 of RFC 9110: @@ -120,19 +121,21 @@ bool isUrlValid(absl::string_view url, bool is_connect) { std::all_of(path_query.begin(), path_query.end(), is_valid_path_query_char); } +// Returns true if `version_input` is a valid HTTP version string as defined at +// https://www.rfc-editor.org/rfc/rfc9112.html#section-2.3, or empty (for HTTP/0.9). bool isVersionValid(absl::string_view version_input) { - // HTTP-version is defined at - // https://www.rfc-editor.org/rfc/rfc9112.html#section-2.3. HTTP/0.9 requests - // have no http-version, so empty `version_input` is also accepted. - - static const auto regex = [] { - envoy::type::matcher::v3::RegexMatcher matcher; - *matcher.mutable_google_re2() = envoy::type::matcher::v3::RegexMatcher::GoogleRE2(); - matcher.set_regex("|HTTP/[0-9]\\.[0-9]"); - return Regex::Utility::parseRegex(matcher); - }(); - - return regex->match(version_input); + if (version_input.empty()) { + return true; + } + + if (!absl::StartsWith(version_input, kHttpVersionPrefix)) { + return false; + } + version_input.remove_prefix(kHttpVersionPrefix.size()); + + // Version number is in the form of "[0-9].[0-9]". + return version_input.size() == 3 && absl::ascii_isdigit(version_input[0]) && + version_input[1] == '.' && absl::ascii_isdigit(version_input[2]); } bool isHeaderNameValid(absl::string_view name) { @@ -150,7 +153,7 @@ BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t m ASSERT(connection_ != nullptr); quiche::HttpValidationPolicy http_validation_policy; - http_validation_policy.disallow_header_continuation_lines = true; + http_validation_policy.disallow_header_continuation_lines = false; http_validation_policy.require_header_colon = true; http_validation_policy.disallow_multiple_content_length = true; http_validation_policy.disallow_transfer_encoding_with_content_length = false; @@ -160,10 +163,10 @@ BalsaParser::BalsaParser(MessageType type, ParserCallbacks* connection, size_t m framer_.set_http_validation_policy(http_validation_policy); framer_.set_balsa_headers(&headers_); - framer_.set_balsa_trailer(&trailers_); framer_.set_balsa_visitor(this); framer_.set_max_header_length(max_header_length); framer_.set_invalid_chars_level(quiche::BalsaFrame::InvalidCharsLevel::kError); + framer_.EnableTrailers(); switch (message_type_) { case MessageType::Request: @@ -270,13 +273,12 @@ void BalsaParser::OnBodyChunkInput(absl::string_view input) { void BalsaParser::OnHeaderInput(absl::string_view /*input*/) {} void BalsaParser::OnTrailerInput(absl::string_view /*input*/) {} -void BalsaParser::OnHeader(absl::string_view /*key*/, absl::string_view /*value*/) {} void BalsaParser::ProcessHeaders(const BalsaHeaders& headers) { validateAndProcessHeadersOrTrailersImpl(headers, /* trailers = */ false); } -void BalsaParser::ProcessTrailers(const BalsaHeaders& trailer) { - validateAndProcessHeadersOrTrailersImpl(trailer, /* trailers = */ true); +void BalsaParser::OnTrailers(std::unique_ptr trailers) { + validateAndProcessHeadersOrTrailersImpl(*trailers, /* trailers = */ true); } void BalsaParser::OnRequestFirstLineInput(absl::string_view /*line_input*/, @@ -393,12 +395,11 @@ void BalsaParser::HandleWarning(BalsaFrameEnums::ErrorCode error_code) { void BalsaParser::validateAndProcessHeadersOrTrailersImpl(const quiche::BalsaHeaders& headers, bool trailers) { - for (const std::pair& key_value : headers.lines()) { + for (const auto& [key, value] : headers.lines()) { if (status_ == ParserStatus::Error) { return; } - absl::string_view key = key_value.first; if (!isHeaderNameValid(key)) { status_ = ParserStatus::Error; error_message_ = "HPE_INVALID_HEADER_TOKEN"; @@ -414,8 +415,22 @@ void BalsaParser::validateAndProcessHeadersOrTrailersImpl(const quiche::BalsaHea return; } - absl::string_view value = key_value.second; - status_ = convertResult(connection_->onHeaderValue(value.data(), value.length())); + // Remove CR and LF characters to match http-parser behavior. + auto is_cr_or_lf = [](char c) { return c == '\r' || c == '\n'; }; + if (std::any_of(value.begin(), value.end(), is_cr_or_lf)) { + std::string value_without_cr_or_lf; + value_without_cr_or_lf.reserve(value.size()); + for (char c : value) { + if (!is_cr_or_lf(c)) { + value_without_cr_or_lf.push_back(c); + } + } + status_ = convertResult(connection_->onHeaderValue(value_without_cr_or_lf.data(), + value_without_cr_or_lf.length())); + } else { + // No need to copy if header value does not contain CR or LF. + status_ = convertResult(connection_->onHeaderValue(value.data(), value.length())); + } } } diff --git a/source/common/http/http1/balsa_parser.h b/source/common/http/http1/balsa_parser.h index 2e8745491a98..b5c4491f7c7e 100644 --- a/source/common/http/http1/balsa_parser.h +++ b/source/common/http/http1/balsa_parser.h @@ -41,11 +41,9 @@ class BalsaParser : public Parser, public quiche::BalsaVisitorInterface { void OnRawBodyInput(absl::string_view input) override; void OnBodyChunkInput(absl::string_view input) override; void OnHeaderInput(absl::string_view input) override; - void OnHeader(absl::string_view key, absl::string_view value) override; void OnTrailerInput(absl::string_view input) override; - void OnTrailers(std::unique_ptr /*trailers*/) override{}; + void OnTrailers(std::unique_ptr trailers) override; void ProcessHeaders(const quiche::BalsaHeaders& headers) override; - void ProcessTrailers(const quiche::BalsaHeaders& trailer) override; void OnRequestFirstLineInput(absl::string_view line_input, absl::string_view method_input, absl::string_view request_uri, absl::string_view version_input) override; @@ -61,7 +59,7 @@ class BalsaParser : public Parser, public quiche::BalsaVisitorInterface { void HandleError(quiche::BalsaFrameEnums::ErrorCode error_code) override; void HandleWarning(quiche::BalsaFrameEnums::ErrorCode error_code) override; - // Shared implementation for ProcessHeaders() and ProcessTrailers(). + // Shared implementation for ProcessHeaders() and OnTrailers(). void validateAndProcessHeadersOrTrailersImpl(const quiche::BalsaHeaders& headers, bool trailers); // Return ParserStatus::Error if `result` is CallbackResult::Error. @@ -71,7 +69,6 @@ class BalsaParser : public Parser, public quiche::BalsaVisitorInterface { quiche::BalsaFrame framer_; quiche::BalsaHeaders headers_; - quiche::BalsaHeaders trailers_; const MessageType message_type_ = MessageType::Request; ParserCallbacks* connection_ = nullptr; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 8a26aaa6adb9..fb2963d6076a 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -661,9 +661,10 @@ ConnectionImpl::StreamDataFrameSource::SelectPayloadLength(size_t max_length) { if (stream_.local_end_stream_ && length == stream_.pending_send_data_->length()) { end_data = true; if (stream_.pending_trailers_to_encode_) { - send_fin_ = false; stream_.submitTrailers(*stream_.pending_trailers_to_encode_); stream_.pending_trailers_to_encode_.reset(); + } else { + send_fin_ = true; } } return {static_cast(length), end_data}; @@ -1782,6 +1783,7 @@ ConnectionImpl::Http2Options::Http2Options( og_options_.max_header_list_bytes = max_headers_kb * 1024; og_options_.max_header_field_size = max_headers_kb * 1024; og_options_.allow_extended_connect = http2_options.allow_connect(); + og_options_.allow_different_host_and_authority = true; #ifdef ENVOY_ENABLE_UHV // UHV - disable header validations in oghttp2 @@ -2127,6 +2129,18 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na // For a server connection, we should never get push promise frames. ASSERT(frame->hd.type == NGHTTP2_HEADERS); ASSERT(frame->headers.cat == NGHTTP2_HCAT_REQUEST || frame->headers.cat == NGHTTP2_HCAT_HEADERS); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.http2_discard_host_header")) { + StreamImpl* stream = getStreamUnchecked(frame->hd.stream_id); + if (stream && name == static_cast(Http::Headers::get().HostLegacy)) { + // Check if there is already the :authority header + const auto result = stream->headers().get(Http::Headers::get().Host); + if (!result.empty()) { + // Discard the host header value + return 0; + } + // Otherwise use host value as :authority + } + } return saveHeader(frame, std::move(name), std::move(value)); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 2cbb5d28814e..042ecf03dff0 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -426,7 +426,7 @@ class ConnectionImpl : public virtual Connection, private: StreamImpl& stream_; - bool send_fin_ = true; + bool send_fin_ = false; }; using StreamImplPtr = std::unique_ptr; diff --git a/source/common/http/http_server_properties_cache_manager_impl.cc b/source/common/http/http_server_properties_cache_manager_impl.cc index 307e8991eb55..a640fcd3be6a 100644 --- a/source/common/http/http_server_properties_cache_manager_impl.cc +++ b/source/common/http/http_server_properties_cache_manager_impl.cc @@ -61,7 +61,7 @@ HttpServerPropertiesCacheSharedPtr HttpServerPropertiesCacheManagerImpl::getCach entry : options.prepopulated_entries()) { const HttpServerPropertiesCacheImpl::Origin origin = {"https", entry.hostname(), entry.port()}; std::vector protocol = { - {"h3", entry.hostname(), entry.port(), + {"h3", "", entry.port(), dispatcher.timeSource().monotonicTime() + std::chrono::hours(168)}}; OptRef> existing_protocols = new_cache->findAlternatives(origin); diff --git a/source/common/http/http_server_properties_cache_manager_impl.h b/source/common/http/http_server_properties_cache_manager_impl.h index 6ab06d2b401b..0cfe8411a9e5 100644 --- a/source/common/http/http_server_properties_cache_manager_impl.h +++ b/source/common/http/http_server_properties_cache_manager_impl.h @@ -13,10 +13,11 @@ namespace Envoy { namespace Http { struct AlternateProtocolsData { - AlternateProtocolsData(Server::Configuration::FactoryContextBase& context) - : dispatcher_(context.mainThreadDispatcher()), - validation_visitor_(context.messageValidationVisitor()), + AlternateProtocolsData(Server::Configuration::ServerFactoryContext& context, + ProtobufMessage::ValidationVisitor& validation_visitor) + : dispatcher_(context.mainThreadDispatcher()), validation_visitor_(validation_visitor), file_system_(context.api().fileSystem()), concurrency_(context.options().concurrency()) {} + Event::Dispatcher& dispatcher_; ProtobufMessage::ValidationVisitor& validation_visitor_; Filesystem::Instance& file_system_; diff --git a/source/common/http/match_delegate/config.cc b/source/common/http/match_delegate/config.cc index 88f8391e7ceb..a96f4a1cea58 100644 --- a/source/common/http/match_delegate/config.cc +++ b/source/common/http/match_delegate/config.cc @@ -280,16 +280,20 @@ Envoy::Http::FilterFactoryCb MatchDelegateConfig::createFilterFactoryFromProtoTy auto message = Config::Utility::translateAnyToFactoryConfig( proto_config.extension_config().typed_config(), context.messageValidationVisitor(), factory); - auto filter_factory = factory.createFilterFactoryFromProto(*message, prefix, context); + auto filter_factory_or_error = factory.createFilterFactoryFromProto(*message, prefix, context); + if (!filter_factory_or_error.ok()) { + throwEnvoyExceptionOrPanic(std::string(filter_factory_or_error.status().message())); + } + auto filter_factory = filter_factory_or_error.value(); Factory::MatchTreeValidationVisitor validation_visitor(*factory.matchingRequirements()); Envoy::Http::Matching::HttpFilterActionContext action_context{prefix, context, - context.getServerFactoryContext()}; + context.serverFactoryContext()}; Matcher::MatchTreeFactory - matcher_factory(action_context, context.getServerFactoryContext(), validation_visitor); + matcher_factory(action_context, context.serverFactoryContext(), validation_visitor); absl::optional> factory_cb = std::nullopt; if (proto_config.has_xds_matcher()) { @@ -300,8 +304,8 @@ Envoy::Http::FilterFactoryCb MatchDelegateConfig::createFilterFactoryFromProtoTy if (!validation_visitor.errors().empty()) { // TODO(snowp): Output all violations. - throw EnvoyException(fmt::format("requirement violation while creating match tree: {}", - validation_visitor.errors()[0])); + throwEnvoyExceptionOrPanic(fmt::format("requirement violation while creating match tree: {}", + validation_visitor.errors()[0])); } Matcher::MatchTreeSharedPtr match_tree = nullptr; diff --git a/source/common/http/matching/inputs.h b/source/common/http/matching/inputs.h index 95849bc5f22e..46220f079b00 100644 --- a/source/common/http/matching/inputs.h +++ b/source/common/http/matching/inputs.h @@ -161,15 +161,15 @@ class HttpRequestQueryParamsDataInput : public Matcher::DataInputvalue().getStringView()); + auto params = + Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(ret->value().getStringView()); - auto ItParam = params.find(query_param_); - if (ItParam == params.end()) { + auto ItParam = params.getFirstValue(query_param_); + if (!ItParam.has_value()) { return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, absl::monostate()}; } return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable, - std::move(ItParam->second)}; + std::move(ItParam.value())}; } private: diff --git a/source/common/http/null_route_impl.h b/source/common/http/null_route_impl.h index 8caef5df4d70..465f9bf3f4dc 100644 --- a/source/common/http/null_route_impl.h +++ b/source/common/http/null_route_impl.h @@ -43,6 +43,12 @@ struct NullCommonConfig : public Router::CommonConfig { bool usesVhds() const override { return false; } bool mostSpecificHeaderMutationsWins() const override { return false; } uint32_t maxDirectResponseBodySizeBytes() const override { return 0; } + const envoy::config::core::v3::Metadata& metadata() const override { + return Router::DefaultRouteMetadataPack::get().proto_metadata_; + } + const Envoy::Config::TypedMetadata& typedMetadata() const override { + return Router::DefaultRouteMetadataPack::get().typed_metadata_; + } static const std::list internal_only_headers_; }; @@ -64,6 +70,13 @@ struct NullVirtualHost : public Router::VirtualHost { void traversePerFilterConfig( const std::string&, std::function) const override {} + const envoy::config::core::v3::Metadata& metadata() const override { + return Router::DefaultRouteMetadataPack::get().proto_metadata_; + } + const Envoy::Config::TypedMetadata& typedMetadata() const override { + return Router::DefaultRouteMetadataPack::get().typed_metadata_; + } + static const NullRateLimitPolicy rate_limit_policy_; static const NullCommonConfig route_configuration_; }; @@ -212,8 +225,7 @@ struct NullRouteImpl : public Router::Route { const Protobuf::RepeatedPtrField& hash_policy = {}, const absl::optional& retry_policy = {}) - : route_entry_(cluster_name, singleton_manager, timeout, hash_policy, retry_policy), - typed_metadata_({}) {} + : route_entry_(cluster_name, singleton_manager, timeout, hash_policy, retry_policy) {} // Router::Route const Router::DirectResponseEntry* directResponseEntry() const override { return nullptr; } @@ -227,14 +239,16 @@ struct NullRouteImpl : public Router::Route { void traversePerFilterConfig( const std::string&, std::function) const override {} - const envoy::config::core::v3::Metadata& metadata() const override { return metadata_; } - const Envoy::Config::TypedMetadata& typedMetadata() const override { return typed_metadata_; } - bool filterDisabled(absl::string_view) const override { return false; } + const envoy::config::core::v3::Metadata& metadata() const override { + return Router::DefaultRouteMetadataPack::get().proto_metadata_; + } + const Envoy::Config::TypedMetadata& typedMetadata() const override { + return Router::DefaultRouteMetadataPack::get().typed_metadata_; + } + absl::optional filterDisabled(absl::string_view) const override { return {}; } const std::string& routeName() const override { return EMPTY_STRING; } RouteEntryImpl route_entry_; - const envoy::config::core::v3::Metadata metadata_; - const Envoy::Config::TypedMetadataImpl typed_metadata_; }; } // namespace Http diff --git a/source/common/http/request_id_extension_impl.cc b/source/common/http/request_id_extension_impl.cc index ed90f218c089..89a55b8d95a4 100644 --- a/source/common/http/request_id_extension_impl.cc +++ b/source/common/http/request_id_extension_impl.cc @@ -8,7 +8,7 @@ namespace Http { absl::StatusOr RequestIDExtensionFactory::fromProto( const envoy::extensions::filters::network::http_connection_manager::v3::RequestIDExtension& config, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { const std::string type{TypeUtil::typeUrlToDescriptorFullName(config.typed_config().type_url())}; auto* factory = Registry::FactoryRegistry::getFactoryByType( diff --git a/source/common/http/request_id_extension_impl.h b/source/common/http/request_id_extension_impl.h index 01cf8984ca00..7d2c43cbeee2 100644 --- a/source/common/http/request_id_extension_impl.h +++ b/source/common/http/request_id_extension_impl.h @@ -18,7 +18,7 @@ class RequestIDExtensionFactory { static absl::StatusOr fromProto( const envoy::extensions::filters::network::http_connection_manager::v3::RequestIDExtension& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::FactoryContext& context); }; } // namespace Http diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index bd9ce8323bd1..558ba87c8b2b 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -32,6 +32,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "quiche/http2/adapter/http2_protocol.h" namespace Envoy { @@ -121,15 +122,16 @@ void validateCustomSettingsParameters( switch (it.identifier().value()) { case http2::adapter::ENABLE_PUSH: if (it.value().value() == 1) { - throw EnvoyException("server push is not supported by Envoy and can not be enabled via a " - "SETTINGS parameter."); + throwEnvoyExceptionOrPanic( + "server push is not supported by Envoy and can not be enabled via a " + "SETTINGS parameter."); } break; case http2::adapter::ENABLE_CONNECT_PROTOCOL: // An exception is made for `allow_connect` which can't be checked for presence due to the // use of a primitive type (bool). - throw EnvoyException("the \"allow_connect\" SETTINGS parameter must only be configured " - "through the named field"); + throwEnvoyExceptionOrPanic("the \"allow_connect\" SETTINGS parameter must only be configured " + "through the named field"); case http2::adapter::HEADER_TABLE_SIZE: if (options.has_hpack_table_size()) { parameter_collisions.push_back("hpack_table_size"); @@ -152,12 +154,12 @@ void validateCustomSettingsParameters( } if (!custom_parameter_collisions.empty()) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "inconsistent HTTP/2 custom SETTINGS parameter(s) detected; identifiers = {{{}}}", absl::StrJoin(custom_parameter_collisions, ","))); } if (!parameter_collisions.empty()) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "the {{{}}} HTTP/2 SETTINGS parameter(s) can not be configured through both named and " "custom parameters", absl::StrJoin(parameter_collisions, ","))); @@ -492,17 +494,6 @@ std::string Utility::createSslRedirectPath(const RequestHeaderMap& headers) { return fmt::format("https://{}{}", headers.getHostValue(), headers.getPathValue()); } -Utility::QueryParams Utility::parseQueryString(absl::string_view url) { - size_t start = url.find('?'); - if (start == std::string::npos) { - QueryParams params; - return params; - } - - start++; - return parseParameters(url, start, /*decode_params=*/false); -} - Utility::QueryParamsMulti Utility::QueryParamsMulti::parseQueryString(absl::string_view url) { size_t start = url.find('?'); if (start == std::string::npos) { @@ -513,17 +504,6 @@ Utility::QueryParamsMulti Utility::QueryParamsMulti::parseQueryString(absl::stri return Utility::QueryParamsMulti::parseParameters(url, start, /*decode_params=*/false); } -Utility::QueryParams Utility::parseAndDecodeQueryString(absl::string_view url) { - size_t start = url.find('?'); - if (start == std::string::npos) { - QueryParams params; - return params; - } - - start++; - return parseParameters(url, start, /*decode_params=*/true); -} - Utility::QueryParamsMulti Utility::QueryParamsMulti::parseAndDecodeQueryString(absl::string_view url) { size_t start = url.find('?'); @@ -535,38 +515,6 @@ Utility::QueryParamsMulti::parseAndDecodeQueryString(absl::string_view url) { return Utility::QueryParamsMulti::parseParameters(url, start, /*decode_params=*/true); } -Utility::QueryParams Utility::parseFromBody(absl::string_view body) { - return parseParameters(body, 0, /*decode_params=*/true); -} - -Utility::QueryParams Utility::parseParameters(absl::string_view data, size_t start, - bool decode_params) { - QueryParams params; - - while (start < data.size()) { - size_t end = data.find('&', start); - if (end == std::string::npos) { - end = data.size(); - } - absl::string_view param(data.data() + start, end - start); - - const size_t equal = param.find('='); - if (equal != std::string::npos) { - const auto param_name = StringUtil::subspan(data, start, start + equal); - const auto param_value = StringUtil::subspan(data, start + equal + 1, end); - params.emplace(decode_params ? PercentEncoding::decode(param_name) : param_name, - decode_params ? PercentEncoding::decode(param_value) : param_value); - } else { - const auto param_name = StringUtil::subspan(data, start, end); - params.emplace(decode_params ? PercentEncoding::decode(param_name) : param_name, ""); - } - - start = end + 1; - } - - return params; -} - Utility::QueryParamsMulti Utility::QueryParamsMulti::parseParameters(absl::string_view data, size_t start, bool decode_params) { @@ -609,6 +557,15 @@ void Utility::QueryParamsMulti::overwrite(absl::string_view key, absl::string_vi this->data_[key] = std::vector{std::string(value)}; } +absl::optional Utility::QueryParamsMulti::getFirstValue(absl::string_view key) const { + auto it = this->data_.find(key); + if (it == this->data_.end()) { + return std::nullopt; + } + + return absl::optional{it->second.at(0)}; +} + absl::string_view Utility::findQueryStringStart(const HeaderString& path) { absl::string_view path_str = path.getStringView(); size_t query_offset = path_str.find('?'); @@ -625,19 +582,7 @@ std::string Utility::stripQueryString(const HeaderString& path) { return {path_str.data(), query_offset != path_str.npos ? query_offset : path_str.size()}; } -std::string Utility::replaceQueryString(const HeaderString& path, - const Utility::QueryParams& params) { - std::string new_path{Http::Utility::stripQueryString(path)}; - - if (!params.empty()) { - const auto new_query_string = Http::Utility::queryParamsToString(params); - absl::StrAppend(&new_path, new_query_string); - } - - return new_path; -} - -std::string Utility::QueryParamsMulti::replaceQueryString(const HeaderString& path) { +std::string Utility::QueryParamsMulti::replaceQueryString(const HeaderString& path) const { std::string new_path{Http::Utility::stripQueryString(path)}; if (!this->data_.empty()) { @@ -845,19 +790,19 @@ Utility::getLastAddressFromXFF(const Http::RequestHeaderMap& request_headers, } absl::string_view xff_string(xff_header->value().getStringView()); - static const std::string separator(","); + static constexpr absl::string_view separator(","); // Ignore the last num_to_skip addresses at the end of XFF. for (uint32_t i = 0; i < num_to_skip; i++) { - const std::string::size_type last_comma = xff_string.rfind(separator); - if (last_comma == std::string::npos) { + const absl::string_view::size_type last_comma = xff_string.rfind(separator); + if (last_comma == absl::string_view::npos) { return {nullptr, false}; } xff_string = xff_string.substr(0, last_comma); } // The text after the last remaining comma, or the entirety of the string if there // is no comma, is the requested IP address. - const std::string::size_type last_comma = xff_string.rfind(separator); - if (last_comma != std::string::npos && last_comma + separator.size() < xff_string.size()) { + const absl::string_view::size_type last_comma = xff_string.rfind(separator); + if (last_comma != absl::string_view::npos && last_comma + separator.size() < xff_string.size()) { xff_string = xff_string.substr(last_comma + separator.size()); } @@ -872,14 +817,14 @@ Utility::getLastAddressFromXFF(const Http::RequestHeaderMap& request_headers, Network::Address::InstanceConstSharedPtr address = Network::Utility::parseInternetAddressNoThrow(std::string(xff_string)); if (address != nullptr) { - return {address, last_comma == std::string::npos && num_to_skip == 0}; + return {address, last_comma == absl::string_view::npos && num_to_skip == 0}; } return {nullptr, false}; } bool Utility::sanitizeConnectionHeader(Http::RequestHeaderMap& headers) { - static const size_t MAX_ALLOWED_NOMINATED_HEADERS = 10; - static const size_t MAX_ALLOWED_TE_VALUE_SIZE = 256; + static constexpr size_t MAX_ALLOWED_NOMINATED_HEADERS = 10; + static constexpr size_t MAX_ALLOWED_TE_VALUE_SIZE = 256; // Remove any headers nominated by the Connection header. The TE header // is sanitized and removed only if it's empty after removing unsupported values @@ -1081,19 +1026,7 @@ RequestMessagePtr Utility::prepareHeaders(const envoy::config::core::v3::HttpUri return message; } -// TODO(jmarantz): make QueryParams a real class and put this serializer there, -// along with proper URL escaping of the name and value. -std::string Utility::queryParamsToString(const QueryParams& params) { - std::string out; - std::string delim = "?"; - for (const auto& p : params) { - absl::StrAppend(&out, delim, p.first, "=", p.second); - delim = "&"; - } - return out; -} - -std::string Utility::QueryParamsMulti::toString() { +std::string Utility::QueryParamsMulti::toString() const { std::string out; std::string delim = "?"; for (const auto& p : this->data_) { @@ -1410,7 +1343,7 @@ void Utility::validateCoreRetryPolicy(const envoy::config::core::v3::RetryPolicy PROTOBUF_GET_MS_OR_DEFAULT(core_back_off, max_interval, base_interval_ms * 10); if (max_interval_ms < base_interval_ms) { - throw EnvoyException("max_interval must be greater than or equal to the base_interval"); + throwEnvoyExceptionOrPanic("max_interval must be greater than or equal to the base_interval"); } } } diff --git a/source/common/http/utility.h b/source/common/http/utility.h index 57b0e435ed03..db051e11c8c6 100644 --- a/source/common/http/utility.h +++ b/source/common/http/utility.h @@ -258,37 +258,6 @@ void updateAuthority(RequestHeaderMap& headers, absl::string_view hostname, bool */ std::string createSslRedirectPath(const RequestHeaderMap& headers); -/** - * Parse a URL into query parameters. - * @param url supplies the url to parse. - * @return QueryParams the parsed parameters, if any. - */ -QueryParams parseQueryString(absl::string_view url); - -/** - * Parse a URL into query parameters. - * @param url supplies the url to parse. - * @return QueryParams the parsed and percent-decoded parameters, if any. - */ -QueryParams parseAndDecodeQueryString(absl::string_view url); - -/** - * Parse a a request body into query parameters. - * @param body supplies the body to parse. - * @return QueryParams the parsed parameters, if any. - */ -QueryParams parseFromBody(absl::string_view body); - -/** - * Parse query parameters from a URL or body. - * @param data supplies the data to parse. - * @param start supplies the offset within the data. - * @param decode_params supplies the flag whether to percent-decode the parsed parameters (both name - * and value). Set to false to keep the parameters encoded. - * @return QueryParams the parsed parameters, if any. - */ -QueryParams parseParameters(absl::string_view data, size_t start, bool decode_params); - /** * Finds the start of the query string in a path * @param path supplies a HeaderString& to search for the query string @@ -305,20 +274,6 @@ absl::string_view findQueryStringStart(const HeaderString& path); */ std::string stripQueryString(const HeaderString& path); -/** - * Replace the query string portion of a given path with a new one. - * - * e.g. replaceQueryString("/foo?key=1", {key:2}) -> "/foo?key=2" - * replaceQueryString("/bar", {hello:there}) -> "/bar?hello=there" - * - * @param path the original path that may or may not contain an existing query string - * @param params the new params whose string representation should be formatted onto - * the `path` above - * @return std::string the new path whose query string has been replaced by `params` and whose path - * portion from `path` remains unchanged. - */ -std::string replaceQueryString(const HeaderString& path, const QueryParams& params); - /** * Parse a particular value out of a cookie * @param headers supplies the headers to get the cookie from. @@ -542,11 +497,6 @@ std::string localPathFromFilePath(const absl::string_view& file_path); */ RequestMessagePtr prepareHeaders(const envoy::config::core::v3::HttpUri& http_uri); -/** - * Serialize query-params into a string. - */ -std::string queryParamsToString(const QueryParams& query_params); - /** * Returns string representation of StreamResetReason. */ diff --git a/source/common/init/manager_impl.cc b/source/common/init/manager_impl.cc index 4ffbf75a2b1b..3e9ce802337c 100644 --- a/source/common/init/manager_impl.cc +++ b/source/common/init/manager_impl.cc @@ -9,7 +9,7 @@ namespace Envoy { namespace Init { ManagerImpl::ManagerImpl(absl::string_view name) - : name_(fmt::format("init manager {}", name)), state_(State::Uninitialized), count_(0), + : name_(fmt::format("init manager {}", name)), watcher_(name_, [this](absl::string_view target_name) { onTargetReady(target_name); }) {} Manager::State ManagerImpl::state() const { return state_; } diff --git a/source/common/init/manager_impl.h b/source/common/init/manager_impl.h index 85d901cd56a6..f780a2420e27 100644 --- a/source/common/init/manager_impl.h +++ b/source/common/init/manager_impl.h @@ -14,7 +14,7 @@ namespace Init { /** * Init::ManagerImpl coordinates initialization of one or more "targets." See comments in - * include/envoy/init/manager.h for an overview. + * envoy/init/manager.h for an overview. * * When the logging level is set to "debug" or "trace," the log will contain entries for all * significant events in the initialization flow: @@ -49,10 +49,10 @@ class ManagerImpl : public Manager, Logger::Loggable { const std::string name_; // Current state. - State state_; + State state_{State::Uninitialized}; // Current number of registered targets that have not yet initialized. - uint32_t count_; + uint32_t count_{0}; // Handle to the watcher passed in `initialize`, to be called when initialization completes. WatcherHandlePtr watcher_handle_; diff --git a/source/common/io/BUILD b/source/common/io/BUILD index 6a680e06c579..1db6f132c481 100644 --- a/source/common/io/BUILD +++ b/source/common/io/BUILD @@ -35,6 +35,8 @@ envoy_cc_library( deps = [ ":io_uring_impl_lib", "//envoy/common/io:io_uring_interface", + "//envoy/event:file_event_interface", + "//source/common/buffer:buffer_lib", "//source/common/common:linked_object", ], ) diff --git a/source/common/io/io_uring_impl.cc b/source/common/io/io_uring_impl.cc index 03c658f6d12c..f5fddf9db681 100644 --- a/source/common/io/io_uring_impl.cc +++ b/source/common/io/io_uring_impl.cc @@ -181,6 +181,36 @@ IoUringResult IoUringImpl::prepareClose(os_fd_t fd, Request* user_data) { return IoUringResult::Ok; } +IoUringResult IoUringImpl::prepareCancel(Request* cancelling_user_data, Request* user_data) { + ENVOY_LOG(trace, "prepare cancels for user data = {}", fmt::ptr(cancelling_user_data)); + // TODO (soulxu): Handling the case of CQ ring is overflow. + ASSERT(!(*(ring_.sq.kflags) & IORING_SQ_CQ_OVERFLOW)); + struct io_uring_sqe* sqe = io_uring_get_sqe(&ring_); + if (sqe == nullptr) { + ENVOY_LOG(trace, "failed to prepare cancel for user data = {}", fmt::ptr(cancelling_user_data)); + return IoUringResult::Failed; + } + + io_uring_prep_cancel(sqe, cancelling_user_data, 0); + io_uring_sqe_set_data(sqe, user_data); + return IoUringResult::Ok; +} + +IoUringResult IoUringImpl::prepareShutdown(os_fd_t fd, int how, Request* user_data) { + ENVOY_LOG(trace, "prepare shutdown for fd = {}, how = {}", fd, how); + // TODO (soulxu): Handling the case of CQ ring is overflow. + ASSERT(!(*(ring_.sq.kflags) & IORING_SQ_CQ_OVERFLOW)); + struct io_uring_sqe* sqe = io_uring_get_sqe(&ring_); + if (sqe == nullptr) { + ENVOY_LOG(trace, "failed to prepare shutdown for fd = {}", fd); + return IoUringResult::Failed; + } + + io_uring_prep_shutdown(sqe, fd, how); + io_uring_sqe_set_data(sqe, user_data); + return IoUringResult::Ok; +} + IoUringResult IoUringImpl::submit() { int res = io_uring_submit(&ring_); RELEASE_ASSERT(res >= 0 || res == -EBUSY, "unable to submit io_uring queue entries"); diff --git a/source/common/io/io_uring_impl.h b/source/common/io/io_uring_impl.h index 5afe05df170e..c5bb1ef1591c 100644 --- a/source/common/io/io_uring_impl.h +++ b/source/common/io/io_uring_impl.h @@ -41,6 +41,8 @@ class IoUringImpl : public IoUring, IoUringResult prepareWritev(os_fd_t fd, const struct iovec* iovecs, unsigned nr_vecs, off_t offset, Request* user_data) override; IoUringResult prepareClose(os_fd_t fd, Request* user_data) override; + IoUringResult prepareCancel(Request* cancelling_user_data, Request* user_data) override; + IoUringResult prepareShutdown(os_fd_t fd, int how, Request* user_data) override; IoUringResult submit() override; void injectCompletion(os_fd_t fd, Request* user_data, int32_t result) override; void removeInjectedCompletion(os_fd_t fd) override; diff --git a/source/common/io/io_uring_worker_impl.cc b/source/common/io/io_uring_worker_impl.cc index 76528c4307eb..7fe3425f49fa 100644 --- a/source/common/io/io_uring_worker_impl.cc +++ b/source/common/io/io_uring_worker_impl.cc @@ -3,8 +3,24 @@ namespace Envoy { namespace Io { -IoUringSocketEntry::IoUringSocketEntry(os_fd_t fd, IoUringWorkerImpl& parent) - : fd_(fd), parent_(parent) {} +ReadRequest::ReadRequest(IoUringSocket& socket, uint32_t size) + : Request(RequestType::Read, socket), buf_(std::make_unique(size)), + iov_(std::make_unique()) { + iov_->iov_base = buf_.get(); + iov_->iov_len = size; +} + +WriteRequest::WriteRequest(IoUringSocket& socket, const Buffer::RawSliceVector& slices) + : Request(RequestType::Write, socket), iov_(std::make_unique(slices.size())) { + for (size_t i = 0; i < slices.size(); i++) { + iov_[i].iov_base = slices[i].mem_; + iov_[i].iov_len = slices[i].len_; + } +} + +IoUringSocketEntry::IoUringSocketEntry(os_fd_t fd, IoUringWorkerImpl& parent, Event::FileReadyCb cb, + bool enable_close_event) + : fd_(fd), parent_(parent), enable_close_event_(enable_close_event), cb_(std::move(cb)) {} void IoUringSocketEntry::cleanup() { IoUringSocketEntryPtr socket = parent_.removeSocket(*this); @@ -24,13 +40,34 @@ void IoUringSocketEntry::injectCompletion(Request::RequestType type) { parent_.injectCompletion(*this, type, -EAGAIN); } +void IoUringSocketEntry::onReadCompleted() { + ENVOY_LOG(trace, + "calling event callback since pending read buf has {} size data, data = {}, " + "fd = {}", + getReadParam()->buf_.length(), getReadParam()->buf_.toString(), fd_); + cb_(Event::FileReadyType::Read); +} + +void IoUringSocketEntry::onWriteCompleted() { + ENVOY_LOG(trace, "call event callback for write since result = {}", getWriteParam()->result_); + cb_(Event::FileReadyType::Write); +} + +void IoUringSocketEntry::onRemoteClose() { + ENVOY_LOG(trace, "onRemoteClose fd = {}", fd_); + cb_(Event::FileReadyType::Closed); +} + IoUringWorkerImpl::IoUringWorkerImpl(uint32_t io_uring_size, bool use_submission_queue_polling, + uint32_t read_buffer_size, uint32_t write_timeout_ms, Event::Dispatcher& dispatcher) : IoUringWorkerImpl(std::make_unique(io_uring_size, use_submission_queue_polling), - dispatcher) {} + read_buffer_size, write_timeout_ms, dispatcher) {} -IoUringWorkerImpl::IoUringWorkerImpl(IoUringPtr&& io_uring, Event::Dispatcher& dispatcher) - : io_uring_(std::move(io_uring)), dispatcher_(dispatcher) { +IoUringWorkerImpl::IoUringWorkerImpl(IoUringPtr&& io_uring, uint32_t read_buffer_size, + uint32_t write_timeout_ms, Event::Dispatcher& dispatcher) + : io_uring_(std::move(io_uring)), read_buffer_size_(read_buffer_size), + write_timeout_ms_(write_timeout_ms), dispatcher_(dispatcher) { const os_fd_t event_fd = io_uring_->registerEventfd(); // We only care about the read event of Eventfd, since we only receive the // event here. @@ -42,9 +79,41 @@ IoUringWorkerImpl::IoUringWorkerImpl(IoUringPtr&& io_uring, Event::Dispatcher& d IoUringWorkerImpl::~IoUringWorkerImpl() { ENVOY_LOG(trace, "destruct io uring worker, existing sockets = {}", sockets_.size()); + for (auto& socket : sockets_) { + if (socket->getStatus() != Closed) { + socket->close(false); + } + } + + while (!sockets_.empty()) { + ENVOY_LOG(trace, "still left {} sockets are not closed", sockets_.size()); + for (auto& socket : sockets_) { + ENVOY_LOG(trace, "the socket fd = {} not closed", socket->fd()); + } + onFileEvent(); + } + dispatcher_.clearDeferredDeleteList(); } +IoUringSocket& IoUringWorkerImpl::addServerSocket(os_fd_t fd, Event::FileReadyCb cb, + bool enable_close_event) { + ENVOY_LOG(trace, "add server socket, fd = {}", fd); + std::unique_ptr socket = std::make_unique( + fd, *this, std::move(cb), write_timeout_ms_, enable_close_event); + socket->enableRead(); + return addSocket(std::move(socket)); +} + +IoUringSocket& IoUringWorkerImpl::addServerSocket(os_fd_t fd, Buffer::Instance& read_buf, + Event::FileReadyCb cb, bool enable_close_event) { + ENVOY_LOG(trace, "add server socket through existing socket, fd = {}", fd); + std::unique_ptr socket = std::make_unique( + fd, read_buf, *this, std::move(cb), write_timeout_ms_, enable_close_event); + socket->enableRead(); + return addSocket(std::move(socket)); +} + Event::Dispatcher& IoUringWorkerImpl::dispatcher() { return dispatcher_; } IoUringSocketEntry& IoUringWorkerImpl::addSocket(IoUringSocketEntryPtr&& socket) { @@ -52,6 +121,89 @@ IoUringSocketEntry& IoUringWorkerImpl::addSocket(IoUringSocketEntryPtr&& socket) return *sockets_.back(); } +Request* IoUringWorkerImpl::submitReadRequest(IoUringSocket& socket) { + ReadRequest* req = new ReadRequest(socket, read_buffer_size_); + + ENVOY_LOG(trace, "submit read request, fd = {}, read req = {}", socket.fd(), fmt::ptr(req)); + + auto res = io_uring_->prepareReadv(socket.fd(), req->iov_.get(), 1, 0, req); + if (res == IoUringResult::Failed) { + // TODO(rojkov): handle `EBUSY` in case the completion queue is never reaped. + submit(); + res = io_uring_->prepareReadv(socket.fd(), req->iov_.get(), 1, 0, req); + RELEASE_ASSERT(res == IoUringResult::Ok, "unable to prepare readv"); + } + submit(); + return req; +} + +Request* IoUringWorkerImpl::submitWriteRequest(IoUringSocket& socket, + const Buffer::RawSliceVector& slices) { + WriteRequest* req = new WriteRequest(socket, slices); + + ENVOY_LOG(trace, "submit write request, fd = {}, req = {}", socket.fd(), fmt::ptr(req)); + + auto res = io_uring_->prepareWritev(socket.fd(), req->iov_.get(), slices.size(), 0, req); + if (res == IoUringResult::Failed) { + // TODO(rojkov): handle `EBUSY` in case the completion queue is never reaped. + submit(); + res = io_uring_->prepareWritev(socket.fd(), req->iov_.get(), slices.size(), 0, req); + RELEASE_ASSERT(res == IoUringResult::Ok, "unable to prepare writev"); + } + submit(); + return req; +} + +Request* IoUringWorkerImpl::submitCloseRequest(IoUringSocket& socket) { + Request* req = new Request(Request::RequestType::Close, socket); + + ENVOY_LOG(trace, "submit close request, fd = {}, close req = {}", socket.fd(), fmt::ptr(req)); + + auto res = io_uring_->prepareClose(socket.fd(), req); + if (res == IoUringResult::Failed) { + // TODO(rojkov): handle `EBUSY` in case the completion queue is never reaped. + submit(); + res = io_uring_->prepareClose(socket.fd(), req); + RELEASE_ASSERT(res == IoUringResult::Ok, "unable to prepare close"); + } + submit(); + return req; +} + +Request* IoUringWorkerImpl::submitCancelRequest(IoUringSocket& socket, Request* request_to_cancel) { + Request* req = new Request(Request::RequestType::Cancel, socket); + + ENVOY_LOG(trace, "submit cancel request, fd = {}, cancel req = {}, req to cancel = {}", + socket.fd(), fmt::ptr(req), fmt::ptr(request_to_cancel)); + + auto res = io_uring_->prepareCancel(request_to_cancel, req); + if (res == IoUringResult::Failed) { + // TODO(rojkov): handle `EBUSY` in case the completion queue is never reaped. + submit(); + res = io_uring_->prepareCancel(request_to_cancel, req); + RELEASE_ASSERT(res == IoUringResult::Ok, "unable to prepare cancel"); + } + submit(); + return req; +} + +Request* IoUringWorkerImpl::submitShutdownRequest(IoUringSocket& socket, int how) { + Request* req = new Request(Request::RequestType::Shutdown, socket); + + ENVOY_LOG(trace, "submit shutdown request, fd = {}, shutdown req = {}", socket.fd(), + fmt::ptr(req)); + + auto res = io_uring_->prepareShutdown(socket.fd(), how, req); + if (res == IoUringResult::Failed) { + // TODO(rojkov): handle `EBUSY` in case the completion queue is never reaped. + submit(); + res = io_uring_->prepareShutdown(socket.fd(), how, req); + RELEASE_ASSERT(res == IoUringResult::Ok, "unable to prepare cancel"); + } + submit(); + return req; +} + IoUringSocketEntryPtr IoUringWorkerImpl::removeSocket(IoUringSocketEntry& socket) { // Remove all the injection completion for this socket. io_uring_->removeInjectedCompletion(socket.fd()); @@ -123,5 +275,341 @@ void IoUringWorkerImpl::submit() { } } +IoUringServerSocket::IoUringServerSocket(os_fd_t fd, IoUringWorkerImpl& parent, + Event::FileReadyCb cb, uint32_t write_timeout_ms, + bool enable_close_event) + : IoUringSocketEntry(fd, parent, std::move(cb), enable_close_event), + write_timeout_ms_(write_timeout_ms) {} + +IoUringServerSocket::IoUringServerSocket(os_fd_t fd, Buffer::Instance& read_buf, + IoUringWorkerImpl& parent, Event::FileReadyCb cb, + uint32_t write_timeout_ms, bool enable_close_event) + : IoUringSocketEntry(fd, parent, std::move(cb), enable_close_event), + write_timeout_ms_(write_timeout_ms) { + read_buf_.move(read_buf); +} + +IoUringServerSocket::~IoUringServerSocket() { + if (write_timeout_timer_) { + write_timeout_timer_->disableTimer(); + } +} + +void IoUringServerSocket::close(bool keep_fd_open, IoUringSocketOnClosedCb cb) { + ENVOY_LOG(trace, "close the socket, fd = {}, status = {}", fd_, status_); + + IoUringSocketEntry::close(keep_fd_open, cb); + keep_fd_open_ = keep_fd_open; + + // Delay close until read request and write (or shutdown) request are drained. + if (read_req_ == nullptr && write_or_shutdown_req_ == nullptr) { + closeInternal(); + return; + } + + if (read_req_ != nullptr) { + ENVOY_LOG(trace, "cancel the read request, fd = {}", fd_); + read_cancel_req_ = parent_.submitCancelRequest(*this, read_req_); + } + + if (write_or_shutdown_req_ != nullptr) { + ENVOY_LOG(trace, "delay cancel the write request, fd = {}", fd_); + if (write_timeout_ms_ > 0) { + write_timeout_timer_ = parent_.dispatcher().createTimer([this]() { + if (write_or_shutdown_req_ != nullptr) { + ENVOY_LOG(trace, "cancel the write or shutdown request, fd = {}", fd_); + write_or_shutdown_cancel_req_ = + parent_.submitCancelRequest(*this, write_or_shutdown_req_); + } + }); + write_timeout_timer_->enableTimer(std::chrono::milliseconds(write_timeout_ms_)); + } + } +} + +void IoUringServerSocket::enableRead() { + IoUringSocketEntry::enableRead(); + ENVOY_LOG(trace, "enable read, fd = {}", fd_); + + // Continue processing read buffer remained by the previous read. + if (read_buf_.length() > 0 || read_error_.has_value()) { + ENVOY_LOG(trace, "continue reading from socket, fd = {}, size = {}", fd_, read_buf_.length()); + injectCompletion(Request::RequestType::Read); + return; + } + + submitReadRequest(); +} + +void IoUringServerSocket::disableRead() { IoUringSocketEntry::disableRead(); } + +void IoUringServerSocket::write(Buffer::Instance& data) { + ENVOY_LOG(trace, "write, buffer size = {}, fd = {}", data.length(), fd_); + ASSERT(!shutdown_.has_value()); + + // We need to reset the drain trackers, since the write and close is async in + // the iouring. When the write is actually finished the above layer may already + // release the drain trackers. + write_buf_.move(data, data.length(), true); + + submitWriteOrShutdownRequest(); +} + +uint64_t IoUringServerSocket::write(const Buffer::RawSlice* slices, uint64_t num_slice) { + ENVOY_LOG(trace, "write, num_slices = {}, fd = {}", num_slice, fd_); + ASSERT(!shutdown_.has_value()); + + uint64_t bytes_written = 0; + for (uint64_t i = 0; i < num_slice; i++) { + write_buf_.add(slices[i].mem_, slices[i].len_); + bytes_written += slices[i].len_; + } + + submitWriteOrShutdownRequest(); + return bytes_written; +} + +void IoUringServerSocket::shutdown(int how) { + ENVOY_LOG(trace, "shutdown the socket, fd = {}, how = {}", fd_, how); + ASSERT(how == SHUT_WR); + shutdown_ = false; + submitWriteOrShutdownRequest(); +} + +void IoUringServerSocket::onClose(Request* req, int32_t result, bool injected) { + IoUringSocketEntry::onClose(req, result, injected); + ASSERT(!injected); + cleanup(); +} + +void IoUringServerSocket::onCancel(Request* req, int32_t result, bool injected) { + IoUringSocketEntry::onCancel(req, result, injected); + ASSERT(!injected); + if (read_cancel_req_ == req) { + read_cancel_req_ = nullptr; + } + if (write_or_shutdown_cancel_req_ == req) { + write_or_shutdown_cancel_req_ = nullptr; + } + if (status_ == Closed && write_or_shutdown_req_ == nullptr && read_req_ == nullptr && + write_or_shutdown_cancel_req_ == nullptr) { + closeInternal(); + } +} + +void IoUringServerSocket::moveReadDataToBuffer(Request* req, size_t data_length) { + ReadRequest* read_req = static_cast(req); + Buffer::BufferFragment* fragment = new Buffer::BufferFragmentImpl( + read_req->buf_.release(), data_length, + [](const void* data, size_t, const Buffer::BufferFragmentImpl* this_fragment) { + delete[] reinterpret_cast(data); + delete this_fragment; + }); + read_buf_.addBufferFragment(*fragment); +} + +void IoUringServerSocket::onReadCompleted(int32_t result) { + ENVOY_LOG(trace, "read from socket, fd = {}, result = {}", fd_, result); + ReadParam param{read_buf_, result}; + read_param_ = param; + IoUringSocketEntry::onReadCompleted(); + read_param_ = absl::nullopt; + ENVOY_LOG(trace, "after read from socket, fd = {}, remain = {}", fd_, read_buf_.length()); +} + +// TODO(zhxie): concern submit multiple read requests or submit read request in advance to improve +// performance in the next iteration. +void IoUringServerSocket::onRead(Request* req, int32_t result, bool injected) { + IoUringSocketEntry::onRead(req, result, injected); + + ENVOY_LOG(trace, + "onRead with result {}, fd = {}, injected = {}, status_ = {}, enable_close_event = {}", + result, fd_, injected, status_, enable_close_event_); + if (!injected) { + read_req_ = nullptr; + // If the socket is going to close, discard all results. + if (status_ == Closed && write_or_shutdown_req_ == nullptr && read_cancel_req_ == nullptr && + write_or_shutdown_cancel_req_ == nullptr) { + if (result > 0 && keep_fd_open_) { + moveReadDataToBuffer(req, result); + } + closeInternal(); + return; + } + } + + // Move read data from request to buffer or store the error. + if (result > 0) { + moveReadDataToBuffer(req, result); + } else { + if (result != -ECANCELED) { + read_error_ = result; + } + } + + // Discard calling back since the socket is not ready or closed. + if (status_ == Initialized || status_ == Closed) { + return; + } + + // If the socket is enabled and there are bytes to read, notify the handler. + if (status_ == ReadEnabled) { + if (read_buf_.length() > 0) { + onReadCompleted(static_cast(read_buf_.length())); + } else if (read_error_.has_value() && read_error_ < 0) { + onReadCompleted(read_error_.value()); + read_error_.reset(); + } + // Handle remote closed at last. + // Depending on the event listened to, calling different event back to handle remote closed. + // * events & (Read | Closed): Callback Closed, + // * events & (Read) : Callback Read, + // * events & (Closed) : Callback Closed, + // * ...else : Callback Write. + if (read_error_.has_value() && read_error_ == 0 && !enable_close_event_) { + ENVOY_LOG(trace, "read remote closed from socket, fd = {}", fd_); + onReadCompleted(read_error_.value()); + read_error_.reset(); + return; + } + } + + // If `enable_close_event_` is true, then deliver the remote close as close event. + if (read_error_.has_value() && read_error_ == 0) { + if (enable_close_event_) { + ENVOY_LOG(trace, + "remote closed and close event enabled, raise the close event, fd = " + "{}, result = {}", + fd_, read_error_.value()); + status_ = RemoteClosed; + IoUringSocketEntry::onRemoteClose(); + read_error_.reset(); + return; + } else { + // In this case, the closed event isn't listened and the status is disabled. + // It means we can't raise the closed or read event. So we only can raise the + // write event. + ENVOY_LOG(trace, + "remote closed and close event disabled, raise the write event, fd = " + "{}, result = {}", + fd_, read_error_.value()); + status_ = RemoteClosed; + onWriteCompleted(0); + read_error_.reset(); + return; + } + } + + // The socket may be not readable during handler onRead callback, check it again here. + if (status_ == ReadEnabled) { + // If the read error is zero, it means remote close, then needn't new request. + if (!read_error_.has_value() || read_error_.value() != 0) { + // Submit a read request for the next read. + submitReadRequest(); + } + } else if (status_ == ReadDisabled) { + // Since error in a disabled socket will not be handled by the handler, stop submit read + // request if there is any error. + if (!read_error_.has_value()) { + // Submit a read request for monitoring the remote close event, otherwise there is no + // way to know the connection is closed by the remote. + submitReadRequest(); + } + } +} + +void IoUringServerSocket::onWriteCompleted(int32_t result) { + WriteParam param{result}; + write_param_ = param; + IoUringSocketEntry::onWriteCompleted(); + write_param_ = absl::nullopt; +} + +void IoUringServerSocket::onWrite(Request* req, int32_t result, bool injected) { + IoUringSocketEntry::onWrite(req, result, injected); + + ENVOY_LOG(trace, "onWrite with result {}, fd = {}, injected = {}, status_ = {}", result, fd_, + injected, status_); + if (!injected) { + write_or_shutdown_req_ = nullptr; + } + + // Notify the handler directly since it is an injected request. + if (injected) { + ENVOY_LOG(trace, + "there is a inject event, and same time we have regular write request, fd = {}", fd_); + // There is case where write injection may come after shutdown or close which should be ignored + // since the I/O handle or connection may be released after closing. + if (!shutdown_.has_value() && status_ != Closed) { + onWriteCompleted(result); + } + return; + } + + if (result > 0) { + write_buf_.drain(result); + ENVOY_LOG(trace, "drain write buf, drain size = {}, fd = {}", result, fd_); + } else { + // Drain all write buf since the write failed. + write_buf_.drain(write_buf_.length()); + if (!shutdown_.has_value() && status_ != Closed) { + status_ = RemoteClosed; + if (result == -EPIPE) { + IoUringSocketEntry::onRemoteClose(); + } else { + onWriteCompleted(result); + } + } + } + + submitWriteOrShutdownRequest(); +} + +void IoUringServerSocket::onShutdown(Request* req, int32_t result, bool injected) { + IoUringSocketEntry::onShutdown(req, result, injected); + + ENVOY_LOG(trace, "onShutdown with result {}, fd = {}, injected = {}", result, fd_, injected); + ASSERT(!injected); + write_or_shutdown_req_ = nullptr; + shutdown_ = true; + + submitWriteOrShutdownRequest(); +} + +void IoUringServerSocket::closeInternal() { + if (keep_fd_open_) { + if (on_closed_cb_) { + on_closed_cb_(read_buf_); + } + cleanup(); + return; + } + if (close_req_ == nullptr) { + close_req_ = parent_.submitCloseRequest(*this); + } +} + +void IoUringServerSocket::submitReadRequest() { + if (!read_req_) { + read_req_ = parent_.submitReadRequest(*this); + } +} + +void IoUringServerSocket::submitWriteOrShutdownRequest() { + if (!write_or_shutdown_req_) { + if (write_buf_.length() > 0) { + Buffer::RawSliceVector slices = write_buf_.getRawSlices(IOV_MAX); + ENVOY_LOG(trace, "submit write request, write_buf size = {}, num_iovecs = {}, fd = {}", + write_buf_.length(), slices.size(), fd_); + write_or_shutdown_req_ = parent_.submitWriteRequest(*this, slices); + } else if (shutdown_.has_value() && !shutdown_.value()) { + write_or_shutdown_req_ = parent_.submitShutdownRequest(*this, SHUT_WR); + } else if (status_ == Closed && read_req_ == nullptr && read_cancel_req_ == nullptr && + write_or_shutdown_cancel_req_ == nullptr) { + closeInternal(); + } + } +} + } // namespace Io } // namespace Envoy diff --git a/source/common/io/io_uring_worker_impl.h b/source/common/io/io_uring_worker_impl.h index 3a70b84835f0..7a003f59060b 100644 --- a/source/common/io/io_uring_worker_impl.h +++ b/source/common/io/io_uring_worker_impl.h @@ -2,23 +2,53 @@ #include "envoy/common/io/io_uring.h" +#include "source/common/buffer/buffer_impl.h" #include "source/common/common/linked_object.h" +#include "source/common/common/logger.h" #include "source/common/io/io_uring_impl.h" namespace Envoy { namespace Io { +class ReadRequest : public Request { +public: + ReadRequest(IoUringSocket& socket, uint32_t size); + + std::unique_ptr buf_; + std::unique_ptr iov_; +}; + +class WriteRequest : public Request { +public: + WriteRequest(IoUringSocket& socket, const Buffer::RawSliceVector& slices); + + std::unique_ptr iov_; +}; + class IoUringSocketEntry; using IoUringSocketEntryPtr = std::unique_ptr; class IoUringWorkerImpl : public IoUringWorker, private Logger::Loggable { public: IoUringWorkerImpl(uint32_t io_uring_size, bool use_submission_queue_polling, + uint32_t read_buffer_size, uint32_t write_timeout_ms, + Event::Dispatcher& dispatcher); + IoUringWorkerImpl(IoUringPtr&& io_uring, uint32_t read_buffer_size, uint32_t write_timeout_ms, Event::Dispatcher& dispatcher); - IoUringWorkerImpl(IoUringPtr&& io_uring, Event::Dispatcher& dispatcher); ~IoUringWorkerImpl() override; // IoUringWorker + IoUringSocket& addServerSocket(os_fd_t fd, Event::FileReadyCb cb, + bool enable_close_event) override; + IoUringSocket& addServerSocket(os_fd_t fd, Buffer::Instance& read_buf, Event::FileReadyCb cb, + bool enable_close_event) override; + + Request* submitReadRequest(IoUringSocket& socket) override; + Request* submitWriteRequest(IoUringSocket& socket, const Buffer::RawSliceVector& slices) override; + Request* submitCloseRequest(IoUringSocket& socket) override; + Request* submitCancelRequest(IoUringSocket& socket, Request* request_to_cancel) override; + Request* submitShutdownRequest(IoUringSocket& socket, int how) override; + Event::Dispatcher& dispatcher() override; // Remove a socket from this worker. @@ -28,7 +58,7 @@ class IoUringWorkerImpl : public IoUringWorker, private Logger::Loggable { public: - IoUringSocketEntry(os_fd_t fd, IoUringWorkerImpl& parent); + IoUringSocketEntry(os_fd_t fd, IoUringWorkerImpl& parent, Event::FileReadyCb cb, + bool enable_close_event); // IoUringSocket IoUringWorker& getIoUringWorker() const override { return parent_; } os_fd_t fd() const override { return fd_; } + + void close(bool, IoUringSocketOnClosedCb cb = nullptr) override { + status_ = Closed; + on_closed_cb_ = cb; + } + void enableRead() override { status_ = ReadEnabled; } + void disableRead() override { status_ = ReadDisabled; } + void enableCloseEvent(bool enable) override { enable_close_event_ = enable; } + void onAccept(Request*, int32_t, bool injected) override { if (injected && (injected_completions_ & static_cast(Request::RequestType::Accept))) { injected_completions_ &= ~static_cast(Request::RequestType::Accept); @@ -98,17 +140,108 @@ class IoUringSocketEntry : public IoUringSocket, } void injectCompletion(Request::RequestType type) override; + IoUringSocketStatus getStatus() const override { return status_; } + + const OptRef& getReadParam() const override { return read_param_; } + const OptRef& getWriteParam() const override { return write_param_; } + + void setFileReadyCb(Event::FileReadyCb cb) override { cb_ = std::move(cb); } + protected: /** * For the socket to remove itself from the IoUringWorker and defer deletion. */ void cleanup(); + void onReadCompleted(); + void onWriteCompleted(); + void onRemoteClose(); os_fd_t fd_{INVALID_SOCKET}; IoUringWorkerImpl& parent_; // This records already injected completion request type to // avoid duplicated injections. uint8_t injected_completions_{0}; + // The current status of socket. + IoUringSocketStatus status_{Initialized}; + // Deliver the remote close as file read event or file close event. + bool enable_close_event_{false}; + // The callback will be invoked when close request is done. + IoUringSocketOnClosedCb on_closed_cb_{nullptr}; + // This object stores the data get from read request. + OptRef read_param_; + // This object stores the data get from write request. + OptRef write_param_; + + Event::FileReadyCb cb_; +}; + +class IoUringServerSocket : public IoUringSocketEntry { +public: + IoUringServerSocket(os_fd_t fd, IoUringWorkerImpl& parent, Event::FileReadyCb cb, + uint32_t write_timeout_ms, bool enable_close_event); + IoUringServerSocket(os_fd_t fd, Buffer::Instance& read_buf, IoUringWorkerImpl& parent, + Event::FileReadyCb cb, uint32_t write_timeout_ms, bool enable_close_event); + ~IoUringServerSocket() override; + + // IoUringSocket + void close(bool keep_fd_open, IoUringSocketOnClosedCb cb = nullptr) override; + void enableRead() override; + void disableRead() override; + void write(Buffer::Instance& data) override; + uint64_t write(const Buffer::RawSlice* slices, uint64_t num_slice) override; + void shutdown(int how) override; + void onClose(Request* req, int32_t result, bool injected) override; + void onRead(Request* req, int32_t result, bool injected) override; + void onWrite(Request* req, int32_t result, bool injected) override; + void onShutdown(Request* req, int32_t result, bool injected) override; + void onCancel(Request* req, int32_t result, bool injected) override; + +protected: + // Since the write of IoUringSocket is async, there may have write request is on the fly when + // close the socket. This timeout is setting for a time to wait the write request done. + const uint32_t write_timeout_ms_; + // For read. iouring socket will read sequentially in the order of buf_ and read_error_. Unless + // the buf_ is empty, the read_error_ will not be past to the handler. There is an exception that + // when enable_close_event_ is set, the remote close read_error_(0) will always be past to the + // handler. + Request* read_req_{}; + // TODO (soulxu): Add water mark here. + Buffer::OwnedImpl read_buf_; + absl::optional read_error_; + + // TODO (soulxu): We need water mark for write buffer. + // The upper layer will think the buffer released when the data copy into this write buffer. + // This leads to the `IntegrationTest.TestFloodUpstreamErrors` timeout, since the http layer + // always think the response is write successful, so flood protection is never kicked. + // + // For write. iouring socket will write sequentially in the order of write_buf_ and shutdown_ + // Unless the write_buf_ is empty, the shutdown operation will not be performed. + Buffer::OwnedImpl write_buf_; + // shutdown_ has 3 states. A absl::nullopt indicates the socket has not been shutdown, a false + // value represents the socket wants to be shutdown but the shutdown has not been performed or + // completed, and a true value means the socket has been shutdown. + absl::optional shutdown_{}; + // If there is in progress write_or_shutdown_req_ during closing, a write timeout timer may be + // setup to cancel the write_or_shutdown_req_, either a write request or a shutdown request. So + // we can make sure all SQEs bounding to the iouring socket is completed and the socket can be + // closed successfully. + Request* write_or_shutdown_req_{nullptr}; + Event::TimerPtr write_timeout_timer_{nullptr}; + // Whether keep the fd open when close the IoUringSocket. + bool keep_fd_open_{false}; + // This is used for tracking the read's cancel request. + Request* read_cancel_req_{nullptr}; + // This is used for tracking the write or shutdown's cancel request. + Request* write_or_shutdown_cancel_req_{nullptr}; + // This is used for tracking the close request. + Request* close_req_{nullptr}; + + void closeInternal(); + void submitReadRequest(); + void submitWriteOrShutdownRequest(); + void moveReadDataToBuffer(Request* req, size_t data_length); + void onReadCompleted(int32_t result); + void onWriteCompleted(int32_t result); }; } // namespace Io diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc index 23d77f5307ef..e7de7479247d 100644 --- a/source/common/json/json_internal.cc +++ b/source/common/json/json_internal.cc @@ -25,6 +25,7 @@ namespace Json { namespace Nlohmann { namespace { + /** * Internal representation of Object. */ @@ -133,9 +134,11 @@ class Field : public Object { bool isType(Type type) const { return type == type_; } void checkType(Type type) const { if (!isType(type)) { - throw Exception(fmt::format( - "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", - line_number_start_, typeAsString(type), typeAsString(type_))); + throwExceptionOrPanic( + Exception, + fmt::format( + "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", + line_number_start_, typeAsString(type), typeAsString(type_))); } } @@ -188,8 +191,9 @@ class ObjectHandler : public nlohmann::json_sax { } bool number_unsigned(uint64_t value) override { if (value > static_cast(std::numeric_limits::max())) { - throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", - line_number_)); + throwExceptionOrPanic( + Exception, fmt::format("JSON value from line {} is larger than int64_t (not supported)", + line_number_)); } return handleValueEvent(Field::createValue(static_cast(value))); } @@ -411,8 +415,9 @@ bool Field::getBoolean(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { - throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, - line_number_start_, line_number_end_)); + throwExceptionOrPanic(Exception, + fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->booleanValue(); } @@ -430,8 +435,9 @@ double Field::getDouble(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { - throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, - line_number_start_, line_number_end_)); + throwExceptionOrPanic(Exception, + fmt::format("key '{}' missing or not a double from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->doubleValue(); } @@ -449,8 +455,9 @@ int64_t Field::getInteger(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { - throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, - line_number_start_, line_number_end_)); + throwExceptionOrPanic(Exception, + fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->integerValue(); } @@ -467,7 +474,7 @@ int64_t Field::getInteger(const std::string& name, int64_t default_value) const ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { auto result = getObjectNoThrow(name, allow_empty); if (!result.ok()) { - throw Exception(std::string(result.status().message())); + throwExceptionOrPanic(Exception, std::string(result.status().message())); } return result.value(); @@ -500,8 +507,9 @@ std::vector Field::getObjectArray(const std::string& name, if (allow_empty && value_itr == value_.object_value_.end()) { return {}; } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); + throwExceptionOrPanic(Exception, + fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); } std::vector array_value = value_itr->second->arrayValue(); @@ -512,8 +520,9 @@ std::string Field::getString(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { - throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, - line_number_start_, line_number_end_)); + throwExceptionOrPanic(Exception, + fmt::format("key '{}' missing or not a string from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->stringValue(); } @@ -535,16 +544,18 @@ std::vector Field::getStringArray(const std::string& name, bool all if (allow_empty && value_itr == value_.object_value_.end()) { return string_array; } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); + throwExceptionOrPanic(Exception, + fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); } std::vector array = value_itr->second->arrayValue(); string_array.reserve(array.size()); for (const auto& element : array) { if (!element->isType(Type::String)) { - throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, - line_number_start_)); + throwExceptionOrPanic(Exception, + fmt::format("JSON array '{}' from line {} does not contain all strings", + name, line_number_start_)); } string_array.push_back(element->stringValue()); } @@ -568,7 +579,8 @@ bool Field::empty() const { } else if (isType(Type::Array)) { return value_.array_value_.empty(); } else { - throw Exception( + throwExceptionOrPanic( + Exception, fmt::format("Json does not support empty() on types other than array and object")); } } @@ -589,7 +601,9 @@ void Field::iterate(const ObjectCallback& callback) const { } } -void Field::validateSchema(const std::string&) const { throw Exception("not implemented"); } +void Field::validateSchema(const std::string&) const { + throwExceptionOrPanic(Exception, "not implemented"); +} bool ObjectHandler::start_object(std::size_t) { FieldSharedPtr object = Field::createObject(); @@ -723,7 +737,7 @@ absl::StatusOr Factory::loadFromStringNoThrow(const std::string ObjectSharedPtr Factory::loadFromString(const std::string& json) { auto result = loadFromStringNoThrow(json); if (!result.ok()) { - throw Exception(std::string(result.status().message())); + throwExceptionOrPanic(Exception, std::string(result.status().message())); } return result.value(); @@ -752,7 +766,7 @@ FieldSharedPtr loadFromProtobufValueInternal(const ProtobufWkt::Value& protobuf_ case ProtobufWkt::Value::kStructValue: return loadFromProtobufStructInternal(protobuf_value.struct_value()); default: - throw Exception("Protobuf value case not implemented"); + throwExceptionOrPanic(Exception, "Protobuf value case not implemented"); } } diff --git a/source/common/json/json_streamer.h b/source/common/json/json_streamer.h index c4def6aafb5e..0da483281e87 100644 --- a/source/common/json/json_streamer.h +++ b/source/common/json/json_streamer.h @@ -141,7 +141,7 @@ class Streamer { void addEntries(const Entries& entries); protected: - virtual void nextField() override; + void nextField() override; private: bool expecting_value_{false}; diff --git a/source/extensions/listener_managers/listener_manager/BUILD b/source/common/listener_manager/BUILD similarity index 94% rename from source/extensions/listener_managers/listener_manager/BUILD rename to source/common/listener_manager/BUILD index fd51badb09d8..1b8e49427033 100644 --- a/source/extensions/listener_managers/listener_manager/BUILD +++ b/source/common/listener_manager/BUILD @@ -1,16 +1,15 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_extension", "envoy_cc_library", - "envoy_extension_package", + "envoy_package", "envoy_select_enable_http3", ) licenses(["notice"]) # Apache 2 -envoy_extension_package() +envoy_package() -envoy_cc_extension( +envoy_cc_library( name = "listener_manager_lib", srcs = [ "listener_impl.cc", @@ -20,18 +19,12 @@ envoy_cc_extension( "listener_impl.h", "listener_manager_impl.h", ], - # any changes to this should be reviewed by mobile maintainers - # to ensure that listener code doesn't leak back into Envoy Mobile. - extra_visibility = [ - "//source/server/admin:__subpackages__", - "//source/server/config_validation:__subpackages__", - "//test:__subpackages__", - ], deps = [ ":active_raw_udp_listener_config", ":connection_handler_lib", ":filter_chain_manager_lib", ":lds_api_lib", + ":listener_info_lib", "//envoy/access_log:access_log_interface", "//envoy/config:typed_metadata_interface", "//envoy/network:connection_interface", @@ -44,7 +37,6 @@ envoy_cc_extension( "//source/common/access_log:access_log_lib", "//source/common/common:basic_resource_lib", "//source/common/common:empty_string", - "//source/common/config:metadata_lib", "//source/common/config:utility_lib", "//source/common/http:conn_manager_lib", "//source/common/init:manager_lib", @@ -66,6 +58,7 @@ envoy_cc_extension( "//source/server:api_listener_lib", "//source/server:configuration_lib", "//source/server:drain_manager_lib", + "//source/server:factory_context_lib", "//source/server:listener_manager_factory_lib", "//source/server:transport_socket_config_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", @@ -78,19 +71,18 @@ envoy_cc_extension( "//source/common/quic:active_quic_listener_lib", "//source/common/quic:client_connection_factory_lib", "//source/common/quic:quic_server_factory_lib", + "//source/common/quic:quic_server_transport_socket_factory_lib", "//source/common/quic:quic_transport_socket_factory_lib", "//source/common/quic:udp_gso_batch_writer_lib", "//source/extensions/udp_packet_writer/gso:config", ]), + alwayslink = True, ) envoy_cc_library( name = "active_raw_udp_listener_config", srcs = ["active_raw_udp_listener_config.cc"], hdrs = ["active_raw_udp_listener_config.h"], - visibility = [ - "//test:__subpackages__", - ], deps = [ ":connection_handler_lib", "//envoy/registry", @@ -155,16 +147,12 @@ envoy_cc_library( ], ) -envoy_cc_extension( +envoy_cc_library( name = "connection_handler_lib", srcs = ["connection_handler_impl.cc"], hdrs = [ "connection_handler_impl.h", ], - # core code. - extra_visibility = [ - "//test:__subpackages__", - ], deps = [ "active_tcp_listener", "//envoy/common:time_interface", @@ -210,6 +198,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:linked_object", "//source/common/network:connection_lib", + "//source/common/network:listener_lib", "//source/common/stats:timespan_lib", "//source/server:active_listener_base", ], @@ -235,6 +224,7 @@ envoy_cc_library( "//envoy/server:listener_manager_interface", "//source/common/common:assert_lib", "//source/common/common:linked_object", + "//source/common/formatter:substitution_formatter_lib", "//source/common/network:connection_lib", "//source/common/network:generic_listener_filter_impl_base_lib", "//source/common/stats:timespan_lib", @@ -242,6 +232,19 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "listener_info_lib", + srcs = ["listener_info_impl.cc"], + hdrs = [ + "listener_info_impl.h", + ], + deps = [ + "//envoy/network:listener_interface", + "//source/common/config:metadata_lib", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "active_tcp_socket", srcs = ["active_tcp_socket.cc"], diff --git a/source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.cc b/source/common/listener_manager/active_raw_udp_listener_config.cc similarity index 83% rename from source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.cc rename to source/common/listener_manager/active_raw_udp_listener_config.cc index d14559914ad9..831ccafcf528 100644 --- a/source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.cc +++ b/source/common/listener_manager/active_raw_udp_listener_config.cc @@ -1,9 +1,9 @@ -#include "source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.h" +#include "source/common/listener_manager/active_raw_udp_listener_config.h" #include #include -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/server/active_udp_listener.h" namespace Envoy { diff --git a/source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.h b/source/common/listener_manager/active_raw_udp_listener_config.h similarity index 100% rename from source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.h rename to source/common/listener_manager/active_raw_udp_listener_config.h diff --git a/source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc b/source/common/listener_manager/active_stream_listener_base.cc similarity index 97% rename from source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc rename to source/common/listener_manager/active_stream_listener_base.cc index b56dbf8bda7f..42a6f413da58 100644 --- a/source/extensions/listener_managers/listener_manager/active_stream_listener_base.cc +++ b/source/common/listener_manager/active_stream_listener_base.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/active_stream_listener_base.h" +#include "source/common/listener_manager/active_stream_listener_base.h" #include "envoy/network/filter.h" @@ -31,7 +31,7 @@ void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info) { stream_info.onRequestComplete(); for (const auto& access_log : config.accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log->log({}, stream_info); } } @@ -51,6 +51,7 @@ void ActiveStreamListenerBase::newConnection(Network::ConnectionSocketPtr&& sock return; } + socket->connectionInfoProvider().setListenerInfo(config_->listenerInfo()); socket->connectionInfoProvider().setFilterChainInfo( std::make_shared(filter_chain->name())); diff --git a/source/extensions/listener_managers/listener_manager/active_stream_listener_base.h b/source/common/listener_manager/active_stream_listener_base.h similarity index 98% rename from source/extensions/listener_managers/listener_manager/active_stream_listener_base.h rename to source/common/listener_manager/active_stream_listener_base.h index 7ec1c509d575..d967ee984cf1 100644 --- a/source/extensions/listener_managers/listener_manager/active_stream_listener_base.h +++ b/source/common/listener_manager/active_stream_listener_base.h @@ -14,7 +14,8 @@ #include "envoy/stream_info/stream_info.h" #include "source/common/common/linked_object.h" -#include "source/extensions/listener_managers/listener_manager/active_tcp_socket.h" +#include "source/common/formatter/http_specific_formatter.h" +#include "source/common/listener_manager/active_tcp_socket.h" #include "source/server/active_listener_base.h" namespace Envoy { diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc b/source/common/listener_manager/active_tcp_listener.cc similarity index 95% rename from source/extensions/listener_managers/listener_manager/active_tcp_listener.cc rename to source/common/listener_manager/active_tcp_listener.cc index 6d972a0ff97a..1f7c88f0a7c3 100644 --- a/source/extensions/listener_managers/listener_manager/active_tcp_listener.cc +++ b/source/common/listener_manager/active_tcp_listener.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/active_tcp_listener.h" +#include "source/common/listener_manager/active_tcp_listener.h" #include @@ -14,12 +14,15 @@ namespace Server { ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerConfig& config, Runtime::Loader& runtime, + Random::RandomGenerator& random, Network::SocketSharedPtr&& socket, Network::Address::InstanceConstSharedPtr& listen_address, - Network::ConnectionBalancer& connection_balancer) + Network::ConnectionBalancer& connection_balancer, + ThreadLocalOverloadStateOptRef overload_state) : OwnedActiveStreamListenerBase( parent, parent.dispatcher(), - parent.dispatcher().createListener(std::move(socket), *this, runtime, config), config), + parent.createListener(std::move(socket), *this, runtime, random, config, overload_state), + config), tcp_conn_handler_(parent), connection_balancer_(connection_balancer), listen_address_(listen_address) { connection_balancer_.registerHandler(*this); diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_listener.h b/source/common/listener_manager/active_tcp_listener.h similarity index 92% rename from source/extensions/listener_managers/listener_manager/active_tcp_listener.h rename to source/common/listener_manager/active_tcp_listener.h index 6d6d9b406044..ebb7a35c0168 100644 --- a/source/extensions/listener_managers/listener_manager/active_tcp_listener.h +++ b/source/common/listener_manager/active_tcp_listener.h @@ -5,8 +5,8 @@ #include "envoy/stream_info/stream_info.h" #include "source/common/common/linked_object.h" -#include "source/extensions/listener_managers/listener_manager/active_stream_listener_base.h" -#include "source/extensions/listener_managers/listener_manager/active_tcp_socket.h" +#include "source/common/listener_manager/active_stream_listener_base.h" +#include "source/common/listener_manager/active_tcp_socket.h" #include "source/server/active_listener_base.h" namespace Envoy { @@ -27,9 +27,11 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, public Network::BalancedConnectionHandler { public: ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerConfig& config, - Runtime::Loader& runtime, Network::SocketSharedPtr&& socket, + Runtime::Loader& runtime, Random::RandomGenerator& random, + Network::SocketSharedPtr&& socket, Network::Address::InstanceConstSharedPtr& listen_address, - Network::ConnectionBalancer& connection_balancer); + Network::ConnectionBalancer& connection_balancer, + ThreadLocalOverloadStateOptRef overload_state); ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerPtr&& listener, Network::Address::InstanceConstSharedPtr& listen_address, Network::ListenerConfig& config, diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_socket.cc b/source/common/listener_manager/active_tcp_socket.cc similarity index 97% rename from source/extensions/listener_managers/listener_manager/active_tcp_socket.cc rename to source/common/listener_manager/active_tcp_socket.cc index c159b7292d7b..a55fdff53a6f 100644 --- a/source/extensions/listener_managers/listener_manager/active_tcp_socket.cc +++ b/source/common/listener_manager/active_tcp_socket.cc @@ -1,9 +1,9 @@ -#include "source/extensions/listener_managers/listener_manager/active_tcp_socket.h" +#include "source/common/listener_manager/active_tcp_socket.h" #include "envoy/network/filter.h" +#include "source/common/listener_manager/active_stream_listener_base.h" #include "source/common/stream_info/stream_info_impl.h" -#include "source/extensions/listener_managers/listener_manager/active_stream_listener_base.h" namespace Envoy { namespace Server { diff --git a/source/extensions/listener_managers/listener_manager/active_tcp_socket.h b/source/common/listener_manager/active_tcp_socket.h similarity index 100% rename from source/extensions/listener_managers/listener_manager/active_tcp_socket.h rename to source/common/listener_manager/active_tcp_socket.h diff --git a/source/extensions/listener_managers/listener_manager/connection_handler_impl.cc b/source/common/listener_manager/connection_handler_impl.cc similarity index 94% rename from source/extensions/listener_managers/listener_manager/connection_handler_impl.cc rename to source/common/listener_manager/connection_handler_impl.cc index 073760cb9088..71f40d5076c2 100644 --- a/source/extensions/listener_managers/listener_manager/connection_handler_impl.cc +++ b/source/common/listener_manager/connection_handler_impl.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include @@ -7,10 +7,11 @@ #include "source/common/common/logger.h" #include "source/common/event/deferred_task.h" +#include "source/common/listener_manager/active_tcp_listener.h" #include "source/common/network/address_impl.h" +#include "source/common/network/tcp_listener_impl.h" #include "source/common/network/utility.h" #include "source/common/runtime/runtime_features.h" -#include "source/extensions/listener_managers/listener_manager/active_tcp_listener.h" #include "source/server/listener_manager_factory.h" namespace Envoy { @@ -35,7 +36,8 @@ void ConnectionHandlerImpl::decNumConnections() { } void ConnectionHandlerImpl::addListener(absl::optional overridden_listener, - Network::ListenerConfig& config, Runtime::Loader& runtime) { + Network::ListenerConfig& config, Runtime::Loader& runtime, + Random::RandomGenerator& random) { if (overridden_listener.has_value()) { ActiveListenerDetailsOptRef listener_detail = findActiveListenerByTag(overridden_listener.value()); @@ -82,9 +84,11 @@ void ConnectionHandlerImpl::addListener(absl::optional overridden_list details->addActiveListener( config, address, listener_reject_fraction_, disable_listeners_, std::make_unique( - *this, config, runtime, + *this, config, runtime, random, socket_factory->getListenSocket(worker_index_.has_value() ? *worker_index_ : 0), - address, config.connectionBalancer(*address)), + address, config.connectionBalancer(*address), + overload_manager_ ? makeOptRef(overload_manager_->getThreadLocalOverloadState()) + : absl::nullopt), overload_manager_); } } else { @@ -347,6 +351,16 @@ ConnectionHandlerImpl::getBalancedHandlerByTag(uint64_t listener_tag, return absl::nullopt; } +Network::ListenerPtr ConnectionHandlerImpl::createListener( + Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime, + Random::RandomGenerator& random, const Network::ListenerConfig& config, + Server::ThreadLocalOverloadStateOptRef overload_state) { + return std::make_unique( + dispatcher(), random, runtime, std::move(socket), cb, config.bindToPort(), + config.ignoreGlobalConnLimit(), config.maxConnectionsToAcceptPerSocketEvent(), + overload_state); +} + Network::BalancedConnectionHandlerOptRef ConnectionHandlerImpl::getBalancedHandlerByAddress(const Network::Address::Instance& address) { // Only Ip address can be restored to original address and redirect. diff --git a/source/extensions/listener_managers/listener_manager/connection_handler_impl.h b/source/common/listener_manager/connection_handler_impl.h similarity index 94% rename from source/extensions/listener_managers/listener_manager/connection_handler_impl.h rename to source/common/listener_manager/connection_handler_impl.h index a8fa4f0b74e3..e0406c7c91da 100644 --- a/source/extensions/listener_managers/listener_manager/connection_handler_impl.h +++ b/source/common/listener_manager/connection_handler_impl.h @@ -45,7 +45,7 @@ class ConnectionHandlerImpl : public ConnectionHandler, void incNumConnections() override; void decNumConnections() override; void addListener(absl::optional overridden_listener, Network::ListenerConfig& config, - Runtime::Loader& runtime) override; + Runtime::Loader& runtime, Random::RandomGenerator& random) override; void removeListeners(uint64_t listener_tag) override; void removeFilterChains(uint64_t listener_tag, const std::list& filter_chains, @@ -65,6 +65,11 @@ class ConnectionHandlerImpl : public ConnectionHandler, const Network::Address::Instance& address) override; Network::BalancedConnectionHandlerOptRef getBalancedHandlerByAddress(const Network::Address::Instance& address) override; + Network::ListenerPtr + createListener(Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state) override; // Network::UdpConnectionHandler Network::UdpListenerCallbacksOptRef @@ -76,6 +81,8 @@ class ConnectionHandlerImpl : public ConnectionHandler, findByAddress(const Network::Address::InstanceConstSharedPtr& listen_address) override; private: + friend class ConnectionHandlerImplPeer; + struct PerAddressActiveListenerDetails { // Strong pointer to the listener, whether TCP, UDP, QUIC, etc. Network::ConnectionHandler::ActiveListenerPtr listener_; diff --git a/source/extensions/listener_managers/listener_manager/filter_chain_factory_context_callback.h b/source/common/listener_manager/filter_chain_factory_context_callback.h similarity index 100% rename from source/extensions/listener_managers/listener_manager/filter_chain_factory_context_callback.h rename to source/common/listener_manager/filter_chain_factory_context_callback.h diff --git a/source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.cc b/source/common/listener_manager/filter_chain_manager_impl.cc similarity index 91% rename from source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.cc rename to source/common/listener_manager/filter_chain_manager_impl.cc index 356072181bb0..f9e62fa9f4fc 100644 --- a/source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.cc +++ b/source/common/listener_manager/filter_chain_manager_impl.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" #include "envoy/config/listener/v3/listener_components.pb.h" @@ -88,98 +88,20 @@ Network::DrainDecision& PerFilterChainFactoryContextImpl::drainDecision() { retu Init::Manager& PerFilterChainFactoryContextImpl::initManager() { return init_manager_; } -ThreadLocal::SlotAllocator& PerFilterChainFactoryContextImpl::threadLocal() { - return parent_context_.threadLocal(); +const Network::ListenerInfo& PerFilterChainFactoryContextImpl::listenerInfo() const { + return parent_context_.listenerInfo(); } -const envoy::config::core::v3::Metadata& -PerFilterChainFactoryContextImpl::listenerMetadata() const { - return parent_context_.listenerMetadata(); -} - -const Envoy::Config::TypedMetadata& -PerFilterChainFactoryContextImpl::listenerTypedMetadata() const { - return parent_context_.listenerTypedMetadata(); -} - -envoy::config::core::v3::TrafficDirection PerFilterChainFactoryContextImpl::direction() const { - return parent_context_.direction(); -} - -ProtobufMessage::ValidationContext& PerFilterChainFactoryContextImpl::messageValidationContext() { - return parent_context_.messageValidationContext(); -} -ProtobufMessage::ValidationVisitor& PerFilterChainFactoryContextImpl::messageValidationVisitor() { +ProtobufMessage::ValidationVisitor& +PerFilterChainFactoryContextImpl::messageValidationVisitor() const { return parent_context_.messageValidationVisitor(); } -AccessLog::AccessLogManager& PerFilterChainFactoryContextImpl::accessLogManager() { - return parent_context_.accessLogManager(); -} - -Upstream::ClusterManager& PerFilterChainFactoryContextImpl::clusterManager() { - return parent_context_.clusterManager(); -} - -Event::Dispatcher& PerFilterChainFactoryContextImpl::mainThreadDispatcher() { - return parent_context_.mainThreadDispatcher(); -} - -const Server::Options& PerFilterChainFactoryContextImpl::options() { - return parent_context_.options(); -} - -Grpc::Context& PerFilterChainFactoryContextImpl::grpcContext() { - return parent_context_.grpcContext(); -} - -bool PerFilterChainFactoryContextImpl::healthCheckFailed() { - return parent_context_.healthCheckFailed(); -} - -Http::Context& PerFilterChainFactoryContextImpl::httpContext() { - return parent_context_.httpContext(); -} - -Router::Context& PerFilterChainFactoryContextImpl::routerContext() { - return parent_context_.routerContext(); -} - -const LocalInfo::LocalInfo& PerFilterChainFactoryContextImpl::localInfo() const { - return parent_context_.localInfo(); -} - -Envoy::Runtime::Loader& PerFilterChainFactoryContextImpl::runtime() { - return parent_context_.runtime(); -} - Stats::Scope& PerFilterChainFactoryContextImpl::scope() { return *scope_; } -Singleton::Manager& PerFilterChainFactoryContextImpl::singletonManager() { - return parent_context_.singletonManager(); -} - -OverloadManager& PerFilterChainFactoryContextImpl::overloadManager() { - return parent_context_.overloadManager(); -} - -OptRef PerFilterChainFactoryContextImpl::admin() { return parent_context_.admin(); } - -TimeSource& PerFilterChainFactoryContextImpl::timeSource() { return api().timeSource(); } - -Api::Api& PerFilterChainFactoryContextImpl::api() { return parent_context_.api(); } - -ServerLifecycleNotifier& PerFilterChainFactoryContextImpl::lifecycleNotifier() { - return parent_context_.lifecycleNotifier(); -} - -ProcessContextOptRef PerFilterChainFactoryContextImpl::processContext() { - return parent_context_.processContext(); -} - Configuration::ServerFactoryContext& -PerFilterChainFactoryContextImpl::getServerFactoryContext() const { - return parent_context_.getServerFactoryContext(); +PerFilterChainFactoryContextImpl::serverFactoryContext() const { + return parent_context_.serverFactoryContext(); } Configuration::TransportSocketFactoryContext& @@ -189,10 +111,6 @@ PerFilterChainFactoryContextImpl::getTransportSocketFactoryContext() const { Stats::Scope& PerFilterChainFactoryContextImpl::listenerScope() { return *filter_chain_scope_; } -bool PerFilterChainFactoryContextImpl::isQuicListener() const { - return parent_context_.isQuicListener(); -} - FilterChainManagerImpl::FilterChainManagerImpl( const std::vector& addresses, Configuration::FactoryContext& factory_context, Init::Manager& init_manager, @@ -314,7 +232,7 @@ void FilterChainManagerImpl::addFilterChains( filter_chains_by_name_ = filter_chains_by_name; FilterChain::FilterChainNameActionValidationVisitor validation_visitor; Matcher::MatchTreeFactory factory( - parent_context_.getServerFactoryContext(), parent_context_.getServerFactoryContext(), + parent_context_.serverFactoryContext(), parent_context_.serverFactoryContext(), validation_visitor); matcher_ = factory.create(*filter_chain_matcher)(); } diff --git a/source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h b/source/common/listener_manager/filter_chain_manager_impl.h similarity index 92% rename from source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h rename to source/common/listener_manager/filter_chain_manager_impl.h index 7addd4f710ff..023fd754e955 100644 --- a/source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h +++ b/source/common/listener_manager/filter_chain_manager_impl.h @@ -18,9 +18,9 @@ #include "source/common/common/logger.h" #include "source/common/init/manager_impl.h" +#include "source/common/listener_manager/filter_chain_factory_context_callback.h" #include "source/common/network/cidr_range.h" #include "source/common/network/lc_trie.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_factory_context_callback.h" #include "source/server/factory_context_impl.h" #include "absl/container/flat_hash_map.h" @@ -58,37 +58,14 @@ class PerFilterChainFactoryContextImpl : public Configuration::FilterChainFactor } // Configuration::FactoryContext - AccessLog::AccessLogManager& accessLogManager() override; - Upstream::ClusterManager& clusterManager() override; - Event::Dispatcher& mainThreadDispatcher() override; - const Server::Options& options() override; Network::DrainDecision& drainDecision() override; - Grpc::Context& grpcContext() override; - Router::Context& routerContext() override; - bool healthCheckFailed() override; - Http::Context& httpContext() override; Init::Manager& initManager() override; - const LocalInfo::LocalInfo& localInfo() const override; - Envoy::Runtime::Loader& runtime() override; Stats::Scope& scope() override; - Stats::Scope& serverScope() override { return parent_context_.serverScope(); } - Singleton::Manager& singletonManager() override; - OverloadManager& overloadManager() override; - ThreadLocal::SlotAllocator& threadLocal() override; - OptRef admin() override; - const envoy::config::core::v3::Metadata& listenerMetadata() const override; - const Envoy::Config::TypedMetadata& listenerTypedMetadata() const override; - envoy::config::core::v3::TrafficDirection direction() const override; - TimeSource& timeSource() override; - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override; - ProtobufMessage::ValidationContext& messageValidationContext() override; - Api::Api& api() override; - ServerLifecycleNotifier& lifecycleNotifier() override; - ProcessContextOptRef processContext() override; - Configuration::ServerFactoryContext& getServerFactoryContext() const override; + const Network::ListenerInfo& listenerInfo() const override; + ProtobufMessage::ValidationVisitor& messageValidationVisitor() const override; + Configuration::ServerFactoryContext& serverFactoryContext() const override; Configuration::TransportSocketFactoryContext& getTransportSocketFactoryContext() const override; Stats::Scope& listenerScope() override; - bool isQuicListener() const override; void startDraining() override { is_draining_.store(true); } diff --git a/source/extensions/listener_managers/listener_manager/lds_api.cc b/source/common/listener_manager/lds_api.cc similarity index 91% rename from source/extensions/listener_managers/listener_manager/lds_api.cc rename to source/common/listener_manager/lds_api.cc index 5d556b31f8f7..ee669d306e5d 100644 --- a/source/extensions/listener_managers/listener_manager/lds_api.cc +++ b/source/common/listener_manager/lds_api.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/lds_api.h" +#include "source/common/listener_manager/lds_api.h" #include "envoy/admin/v3/config_dump.pb.h" #include "envoy/config/core/v3/config_source.pb.h" @@ -68,20 +68,32 @@ LdsApiImpl::onConfigUpdate(const std::vector& added_ ListenerManager::FailureStates failure_state; absl::node_hash_set listener_names; std::string message; + for (const auto& resource : added_resources) { envoy::config::listener::v3::Listener listener; + + auto onError = [&](std::string error_message) { + failure_state.push_back(std::make_unique()); + auto& state = failure_state.back(); + state->set_details(error_message); + state->mutable_failed_configuration()->PackFrom(resource.get().resource()); + absl::StrAppend(&message, listener.name(), ": ", error_message, "\n"); + }; + TRY_ASSERT_MAIN_THREAD { listener = dynamic_cast(resource.get().resource()); if (!listener_names.insert(listener.name()).second) { // NOTE: at this point, the first of these duplicates has already been successfully // applied. - throw EnvoyException(fmt::format("duplicate listener {} found", listener.name())); + onError(fmt::format("duplicate listener {} found", listener.name())); + continue; } absl::StatusOr update_or_error = listener_manager_.addOrUpdateListener(listener, resource.get().version(), true); if (!update_or_error.status().ok()) { - throw EnvoyException(std::string(update_or_error.status().message())); + onError(std::string(update_or_error.status().message())); + continue; } if (update_or_error.value()) { ENVOY_LOG(info, "lds: add/update listener '{}'", listener.name()); @@ -92,11 +104,7 @@ LdsApiImpl::onConfigUpdate(const std::vector& added_ } END_TRY catch (const EnvoyException& e) { - failure_state.push_back(std::make_unique()); - auto& state = failure_state.back(); - state->set_details(e.what()); - state->mutable_failed_configuration()->PackFrom(resource.get().resource()); - absl::StrAppend(&message, listener.name(), ": ", e.what(), "\n"); + onError(e.what()); } } listener_manager_.endListenerUpdate(std::move(failure_state)); @@ -106,7 +114,7 @@ LdsApiImpl::onConfigUpdate(const std::vector& added_ } init_target_.ready(); if (!message.empty()) { - throw EnvoyException(fmt::format("Error adding/updating listener(s) {}", message)); + return absl::InvalidArgumentError(fmt::format("Error adding/updating listener(s) {}", message)); } return absl::OkStatus(); } diff --git a/source/extensions/listener_managers/listener_manager/lds_api.h b/source/common/listener_manager/lds_api.h similarity index 100% rename from source/extensions/listener_managers/listener_manager/lds_api.h rename to source/common/listener_manager/lds_api.h diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.cc b/source/common/listener_manager/listener_impl.cc similarity index 80% rename from source/extensions/listener_managers/listener_manager/listener_impl.cc rename to source/common/listener_manager/listener_impl.cc index de199ffe5a31..984ebca0a7b9 100644 --- a/source/extensions/listener_managers/listener_manager/listener_impl.cc +++ b/source/common/listener_manager/listener_impl.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/listener_impl.h" +#include "source/common/listener_manager/listener_impl.h" #include @@ -18,6 +18,9 @@ #include "source/common/api/os_sys_calls_impl.h" #include "source/common/common/assert.h" #include "source/common/config/utility.h" +#include "source/common/listener_manager/active_raw_udp_listener_config.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" +#include "source/common/listener_manager/listener_manager_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/resolver_impl.h" #include "source/common/network/socket_option_factory.h" @@ -27,9 +30,6 @@ #include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" #include "source/common/runtime/runtime_features.h" -#include "source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" -#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include "source/server/configuration_impl.h" #include "source/server/drain_manager_impl.h" #include "source/server/transport_socket_config_impl.h" @@ -230,75 +230,12 @@ std::string listenerStatsScope(const envoy::config::listener::v3::Listener& conf ListenerFactoryContextBaseImpl::ListenerFactoryContextBaseImpl( Envoy::Server::Instance& server, ProtobufMessage::ValidationVisitor& validation_visitor, const envoy::config::listener::v3::Listener& config, DrainManagerPtr drain_manager) - : server_(server), metadata_(config.metadata()), typed_metadata_(config.metadata()), - direction_(config.traffic_direction()), global_scope_(server.stats().createScope("")), - listener_scope_( - server_.stats().createScope(fmt::format("listener.{}.", listenerStatsScope(config)))), - validation_visitor_(validation_visitor), drain_manager_(std::move(drain_manager)), - is_quic_(config.udp_listener_config().has_quic_options()) {} - -AccessLog::AccessLogManager& ListenerFactoryContextBaseImpl::accessLogManager() { - return server_.accessLogManager(); -} -Upstream::ClusterManager& ListenerFactoryContextBaseImpl::clusterManager() { - return server_.clusterManager(); -} -Event::Dispatcher& ListenerFactoryContextBaseImpl::mainThreadDispatcher() { - return server_.dispatcher(); -} -const Server::Options& ListenerFactoryContextBaseImpl::options() { return server_.options(); } -Grpc::Context& ListenerFactoryContextBaseImpl::grpcContext() { return server_.grpcContext(); } -bool ListenerFactoryContextBaseImpl::healthCheckFailed() { return server_.healthCheckFailed(); } -Http::Context& ListenerFactoryContextBaseImpl::httpContext() { return server_.httpContext(); } -Router::Context& ListenerFactoryContextBaseImpl::routerContext() { return server_.routerContext(); } -const LocalInfo::LocalInfo& ListenerFactoryContextBaseImpl::localInfo() const { - return server_.localInfo(); -} -Envoy::Runtime::Loader& ListenerFactoryContextBaseImpl::runtime() { return server_.runtime(); } -Stats::Scope& ListenerFactoryContextBaseImpl::scope() { return *global_scope_; } -Singleton::Manager& ListenerFactoryContextBaseImpl::singletonManager() { - return server_.singletonManager(); -} -OverloadManager& ListenerFactoryContextBaseImpl::overloadManager() { - return server_.overloadManager(); -} -ThreadLocal::Instance& ListenerFactoryContextBaseImpl::threadLocal() { - return server_.threadLocal(); -} -OptRef ListenerFactoryContextBaseImpl::admin() { return server_.admin(); } -const envoy::config::core::v3::Metadata& ListenerFactoryContextBaseImpl::listenerMetadata() const { - return metadata_; -}; -const Envoy::Config::TypedMetadata& ListenerFactoryContextBaseImpl::listenerTypedMetadata() const { - return typed_metadata_; -} -envoy::config::core::v3::TrafficDirection ListenerFactoryContextBaseImpl::direction() const { - return direction_; -}; -TimeSource& ListenerFactoryContextBaseImpl::timeSource() { return api().timeSource(); } -ProtobufMessage::ValidationContext& ListenerFactoryContextBaseImpl::messageValidationContext() { - return server_.messageValidationContext(); -} -ProtobufMessage::ValidationVisitor& ListenerFactoryContextBaseImpl::messageValidationVisitor() { - return validation_visitor_; -} -Api::Api& ListenerFactoryContextBaseImpl::api() { return server_.api(); } -ServerLifecycleNotifier& ListenerFactoryContextBaseImpl::lifecycleNotifier() { - return server_.lifecycleNotifier(); -} -ProcessContextOptRef ListenerFactoryContextBaseImpl::processContext() { - return server_.processContext(); -} -Configuration::ServerFactoryContext& -ListenerFactoryContextBaseImpl::getServerFactoryContext() const { - return server_.serverFactoryContext(); -} -Configuration::TransportSocketFactoryContext& -ListenerFactoryContextBaseImpl::getTransportSocketFactoryContext() const { - return server_.transportSocketFactoryContext(); -} -Stats::Scope& ListenerFactoryContextBaseImpl::listenerScope() { return *listener_scope_; } -bool ListenerFactoryContextBaseImpl::isQuicListener() const { return is_quic_; } + : Server::FactoryContextImplBase( + server, validation_visitor, server.stats().createScope(""), + server.stats().createScope(fmt::format("listener.{}.", listenerStatsScope(config))), + std::make_shared(config)), + drain_manager_(std::move(drain_manager)) {} + Network::DrainDecision& ListenerFactoryContextBaseImpl::drainDecision() { return *this; } Server::DrainManager& ListenerFactoryContextBaseImpl::drainManager() { return *drain_manager_; } Init::Manager& ListenerFactoryContextBaseImpl::initManager() { return server_.initManager(); } @@ -336,14 +273,14 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, PROTOBUF_GET_MS_OR_DEFAULT(config, listener_filters_timeout, 15000)), continue_on_listener_filters_timeout_(config.continue_on_listener_filters_timeout()), listener_factory_context_(std::make_shared( - parent.server_, validation_visitor_, config, this, *this, + parent.server_, validation_visitor_, config, *this, parent_.factory_->createDrainManager(config.drain_type()))), - reuse_port_(getReusePortOrDefault(parent_.server_, config_, socket_type_)), - cx_limit_runtime_key_("envoy.resource_limits.listener." + config_.name() + + reuse_port_(getReusePortOrDefault(parent_.server_, config, socket_type_)), + cx_limit_runtime_key_("envoy.resource_limits.listener." + config.name() + ".connection_limit"), open_connections_(std::make_shared( - std::numeric_limits::max(), listener_factory_context_->runtime(), - cx_limit_runtime_key_)), + std::numeric_limits::max(), + listener_factory_context_->serverFactoryContext().runtime(), cx_limit_runtime_key_)), local_init_watcher_(fmt::format("Listener-local-init-watcher {}", name), [this] { if (workers_started_) { @@ -367,14 +304,14 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, if (config.has_internal_listener()) { addresses_.emplace_back( std::make_shared(config.name())); - address_opts_list.emplace_back(std::ref(config_.socket_options())); + address_opts_list.emplace_back(std::ref(config.socket_options())); } else { // All the addresses should be same socket type, so get the first address's socket type is // enough. auto address = Network::Address::resolveProtoAddress(config.address()); checkIpv4CompatAddress(address, config.address()); addresses_.emplace_back(address); - address_opts_list.emplace_back(std::ref(config_.socket_options())); + address_opts_list.emplace_back(std::ref(config.socket_options())); for (auto i = 0; i < config.additional_addresses_size(); i++) { if (socket_type_ != @@ -392,38 +329,39 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, address_opts_list.emplace_back( std::ref(config.additional_addresses(i).socket_options().socket_options())); } else { - address_opts_list.emplace_back(std::ref(config_.socket_options())); + address_opts_list.emplace_back(std::ref(config.socket_options())); } } } const absl::optional runtime_val = - listener_factory_context_->runtime().snapshot().get(cx_limit_runtime_key_); + listener_factory_context_->serverFactoryContext().runtime().snapshot().get( + cx_limit_runtime_key_); if (runtime_val && runtime_val->empty()) { ENVOY_LOG(warn, "Listener connection limit runtime key {} is empty. There are currently no " "limitations on the number of accepted connections for listener {}.", - cx_limit_runtime_key_, config_.name()); + cx_limit_runtime_key_, config.name()); } filter_chain_manager_ = std::make_unique( addresses_, listener_factory_context_->parentFactoryContext(), initManager()), - buildAccessLog(); + buildAccessLog(config); validateConfig(); // buildUdpListenerFactory() must come before buildListenSocketOptions() because the UDP // listener factory can provide additional options. - buildUdpListenerFactory(parent_.server_.options().concurrency()); - buildListenSocketOptions(address_opts_list); - createListenerFilterFactories(); - validateFilterChains(); - buildFilterChains(); + buildUdpListenerFactory(config, parent_.server_.options().concurrency()); + buildListenSocketOptions(config, address_opts_list); + createListenerFilterFactories(config); + validateFilterChains(config); + buildFilterChains(config); if (socket_type_ != Network::Socket::Type::Datagram) { - buildSocketOptions(); - buildOriginalDstListenerFilter(); - buildProxyProtocolListenerFilter(); - buildInternalListener(); + buildSocketOptions(config); + buildOriginalDstListenerFilter(config); + buildProxyProtocolListenerFilter(config); + buildInternalListener(config); } if (!workers_started_) { // Initialize dynamic_init_manager_ from Server's init manager if it's not initialized. @@ -467,8 +405,10 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, continue_on_listener_filters_timeout_(config.continue_on_listener_filters_timeout()), udp_listener_config_(origin.udp_listener_config_), connection_balancers_(origin.connection_balancers_), + // Reuse the listener_factory_context_base_ from the origin listener because the filter chain + // only updates will not change the listener_factory_context_base_. listener_factory_context_(std::make_shared( - origin.listener_factory_context_->listener_factory_context_base_, this, *this)), + origin.listener_factory_context_->listener_factory_context_base_, *this)), filter_chain_manager_(std::make_unique( addresses_, origin.listener_factory_context_->parentFactoryContext(), initManager(), *origin.filter_chain_manager_)), @@ -482,17 +422,17 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, quic_stat_names_(parent_.quicStatNames()), missing_listener_config_stats_({ALL_MISSING_LISTENER_CONFIG_STATS( POOL_COUNTER(listener_factory_context_->listenerScope()))}) { - buildAccessLog(); + buildAccessLog(config); validateConfig(); - createListenerFilterFactories(); - validateFilterChains(); - buildFilterChains(); - buildInternalListener(); + createListenerFilterFactories(config); + validateFilterChains(config); + buildFilterChains(config); + buildInternalListener(config); if (socket_type_ == Network::Socket::Type::Stream) { // Apply the options below only for TCP. - buildSocketOptions(); - buildOriginalDstListenerFilter(); - buildProxyProtocolListenerFilter(); + buildSocketOptions(config); + buildOriginalDstListenerFilter(config); + buildProxyProtocolListenerFilter(config); open_connections_ = origin.open_connections_; } } @@ -530,32 +470,32 @@ void ListenerImpl::validateConfig() { } } -void ListenerImpl::buildAccessLog() { - for (const auto& access_log : config_.access_log()) { +void ListenerImpl::buildAccessLog(const envoy::config::listener::v3::Listener& config) { + for (const auto& access_log : config.access_log()) { AccessLog::InstanceSharedPtr current_access_log = AccessLog::AccessLogFactory::fromProto(access_log, *listener_factory_context_); access_logs_.push_back(current_access_log); } } -void ListenerImpl::buildInternalListener() { - if (config_.has_internal_listener()) { - if (config_.has_address() || !config_.additional_addresses().empty()) { +void ListenerImpl::buildInternalListener(const envoy::config::listener::v3::Listener& config) { + if (config.has_internal_listener()) { + if (config.has_address() || !config.additional_addresses().empty()) { throw EnvoyException(fmt::format("error adding listener '{}': address should not be used " "when an internal listener config is provided", name_)); } - if ((config_.has_connection_balance_config() && - config_.connection_balance_config().has_exact_balance()) || - config_.enable_mptcp() || - config_.has_enable_reuse_port() // internal listener doesn't use physical l4 port. - || (config_.has_freebind() && config_.freebind().value()) || - config_.has_tcp_backlog_size() || config_.has_tcp_fast_open_queue_length() || - (config_.has_transparent() && config_.transparent().value())) { + if ((config.has_connection_balance_config() && + config.connection_balance_config().has_exact_balance()) || + config.enable_mptcp() || + config.has_enable_reuse_port() // internal listener doesn't use physical l4 port. + || (config.has_freebind() && config.freebind().value()) || config.has_tcp_backlog_size() || + config.has_tcp_fast_open_queue_length() || + (config.has_transparent() && config.transparent().value())) { throw EnvoyException(fmt::format( "error adding listener named '{}': has unsupported tcp listener feature", name_)); } - if (!config_.socket_options().empty()) { + if (!config.socket_options().empty()) { throw EnvoyException( fmt::format("error adding listener named '{}': does not support socket option", name_)); } @@ -569,9 +509,8 @@ void ListenerImpl::buildInternalListener() { } internal_listener_config_ = std::make_unique(*internal_listener_registry); - } else if (config_.address().has_envoy_internal_address() || - std::any_of(config_.additional_addresses().begin(), - config_.additional_addresses().end(), + } else if (config.address().has_envoy_internal_address() || + std::any_of(config.additional_addresses().begin(), config.additional_addresses().end(), [](const envoy::config::listener::v3::AdditionalAddress& proto_address) { return proto_address.address().has_envoy_internal_address(); })) { @@ -595,7 +534,8 @@ bool ListenerImpl::buildUdpListenerWorkerRouter(const Network::Address::Instance return true; } -void ListenerImpl::buildUdpListenerFactory(uint32_t concurrency) { +void ListenerImpl::buildUdpListenerFactory(const envoy::config::listener::v3::Listener& config, + uint32_t concurrency) { if (socket_type_ != Network::Socket::Type::Datagram) { return; } @@ -606,23 +546,23 @@ void ListenerImpl::buildUdpListenerFactory(uint32_t concurrency) { "set concurrency = 1."); } - udp_listener_config_ = std::make_shared(config_.udp_listener_config()); + udp_listener_config_ = std::make_shared(config.udp_listener_config()); ProtobufTypes::MessagePtr udp_packet_packet_writer_config; - if (config_.udp_listener_config().has_udp_packet_packet_writer_config()) { + if (config.udp_listener_config().has_udp_packet_packet_writer_config()) { auto* factory_factory = Config::Utility::getFactory( - config_.udp_listener_config().udp_packet_packet_writer_config()); + config.udp_listener_config().udp_packet_packet_writer_config()); udp_listener_config_->writer_factory_ = factory_factory->createUdpPacketWriterFactory( - config_.udp_listener_config().udp_packet_packet_writer_config()); + config.udp_listener_config().udp_packet_packet_writer_config()); } - if (config_.udp_listener_config().has_quic_options()) { + if (config.udp_listener_config().has_quic_options()) { #ifdef ENVOY_ENABLE_QUIC - if (config_.has_connection_balance_config()) { + if (config.has_connection_balance_config()) { throw EnvoyException("connection_balance_config is configured for QUIC listener which " "doesn't work with connection balancer."); } udp_listener_config_->listener_factory_ = std::make_unique( - config_.udp_listener_config().quic_options(), concurrency, quic_stat_names_, - validation_visitor_, listener_factory_context_->processContext()); + config.udp_listener_config().quic_options(), concurrency, quic_stat_names_, + validation_visitor_, listener_factory_context_->serverFactoryContext().processContext()); #if UDP_GSO_BATCH_WRITER_COMPILETIME_SUPPORT // TODO(mattklein123): We should be able to use GSO without QUICHE/QUIC. Right now this causes // non-QUIC integration tests to fail, which I haven't investigated yet. Additionally, from @@ -647,6 +587,7 @@ void ListenerImpl::buildUdpListenerFactory(uint32_t concurrency) { } void ListenerImpl::buildListenSocketOptions( + const envoy::config::listener::v3::Listener& config, std::vector>>& address_opts_list) { @@ -662,11 +603,11 @@ void ListenerImpl::buildListenSocketOptions( addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildSocketNoSigpipeOptions()); } - if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config_, transparent, false)) { + if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, transparent, false)) { addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildIpTransparentOptions()); } - if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config_, freebind, false)) { + if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, freebind, false)) { addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildIpFreebindOptions()); } @@ -674,7 +615,7 @@ void ListenerImpl::buildListenSocketOptions( addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildReusePortOptions()); } - if (!config_.socket_options().empty()) { + if (!config.socket_options().empty()) { addListenSocketOptions( listen_socket_options_list_[i], Network::SocketOptionFactory::buildLiteralOptions(address_opts_list[i])); @@ -702,30 +643,31 @@ void ListenerImpl::buildListenSocketOptions( } } -void ListenerImpl::createListenerFilterFactories() { - if (!config_.listener_filters().empty()) { +void ListenerImpl::createListenerFilterFactories( + const envoy::config::listener::v3::Listener& config) { + if (!config.listener_filters().empty()) { switch (socket_type_) { case Network::Socket::Type::Datagram: { if (udp_listener_config_->listener_factory_->isTransportConnectionless()) { udp_listener_filter_factories_ = parent_.factory_->createUdpListenerFilterFactoryList( - config_.listener_filters(), *listener_factory_context_); + config.listener_filters(), *listener_factory_context_); } else { // This is a QUIC listener. quic_listener_filter_factories_ = parent_.factory_->createQuicListenerFilterFactoryList( - config_.listener_filters(), *listener_factory_context_); + config.listener_filters(), *listener_factory_context_); } break; } case Network::Socket::Type::Stream: listener_filter_factories_ = parent_.factory_->createListenerFilterFactoryList( - config_.listener_filters(), *listener_factory_context_); + config.listener_filters(), *listener_factory_context_); break; } } } -void ListenerImpl::validateFilterChains() { - if (config_.filter_chains().empty() && !config_.has_default_filter_chain() && +void ListenerImpl::validateFilterChains(const envoy::config::listener::v3::Listener& config) { + if (config.filter_chains().empty() && !config.has_default_filter_chain() && (socket_type_ == Network::Socket::Type::Stream || !udp_listener_config_->listener_factory_->isTransportConnectionless())) { // If we got here, this is a tcp listener or connection-oriented udp listener, so ensure there @@ -736,7 +678,7 @@ void ListenerImpl::validateFilterChains() { } else if (udp_listener_config_ != nullptr && !udp_listener_config_->listener_factory_->isTransportConnectionless()) { // Early fail if any filter chain doesn't have transport socket configured. - if (anyFilterChain(config_, [](const auto& filter_chain) { + if (anyFilterChain(config, [](const auto& filter_chain) { return !filter_chain.has_transport_socket(); })) { throw EnvoyException( @@ -744,28 +686,29 @@ void ListenerImpl::validateFilterChains() { "specified for connection oriented UDP listener", absl::StrJoin(addresses_, ",", Network::AddressStrFormatter()))); } - } else if ((!config_.filter_chains().empty() || config_.has_default_filter_chain()) && + } else if ((!config.filter_chains().empty() || config.has_default_filter_chain()) && udp_listener_config_ != nullptr && udp_listener_config_->listener_factory_->isTransportConnectionless()) { throw EnvoyException(fmt::format("error adding listener '{}': {} filter chain(s) specified for " "connection-less UDP listener.", absl::StrJoin(addresses_, ",", Network::AddressStrFormatter()), - config_.filter_chains_size())); + config.filter_chains_size())); } } -void ListenerImpl::buildFilterChains() { +void ListenerImpl::buildFilterChains(const envoy::config::listener::v3::Listener& config) { transport_factory_context_->setInitManager(*dynamic_init_manager_); ListenerFilterChainFactoryBuilder builder(*this, *transport_factory_context_); filter_chain_manager_->addFilterChains( - config_.has_filter_chain_matcher() ? &config_.filter_chain_matcher() : nullptr, - config_.filter_chains(), - config_.has_default_filter_chain() ? &config_.default_filter_chain() : nullptr, builder, + config.has_filter_chain_matcher() ? &config.filter_chain_matcher() : nullptr, + config.filter_chains(), + config.has_default_filter_chain() ? &config.default_filter_chain() : nullptr, builder, *filter_chain_manager_); } -void ListenerImpl::buildConnectionBalancer(const Network::Address::Instance& address) { +void ListenerImpl::buildConnectionBalancer(const envoy::config::listener::v3::Listener& config, + const Network::Address::Instance& address) { auto iter = connection_balancers_.find(address.asString()); if (iter == connection_balancers_.end() && socket_type_ == Network::Socket::Type::Stream) { #ifdef WIN32 @@ -778,20 +721,20 @@ void ListenerImpl::buildConnectionBalancer(const Network::Address::Instance& add "ExactBalance was forced enabled for TCP listener '{}' because " "Envoy is running on Windows." "ExactBalance is used to load balance connections between workers on Windows.", - config_.name()); + config.name()); connection_balancers_.emplace(address.asString(), std::make_shared()); #else // Not in place listener update. - if (config_.has_connection_balance_config()) { - switch (config_.connection_balance_config().balance_type_case()) { + if (config.has_connection_balance_config()) { + switch (config.connection_balance_config().balance_type_case()) { case envoy::config::listener::v3::Listener_ConnectionBalanceConfig::kExactBalance: connection_balancers_.emplace(address.asString(), std::make_shared()); break; case envoy::config::listener::v3::Listener_ConnectionBalanceConfig::kExtendBalance: { const std::string connection_balance_library_type{TypeUtil::typeUrlToDescriptorFullName( - config_.connection_balance_config().extend_balance().typed_config().type_url())}; + config.connection_balance_config().extend_balance().typed_config().type_url())}; auto factory = Envoy::Registry::FactoryRegistry::getFactoryByType( connection_balance_library_type); @@ -802,7 +745,7 @@ void ListenerImpl::buildConnectionBalancer(const Network::Address::Instance& add connection_balancers_.emplace( address.asString(), factory->createConnectionBalancerFromProto( - config_.connection_balance_config().extend_balance(), *listener_factory_context_)); + config.connection_balance_config().extend_balance(), *listener_factory_context_)); break; } case envoy::config::listener::v3::Listener_ConnectionBalanceConfig::BALANCE_TYPE_NOT_SET: { @@ -817,20 +760,21 @@ void ListenerImpl::buildConnectionBalancer(const Network::Address::Instance& add } } -void ListenerImpl::buildSocketOptions() { - if (config_.has_tcp_fast_open_queue_length()) { +void ListenerImpl::buildSocketOptions(const envoy::config::listener::v3::Listener& config) { + if (config.has_tcp_fast_open_queue_length()) { for (std::vector::size_type i = 0; i < addresses_.size(); i++) { addListenSocketOptions(listen_socket_options_list_[i], Network::SocketOptionFactory::buildTcpFastOpenOptions( - config_.tcp_fast_open_queue_length().value())); + config.tcp_fast_open_queue_length().value())); } } } -void ListenerImpl::buildOriginalDstListenerFilter() { +void ListenerImpl::buildOriginalDstListenerFilter( + const envoy::config::listener::v3::Listener& config) { // Add original dst listener filter if 'use_original_dst' flag is set. - if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config_, use_original_dst, false)) { + if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, use_original_dst, false)) { auto& factory = Config::Utility::getAndCheckFactoryByName( "envoy.filters.listener.original_dst"); @@ -844,12 +788,13 @@ void ListenerImpl::buildOriginalDstListenerFilter() { } } -void ListenerImpl::buildProxyProtocolListenerFilter() { +void ListenerImpl::buildProxyProtocolListenerFilter( + const envoy::config::listener::v3::Listener& config) { // Add proxy protocol listener filter if 'use_proxy_proto' flag is set. // TODO(jrajahalme): This is the last listener filter on purpose. When filter chain matching // is implemented, this needs to be run after the filter chain has been // selected. - if (usesProxyProto(config_)) { + if (usesProxyProto(config)) { auto& factory = Config::Utility::getAndCheckFactoryByName( "envoy.filters.listener.proxy_protocol"); @@ -863,82 +808,30 @@ void ListenerImpl::buildProxyProtocolListenerFilter() { listener_filter_factories_.push_back(std::move(filter_config_provider)); } } +PerListenerFactoryContextImpl::PerListenerFactoryContextImpl( + Envoy::Server::Instance& server, ProtobufMessage::ValidationVisitor& validation_visitor, + const envoy::config::listener::v3::Listener& config_message, ListenerImpl& listener_impl, + DrainManagerPtr drain_manager) + : listener_factory_context_base_(std::make_shared( + server, validation_visitor, config_message, std::move(drain_manager))), + listener_impl_(listener_impl) {} -AccessLog::AccessLogManager& PerListenerFactoryContextImpl::accessLogManager() { - return listener_factory_context_base_->accessLogManager(); -} -Upstream::ClusterManager& PerListenerFactoryContextImpl::clusterManager() { - return listener_factory_context_base_->clusterManager(); -} -Event::Dispatcher& PerListenerFactoryContextImpl::mainThreadDispatcher() { - return listener_factory_context_base_->mainThreadDispatcher(); -} -const Server::Options& PerListenerFactoryContextImpl::options() { - return listener_factory_context_base_->options(); -} Network::DrainDecision& PerListenerFactoryContextImpl::drainDecision() { PANIC("not implemented"); } -Grpc::Context& PerListenerFactoryContextImpl::grpcContext() { - return listener_factory_context_base_->grpcContext(); -} -bool PerListenerFactoryContextImpl::healthCheckFailed() { - return listener_factory_context_base_->healthCheckFailed(); -} -Http::Context& PerListenerFactoryContextImpl::httpContext() { - return listener_factory_context_base_->httpContext(); -} -Router::Context& PerListenerFactoryContextImpl::routerContext() { - return listener_factory_context_base_->routerContext(); -} -const LocalInfo::LocalInfo& PerListenerFactoryContextImpl::localInfo() const { - return listener_factory_context_base_->localInfo(); -} -Envoy::Runtime::Loader& PerListenerFactoryContextImpl::runtime() { - return listener_factory_context_base_->runtime(); -} + Stats::Scope& PerListenerFactoryContextImpl::scope() { return listener_factory_context_base_->scope(); } -Singleton::Manager& PerListenerFactoryContextImpl::singletonManager() { - return listener_factory_context_base_->singletonManager(); -} -OverloadManager& PerListenerFactoryContextImpl::overloadManager() { - return listener_factory_context_base_->overloadManager(); -} -ThreadLocal::Instance& PerListenerFactoryContextImpl::threadLocal() { - return listener_factory_context_base_->threadLocal(); -} -OptRef PerListenerFactoryContextImpl::admin() { - return listener_factory_context_base_->admin(); -} -const envoy::config::core::v3::Metadata& PerListenerFactoryContextImpl::listenerMetadata() const { - return listener_factory_context_base_->listenerMetadata(); -}; -const Envoy::Config::TypedMetadata& PerListenerFactoryContextImpl::listenerTypedMetadata() const { - return listener_factory_context_base_->listenerTypedMetadata(); -} -envoy::config::core::v3::TrafficDirection PerListenerFactoryContextImpl::direction() const { - return listener_factory_context_base_->direction(); -}; -TimeSource& PerListenerFactoryContextImpl::timeSource() { return api().timeSource(); } -const Network::ListenerConfig& PerListenerFactoryContextImpl::listenerConfig() const { - return *listener_config_; -} -ProtobufMessage::ValidationContext& PerListenerFactoryContextImpl::messageValidationContext() { - return getServerFactoryContext().messageValidationContext(); + +const Network::ListenerInfo& PerListenerFactoryContextImpl::listenerInfo() const { + return listener_factory_context_base_->listenerInfo(); } -ProtobufMessage::ValidationVisitor& PerListenerFactoryContextImpl::messageValidationVisitor() { + +ProtobufMessage::ValidationVisitor& +PerListenerFactoryContextImpl::messageValidationVisitor() const { return listener_factory_context_base_->messageValidationVisitor(); } -Api::Api& PerListenerFactoryContextImpl::api() { return listener_factory_context_base_->api(); } -ServerLifecycleNotifier& PerListenerFactoryContextImpl::lifecycleNotifier() { - return listener_factory_context_base_->lifecycleNotifier(); -} -ProcessContextOptRef PerListenerFactoryContextImpl::processContext() { - return listener_factory_context_base_->processContext(); -} -Configuration::ServerFactoryContext& -PerListenerFactoryContextImpl::getServerFactoryContext() const { - return listener_factory_context_base_->getServerFactoryContext(); +Configuration::ServerFactoryContext& PerListenerFactoryContextImpl::serverFactoryContext() const { + return listener_factory_context_base_->serverFactoryContext(); } Configuration::TransportSocketFactoryContext& PerListenerFactoryContextImpl::getTransportSocketFactoryContext() const { @@ -947,9 +840,6 @@ PerListenerFactoryContextImpl::getTransportSocketFactoryContext() const { Stats::Scope& PerListenerFactoryContextImpl::listenerScope() { return listener_factory_context_base_->listenerScope(); } -bool PerListenerFactoryContextImpl::isQuicListener() const { - return listener_factory_context_base_->isQuicListener(); -} Init::Manager& PerListenerFactoryContextImpl::initManager() { return listener_impl_.initManager(); } bool ListenerImpl::createNetworkFilterChain( @@ -999,7 +889,7 @@ void ListenerImpl::debugLog(const std::string& message) { } void ListenerImpl::initialize() { - last_updated_ = listener_factory_context_->timeSource().systemTime(); + last_updated_ = listener_factory_context_->serverFactoryContext().timeSource().systemTime(); // If workers have already started, we shift from using the global init manager to using a local // per listener init manager. See ~ListenerImpl() for why we gate the onListenerWarmed() call // by resetting the watcher. @@ -1022,7 +912,7 @@ ListenerImpl::~ListenerImpl() { Init::Manager& ListenerImpl::initManager() { return *dynamic_init_manager_; } void ListenerImpl::addSocketFactory(Network::ListenSocketFactoryPtr&& socket_factory) { - buildConnectionBalancer(*socket_factory->localAddress()); + buildConnectionBalancer(config(), *socket_factory->localAddress()); if (buildUdpListenerWorkerRouter(*socket_factory->localAddress(), parent_.server_.options().concurrency())) { parent_.server_.hotRestart().registerUdpForwardingListener(socket_factory->localAddress(), @@ -1031,7 +921,7 @@ void ListenerImpl::addSocketFactory(Network::ListenSocketFactoryPtr&& socket_fac socket_factories_.emplace_back(std::move(socket_factory)); } -bool ListenerImpl::supportUpdateFilterChain(const envoy::config::listener::v3::Listener& config, +bool ListenerImpl::supportUpdateFilterChain(const envoy::config::listener::v3::Listener& new_config, bool worker_started) { // The in place update needs the active listener in worker thread. worker_started guarantees the // existence of that active listener. @@ -1042,21 +932,21 @@ bool ListenerImpl::supportUpdateFilterChain(const envoy::config::listener::v3::L // Full listener update currently rejects tcp listener having 0 filter chain. // In place filter chain update could survive under zero filter chain but we should keep the // same behavior for now. This also guards the below filter chain access. - if (config.filter_chains_size() == 0) { + if (new_config.filter_chains_size() == 0) { return false; } // See buildProxyProtocolListenerFilter(). - if (usesProxyProto(config_) ^ usesProxyProto(config)) { + if (usesProxyProto(config()) ^ usesProxyProto(new_config)) { return false; } - if (ListenerMessageUtil::filterChainOnlyChange(config_, config)) { + if (ListenerMessageUtil::filterChainOnlyChange(config(), new_config)) { // We need to calculate the reuse port's default value then ensure whether it is changed or not. // Since reuse port's default value isn't the YAML bool field default value. When // `enable_reuse_port` is specified, `ListenerMessageUtil::filterChainOnlyChange` use the YAML // default value to do the comparison. - return reuse_port_ == getReusePortOrDefault(parent_.server_, config, socket_type_); + return reuse_port_ == getReusePortOrDefault(parent_.server_, new_config, socket_type_); } return false; @@ -1136,7 +1026,7 @@ bool ListenerImpl::getReusePortOrDefault(Server::Instance& server, } bool ListenerImpl::socketOptionsEqual(const ListenerImpl& other) const { - return ListenerMessageUtil::socketOptionsEqual(config_, other.config_); + return ListenerMessageUtil::socketOptionsEqual(config(), other.config()); } bool ListenerImpl::hasCompatibleAddress(const ListenerImpl& other) const { @@ -1166,7 +1056,8 @@ bool ListenerImpl::hasCompatibleAddress(const ListenerImpl& other) const { bool ListenerImpl::hasDuplicatedAddress(const ListenerImpl& other) const { // Skip the duplicate address check if this is the case of a listener update with new socket // options. - if ((name_ == other.name_) && !ListenerMessageUtil::socketOptionsEqual(config_, other.config_)) { + if ((name_ == other.name_) && + !ListenerMessageUtil::socketOptionsEqual(config(), other.config())) { return false; } diff --git a/source/extensions/listener_managers/listener_manager/listener_impl.h b/source/common/listener_manager/listener_impl.h similarity index 80% rename from source/extensions/listener_managers/listener_manager/listener_impl.h rename to source/common/listener_manager/listener_impl.h index 08808e152927..9bb5b8702411 100644 --- a/source/extensions/listener_managers/listener_manager/listener_impl.h +++ b/source/common/listener_manager/listener_impl.h @@ -18,11 +18,12 @@ #include "source/common/common/basic_resource_impl.h" #include "source/common/common/logger.h" -#include "source/common/config/metadata.h" #include "source/common/init/manager_impl.h" #include "source/common/init/target_impl.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" +#include "source/common/listener_manager/listener_info_impl.h" #include "source/common/quic/quic_stat_names.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" +#include "source/server/factory_context_impl.h" #include "source/server/transport_socket_config_impl.h" namespace Envoy { @@ -118,48 +119,22 @@ class ListenSocketFactoryImpl : public Network::ListenSocketFactory, // TODO(mattklein123): Consider getting rid of pre-worker start and post-worker start code by // initializing all listeners after workers are started. +class ListenerImpl; + /** * The common functionality shared by PerListenerFilterFactoryContexts and * PerFilterChainFactoryFactoryContexts. */ -class ListenerFactoryContextBaseImpl final : public Configuration::FactoryContext, +class ListenerFactoryContextBaseImpl final : public Server::FactoryContextImplBase, public Network::DrainDecision { public: ListenerFactoryContextBaseImpl(Envoy::Server::Instance& server, ProtobufMessage::ValidationVisitor& validation_visitor, const envoy::config::listener::v3::Listener& config, Server::DrainManagerPtr drain_manager); - AccessLog::AccessLogManager& accessLogManager() override; - Upstream::ClusterManager& clusterManager() override; - Event::Dispatcher& mainThreadDispatcher() override; - const Server::Options& options() override; - Network::DrainDecision& drainDecision() override; - Grpc::Context& grpcContext() override; - bool healthCheckFailed() override; - Http::Context& httpContext() override; - Router::Context& routerContext() override; + Init::Manager& initManager() override; - const LocalInfo::LocalInfo& localInfo() const override; - Envoy::Runtime::Loader& runtime() override; - Stats::Scope& serverScope() override { return *server_.stats().rootScope(); } - Stats::Scope& scope() override; - Singleton::Manager& singletonManager() override; - OverloadManager& overloadManager() override; - ThreadLocal::Instance& threadLocal() override; - OptRef admin() override; - const envoy::config::core::v3::Metadata& listenerMetadata() const override; - const Envoy::Config::TypedMetadata& listenerTypedMetadata() const override; - envoy::config::core::v3::TrafficDirection direction() const override; - TimeSource& timeSource() override; - ProtobufMessage::ValidationContext& messageValidationContext() override; - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override; - Api::Api& api() override; - ServerLifecycleNotifier& lifecycleNotifier() override; - ProcessContextOptRef processContext() override; - Configuration::ServerFactoryContext& getServerFactoryContext() const override; - Configuration::TransportSocketFactoryContext& getTransportSocketFactoryContext() const override; - Stats::Scope& listenerScope() override; - bool isQuicListener() const override; + Network::DrainDecision& drainDecision() override; // DrainDecision bool drainClose() const override { @@ -170,22 +145,12 @@ class ListenerFactoryContextBaseImpl final : public Configuration::FactoryContex return nullptr; } Server::DrainManager& drainManager(); + friend class ListenerImpl; private: - Envoy::Server::Instance& server_; - const envoy::config::core::v3::Metadata metadata_; - const Envoy::Config::TypedMetadataImpl - typed_metadata_; - envoy::config::core::v3::TrafficDirection direction_; - Stats::ScopeSharedPtr global_scope_; - Stats::ScopeSharedPtr listener_scope_; // Stats with listener named scope. - ProtobufMessage::ValidationVisitor& validation_visitor_; const Server::DrainManagerPtr drain_manager_; - bool is_quic_; }; -class ListenerImpl; - // TODO(lambdai): Strip the interface since ListenerFactoryContext only need to support // ListenerFilterChain creation. e.g, Is listenerMetaData() required? Is it required only at // listener update or during the lifetime of listener? @@ -194,60 +159,30 @@ class PerListenerFactoryContextImpl : public Configuration::ListenerFactoryConte PerListenerFactoryContextImpl(Envoy::Server::Instance& server, ProtobufMessage::ValidationVisitor& validation_visitor, const envoy::config::listener::v3::Listener& config_message, - const Network::ListenerConfig* listener_config, - ListenerImpl& listener_impl, DrainManagerPtr drain_manager) - : listener_factory_context_base_(std::make_shared( - server, validation_visitor, config_message, std::move(drain_manager))), - listener_config_(listener_config), listener_impl_(listener_impl) {} + ListenerImpl& listener_impl, DrainManagerPtr drain_manager); + PerListenerFactoryContextImpl( std::shared_ptr listener_factory_context_base, - const Network::ListenerConfig* listener_config, ListenerImpl& listener_impl) + ListenerImpl& listener_impl) : listener_factory_context_base_(listener_factory_context_base), - listener_config_(listener_config), listener_impl_(listener_impl) {} + listener_impl_(listener_impl) {} // FactoryContext - AccessLog::AccessLogManager& accessLogManager() override; - Upstream::ClusterManager& clusterManager() override; - Event::Dispatcher& mainThreadDispatcher() override; - const Options& options() override; Network::DrainDecision& drainDecision() override; - Grpc::Context& grpcContext() override; - bool healthCheckFailed() override; - Http::Context& httpContext() override; - Router::Context& routerContext() override; Init::Manager& initManager() override; - const LocalInfo::LocalInfo& localInfo() const override; - Envoy::Runtime::Loader& runtime() override; Stats::Scope& scope() override; - Stats::Scope& serverScope() override { return listener_factory_context_base_->serverScope(); } - Singleton::Manager& singletonManager() override; - OverloadManager& overloadManager() override; - ThreadLocal::Instance& threadLocal() override; - OptRef admin() override; - const envoy::config::core::v3::Metadata& listenerMetadata() const override; - const Envoy::Config::TypedMetadata& listenerTypedMetadata() const override; - envoy::config::core::v3::TrafficDirection direction() const override; - TimeSource& timeSource() override; - ProtobufMessage::ValidationContext& messageValidationContext() override; - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override; - Api::Api& api() override; - ServerLifecycleNotifier& lifecycleNotifier() override; - ProcessContextOptRef processContext() override; - Configuration::ServerFactoryContext& getServerFactoryContext() const override; + const Network::ListenerInfo& listenerInfo() const override; + ProtobufMessage::ValidationVisitor& messageValidationVisitor() const override; + Configuration::ServerFactoryContext& serverFactoryContext() const override; Configuration::TransportSocketFactoryContext& getTransportSocketFactoryContext() const override; Stats::Scope& listenerScope() override; - bool isQuicListener() const override; - - // ListenerFactoryContext - const Network::ListenerConfig& listenerConfig() const override; ListenerFactoryContextBaseImpl& parentFactoryContext() { return *listener_factory_context_base_; } friend class ListenerImpl; private: std::shared_ptr listener_factory_context_base_; - const Network::ListenerConfig* listener_config_; ListenerImpl& listener_impl_; }; @@ -287,7 +222,7 @@ class ListenerImpl final : public Network::ListenerConfig, /** * Determine if in place filter chain update could be executed at this moment. */ - bool supportUpdateFilterChain(const envoy::config::listener::v3::Listener& config, + bool supportUpdateFilterChain(const envoy::config::listener::v3::Listener& new_config, bool worker_started); /** @@ -381,8 +316,9 @@ class ListenerImpl final : public Network::ListenerConfig, } Init::Manager& initManager() override; bool ignoreGlobalConnLimit() const override { return ignore_global_conn_limit_; } - envoy::config::core::v3::TrafficDirection direction() const override { - return config().traffic_direction(); + const Network::ListenerInfoConstSharedPtr& listenerInfo() const override { + ASSERT(listener_factory_context_ != nullptr); + return listener_factory_context_->listener_factory_context_base_->listener_info_; } void ensureSocketOptions(Network::Socket::OptionsSharedPtr& options) { @@ -449,21 +385,24 @@ class ListenerImpl final : public Network::ListenerConfig, const std::string& version_info, ListenerManagerImpl& parent, const std::string& name, bool added_via_api, bool workers_started, uint64_t hash); // Helpers for constructor. - void buildAccessLog(); - void buildInternalListener(); + void buildAccessLog(const envoy::config::listener::v3::Listener& config); + void buildInternalListener(const envoy::config::listener::v3::Listener& config); void validateConfig(); bool buildUdpListenerWorkerRouter(const Network::Address::Instance& address, uint32_t concurrency); - void buildUdpListenerFactory(uint32_t concurrency); - void buildListenSocketOptions(std::vector>>& address_opts_list); - void createListenerFilterFactories(); - void validateFilterChains(); - void buildFilterChains(); - void buildConnectionBalancer(const Network::Address::Instance& address); - void buildSocketOptions(); - void buildOriginalDstListenerFilter(); - void buildProxyProtocolListenerFilter(); + void createListenerFilterFactories(const envoy::config::listener::v3::Listener& config); + void validateFilterChains(const envoy::config::listener::v3::Listener& config); + void buildFilterChains(const envoy::config::listener::v3::Listener& config); + void buildConnectionBalancer(const envoy::config::listener::v3::Listener& config, + const Network::Address::Instance& address); + void buildSocketOptions(const envoy::config::listener::v3::Listener& config); + void buildOriginalDstListenerFilter(const envoy::config::listener::v3::Listener& config); + void buildProxyProtocolListenerFilter(const envoy::config::listener::v3::Listener& config); void checkIpv4CompatAddress(const Network::Address::InstanceConstSharedPtr& address, const envoy::config::core::v3::Address& proto_address); @@ -525,8 +464,8 @@ class ListenerImpl final : public Network::ListenerConfig, // This init watcher, if workers_started_ is false, notifies the "parent" listener manager when // listener initialization is complete. - // Important: local_init_watcher_ must be the last field in the class to avoid unexpected watcher - // callback during the destroy of ListenerImpl. + // Important: local_init_watcher_ must be the last field in the class to avoid unexpected + // watcher callback during the destroy of ListenerImpl. Init::WatcherImpl local_init_watcher_; std::shared_ptr transport_factory_context_; diff --git a/source/common/listener_manager/listener_info_impl.cc b/source/common/listener_manager/listener_info_impl.cc new file mode 100644 index 000000000000..ce5fc6991ded --- /dev/null +++ b/source/common/listener_manager/listener_info_impl.cc @@ -0,0 +1,14 @@ +#include "source/common/listener_manager/listener_info_impl.h" + +namespace Envoy { +namespace Server { + +const envoy::config::core::v3::Metadata& ListenerInfoImpl::metadata() const { + return metadata_.proto_metadata_; +} +const Envoy::Config::TypedMetadata& ListenerInfoImpl::typedMetadata() const { + return metadata_.typed_metadata_; +} + +} // namespace Server +} // namespace Envoy diff --git a/source/common/listener_manager/listener_info_impl.h b/source/common/listener_manager/listener_info_impl.h new file mode 100644 index 000000000000..d54e38ceda7c --- /dev/null +++ b/source/common/listener_manager/listener_info_impl.h @@ -0,0 +1,34 @@ +#pragma once + +#include "envoy/config/listener/v3/listener.pb.h" +#include "envoy/network/listener.h" + +#include "source/common/config/metadata.h" + +namespace Envoy { +namespace Server { + +using ListenerMetadataPack = + Envoy::Config::MetadataPack; + +class ListenerInfoImpl : public Network::ListenerInfo { +public: + explicit ListenerInfoImpl(const envoy::config::listener::v3::Listener& config) + : metadata_(config.metadata()), direction_(config.traffic_direction()), + is_quic_(config.udp_listener_config().has_quic_options()) {} + ListenerInfoImpl() = default; + + // Network::ListenerInfo + const envoy::config::core::v3::Metadata& metadata() const override; + const Envoy::Config::TypedMetadata& typedMetadata() const override; + envoy::config::core::v3::TrafficDirection direction() const override { return direction_; } + bool isQuic() const override { return is_quic_; } + +private: + const ListenerMetadataPack metadata_; + const envoy::config::core::v3::TrafficDirection direction_{}; + const bool is_quic_{}; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc b/source/common/listener_manager/listener_manager_impl.cc similarity index 97% rename from source/extensions/listener_managers/listener_manager/listener_manager_impl.cc rename to source/common/listener_manager/listener_manager_impl.cc index 66f4f777d5aa..3982b72eed00 100644 --- a/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc +++ b/source/common/listener_manager/listener_manager_impl.cc @@ -1,4 +1,4 @@ -#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" +#include "source/common/listener_manager/listener_manager_impl.h" #include @@ -27,13 +27,13 @@ #include "absl/synchronization/blocking_counter.h" #if defined(ENVOY_ENABLE_QUIC) -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_server_transport_socket_factory.h" #endif #include "source/server/api_listener_impl.h" #include "source/server/configuration_impl.h" #include "source/server/drain_manager_impl.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" #include "source/server/transport_socket_config_impl.h" namespace Envoy { @@ -93,14 +93,15 @@ Filter::NetworkFilterFactoriesList ProdListenerComponentFactory::createNetworkFi ENVOY_LOG(debug, " dynamic filter name: {}", proto_config.name()); ret.push_back(config_provider_manager.createDynamicFilterConfigProvider( proto_config.config_discovery(), proto_config.name(), - filter_chain_factory_context.getServerFactoryContext(), filter_chain_factory_context, - filter_chain_factory_context.clusterManager(), is_terminal, "network", nullptr)); + filter_chain_factory_context.serverFactoryContext(), filter_chain_factory_context, + filter_chain_factory_context.serverFactoryContext().clusterManager(), is_terminal, + "network", nullptr)); continue; } ENVOY_LOG(debug, " name: {}", proto_config.name()); ENVOY_LOG(debug, " config: {}", - MessageUtil::getJsonStringFromMessageOrError( + MessageUtil::convertToStringForLogs( static_cast(proto_config.typed_config()))); // Now see if there is a factory that will accept the config. @@ -113,7 +114,7 @@ Filter::NetworkFilterFactoriesList ProdListenerComponentFactory::createNetworkFi Config::Utility::validateTerminalFilters( filters[i].name(), factory.name(), "network", factory.isTerminalFilterByProto(*message, - filter_chain_factory_context.getServerFactoryContext()), + filter_chain_factory_context.serverFactoryContext()), is_terminal); Network::FilterFactoryCb callback = factory.createFilterFactoryFromProto(*message, filter_chain_factory_context); @@ -157,13 +158,13 @@ ProdListenerComponentFactory::createListenerFilterFactoryListImpl( } } auto filter_config_provider = config_provider_manager.createDynamicFilterConfigProvider( - config_discovery, name, context.getServerFactoryContext(), context, - context.clusterManager(), false, "tcp-listener", + config_discovery, name, context.serverFactoryContext(), context, + context.serverFactoryContext().clusterManager(), false, "tcp-listener", createListenerFilterMatcher(proto_config)); ret.push_back(std::move(filter_config_provider)); } else { ENVOY_LOG(debug, " config: {}", - MessageUtil::getJsonStringFromMessageOrError( + MessageUtil::convertToStringForLogs( static_cast(proto_config.typed_config()))); // For static configuration, now see if there is a factory that will accept the config. auto& factory = @@ -191,7 +192,7 @@ ProdListenerComponentFactory::createUdpListenerFilterFactoryListImpl( ENVOY_LOG(debug, " filter #{}:", i); ENVOY_LOG(debug, " name: {}", proto_config.name()); ENVOY_LOG(debug, " config: {}", - MessageUtil::getJsonStringFromMessageOrError( + MessageUtil::convertToStringForLogs( static_cast(proto_config.typed_config()))); if (proto_config.config_type_case() == envoy::config::listener::v3::ListenerFilter::ConfigTypeCase::kConfigDiscovery) { @@ -245,12 +246,12 @@ ProdListenerComponentFactory::createQuicListenerFilterFactoryListImpl( } } ret.push_back(config_provider_manager.createDynamicFilterConfigProvider( - config_discovery, name, context.getServerFactoryContext(), context, - context.clusterManager(), false, "quic-listener", + config_discovery, name, context.serverFactoryContext(), context, + context.serverFactoryContext().clusterManager(), false, "quic-listener", createListenerFilterMatcher(proto_config))); } else { ENVOY_LOG(debug, " config: {}", - MessageUtil::getJsonStringFromMessageOrError( + MessageUtil::convertToStringForLogs( static_cast(proto_config.typed_config()))); // For static configuration, now see if there is a factory that will accept the config. auto& factory = @@ -755,7 +756,7 @@ void ListenerManagerImpl::addListenerToWorker(Worker& worker, } }); }, - server_.runtime()); + server_.runtime(), server_.api().randomGenerator()); } void ListenerManagerImpl::onListenerWarmed(ListenerImpl& listener) { @@ -919,7 +920,7 @@ bool ListenerManagerImpl::removeListenerInternal(const std::string& name, return true; } -void ListenerManagerImpl::startWorkers(GuardDog& guard_dog, std::function callback) { +void ListenerManagerImpl::startWorkers(OptRef guard_dog, std::function callback) { ENVOY_LOG(info, "all dependencies initialized. starting workers"); ASSERT(!workers_started_); workers_started_ = true; @@ -992,7 +993,7 @@ void ListenerManagerImpl::stopListeners(StopListenersType stop_listeners_type, stop_listeners_type_ = stop_listeners_type; for (Network::ListenerConfig& listener : listeners()) { if (stop_listeners_type != StopListenersType::InboundOnly || - listener.direction() == envoy::config::core::v3::INBOUND) { + listener.listenerInfo()->direction() == envoy::config::core::v3::INBOUND) { ENVOY_LOG(debug, "begin stop listener: name={}", listener.name()); auto existing_warming_listener = getListenerByName(warming_listeners_, listener.name()); // Destroy a warming listener directly. diff --git a/source/extensions/listener_managers/listener_manager/listener_manager_impl.h b/source/common/listener_manager/listener_manager_impl.h similarity index 97% rename from source/extensions/listener_managers/listener_manager/listener_manager_impl.h rename to source/common/listener_manager/listener_manager_impl.h index 56bd78050836..92a921489d1c 100644 --- a/source/extensions/listener_managers/listener_manager/listener_manager_impl.h +++ b/source/common/listener_manager/listener_manager_impl.h @@ -20,11 +20,11 @@ #include "source/common/config/well_known_names.h" #include "source/common/filter/config_discovery_impl.h" +#include "source/common/listener_manager/filter_chain_factory_context_callback.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" +#include "source/common/listener_manager/lds_api.h" +#include "source/common/listener_manager/listener_impl.h" #include "source/common/quic/quic_stat_names.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_factory_context_callback.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" -#include "source/extensions/listener_managers/listener_manager/lds_api.h" -#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include "source/server/listener_manager_factory.h" namespace Envoy { @@ -222,7 +222,7 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable callback) override; + void startWorkers(OptRef guard_dog, std::function callback) override; void stopListeners(StopListenersType stop_listeners_type, const Network::ExtraShutdownListenerOptions& options) override; void stopWorkers() override; diff --git a/source/common/local_reply/local_reply.cc b/source/common/local_reply/local_reply.cc index 08ae67506666..1f745e69a8d1 100644 --- a/source/common/local_reply/local_reply.cc +++ b/source/common/local_reply/local_reply.cc @@ -19,11 +19,11 @@ namespace LocalReply { class BodyFormatter { public: BodyFormatter() - : formatter_(std::make_unique("%LOCAL_REPLY_BODY%")), + : formatter_(std::make_unique("%LOCAL_REPLY_BODY%", false)), content_type_(Http::Headers::get().ContentTypeValues.Text) {} BodyFormatter(const envoy::config::core::v3::SubstitutionFormatString& config, - Server::Configuration::CommonFactoryContext& context) + Server::Configuration::GenericFactoryContext& context) : formatter_(Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, context)), content_type_( !config.content_type().empty() ? config.content_type() @@ -61,7 +61,7 @@ class ResponseMapper { status_code_ = static_cast(config.status_code().value()); } if (config.has_body()) { - body_ = Config::DataSource::read(config.body(), true, context.api()); + body_ = Config::DataSource::read(config.body(), true, context.serverFactoryContext().api()); } if (config.has_body_format_override()) { @@ -78,8 +78,8 @@ class ResponseMapper { BodyFormatter*& final_formatter) const { // If not matched, just bail out. if (filter_ == nullptr || - !filter_->evaluate(stream_info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)) { + !filter_->evaluate({&request_headers, &response_headers, &response_trailers}, + stream_info)) { return false; } @@ -87,7 +87,7 @@ class ResponseMapper { body = body_.value(); } - header_parser_->evaluateHeaders(response_headers, request_headers, response_headers, + header_parser_->evaluateHeaders(response_headers, {&request_headers, &response_headers}, stream_info); if (status_code_.has_value() && code != status_code_.value()) { diff --git a/source/common/matcher/field_matcher.h b/source/common/matcher/field_matcher.h index e27c1aa444e8..ddda93d0513c 100644 --- a/source/common/matcher/field_matcher.h +++ b/source/common/matcher/field_matcher.h @@ -150,7 +150,7 @@ class SingleFieldMatcher : public FieldMatcher, Logger::LoggabledataInputType()) == supported_input_types.end()) { std::string supported_types = absl::StrJoin(supported_input_types.begin(), supported_input_types.end(), ", "); - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Unsupported data input type: ", data_input_->dataInputType(), ". The matcher supports input type: ", supported_types)); } diff --git a/source/common/matcher/map_matcher.h b/source/common/matcher/map_matcher.h index 4cb50e542f99..1e2c73bfd60c 100644 --- a/source/common/matcher/map_matcher.h +++ b/source/common/matcher/map_matcher.h @@ -20,7 +20,7 @@ class MapMatcher : public MatchTree, Logger::LoggabledataInputType(); if (input_type != DefaultMatchingDataType) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Unsupported data input type: ", input_type, ", currently only string type is supported in map matcher")); } diff --git a/source/common/memory/heap_shrinker.cc b/source/common/memory/heap_shrinker.cc index 0fe1cac2c20b..d8353f7cf216 100644 --- a/source/common/memory/heap_shrinker.cc +++ b/source/common/memory/heap_shrinker.cc @@ -12,8 +12,7 @@ namespace Memory { constexpr std::chrono::milliseconds kTimerInterval = std::chrono::milliseconds(10000); HeapShrinker::HeapShrinker(Event::Dispatcher& dispatcher, Server::OverloadManager& overload_manager, - Stats::Scope& stats) - : active_(false) { + Stats::Scope& stats) { const auto action_name = Server::OverloadActionNames::get().ShrinkHeap; if (overload_manager.registerForAction( action_name, dispatcher, diff --git a/source/common/memory/heap_shrinker.h b/source/common/memory/heap_shrinker.h index 9b1930f2bb06..bb292bff9137 100644 --- a/source/common/memory/heap_shrinker.h +++ b/source/common/memory/heap_shrinker.h @@ -20,7 +20,7 @@ class HeapShrinker { private: void shrinkHeap(); - bool active_; + bool active_{false}; Envoy::Stats::Counter* shrink_counter_; Envoy::Event::TimerPtr timer_; }; diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 63662c0128c9..f91ba321c879 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -32,6 +32,7 @@ envoy_cc_library( srcs = ["application_protocol.cc"], hdrs = ["application_protocol.h"], deps = [ + "//envoy/registry", "//envoy/stream_info:filter_state_interface", "//source/common/common:macros", ], @@ -78,8 +79,8 @@ envoy_cc_library( srcs = ["connection_impl_base.cc"], hdrs = ["connection_impl_base.h"], deps = [ + ":connection_socket_lib", ":filter_manager_lib", - ":listen_socket_lib", "//envoy/common:scope_tracker_interface", "//envoy/event:dispatcher_interface", "//source/common/common:assert_lib", @@ -117,7 +118,6 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:minimal_logger_lib", "//source/common/event:libevent_lib", - "//source/common/network:listen_socket_lib", "//source/common/network:socket_option_factory_lib", "//source/common/runtime:runtime_features_lib", "//source/common/stream_info:stream_info_lib", @@ -243,11 +243,13 @@ envoy_cc_library( envoy_cc_library( name = "default_socket_interface_lib", srcs = [ + "io_socket_handle_base_impl.cc", "io_socket_handle_impl.cc", "socket_interface_impl.cc", "win32_socket_handle_impl.cc", ], hdrs = [ + "io_socket_handle_base_impl.h", "io_socket_handle_impl.h", "socket_interface_impl.h", "win32_socket_handle_impl.h", @@ -272,6 +274,7 @@ envoy_cc_library( srcs = ["socket_impl.cc"], hdrs = ["socket_impl.h"], deps = [ + "//envoy/network:listener_interface", "//envoy/network:socket_interface", "//envoy/network:socket_interface_interface", "//source/common/api:os_sys_calls_lib", @@ -281,11 +284,23 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "connection_socket_lib", + hdrs = ["connection_socket_impl.h"], + deps = [ + ":socket_lib", + ":utility_lib", + "//envoy/network:exception_interface", + "//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "listen_socket_lib", srcs = ["listen_socket_impl.cc"], hdrs = ["listen_socket_impl.h"], deps = [ + ":connection_socket_lib", ":socket_lib", ":utility_lib", "//envoy/network:exception_interface", @@ -334,6 +349,7 @@ envoy_cc_library( "//source/common/common:empty_string", "//source/common/common:linked_object", "//source/common/event:dispatcher_includes", + "//source/common/runtime:runtime_keys_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) @@ -496,9 +512,10 @@ envoy_cc_library( hdrs = ["filter_state_dst_address.h"], deps = [ ":utility_lib", + "//envoy/common:hashable_interface", "//envoy/network:address_interface", - "//envoy/registry", "//envoy/stream_info:filter_state_interface", + "//source/common/common:hash_lib", "//source/common/common:macros", ], ) @@ -508,6 +525,7 @@ envoy_cc_library( srcs = ["upstream_server_name.cc"], hdrs = ["upstream_server_name.h"], deps = [ + "//envoy/registry", "//envoy/stream_info:filter_state_interface", "//source/common/common:macros", ], @@ -518,6 +536,7 @@ envoy_cc_library( srcs = ["upstream_subject_alt_names.cc"], hdrs = ["upstream_subject_alt_names.h"], deps = [ + "//envoy/registry", "//envoy/stream_info:filter_state_interface", "//source/common/common:macros", ], diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index b165c2bcec9d..553e5e646b63 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -34,7 +34,7 @@ const SocketInterface* sockInterfaceOrDefault(const SocketInterface* sock_interf void throwOnError(absl::Status status) { if (!status.ok()) { - throw EnvoyException(status.ToString()); + throwEnvoyExceptionOrPanic(status.ToString()); } } @@ -151,7 +151,7 @@ Ipv4Instance::Ipv4Instance(const std::string& address, uint32_t port, ip_.ipv4_.address_.sin_port = htons(port); int rc = inet_pton(AF_INET, address.c_str(), &ip_.ipv4_.address_.sin_addr); if (1 != rc) { - throw EnvoyException(fmt::format("invalid ipv4 address '{}'", address)); + throwEnvoyExceptionOrPanic(fmt::format("invalid ipv4 address '{}'", address)); } friendly_name_ = absl::StrCat(address, ":", port); @@ -296,7 +296,7 @@ Ipv6Instance::Ipv6Instance(const std::string& address, uint32_t port, addr_in.sin6_port = htons(port); if (!address.empty()) { if (1 != inet_pton(AF_INET6, address.c_str(), &addr_in.sin6_addr)) { - throw EnvoyException(fmt::format("invalid ipv6 address '{}'", address)); + throwEnvoyExceptionOrPanic(fmt::format("invalid ipv6 address '{}'", address)); } } else { addr_in.sin6_addr = in6addr_any; @@ -344,7 +344,7 @@ PipeInstance::PipeInstance(const sockaddr_un* address, socklen_t ss_len, mode_t : InstanceBase(Type::Pipe, sockInterfaceOrDefault(sock_interface)) { if (address->sun_path[0] == '\0') { #if !defined(__linux__) - throw EnvoyException("Abstract AF_UNIX sockets are only supported on linux."); + throwEnvoyExceptionOrPanic("Abstract AF_UNIX sockets are only supported on linux."); #endif RELEASE_ASSERT(static_cast(ss_len) >= offsetof(struct sockaddr_un, sun_path) + 1, ""); @@ -359,7 +359,7 @@ PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, const SocketInterface* sock_interface) : InstanceBase(Type::Pipe, sockInterfaceOrDefault(sock_interface)) { if (pipe_path.size() >= sizeof(pipe_.address_.sun_path)) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Path \"{}\" exceeds maximum UNIX domain socket path size of {}.", pipe_path, sizeof(pipe_.address_.sun_path))); } @@ -372,10 +372,10 @@ PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, // be null terminated. The friendly name is the address path with embedded nulls replaced with // '@' for consistency with the first character. #if !defined(__linux__) - throw EnvoyException("Abstract AF_UNIX sockets are only supported on linux."); + throwEnvoyExceptionOrPanic("Abstract AF_UNIX sockets are only supported on linux."); #endif if (mode != 0) { - throw EnvoyException("Cannot set mode for Abstract AF_UNIX sockets"); + throwEnvoyExceptionOrPanic("Cannot set mode for Abstract AF_UNIX sockets"); } pipe_.abstract_namespace_ = true; pipe_.address_length_ = pipe_path.size(); @@ -389,7 +389,7 @@ PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, } else { // Throw an error if the pipe path has an embedded null character. if (pipe_path.size() != strlen(pipe_path.c_str())) { - throw EnvoyException("UNIX domain socket pathname contains embedded null characters"); + throwEnvoyExceptionOrPanic("UNIX domain socket pathname contains embedded null characters"); } StringUtil::strlcpy(&pipe_.address_.sun_path[0], pipe_path.c_str(), sizeof(pipe_.address_.sun_path)); diff --git a/source/common/network/application_protocol.cc b/source/common/network/application_protocol.cc index 6794194ac90d..047c72d3c3fc 100644 --- a/source/common/network/application_protocol.cc +++ b/source/common/network/application_protocol.cc @@ -1,5 +1,8 @@ #include "source/common/network/application_protocol.h" +#include "envoy/registry/registry.h" +#include "envoy/stream_info/filter_state.h" + #include "source/common/common/macros.h" namespace Envoy { @@ -8,5 +11,17 @@ namespace Network { const std::string& ApplicationProtocols::key() { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.network.application_protocols"); } + +class ApplicationProtocolsObjectFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return ApplicationProtocols::key(); } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + const std::vector parts = absl::StrSplit(data, ','); + return std::make_unique(parts); + } +}; + +REGISTER_FACTORY(ApplicationProtocolsObjectFactory, StreamInfo::FilterState::ObjectFactory); } // namespace Network } // namespace Envoy diff --git a/source/common/network/base_listener_impl.cc b/source/common/network/base_listener_impl.cc index aa726efccc31..64eff0466918 100644 --- a/source/common/network/base_listener_impl.cc +++ b/source/common/network/base_listener_impl.cc @@ -15,7 +15,7 @@ namespace Envoy { namespace Network { -BaseListenerImpl::BaseListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket) +BaseListenerImpl::BaseListenerImpl(Event::Dispatcher& dispatcher, SocketSharedPtr socket) : local_address_(nullptr), dispatcher_(dispatcher), socket_(std::move(socket)) { const auto ip = socket_->connectionInfoProvider().localAddress()->ip(); diff --git a/source/common/network/base_listener_impl.h b/source/common/network/base_listener_impl.h index 62cebdbd4dd5..ab343d74451e 100644 --- a/source/common/network/base_listener_impl.h +++ b/source/common/network/base_listener_impl.h @@ -17,11 +17,11 @@ class BaseListenerImpl : public virtual Listener { * @param socket the listening socket for this listener. It might be shared * with other listeners if all listeners use single listen socket. */ - BaseListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket); + BaseListenerImpl(Event::Dispatcher& dispatcher, SocketSharedPtr socket); protected: Address::InstanceConstSharedPtr local_address_; - Event::DispatcherImpl& dispatcher_; + Event::Dispatcher& dispatcher_; const SocketSharedPtr socket_; }; diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index eabc4c88ceb5..7fc622644d52 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -194,6 +194,14 @@ InstanceConstSharedPtr CidrRange::truncateIpAddressAndLength(InstanceConstShared PANIC_DUE_TO_CORRUPT_ENUM; } +absl::StatusOr> +IpList::create(const Protobuf::RepeatedPtrField& cidrs) { + std::unique_ptr ret = std::unique_ptr(new IpList(cidrs)); + if (!ret->error_.empty()) { + return absl::InvalidArgumentError(ret->error_); + } + return ret; +} IpList::IpList(const Protobuf::RepeatedPtrField& cidrs) { ip_list_.reserve(cidrs.size()); for (const envoy::config::core::v3::CidrRange& entry : cidrs) { @@ -201,9 +209,8 @@ IpList::IpList(const Protobuf::RepeatedPtrField/<# mask bits>)", - entry.address_prefix(), entry.prefix_len().value())); + error_ = fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", + entry.address_prefix(), entry.prefix_len().value()); } } } diff --git a/source/common/network/cidr_range.h b/source/common/network/cidr_range.h index 8de06cdb7a9d..b78555dc41f6 100644 --- a/source/common/network/cidr_range.h +++ b/source/common/network/cidr_range.h @@ -132,14 +132,19 @@ class CidrRange { */ class IpList { public: - explicit IpList(const Protobuf::RepeatedPtrField& cidrs); + static absl::StatusOr> + create(const Protobuf::RepeatedPtrField& cidrs); + IpList() = default; bool contains(const Instance& address) const; size_t getIpListSize() const { return ip_list_.size(); }; + const std::string& error() const { return error_; } private: + explicit IpList(const Protobuf::RepeatedPtrField& cidrs); std::vector ip_list_; + std::string error_; }; } // namespace Address diff --git a/source/common/network/connection_balancer_impl.cc b/source/common/network/connection_balancer_impl.cc index 309628073f19..fc0471c675a5 100644 --- a/source/common/network/connection_balancer_impl.cc +++ b/source/common/network/connection_balancer_impl.cc @@ -28,7 +28,7 @@ ExactConnectionBalancerImpl::pickTargetHandler(BalancedConnectionHandler&) { } } - min_connection_handler->incNumConnections(); + min_connection_handler->incNumConnections(); // NOLINT(clang-analyzer-core.CallAndMessage) } return *min_connection_handler; diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 22b8a7c4055b..b6858939f517 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -18,7 +18,7 @@ #include "source/common/common/enum_to_int.h" #include "source/common/common/scope_tracker.h" #include "source/common/network/address_impl.h" -#include "source/common/network/listen_socket_impl.h" +#include "source/common/network/connection_socket_impl.h" #include "source/common/network/raw_buffer_socket.h" #include "source/common/network/socket_option_factory.h" #include "source/common/network/socket_option_impl.h" diff --git a/source/common/network/connection_socket_impl.h b/source/common/network/connection_socket_impl.h new file mode 100644 index 000000000000..51589d8ac4a3 --- /dev/null +++ b/source/common/network/connection_socket_impl.h @@ -0,0 +1,110 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/platform.h" +#include "envoy/network/connection.h" +#include "envoy/network/listen_socket.h" +#include "envoy/network/socket.h" +#include "envoy/network/socket_interface.h" + +#include "source/common/common/assert.h" +#include "source/common/common/dump_state_utils.h" +#include "source/common/network/socket_impl.h" +#include "source/common/network/socket_interface.h" + +namespace Envoy { +namespace Network { + +/** + * Wraps a unix socket. + */ +template struct NetworkSocketTrait {}; + +template <> struct NetworkSocketTrait { + static constexpr Socket::Type type = Socket::Type::Stream; +}; + +template <> struct NetworkSocketTrait { + static constexpr Socket::Type type = Socket::Type::Datagram; +}; + +class ConnectionSocketImpl : public SocketImpl, public ConnectionSocket { +public: + ConnectionSocketImpl(IoHandlePtr&& io_handle, + const Address::InstanceConstSharedPtr& local_address, + const Address::InstanceConstSharedPtr& remote_address) + : SocketImpl(std::move(io_handle), local_address, remote_address) {} + + ConnectionSocketImpl(Socket::Type type, const Address::InstanceConstSharedPtr& local_address, + const Address::InstanceConstSharedPtr& remote_address, + const SocketCreationOptions& options) + : SocketImpl(type, local_address, remote_address, options) { + connection_info_provider_->setLocalAddress(local_address); + } + + // Network::ConnectionSocket + void setDetectedTransportProtocol(absl::string_view protocol) override { + transport_protocol_ = std::string(protocol); + } + absl::string_view detectedTransportProtocol() const override { return transport_protocol_; } + + void setRequestedApplicationProtocols(const std::vector& protocols) override { + application_protocols_.clear(); + for (const auto& protocol : protocols) { + application_protocols_.emplace_back(protocol); + } + } + const std::vector& requestedApplicationProtocols() const override { + return application_protocols_; + } + + void setRequestedServerName(absl::string_view server_name) override { + // Always keep the server_name_ as lower case. + connectionInfoProvider().setRequestedServerName(absl::AsciiStrToLower(server_name)); + } + absl::string_view requestedServerName() const override { + return connectionInfoProvider().requestedServerName(); + } + + void setJA3Hash(absl::string_view ja3_hash) override { + connectionInfoProvider().setJA3Hash(ja3_hash); + } + absl::string_view ja3Hash() const override { return connectionInfoProvider().ja3Hash(); } + + absl::optional lastRoundTripTime() override { + return ioHandle().lastRoundTripTime(); + } + + absl::optional congestionWindowInBytes() const override { + return ioHandle().congestionWindowInBytes(); + } + + void dumpState(std::ostream& os, int indent_level) const override { + const char* spaces = spacesForLevel(indent_level); + os << spaces << "ListenSocketImpl " << this << DUMP_MEMBER(transport_protocol_) << "\n"; + DUMP_DETAILS(connection_info_provider_); + } + +protected: + std::string transport_protocol_; + std::vector application_protocols_; +}; + +// ConnectionSocket used with client connections. +class ClientSocketImpl : public ConnectionSocketImpl { +public: + ClientSocketImpl(const Address::InstanceConstSharedPtr& remote_address, + const OptionsSharedPtr& options) + : ConnectionSocketImpl(Network::ioHandleForAddr(Socket::Type::Stream, remote_address, {}), + nullptr, remote_address) { + if (options) { + addOptions(options); + } + } +}; + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/filter_state_dst_address.cc b/source/common/network/filter_state_dst_address.cc index 69e375a65137..b164573e5f7a 100644 --- a/source/common/network/filter_state_dst_address.cc +++ b/source/common/network/filter_state_dst_address.cc @@ -1,19 +1,17 @@ #include "source/common/network/filter_state_dst_address.h" -#include "envoy/registry/registry.h" - #include "source/common/network/utility.h" namespace Envoy { namespace Network { -const std::string& DestinationAddress::key() { - CONSTRUCT_ON_FIRST_USE(std::string, "envoy.network.transport_socket.original_dst_address"); +absl::optional AddressObject::hash() const { + return HashUtil::xxHash64(address_->asStringView()); } -class DestinationAddressReflection : public StreamInfo::FilterState::ObjectReflection { +class AddressObjectReflection : public StreamInfo::FilterState::ObjectReflection { public: - DestinationAddressReflection(const DestinationAddress* object) : object_(object) {} + AddressObjectReflection(const AddressObject* object) : object_(object) {} FieldType getField(absl::string_view field_name) const override { const auto* ip = object_->address_->ip(); if (!ip) { @@ -28,28 +26,22 @@ class DestinationAddressReflection : public StreamInfo::FilterState::ObjectRefle } private: - const DestinationAddress* object_; + const AddressObject* object_; }; -class DestinationAddressFactory : public StreamInfo::FilterState::ObjectFactory { -public: - std::string name() const override { return DestinationAddress::key(); } - std::unique_ptr - createFromBytes(absl::string_view data) const override { - const auto address = Utility::parseInternetAddressAndPortNoThrow(std::string(data)); - return address ? std::make_unique(address) : nullptr; - } - std::unique_ptr - reflect(const StreamInfo::FilterState::Object* data) const override { - const auto* object = dynamic_cast(data); - if (object) { - return std::make_unique(object); - } - return nullptr; +std::unique_ptr +BaseAddressObjectFactory::createFromBytes(absl::string_view data) const { + const auto address = Utility::parseInternetAddressAndPortNoThrow(std::string(data)); + return address ? std::make_unique(address) : nullptr; +} +std::unique_ptr +BaseAddressObjectFactory::reflect(const StreamInfo::FilterState::Object* data) const { + const auto* object = dynamic_cast(data); + if (object) { + return std::make_unique(object); } -}; - -REGISTER_FACTORY(DestinationAddressFactory, StreamInfo::FilterState::ObjectFactory); + return nullptr; +} } // namespace Network } // namespace Envoy diff --git a/source/common/network/filter_state_dst_address.h b/source/common/network/filter_state_dst_address.h index e9584ac988d1..ec6c565fd689 100644 --- a/source/common/network/filter_state_dst_address.h +++ b/source/common/network/filter_state_dst_address.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/common/hashable.h" #include "envoy/network/address.h" #include "envoy/stream_info/filter_state.h" @@ -7,22 +8,33 @@ namespace Envoy { namespace Network { /** - * Overrides the destination host address selection for ORIGINAL_DST cluster. + * Overrides the address selection for extensions, e.g. ORIGINAL_DST cluster. */ -class DestinationAddress : public StreamInfo::FilterState::Object { +class AddressObject : public StreamInfo::FilterState::Object, public Hashable { public: - // Returns the key for looking up in the FilterState. - static const std::string& key(); - - DestinationAddress(Network::Address::InstanceConstSharedPtr address) : address_(address) {} + AddressObject(Network::Address::InstanceConstSharedPtr address) : address_(address) {} Network::Address::InstanceConstSharedPtr address() const { return address_; } absl::optional serializeAsString() const override { return address_ ? absl::make_optional(address_->asString()) : absl::nullopt; } + // Implements hashing interface because the value is applied once per upstream connection. + // Multiple streams sharing the upstream connection must have the same address object. + absl::optional hash() const override; private: const Network::Address::InstanceConstSharedPtr address_; - friend class DestinationAddressReflection; + friend class AddressObjectReflection; +}; + +/** + * Registers the filter state object for the dynamic extension support. + */ +class BaseAddressObjectFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::unique_ptr + createFromBytes(absl::string_view data) const override; + std::unique_ptr + reflect(const StreamInfo::FilterState::Object* data) const override; }; } // namespace Network diff --git a/source/common/network/io_socket_error_impl.cc b/source/common/network/io_socket_error_impl.cc index b060267b538e..d9f54cb9f59e 100644 --- a/source/common/network/io_socket_error_impl.cc +++ b/source/common/network/io_socket_error_impl.cc @@ -12,33 +12,26 @@ Api::IoError::IoErrorCode IoSocketError::getErrorCode() const { return error_cod std::string IoSocketError::getErrorDetails() const { return errorDetails(errno_); } -IoSocketError* IoSocketError::getIoSocketInvalidAddressInstance() { - static auto* instance = - new IoSocketError(SOCKET_ERROR_NOT_SUP, Api::IoError::IoErrorCode::NoSupport); - return instance; +Api::IoErrorPtr IoSocketError::getIoSocketInvalidAddressError() { + return Api::IoError::wrap( + new IoSocketError(SOCKET_ERROR_NOT_SUP, Api::IoError::IoErrorCode::NoSupport)); } -IoSocketError* IoSocketError::getIoSocketEbadfInstance() { - static auto* instance = - new IoSocketError(SOCKET_ERROR_BADF, Api::IoError::IoErrorCode::NoSupport); - return instance; +Api::IoErrorPtr IoSocketError::create(int sys_errno) { + return Api::IoError::wrap(new IoSocketError(sys_errno)); } -IoSocketError* IoSocketError::getIoSocketEagainInstance() { - static auto* instance = new IoSocketError(SOCKET_ERROR_AGAIN, Api::IoError::IoErrorCode::Again); - return instance; +Api::IoErrorPtr IoSocketError::getIoSocketEbadfError() { + return Api::IoError::wrap(new IoSocketError(SOCKET_ERROR_BADF, Api::IoError::IoErrorCode::BadFd)); } -void IoSocketError::deleteIoError(Api::IoError* err) { - ASSERT(err != nullptr); - ASSERT(err != getIoSocketInvalidAddressInstance()); - if (err != getIoSocketEagainInstance()) { - delete err; - } +Api::IoErrorPtr IoSocketError::getIoSocketEagainError() { + static auto* instance = new IoSocketError(SOCKET_ERROR_AGAIN, Api::IoError::IoErrorCode::Again); + return Api::IoError::reusedStatic(instance); } Api::IoCallUint64Result IoSocketError::ioResultSocketInvalidAddress() { - return {0, Api::IoErrorPtr(getIoSocketInvalidAddressInstance(), [](IoError*) {})}; + return {0, getIoSocketInvalidAddressError()}; } Api::IoError::IoErrorCode IoSocketError::errorCodeFromErrno(int sys_errno) { diff --git a/source/common/network/io_socket_error_impl.h b/source/common/network/io_socket_error_impl.h index 392dd6129ebb..f5eaf6ac94ae 100644 --- a/source/common/network/io_socket_error_impl.h +++ b/source/common/network/io_socket_error_impl.h @@ -9,39 +9,35 @@ namespace Network { class IoSocketError : public Api::IoError { public: - explicit IoSocketError(int sys_errno) - : errno_(sys_errno), error_code_(errorCodeFromErrno(errno_)) { - ASSERT(error_code_ != IoErrorCode::Again, - "Didn't use getIoSocketEagainInstance() to generate `Again`."); - } + static Api::IoErrorPtr create(int sys_errno); ~IoSocketError() override = default; Api::IoError::IoErrorCode getErrorCode() const override; std::string getErrorDetails() const override; int getSystemErrorCode() const override { return errno_; } - // IoErrorCode::Again is used frequently. Define it to be a singleton to avoid frequent memory - // allocation of such instance. If this is used, IoHandleCallResult has to be instantiated with - // deleter deleteIoError() below to avoid deallocating memory for this error. - static IoSocketError* getIoSocketEagainInstance(); - - static IoSocketError* getIoSocketEbadfInstance(); + // IoErrorCode::Again is used frequently. This custom error returns a + // reusable singleton to avoid repeated allocation. + static Api::IoErrorPtr getIoSocketEagainError(); + static Api::IoErrorPtr getIoSocketEbadfError(); // This error is introduced when Envoy create socket for unsupported address. It is either a bug, // or this Envoy instance received config which is not yet supported. This should not be fatal // error. static Api::IoCallUint64Result ioResultSocketInvalidAddress(); - // Deallocate memory only if the error is not Again. - static void deleteIoError(Api::IoError* err); - private: + explicit IoSocketError(int sys_errno) + : errno_(sys_errno), error_code_(errorCodeFromErrno(errno_)) { + ASSERT(error_code_ != IoErrorCode::Again, + "Didn't use getIoSocketEagainError() to generate `Again`."); + } explicit IoSocketError(int sys_errno, Api::IoError::IoErrorCode error_code) : errno_(sys_errno), error_code_(error_code) {} static Api::IoError::IoErrorCode errorCodeFromErrno(int sys_errno); - static IoSocketError* getIoSocketInvalidAddressInstance(); + static Api::IoErrorPtr getIoSocketInvalidAddressError(); const int errno_; const Api::IoError::IoErrorCode error_code_; diff --git a/source/common/network/io_socket_handle_base_impl.cc b/source/common/network/io_socket_handle_base_impl.cc new file mode 100644 index 000000000000..243285147979 --- /dev/null +++ b/source/common/network/io_socket_handle_base_impl.cc @@ -0,0 +1,175 @@ +#include "io_socket_handle_base_impl.h" + +#include "envoy/buffer/buffer.h" +#include "envoy/common/exception.h" +#include "envoy/event/dispatcher.h" + +#include "source/common/api/os_sys_calls_impl.h" +#include "source/common/common/assert.h" +#include "source/common/common/utility.h" +#include "source/common/network/address_impl.h" +#include "source/common/network/io_socket_error_impl.h" +#include "source/common/network/io_socket_handle_impl.h" +#include "source/common/network/socket_interface_impl.h" + +namespace Envoy { +namespace Network { + +IoSocketHandleBaseImpl::IoSocketHandleBaseImpl(os_fd_t fd, bool socket_v6only, + absl::optional domain) + : fd_(fd), socket_v6only_(socket_v6only), domain_(domain) {} + +IoSocketHandleBaseImpl::~IoSocketHandleBaseImpl() { + if (SOCKET_VALID(fd_)) { + // The TLS slot has been shut down by this moment with IoUring wiped out, thus + // better use this posix system call instead of IoSocketHandleBaseImpl::close(). + ::close(fd_); + } +} + +bool IoSocketHandleBaseImpl::isOpen() const { return SOCKET_VALID(fd_); } + +bool IoSocketHandleBaseImpl::supportsMmsg() const { + return Api::OsSysCallsSingleton::get().supportsMmsg(); +} + +bool IoSocketHandleBaseImpl::supportsUdpGro() const { + return Api::OsSysCallsSingleton::get().supportsUdpGro(); +} + +Api::SysCallIntResult IoSocketHandleBaseImpl::setOption(int level, int optname, const void* optval, + socklen_t optlen) { + return Api::OsSysCallsSingleton::get().setsockopt(fd_, level, optname, optval, optlen); +} + +Api::SysCallIntResult IoSocketHandleBaseImpl::getOption(int level, int optname, void* optval, + socklen_t* optlen) { + return Api::OsSysCallsSingleton::get().getsockopt(fd_, level, optname, optval, optlen); +} + +Api::SysCallIntResult IoSocketHandleBaseImpl::ioctl(unsigned long control_code, void* in_buffer, + unsigned long in_buffer_len, void* out_buffer, + unsigned long out_buffer_len, + unsigned long* bytes_returned) { + return Api::OsSysCallsSingleton::get().ioctl(fd_, control_code, in_buffer, in_buffer_len, + out_buffer, out_buffer_len, bytes_returned); +} + +Api::SysCallIntResult IoSocketHandleBaseImpl::setBlocking(bool blocking) { + return Api::OsSysCallsSingleton::get().setsocketblocking(fd_, blocking); +} + +absl::optional IoSocketHandleBaseImpl::domain() { return domain_; } + +Address::InstanceConstSharedPtr IoSocketHandleBaseImpl::localAddress() { + sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + memset(&ss, 0, ss_len); + auto& os_sys_calls = Api::OsSysCallsSingleton::get(); + Api::SysCallIntResult result = + os_sys_calls.getsockname(fd_, reinterpret_cast(&ss), &ss_len); + if (result.return_value_ != 0) { + throwEnvoyExceptionOrPanic(fmt::format("getsockname failed for '{}': ({}) {}", fd_, + result.errno_, errorDetails(result.errno_))); + } + return Address::addressFromSockAddrOrThrow(ss, ss_len, socket_v6only_); +} + +Address::InstanceConstSharedPtr IoSocketHandleBaseImpl::peerAddress() { + sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + memset(&ss, 0, ss_len); + auto& os_sys_calls = Api::OsSysCallsSingleton::get(); + Api::SysCallIntResult result = + os_sys_calls.getpeername(fd_, reinterpret_cast(&ss), &ss_len); + if (result.return_value_ != 0) { + throwEnvoyExceptionOrPanic( + fmt::format("getpeername failed for '{}': {}", fd_, errorDetails(result.errno_))); + } + + if (static_cast(ss_len) >= + (offsetof(sockaddr_storage, ss_family) + sizeof(ss.ss_family)) && + ss.ss_family == AF_UNIX) { + // For Unix domain sockets, can't find out the peer name, but it should match our own + // name for the socket (i.e. the path should match, barring any namespace or other + // mechanisms to hide things, of which there are many). + ss_len = sizeof(ss); + result = os_sys_calls.getsockname(fd_, reinterpret_cast(&ss), &ss_len); + if (result.return_value_ != 0) { + throwEnvoyExceptionOrPanic( + fmt::format("getsockname failed for '{}': {}", fd_, errorDetails(result.errno_))); + } + } + return Address::addressFromSockAddrOrThrow(ss, ss_len, socket_v6only_); +} + +absl::optional IoSocketHandleBaseImpl::lastRoundTripTime() { + Api::EnvoyTcpInfo info; + auto result = Api::OsSysCallsSingleton::get().socketTcpInfo(fd_, &info); + if (!result.return_value_) { + return {}; + } + return std::chrono::duration_cast(info.tcpi_rtt); +} + +absl::optional IoSocketHandleBaseImpl::congestionWindowInBytes() const { + Api::EnvoyTcpInfo info; + auto result = Api::OsSysCallsSingleton::get().socketTcpInfo(fd_, &info); + if (!result.return_value_) { + return {}; + } + return info.tcpi_snd_cwnd; +} + +absl::optional IoSocketHandleBaseImpl::interfaceName() { + auto& os_syscalls_singleton = Api::OsSysCallsSingleton::get(); + if (!os_syscalls_singleton.supportsGetifaddrs()) { + return absl::nullopt; + } + + Address::InstanceConstSharedPtr socket_address = localAddress(); + if (!socket_address || socket_address->type() != Address::Type::Ip) { + return absl::nullopt; + } + + Api::InterfaceAddressVector interface_addresses{}; + const Api::SysCallIntResult rc = os_syscalls_singleton.getifaddrs(interface_addresses); + RELEASE_ASSERT(!rc.return_value_, fmt::format("getifaddrs error: {}", rc.errno_)); + + absl::optional selected_interface_name{}; + for (const auto& interface_address : interface_addresses) { + if (!interface_address.interface_addr_) { + continue; + } + + if (socket_address->ip()->version() == interface_address.interface_addr_->ip()->version()) { + // Compare address _without port_. + // TODO: create common addressAsStringWithoutPort method to simplify code here. + absl::uint128 socket_address_value; + absl::uint128 interface_address_value; + switch (socket_address->ip()->version()) { + case Address::IpVersion::v4: + socket_address_value = socket_address->ip()->ipv4()->address(); + interface_address_value = interface_address.interface_addr_->ip()->ipv4()->address(); + break; + case Address::IpVersion::v6: + socket_address_value = socket_address->ip()->ipv6()->address(); + interface_address_value = interface_address.interface_addr_->ip()->ipv6()->address(); + break; + default: + ENVOY_BUG(false, fmt::format("unexpected IP family {}", + static_cast(socket_address->ip()->version()))); + } + + if (socket_address_value == interface_address_value) { + selected_interface_name = interface_address.interface_name_; + break; + } + } + } + + return selected_interface_name; +} + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/io_socket_handle_base_impl.h b/source/common/network/io_socket_handle_base_impl.h new file mode 100644 index 000000000000..7873154c0c0f --- /dev/null +++ b/source/common/network/io_socket_handle_base_impl.h @@ -0,0 +1,44 @@ +#pragma once + +#include "envoy/network/io_handle.h" + +#include "source/common/common/logger.h" + +namespace Envoy { +namespace Network { + +/** + * IoHandle derivative for sockets. + */ +class IoSocketHandleBaseImpl : public IoHandle, protected Logger::Loggable { +public: + IoSocketHandleBaseImpl(os_fd_t fd = INVALID_SOCKET, bool socket_v6only = false, + absl::optional domain = absl::nullopt); + ~IoSocketHandleBaseImpl() override; + + // TODO(sbelair2) To be removed when the fd is fully abstracted from clients. + os_fd_t fdDoNotUse() const override { return fd_; } + bool isOpen() const override; + bool supportsMmsg() const override; + bool supportsUdpGro() const override; + Api::SysCallIntResult setOption(int level, int optname, const void* optval, + socklen_t optlen) override; + Api::SysCallIntResult getOption(int level, int optname, void* optval, socklen_t* optlen) override; + Api::SysCallIntResult ioctl(unsigned long, void*, unsigned long, void*, unsigned long, + unsigned long*) override; + Api::SysCallIntResult setBlocking(bool blocking) override; + absl::optional domain() override; + Address::InstanceConstSharedPtr localAddress() override; + Address::InstanceConstSharedPtr peerAddress() override; + absl::optional lastRoundTripTime() override; + absl::optional congestionWindowInBytes() const override; + absl::optional interfaceName() override; + +protected: + os_fd_t fd_; + int socket_v6only_; + const absl::optional domain_; +}; + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index 0ccd658e7df8..91b8f9914fb6 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -17,6 +17,7 @@ using Envoy::Api::SysCallSizeResult; namespace Envoy { namespace { + constexpr int messageTypeContainsIP() { #ifdef IP_RECVDSTADDR return IP_RECVDSTADDR; @@ -62,11 +63,9 @@ Api::IoCallUint64Result IoSocketHandleImpl::close() { ASSERT(SOCKET_VALID(fd_)); const int rc = Api::OsSysCallsSingleton::get().close(fd_).return_value_; SET_SOCKET_INVALID(fd_); - return {static_cast(rc), Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)}; + return {static_cast(rc), Api::IoError::none()}; } -bool IoSocketHandleImpl::isOpen() const { return SOCKET_VALID(fd_); } - Api::IoCallUint64Result IoSocketHandleImpl::readv(uint64_t max_length, Buffer::RawSlice* slices, uint64_t num_slice) { absl::FixedArray iov(num_slice); @@ -446,14 +445,6 @@ Api::IoCallUint64Result IoSocketHandleImpl::recv(void* buffer, size_t length, in return sysCallResultToIoCallResult(result); } -bool IoSocketHandleImpl::supportsMmsg() const { - return Api::OsSysCallsSingleton::get().supportsMmsg(); -} - -bool IoSocketHandleImpl::supportsUdpGro() const { - return Api::OsSysCallsSingleton::get().supportsUdpGro(); -} - Api::SysCallIntResult IoSocketHandleImpl::bind(Address::InstanceConstSharedPtr address) { return Api::OsSysCallsSingleton::get().bind(fd_, address->sockAddr(), address->sockAddrLen()); } @@ -501,28 +492,6 @@ Api::SysCallIntResult IoSocketHandleImpl::connect(Address::InstanceConstSharedPt return Api::OsSysCallsSingleton::get().connect(fd_, sockaddr_to_use, sockaddr_len_to_use); } -Api::SysCallIntResult IoSocketHandleImpl::setOption(int level, int optname, const void* optval, - socklen_t optlen) { - return Api::OsSysCallsSingleton::get().setsockopt(fd_, level, optname, optval, optlen); -} - -Api::SysCallIntResult IoSocketHandleImpl::getOption(int level, int optname, void* optval, - socklen_t* optlen) { - return Api::OsSysCallsSingleton::get().getsockopt(fd_, level, optname, optval, optlen); -} - -Api::SysCallIntResult IoSocketHandleImpl::ioctl(unsigned long control_code, void* in_buffer, - unsigned long in_buffer_len, void* out_buffer, - unsigned long out_buffer_len, - unsigned long* bytes_returned) { - return Api::OsSysCallsSingleton::get().ioctl(fd_, control_code, in_buffer, in_buffer_len, - out_buffer, out_buffer_len, bytes_returned); -} - -Api::SysCallIntResult IoSocketHandleImpl::setBlocking(bool blocking) { - return Api::OsSysCallsSingleton::get().setsocketblocking(fd_, blocking); -} - IoHandlePtr IoSocketHandleImpl::duplicate() { auto result = Api::OsSysCallsSingleton::get().duplicate(fd_); RELEASE_ASSERT(result.return_value_ != -1, @@ -532,50 +501,6 @@ IoHandlePtr IoSocketHandleImpl::duplicate() { domain_); } -absl::optional IoSocketHandleImpl::domain() { return domain_; } - -Address::InstanceConstSharedPtr IoSocketHandleImpl::localAddress() { - sockaddr_storage ss; - socklen_t ss_len = sizeof(ss); - memset(&ss, 0, ss_len); - auto& os_sys_calls = Api::OsSysCallsSingleton::get(); - Api::SysCallIntResult result = - os_sys_calls.getsockname(fd_, reinterpret_cast(&ss), &ss_len); - if (result.return_value_ != 0) { - throw EnvoyException(fmt::format("getsockname failed for '{}': ({}) {}", fd_, result.errno_, - errorDetails(result.errno_))); - } - return Address::addressFromSockAddrOrThrow(ss, ss_len, socket_v6only_); -} - -Address::InstanceConstSharedPtr IoSocketHandleImpl::peerAddress() { - sockaddr_storage ss; - socklen_t ss_len = sizeof(ss); - memset(&ss, 0, ss_len); - auto& os_sys_calls = Api::OsSysCallsSingleton::get(); - Api::SysCallIntResult result = - os_sys_calls.getpeername(fd_, reinterpret_cast(&ss), &ss_len); - if (result.return_value_ != 0) { - throw EnvoyException( - fmt::format("getpeername failed for '{}': {}", fd_, errorDetails(result.errno_))); - } - - if (static_cast(ss_len) >= - (offsetof(sockaddr_storage, ss_family) + sizeof(ss.ss_family)) && - ss.ss_family == AF_UNIX) { - // For Unix domain sockets, can't find out the peer name, but it should match our own - // name for the socket (i.e. the path should match, barring any namespace or other - // mechanisms to hide things, of which there are many). - ss_len = sizeof(ss); - result = os_sys_calls.getsockname(fd_, reinterpret_cast(&ss), &ss_len); - if (result.return_value_ != 0) { - throw EnvoyException( - fmt::format("getsockname failed for '{}': {}", fd_, errorDetails(result.errno_))); - } - } - return Address::addressFromSockAddrOrThrow(ss, ss_len, socket_v6only_); -} - void IoSocketHandleImpl::initializeFileEvent(Event::Dispatcher& dispatcher, Event::FileReadyCb cb, Event::FileTriggerType trigger, uint32_t events) { ASSERT(file_event_ == nullptr, "Attempting to initialize two `file_event_` for the same " @@ -603,73 +528,5 @@ Api::SysCallIntResult IoSocketHandleImpl::shutdown(int how) { return Api::OsSysCallsSingleton::get().shutdown(fd_, how); } -absl::optional IoSocketHandleImpl::lastRoundTripTime() { - Api::EnvoyTcpInfo info; - auto result = Api::OsSysCallsSingleton::get().socketTcpInfo(fd_, &info); - if (!result.return_value_) { - return {}; - } - return std::chrono::duration_cast(info.tcpi_rtt); -} - -absl::optional IoSocketHandleImpl::congestionWindowInBytes() const { - Api::EnvoyTcpInfo info; - auto result = Api::OsSysCallsSingleton::get().socketTcpInfo(fd_, &info); - if (!result.return_value_) { - return {}; - } - return info.tcpi_snd_cwnd; -} - -absl::optional IoSocketHandleImpl::interfaceName() { - auto& os_syscalls_singleton = Api::OsSysCallsSingleton::get(); - if (!os_syscalls_singleton.supportsGetifaddrs()) { - return absl::nullopt; - } - - Address::InstanceConstSharedPtr socket_address = localAddress(); - if (!socket_address || socket_address->type() != Address::Type::Ip) { - return absl::nullopt; - } - - Api::InterfaceAddressVector interface_addresses{}; - const Api::SysCallIntResult rc = os_syscalls_singleton.getifaddrs(interface_addresses); - RELEASE_ASSERT(!rc.return_value_, fmt::format("getifaddrs error: {}", rc.errno_)); - - absl::optional selected_interface_name{}; - for (const auto& interface_address : interface_addresses) { - if (!interface_address.interface_addr_) { - continue; - } - - if (socket_address->ip()->version() == interface_address.interface_addr_->ip()->version()) { - // Compare address _without port_. - // TODO: create common addressAsStringWithoutPort method to simplify code here. - absl::uint128 socket_address_value; - absl::uint128 interface_address_value; - switch (socket_address->ip()->version()) { - case Address::IpVersion::v4: - socket_address_value = socket_address->ip()->ipv4()->address(); - interface_address_value = interface_address.interface_addr_->ip()->ipv4()->address(); - break; - case Address::IpVersion::v6: - socket_address_value = socket_address->ip()->ipv6()->address(); - interface_address_value = interface_address.interface_addr_->ip()->ipv6()->address(); - break; - default: - ENVOY_BUG(false, fmt::format("unexpected IP family {}", - static_cast(socket_address->ip()->version()))); - } - - if (socket_address_value == interface_address_value) { - selected_interface_name = interface_address.interface_name_; - break; - } - } - } - - return selected_interface_name; -} - } // namespace Network } // namespace Envoy diff --git a/source/common/network/io_socket_handle_impl.h b/source/common/network/io_socket_handle_impl.h index fef3158240d8..067035641564 100644 --- a/source/common/network/io_socket_handle_impl.h +++ b/source/common/network/io_socket_handle_impl.h @@ -8,6 +8,7 @@ #include "source/common/common/logger.h" #include "source/common/network/io_socket_error_impl.h" +#include "source/common/network/io_socket_handle_base_impl.h" #include "source/common/runtime/runtime_features.h" namespace Envoy { @@ -16,11 +17,11 @@ namespace Network { /** * IoHandle derivative for sockets. */ -class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable { +class IoSocketHandleImpl : public IoSocketHandleBaseImpl { public: explicit IoSocketHandleImpl(os_fd_t fd = INVALID_SOCKET, bool socket_v6only = false, absl::optional domain = absl::nullopt) - : fd_(fd), socket_v6only_(socket_v6only), domain_(domain), + : IoSocketHandleBaseImpl(fd, socket_v6only, domain), udp_read_normalize_addresses_( Runtime::runtimeFeatureEnabled("envoy.restart_features.udp_read_normalize_addresses")) { } @@ -28,13 +29,8 @@ class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable domain() override; - Address::InstanceConstSharedPtr localAddress() override; - Address::InstanceConstSharedPtr peerAddress() override; void initializeFileEvent(Event::Dispatcher& dispatcher, Event::FileReadyCb cb, Event::FileTriggerType trigger, uint32_t events) override; @@ -83,9 +66,6 @@ class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable lastRoundTripTime() override; - absl::optional congestionWindowInBytes() const override; - absl::optional interfaceName() override; protected: // Converts a SysCallSizeResult to IoCallUint64Result. @@ -93,24 +73,18 @@ class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable& result) { if (result.return_value_ >= 0) { // Return nullptr as IoError upon success. - return Api::IoCallUint64Result(result.return_value_, - Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); + return Api::IoCallUint64Result(result.return_value_, Api::IoError::none()); } if (result.errno_ == SOCKET_ERROR_INVAL) { ENVOY_LOG(error, "Invalid argument passed in."); } return Api::IoCallUint64Result( - /*rc=*/0, - (result.errno_ == SOCKET_ERROR_AGAIN - // EAGAIN is frequent enough that its memory allocation should be avoided. - ? Api::IoErrorPtr(IoSocketError::getIoSocketEagainInstance(), - IoSocketError::deleteIoError) - : Api::IoErrorPtr(new IoSocketError(result.errno_), IoSocketError::deleteIoError))); + /*rc=*/0, (result.errno_ == SOCKET_ERROR_AGAIN + // EAGAIN is frequent enough that its memory allocation should be avoided. + ? IoSocketError::getIoSocketEagainError() + : IoSocketError::create(result.errno_))); } - os_fd_t fd_; - int socket_v6only_{false}; - const absl::optional domain_; Event::FileEventPtr file_event_{nullptr}; // The minimum cmsg buffer size to filled in destination address, packets dropped and gso diff --git a/source/common/network/lc_trie.h b/source/common/network/lc_trie.h index db523fe980a5..253605d183d5 100644 --- a/source/common/network/lc_trie.h +++ b/source/common/network/lc_trie.h @@ -137,8 +137,8 @@ template class LcTrie { // step 1. But it has a useful new property: now that all the prefixes // are at the leaves, they are disjoint: no prefix is nested under another. - std::vector> ipv4_prefixes = ipv4_temp.push_leaves(); - std::vector> ipv6_prefixes = ipv6_temp.push_leaves(); + std::vector> ipv4_prefixes = ipv4_temp.pushLeaves(); + std::vector> ipv6_prefixes = ipv6_temp.pushLeaves(); // Step 3: take the disjoint prefixes from the leaves of each Binary Trie // and use them to construct an LC Trie. @@ -345,7 +345,7 @@ template class LcTrie { * trie can be nested under another leaf) * @return the prefixes associated with the leaf nodes. */ - std::vector> push_leaves() { + std::vector> pushLeaves() { std::vector> prefixes; std::function visit = [&](Node* node, DataSetSharedPtr data, unsigned depth, IpType prefix) { diff --git a/source/common/network/listen_socket_impl.cc b/source/common/network/listen_socket_impl.cc index c24f024ad34c..596d2c0846f6 100644 --- a/source/common/network/listen_socket_impl.cc +++ b/source/common/network/listen_socket_impl.cc @@ -24,10 +24,10 @@ Api::SysCallIntResult ListenSocketImpl::bind(Network::Address::InstanceConstShar const Api::SysCallIntResult result = SocketImpl::bind(connection_info_provider_->localAddress()); if (SOCKET_FAILURE(result.return_value_)) { close(); - throw SocketBindException(fmt::format("cannot bind '{}': {}", - connection_info_provider_->localAddress()->asString(), - errorDetails(result.errno_)), - result.errno_); + const std::string error = + fmt::format("cannot bind '{}': {}", connection_info_provider_->localAddress()->asString(), + errorDetails(result.errno_)); + throw SocketBindException(error, result.errno_); } return {0, 0}; } diff --git a/source/common/network/listen_socket_impl.h b/source/common/network/listen_socket_impl.h index 9fb220c47edd..1dd9a6f668d7 100644 --- a/source/common/network/listen_socket_impl.h +++ b/source/common/network/listen_socket_impl.h @@ -12,6 +12,7 @@ #include "source/common/common/assert.h" #include "source/common/common/dump_state_utils.h" +#include "source/common/network/connection_socket_impl.h" #include "source/common/network/socket_impl.h" #include "source/common/network/socket_interface.h" @@ -42,19 +43,6 @@ class ListenSocketImpl : public SocketImpl { bool isOpen() const override { return io_handle_ != nullptr && io_handle_->isOpen(); } }; -/** - * Wraps a unix socket. - */ -template struct NetworkSocketTrait {}; - -template <> struct NetworkSocketTrait { - static constexpr Socket::Type type = Socket::Type::Stream; -}; - -template <> struct NetworkSocketTrait { - static constexpr Socket::Type type = Socket::Type::Datagram; -}; - template class NetworkListenSocket : public ListenSocketImpl { public: NetworkListenSocket(const Address::InstanceConstSharedPtr& address, @@ -178,80 +166,34 @@ class InternalListenSocket : public ListenSocketImpl { } }; -class ConnectionSocketImpl : public SocketImpl, public ConnectionSocket { -public: - ConnectionSocketImpl(IoHandlePtr&& io_handle, - const Address::InstanceConstSharedPtr& local_address, - const Address::InstanceConstSharedPtr& remote_address) - : SocketImpl(std::move(io_handle), local_address, remote_address) {} - - ConnectionSocketImpl(Socket::Type type, const Address::InstanceConstSharedPtr& local_address, - const Address::InstanceConstSharedPtr& remote_address, - const SocketCreationOptions& options) - : SocketImpl(type, local_address, remote_address, options) { - connection_info_provider_->setLocalAddress(local_address); - } - - // Network::ConnectionSocket - void setDetectedTransportProtocol(absl::string_view protocol) override { - transport_protocol_ = std::string(protocol); - } - absl::string_view detectedTransportProtocol() const override { return transport_protocol_; } - - void setRequestedApplicationProtocols(const std::vector& protocols) override { - application_protocols_.clear(); - for (const auto& protocol : protocols) { - application_protocols_.emplace_back(protocol); - } - } - const std::vector& requestedApplicationProtocols() const override { - return application_protocols_; - } - - void setRequestedServerName(absl::string_view server_name) override { - // Always keep the server_name_ as lower case. - connectionInfoProvider().setRequestedServerName(absl::AsciiStrToLower(server_name)); - } - absl::string_view requestedServerName() const override { - return connectionInfoProvider().requestedServerName(); - } - - void setJA3Hash(absl::string_view ja3_hash) override { - connectionInfoProvider().setJA3Hash(ja3_hash); - } - absl::string_view ja3Hash() const override { return connectionInfoProvider().ja3Hash(); } - - absl::optional lastRoundTripTime() override { - return ioHandle().lastRoundTripTime(); - } - - absl::optional congestionWindowInBytes() const override { - return ioHandle().congestionWindowInBytes(); - } - - void dumpState(std::ostream& os, int indent_level) const override { - const char* spaces = spacesForLevel(indent_level); - os << spaces << "ListenSocketImpl " << this << DUMP_MEMBER(transport_protocol_) << "\n"; - DUMP_DETAILS(connection_info_provider_); - } - -protected: - std::string transport_protocol_; - std::vector application_protocols_; -}; - // ConnectionSocket used with server connections. class AcceptedSocketImpl : public ConnectionSocketImpl { public: AcceptedSocketImpl(IoHandlePtr&& io_handle, const Address::InstanceConstSharedPtr& local_address, - const Address::InstanceConstSharedPtr& remote_address) - : ConnectionSocketImpl(std::move(io_handle), local_address, remote_address) { - ++global_accepted_socket_count_; + const Address::InstanceConstSharedPtr& remote_address, + Server::ThreadLocalOverloadStateOptRef overload_state, + bool track_global_cx_limit_in_overload_manager) + : ConnectionSocketImpl(std::move(io_handle), local_address, remote_address), + overload_state_(overload_state), + track_global_cx_limit_in_overload_manager_(track_global_cx_limit_in_overload_manager) { + // In case when tracking of global connection limit is enabled in the overload manager, the + // global connection limit usage will be incremented in + // TcpListenerImpl::rejectCxOverGlobalLimit() to avoid race conditions (between checking if it + // is possible to increment current usage in TcpListenerImpl::rejectCxOverGlobalLimit() and + // actually incrementing it in the current method). + if (!track_global_cx_limit_in_overload_manager_) { + ++global_accepted_socket_count_; + } } ~AcceptedSocketImpl() override { - ASSERT(global_accepted_socket_count_.load() > 0); - --global_accepted_socket_count_; + if (track_global_cx_limit_in_overload_manager_) { + overload_state_->tryDeallocateResource( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 1); + } else { + ASSERT(global_accepted_socket_count_.load() > 0); + --global_accepted_socket_count_; + } } // TODO (tonya11en): Global connection count tracking is temporarily performed via a static @@ -260,19 +202,8 @@ class AcceptedSocketImpl : public ConnectionSocketImpl { private: static std::atomic global_accepted_socket_count_; -}; - -// ConnectionSocket used with client connections. -class ClientSocketImpl : public ConnectionSocketImpl { -public: - ClientSocketImpl(const Address::InstanceConstSharedPtr& remote_address, - const OptionsSharedPtr& options) - : ConnectionSocketImpl(Network::ioHandleForAddr(Socket::Type::Stream, remote_address, {}), - nullptr, remote_address) { - if (options) { - addOptions(options); - } - } + Server::ThreadLocalOverloadStateOptRef overload_state_; + const bool track_global_cx_limit_in_overload_manager_; }; } // namespace Network diff --git a/source/common/network/resolver_impl.cc b/source/common/network/resolver_impl.cc index 02effebc873d..bd51bcfd2272 100644 --- a/source/common/network/resolver_impl.cc +++ b/source/common/network/resolver_impl.cc @@ -31,8 +31,8 @@ class IpResolver : public Resolver { case envoy::config::core::v3::SocketAddress::PortSpecifierCase::kNamedPort: break; } - throw EnvoyException(fmt::format("IP resolver can't handle port specifier type {}", - socket_address.port_specifier_case())); + throwEnvoyExceptionOrPanic(fmt::format("IP resolver can't handle port specifier type {}", + socket_address.port_specifier_case())); } std::string name() const override { return Config::AddressResolverNames::get().IP; } @@ -46,7 +46,7 @@ REGISTER_FACTORY(IpResolver, Resolver); InstanceConstSharedPtr resolveProtoAddress(const envoy::config::core::v3::Address& address) { switch (address.address_case()) { case envoy::config::core::v3::Address::AddressCase::ADDRESS_NOT_SET: - throw EnvoyException("Address must be set: " + address.DebugString()); + throwEnvoyExceptionOrPanic("Address must be set: " + address.DebugString()); case envoy::config::core::v3::Address::AddressCase::kSocketAddress: return resolveProtoSocketAddress(address.socket_address()); case envoy::config::core::v3::Address::AddressCase::kPipe: @@ -63,7 +63,7 @@ InstanceConstSharedPtr resolveProtoAddress(const envoy::config::core::v3::Addres break; } } - throw EnvoyException("Failed to resolve address:" + address.DebugString()); + throwEnvoyExceptionOrPanic("Failed to resolve address:" + address.DebugString()); } InstanceConstSharedPtr @@ -77,7 +77,7 @@ resolveProtoSocketAddress(const envoy::config::core::v3::SocketAddress& socket_a resolver = Registry::FactoryRegistry::getFactory(resolver_name); } if (resolver == nullptr) { - throw EnvoyException(fmt::format("Unknown address resolver: {}", resolver_name)); + throwEnvoyExceptionOrPanic(fmt::format("Unknown address resolver: {}", resolver_name)); } return resolver->resolve(socket_address); } diff --git a/source/common/network/socket_impl.cc b/source/common/network/socket_impl.cc index 3d96d964e93a..14d9d2c781e8 100644 --- a/source/common/network/socket_impl.cc +++ b/source/common/network/socket_impl.cc @@ -68,9 +68,9 @@ Api::SysCallIntResult SocketImpl::bind(Network::Address::InstanceConstSharedPtr if (pipe->mode() != 0 && !abstract_namespace && bind_result.return_value_ == 0) { auto set_permissions = Api::OsSysCallsSingleton::get().chmod(pipe_sa->sun_path, pipe->mode()); if (set_permissions.return_value_ != 0) { - throw EnvoyException(fmt::format("Failed to create socket with mode {}: {}", - std::to_string(pipe->mode()), - errorDetails(set_permissions.errno_))); + throwEnvoyExceptionOrPanic(fmt::format("Failed to create socket with mode {}: {}", + std::to_string(pipe->mode()), + errorDetails(set_permissions.errno_))); } } return bind_result; diff --git a/source/common/network/socket_impl.h b/source/common/network/socket_impl.h index 62cdc572c2ff..2becca95532d 100644 --- a/source/common/network/socket_impl.h +++ b/source/common/network/socket_impl.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/network/listener.h" #include "envoy/network/socket.h" #include "envoy/network/socket_interface.h" @@ -79,6 +80,12 @@ class ConnectionInfoSetterImpl : public ConnectionInfoSetter { void setFilterChainInfo(FilterChainInfoConstSharedPtr filter_chain_info) override { filter_chain_info_ = std::move(filter_chain_info); } + OptRef listenerInfo() const override { + return makeOptRefFromPtr(listener_info_.get()); + } + void setListenerInfo(ListenerInfoConstSharedPtr listener_info) override { + listener_info_ = std::move(listener_info); + } private: Address::InstanceConstSharedPtr local_address_; @@ -93,6 +100,7 @@ class ConnectionInfoSetterImpl : public ConnectionInfoSetter { std::string ja3_hash_; absl::optional round_trip_time_; FilterChainInfoConstSharedPtr filter_chain_info_; + ListenerInfoConstSharedPtr listener_info_; }; class SocketImpl : public virtual Socket { diff --git a/source/common/network/tcp_listener_impl.cc b/source/common/network/tcp_listener_impl.cc index 2ebef7a30c1d..29b100928521 100644 --- a/source/common/network/tcp_listener_impl.cc +++ b/source/common/network/tcp_listener_impl.cc @@ -13,27 +13,44 @@ #include "source/common/event/file_event_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/io_socket_handle_impl.h" +#include "source/common/runtime/runtime_keys.h" namespace Envoy { namespace Network { -const absl::string_view TcpListenerImpl::GlobalMaxCxRuntimeKey = - "overload.global_downstream_max_connections"; - bool TcpListenerImpl::rejectCxOverGlobalLimit() const { // Enforce the global connection limit if necessary, immediately closing the accepted connection. if (ignore_global_conn_limit_) { return false; } - - // If the connection limit is not set, don't limit the connections, but still track them. - // TODO(tonya11en): In integration tests, threadsafeSnapshot is necessary since the FakeUpstreams - // use a listener and do not run in a worker thread. In practice, this code path will always be - // run on a worker thread, but to prevent failed assertions in test environments, threadsafe - // snapshots must be used. This must be revisited. - const uint64_t global_cx_limit = runtime_.threadsafeSnapshot()->getInteger( - GlobalMaxCxRuntimeKey, std::numeric_limits::max()); - return AcceptedSocketImpl::acceptedSocketCount() >= global_cx_limit; + // TODO(nezdolik): deprecate `overload.global_downstream_max_connections` key once + // downstream connections monitor extension is stable. + if (track_global_cx_limit_in_overload_manager_) { + // Check if runtime flag `overload.global_downstream_max_connections` is configured + // simultaneously with downstream connections monitor in overload manager. + if (runtime_.threadsafeSnapshot()->get(Runtime::Keys::GlobalMaxCxRuntimeKey)) { + ENVOY_LOG_ONCE_MISC( + warn, + "Global downstream connections limits is configured via runtime key {} and in " + "{}. Using overload manager config.", + Runtime::Keys::GlobalMaxCxRuntimeKey, + Server::OverloadProactiveResources::get().GlobalDownstreamMaxConnections); + } + // Try to allocate resource within overload manager. We do it once here, instead of checking if + // it is possible to allocate resource in this method and then actually allocating it later in + // the code to avoid race conditions. + return !(overload_state_->tryAllocateResource( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 1)); + } else { + // If the connection limit is not set, don't limit the connections, but still track them. + // TODO(tonya11en): In integration tests, threadsafeSnapshot is necessary since the + // FakeUpstreams use a listener and do not run in a worker thread. In practice, this code path + // will always be run on a worker thread, but to prevent failed assertions in test environments, + // threadsafe snapshots must be used. This must be revisited. + const uint64_t global_cx_limit = runtime_.threadsafeSnapshot()->getInteger( + Runtime::Keys::GlobalMaxCxRuntimeKey, std::numeric_limits::max()); + return AcceptedSocketImpl::acceptedSocketCount() >= global_cx_limit; + } } void TcpListenerImpl::onSocketEvent(short flags) { @@ -88,8 +105,9 @@ void TcpListenerImpl::onSocketEvent(short flags) { local_address->ip()->version() == Address::IpVersion::v6); - cb_.onAccept( - std::make_unique(std::move(io_handle), local_address, remote_address)); + cb_.onAccept(std::make_unique(std::move(io_handle), local_address, + remote_address, overload_state_, + track_global_cx_limit_in_overload_manager_)); } ENVOY_LOG_MISC(trace, "TcpListener accepted {} new connections.", @@ -97,15 +115,22 @@ void TcpListenerImpl::onSocketEvent(short flags) { cb_.recordConnectionsAcceptedOnSocketEvent(connections_accepted_from_kernel_count); } -TcpListenerImpl::TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random, +TcpListenerImpl::TcpListenerImpl(Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port, bool ignore_global_conn_limit, - uint32_t max_connections_to_accept_per_socket_event) + uint32_t max_connections_to_accept_per_socket_event, + Server::ThreadLocalOverloadStateOptRef overload_state) : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), random_(random), runtime_(runtime), bind_to_port_(bind_to_port), reject_fraction_(0.0), ignore_global_conn_limit_(ignore_global_conn_limit), - max_connections_to_accept_per_socket_event_(max_connections_to_accept_per_socket_event) { + max_connections_to_accept_per_socket_event_(max_connections_to_accept_per_socket_event), + overload_state_(overload_state), + track_global_cx_limit_in_overload_manager_( + overload_state_ + ? overload_state_->isResourceMonitorEnabled( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections) + : false) { if (bind_to_port) { // Use level triggered mode to avoid potential loss of the trigger due to // transient accept errors or early termination due to accepting diff --git a/source/common/network/tcp_listener_impl.h b/source/common/network/tcp_listener_impl.h index 5cd08691665d..d885276cc12d 100644 --- a/source/common/network/tcp_listener_impl.h +++ b/source/common/network/tcp_listener_impl.h @@ -16,10 +16,11 @@ namespace Network { */ class TcpListenerImpl : public BaseListenerImpl { public: - TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random, + TcpListenerImpl(Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port, bool ignore_global_conn_limit, - uint32_t max_connections_to_accept_per_socket_event); + uint32_t max_connections_to_accept_per_socket_event, + Server::ThreadLocalOverloadStateOptRef overload_state); ~TcpListenerImpl() override { if (bind_to_port_) { socket_->ioHandle().resetFileEvents(); @@ -30,8 +31,6 @@ class TcpListenerImpl : public BaseListenerImpl { void setRejectFraction(UnitFloat reject_fraction) override; void configureLoadShedPoints(Server::LoadShedPointProvider& load_shed_point_provider) override; - static const absl::string_view GlobalMaxCxRuntimeKey; - protected: TcpListenerCallbacks& cb_; @@ -49,6 +48,8 @@ class TcpListenerImpl : public BaseListenerImpl { const bool ignore_global_conn_limit_; const uint32_t max_connections_to_accept_per_socket_event_; Server::LoadShedPoint* listener_accept_{nullptr}; + Server::ThreadLocalOverloadStateOptRef overload_state_; + const bool track_global_cx_limit_in_overload_manager_; }; } // namespace Network diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 764da61e9402..62c5b273db96 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -28,7 +28,7 @@ namespace Envoy { namespace Network { -UdpListenerImpl::UdpListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, +UdpListenerImpl::UdpListenerImpl(Event::Dispatcher& dispatcher, SocketSharedPtr socket, UdpListenerCallbacks& cb, TimeSource& time_source, const envoy::config::core::v3::UdpSocketConfig& config) : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), time_source_(time_source), diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index c9865d137e2c..723c3c74de75 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -22,9 +22,8 @@ class UdpListenerImpl : public BaseListenerImpl, public UdpPacketProcessor, protected Logger::Loggable { public: - UdpListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, - UdpListenerCallbacks& cb, TimeSource& time_source, - const envoy::config::core::v3::UdpSocketConfig& config); + UdpListenerImpl(Event::Dispatcher& dispatcher, SocketSharedPtr socket, UdpListenerCallbacks& cb, + TimeSource& time_source, const envoy::config::core::v3::UdpSocketConfig& config); ~UdpListenerImpl() override; uint32_t packetsDropped() { return packets_dropped_; } diff --git a/source/common/network/udp_packet_writer_handler_impl.cc b/source/common/network/udp_packet_writer_handler_impl.cc index cb0e8991e0ae..813c549f7c70 100644 --- a/source/common/network/udp_packet_writer_handler_impl.cc +++ b/source/common/network/udp_packet_writer_handler_impl.cc @@ -6,8 +6,7 @@ namespace Envoy { namespace Network { -UdpDefaultWriter::UdpDefaultWriter(Network::IoHandle& io_handle) - : write_blocked_(false), io_handle_(io_handle) {} +UdpDefaultWriter::UdpDefaultWriter(Network::IoHandle& io_handle) : io_handle_(io_handle) {} UdpDefaultWriter::~UdpDefaultWriter() = default; diff --git a/source/common/network/udp_packet_writer_handler_impl.h b/source/common/network/udp_packet_writer_handler_impl.h index ba33012b81a8..d4a48fd56a53 100644 --- a/source/common/network/udp_packet_writer_handler_impl.h +++ b/source/common/network/udp_packet_writer_handler_impl.h @@ -32,11 +32,11 @@ class UdpDefaultWriter : public UdpPacketWriter { } Api::IoCallUint64Result flush() override { return {/*rc=*/0, - /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + /*err=*/Api::IoError::none()}; } private: - bool write_blocked_; + bool write_blocked_{false}; Network::IoHandle& io_handle_; }; diff --git a/source/common/network/upstream_server_name.cc b/source/common/network/upstream_server_name.cc index c6bc2559679e..941d7648e2e2 100644 --- a/source/common/network/upstream_server_name.cc +++ b/source/common/network/upstream_server_name.cc @@ -1,5 +1,8 @@ #include "source/common/network/upstream_server_name.h" +#include "envoy/registry/registry.h" +#include "envoy/stream_info/filter_state.h" + #include "source/common/common/macros.h" namespace Envoy { @@ -8,5 +11,17 @@ namespace Network { const std::string& UpstreamServerName::key() { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.network.upstream_server_name"); } + +class UpstreamServerNameObjectFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return UpstreamServerName::key(); } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +REGISTER_FACTORY(UpstreamServerNameObjectFactory, StreamInfo::FilterState::ObjectFactory); + } // namespace Network } // namespace Envoy diff --git a/source/common/network/upstream_subject_alt_names.cc b/source/common/network/upstream_subject_alt_names.cc index df8aa9adb184..d4126444d110 100644 --- a/source/common/network/upstream_subject_alt_names.cc +++ b/source/common/network/upstream_subject_alt_names.cc @@ -1,5 +1,8 @@ #include "source/common/network/upstream_subject_alt_names.h" +#include "envoy/registry/registry.h" +#include "envoy/stream_info/filter_state.h" + #include "source/common/common/macros.h" namespace Envoy { @@ -8,5 +11,17 @@ namespace Network { const std::string& UpstreamSubjectAltNames::key() { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.network.upstream_subject_alt_names"); } + +class UpstreamSubjectAltNamesObjectFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return UpstreamSubjectAltNames::key(); } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + const std::vector parts = absl::StrSplit(data, ','); + return std::make_unique(parts); + } +}; + +REGISTER_FACTORY(UpstreamSubjectAltNamesObjectFactory, StreamInfo::FilterState::ObjectFactory); } // namespace Network } // namespace Envoy diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 3d05410ce45d..6e9f3188d244 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -41,6 +41,14 @@ Address::InstanceConstSharedPtr instanceOrNull(StatusOr(url.substr(UNIX_SCHEME.size())); } else { - throw EnvoyException(absl::StrCat("unknown protocol scheme: ", url)); + throwEnvoyExceptionOrPanic(absl::StrCat("unknown protocol scheme: ", url)); } } @@ -215,7 +223,7 @@ Address::InstanceConstSharedPtr Utility::copyInternetAddressAndPort(const Addres } void Utility::throwWithMalformedIp(absl::string_view ip_address) { - throw EnvoyException(absl::StrCat("malformed IP address: ", ip_address)); + throwEnvoyExceptionOrPanic(absl::StrCat("malformed IP address: ", ip_address)); } // TODO(hennna): Currently getLocalAddress does not support choosing between @@ -435,7 +443,7 @@ void Utility::parsePortRangeList(absl::string_view string, std::list& } if (s.empty() || (min > 65535) || (max > 65535) || ss.fail() || !ss.eof()) { - throw EnvoyException(fmt::format("invalid port number or range '{}'", s_string)); + throwEnvoyExceptionOrPanic(fmt::format("invalid port number or range '{}'", s_string)); } list.emplace_back(PortRange(min, max)); @@ -554,7 +562,7 @@ Api::IoCallUint64Result Utility::writeToSocket(IoHandle& handle, Buffer::RawSlic uint64_t num_slices, const Address::Ip* local_ip, const Address::Instance& peer_address) { Api::IoCallUint64Result send_result( - /*rc=*/0, /*err=*/Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); + /*rc=*/0, /*err=*/Api::IoError::none()); do { send_result = handle.sendmsg(slices, num_slices, 0, local_ip, peer_address); } while (!send_result.ok() && diff --git a/source/common/network/utility.h b/source/common/network/utility.h index c36d6b2e0a04..1d2352549191 100644 --- a/source/common/network/utility.h +++ b/source/common/network/utility.h @@ -94,6 +94,16 @@ class Utility { static constexpr absl::string_view UDP_SCHEME{"udp://"}; static constexpr absl::string_view UNIX_SCHEME{"unix://"}; + /** + * Make a URL from a datagram Address::Instance; will be udp:// prefix for + * an IP address, and unix:// prefix otherwise. Giving a tcp address to this + * function will result in incorrect behavior (addresses don't know if they + * are datagram or stream). + * @param addr supplies the address to convert to string. + * @return The appropriate url string compatible with resolveUrl. + */ + static std::string urlFromDatagramAddress(const Address::Instance& addr); + /** * Resolve a URL. * @param url supplies the url to resolve. diff --git a/source/common/network/win32_socket_handle_impl.cc b/source/common/network/win32_socket_handle_impl.cc index e94eb718de8c..413ce9ddd361 100644 --- a/source/common/network/win32_socket_handle_impl.cc +++ b/source/common/network/win32_socket_handle_impl.cc @@ -138,14 +138,14 @@ Api::IoCallUint64Result Win32SocketHandleImpl::drainToPeekBuffer(size_t length) return result; } } - return {total_bytes_read, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})}; + return {total_bytes_read, Api::IoError::none()}; } Api::IoCallUint64Result Win32SocketHandleImpl::readFromPeekBuffer(void* buffer, size_t length) { uint64_t copy_size = std::min(peek_buffer_.length(), static_cast(length)); peek_buffer_.copyOut(0, copy_size, buffer); peek_buffer_.drain(copy_size); - return {copy_size, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})}; + return {copy_size, Api::IoError::none()}; } Api::IoCallUint64Result Win32SocketHandleImpl::readvFromPeekBuffer(uint64_t max_length, @@ -153,20 +153,20 @@ Api::IoCallUint64Result Win32SocketHandleImpl::readvFromPeekBuffer(uint64_t max_ uint64_t num_slice) { uint64_t bytes_read = peek_buffer_.copyOutToSlices(max_length, slices, num_slice); peek_buffer_.drain(bytes_read); - return {bytes_read, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})}; + return {bytes_read, Api::IoError::none()}; } Api::IoCallUint64Result Win32SocketHandleImpl::readFromPeekBuffer(Buffer::Instance& buffer, size_t length) { auto length_to_move = std::min(peek_buffer_.length(), static_cast(length)); buffer.move(peek_buffer_, length_to_move); - return {length_to_move, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})}; + return {length_to_move, Api::IoError::none()}; } Api::IoCallUint64Result Win32SocketHandleImpl::peekFromPeekBuffer(void* buffer, size_t length) { uint64_t copy_size = std::min(peek_buffer_.length(), static_cast(length)); peek_buffer_.copyOut(0, copy_size, buffer); - return {copy_size, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})}; + return {copy_size, Api::IoError::none()}; } void Win32SocketHandleImpl::initializeFileEvent(Event::Dispatcher& dispatcher, diff --git a/source/common/protobuf/BUILD b/source/common/protobuf/BUILD index ba96f6cb5c95..fa252699ff10 100644 --- a/source/common/protobuf/BUILD +++ b/source/common/protobuf/BUILD @@ -87,7 +87,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/protobuf:visitor_lib", "//source/common/runtime:runtime_features_lib", - "@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_cc_proto", "@envoy_api//envoy/annotations:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", "@utf8_range//:utf8_validity", @@ -105,8 +105,8 @@ envoy_cc_library( deps = [ "utility_lib_header", ] + envoy_select_enable_lite_protos([ - "//bazel/cc_proto_descriptor_library:create_dynamic_message", - "//bazel/cc_proto_descriptor_library:text_format_transcoder", + "@envoy_api//bazel/cc_proto_descriptor_library:create_dynamic_message", + "@envoy_api//bazel/cc_proto_descriptor_library:text_format_transcoder", "@envoy_api//envoy/admin/v3:pkg_cc_proto_descriptor", "@envoy_api//envoy/annotations:pkg_cc_proto_descriptor", "@envoy_api//envoy/config/accesslog/v3:pkg_cc_proto_descriptor", @@ -154,6 +154,7 @@ envoy_cc_library( "@envoy_api//envoy/extensions/http/header_validators/envoy_default/v3:pkg_cc_proto_descriptor", "@envoy_api//envoy/extensions/http/original_ip_detection/xff/v3:pkg_cc_proto_descriptor", "@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto_descriptor", + "@envoy_api//envoy/extensions/load_balancing_policies/cluster_provided/v3:pkg_cc_proto_descriptor", "@envoy_api//envoy/extensions/load_balancing_policies/least_request/v3:pkg_cc_proto_descriptor", "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto_descriptor", "@envoy_api//envoy/extensions/load_balancing_policies/round_robin/v3:pkg_cc_proto_descriptor", @@ -198,6 +199,17 @@ envoy_cc_library( ]), ) +envoy_cc_library( + name = "deterministic_hash_lib", + srcs = ["deterministic_hash.cc"], + hdrs = ["deterministic_hash.h"], + deps = [ + ":protobuf", + "//source/common/common:assert_lib", + "//source/common/common:hash_lib", + ], +) + envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], @@ -205,6 +217,7 @@ envoy_cc_library( "protobuf", ], deps = [ + ":deterministic_hash_lib", ":message_validator_lib", ":protobuf", ":utility_lib_header", @@ -215,7 +228,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/protobuf:visitor_lib", "//source/common/runtime:runtime_features_lib", - "@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_cc_proto", "@envoy_api//envoy/annotations:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", ] + envoy_select_enable_yaml(["yaml_utility_lib"]), @@ -236,7 +249,7 @@ envoy_cc_library( ":message_validator_lib", ":protobuf", ":utility_lib_header", - "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", - "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_github_cncf_xds//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/v3:pkg_cc_proto", ], ) diff --git a/source/common/protobuf/create_reflectable_message.cc b/source/common/protobuf/create_reflectable_message.cc index 41b82e0cdf36..a7614d1b6b78 100644 --- a/source/common/protobuf/create_reflectable_message.cc +++ b/source/common/protobuf/create_reflectable_message.cc @@ -110,6 +110,7 @@ Protobuf::ReflectableMessage createReflectableMessage(const Protobuf::Message& m #include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator_descriptor.pb.h" #include "envoy/extensions/http/original_ip_detection/xff/v3/xff_descriptor.pb.h" #include "envoy/extensions/load_balancing_policies/common/v3/common_descriptor.pb.h" +#include "envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided_descriptor.pb.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request_descriptor.pb.h" #include "envoy/extensions/load_balancing_policies/random/v3/random_descriptor.pb.h" #include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin_descriptor.pb.h" @@ -318,6 +319,9 @@ std::unique_ptr createTranscoder() { kFileDescriptorInfo, protobuf::reflection::envoy_extensions_http_original_ip_detection_xff_v3_xff:: kFileDescriptorInfo, + protobuf::reflection:: + envoy_extensions_load_balancing_policies_cluster_provided_v3_cluster_provided:: + kFileDescriptorInfo, protobuf::reflection::envoy_extensions_load_balancing_policies_common_v3_common:: kFileDescriptorInfo, protobuf::reflection:: diff --git a/source/common/protobuf/deterministic_hash.cc b/source/common/protobuf/deterministic_hash.cc new file mode 100644 index 000000000000..52e18c553297 --- /dev/null +++ b/source/common/protobuf/deterministic_hash.cc @@ -0,0 +1,233 @@ +#if defined(ENVOY_ENABLE_FULL_PROTOS) +#include "source/common/protobuf/deterministic_hash.h" + +#include "source/common/common/assert.h" +#include "source/common/common/hash.h" + +namespace Envoy { +namespace DeterministicProtoHash { +namespace { + +// Get a scalar field from protobuf reflection field definition. The return +// type must be specified by the caller. Every implementation is a specialization +// because the reflection interface did separate named functions instead of a +// template. +template +T reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field); + +template <> +uint32_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetUInt32(message, &field); +} + +template <> +int32_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetInt32(message, &field); +} + +template <> +uint64_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetUInt64(message, &field); +} + +template <> +int64_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetInt64(message, &field); +} + +template <> +float reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetFloat(message, &field); +} + +template <> +double reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetDouble(message, &field); +} + +template <> +bool reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field) { + return reflection.GetBool(message, &field); +} + +// Takes a field of scalar type, and hashes it. In case the field is a repeated field, +// the function hashes each of its elements. +template , bool> = true> +uint64_t hashScalarField(const Protobuf::Reflection& reflection, const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field, uint64_t seed) { + if (field.is_repeated()) { + for (const T& scalar : reflection.GetRepeatedFieldRef(message, &field)) { + seed = HashUtil::xxHash64Value(scalar, seed); + } + } else { + seed = HashUtil::xxHash64Value(reflectionGet(reflection, message, field), seed); + } + return seed; +} + +uint64_t reflectionHashMessage(const Protobuf::Message& message, uint64_t seed = 0); +uint64_t reflectionHashField(const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field, uint64_t seed); + +// To make a map serialize deterministically we need to ignore the order of +// the map fields. To do that, we simply combine the hashes of each entry +// using an unordered operator (addition), and then apply that combined hash to +// the seed. +uint64_t reflectionHashMapField(const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field, uint64_t seed) { + const Protobuf::Reflection& reflection = *message.GetReflection(); + ASSERT(field.is_map()); + const auto& entries = reflection.GetRepeatedFieldRef(message, &field); + ASSERT(!entries.empty()); + const Protobuf::Descriptor& map_descriptor = *entries.begin()->GetDescriptor(); + const Protobuf::FieldDescriptor& key_field = *map_descriptor.map_key(); + const Protobuf::FieldDescriptor& value_field = *map_descriptor.map_value(); + uint64_t combined_hash = 0; + for (const Protobuf::Message& entry : entries) { + uint64_t entry_hash = reflectionHashField(entry, key_field, 0); + entry_hash = reflectionHashField(entry, value_field, entry_hash); + combined_hash += entry_hash; + } + return HashUtil::xxHash64Value(combined_hash, seed); +} + +uint64_t reflectionHashField(const Protobuf::Message& message, + const Protobuf::FieldDescriptor& field, uint64_t seed) { + using Protobuf::FieldDescriptor; + const Protobuf::Reflection& reflection = *message.GetReflection(); + seed = HashUtil::xxHash64Value(field.number(), seed); + switch (field.cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_UINT32: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_INT64: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_UINT64: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_BOOL: + seed = hashScalarField(reflection, message, field, seed); + break; + case FieldDescriptor::CPPTYPE_ENUM: + if (field.is_repeated()) { + const int c = reflection.FieldSize(message, &field); + for (int i = 0; i < c; i++) { + seed = HashUtil::xxHash64Value(reflection.GetRepeatedEnumValue(message, &field, i), seed); + } + } else { + seed = HashUtil::xxHash64Value(reflection.GetEnumValue(message, &field), seed); + } + break; + case FieldDescriptor::CPPTYPE_STRING: + if (field.is_repeated()) { + for (const std::string& str : reflection.GetRepeatedFieldRef(message, &field)) { + seed = HashUtil::xxHash64(str, seed); + } + } else { + // Scratch may be used by GetStringReference if the field is not already a std::string. + std::string scratch; + seed = HashUtil::xxHash64(reflection.GetStringReference(message, &field, &scratch), seed); + } + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + if (field.is_map()) { + seed = reflectionHashMapField(message, field, seed); + } else if (field.is_repeated()) { + for (const Protobuf::Message& submsg : + reflection.GetRepeatedFieldRef(message, &field)) { + seed = reflectionHashMessage(submsg, seed); + } + } else { + seed = reflectionHashMessage(reflection.GetMessage(message, &field), seed); + } + break; + } + return seed; +} + +// Converts from type urls OR descriptor full names to descriptor full names. +// Type urls are as used in envoy yaml config, e.g. +// "type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig" +// becomes +// "envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig" +absl::string_view typeUrlToDescriptorFullName(absl::string_view url) { + const size_t pos = url.rfind('/'); + if (pos != absl::string_view::npos) { + return url.substr(pos + 1); + } + return url; +} + +std::unique_ptr unpackAnyForReflection(const ProtobufWkt::Any& any) { + const Protobuf::Descriptor* descriptor = + Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( + typeUrlToDescriptorFullName(any.type_url())); + // If the type name refers to an unknown type, we treat it the same as other + // unknown fields - not including its contents in the hash. + if (descriptor == nullptr) { + return nullptr; + } + const Protobuf::Message* prototype = + Protobuf::MessageFactory::generated_factory()->GetPrototype(descriptor); + ASSERT(prototype != nullptr, "should be impossible since the descriptor is known"); + std::unique_ptr msg(prototype->New()); + any.UnpackTo(msg.get()); + return msg; +} + +// This is intentionally ignoring unknown fields. +uint64_t reflectionHashMessage(const Protobuf::Message& message, uint64_t seed) { + using Protobuf::FieldDescriptor; + std::string scratch; + const Protobuf::Reflection* reflection = message.GetReflection(); + const Protobuf::Descriptor* descriptor = message.GetDescriptor(); + seed = HashUtil::xxHash64(descriptor->full_name(), seed); + if (descriptor->well_known_type() == Protobuf::Descriptor::WELLKNOWNTYPE_ANY) { + const ProtobufWkt::Any* any = Protobuf::DynamicCastToGenerated(&message); + ASSERT(any != nullptr, "casting to any should always work for WELLKNOWNTYPE_ANY"); + std::unique_ptr submsg = unpackAnyForReflection(*any); + if (submsg == nullptr) { + // If we wanted to handle unknown types in Any, this is where we'd have to do it. + // Since we don't know the type to introspect it, we hash just its type name. + return HashUtil::xxHash64(any->type_url(), seed); + } + return reflectionHashMessage(*submsg, seed); + } + std::vector fields; + // ListFields returned the fields ordered by field number. + reflection->ListFields(message, &fields); + // If we wanted to handle unknown fields, we'd need to also GetUnknownFields here. + for (const FieldDescriptor* field : fields) { + seed = reflectionHashField(message, *field, seed); + } + // Hash one extra character to signify end of message, so that + // msg{} field2=2 + // hashes differently from + // msg{field2=2} + return HashUtil::xxHash64("\x17", seed); +} +} // namespace + +uint64_t hash(const Protobuf::Message& message) { return reflectionHashMessage(message, 0); } + +} // namespace DeterministicProtoHash +} // namespace Envoy +#endif diff --git a/source/common/protobuf/deterministic_hash.h b/source/common/protobuf/deterministic_hash.h new file mode 100644 index 000000000000..64ea0ff76bc1 --- /dev/null +++ b/source/common/protobuf/deterministic_hash.h @@ -0,0 +1,26 @@ +#pragma once + +#include "source/common/protobuf/protobuf.h" + +#if defined(ENVOY_ENABLE_FULL_PROTOS) +namespace Envoy { +namespace DeterministicProtoHash { + +// Note: this ignores unknown fields and unrecognized types in Any fields. +// An alternative approach might treat such fields as "raw data" and include +// them in the hash, which would risk breaking the deterministic behavior, +// versus this way risks ignoring significant data. +// +// Ignoring unknown fields was chosen as the implementation because the +// TextFormat-based hashing this replaces was explicitly ignoring unknown +// fields. +// +// If this is used as part of making a hash table, it may result in +// collisions if unknown fields are present and are not ignored by the +// corresponding comparator. A `MessageDifferencer` can be configured to +// ignore unknown fields, or not to. +uint64_t hash(const Protobuf::Message& message); + +} // namespace DeterministicProtoHash +} // namespace Envoy +#endif diff --git a/source/common/protobuf/message_validator_impl.cc b/source/common/protobuf/message_validator_impl.cc index 7908f688d3ac..15cebb197654 100644 --- a/source/common/protobuf/message_validator_impl.cc +++ b/source/common/protobuf/message_validator_impl.cc @@ -20,7 +20,8 @@ void onDeprecatedFieldCommon(absl::string_view description, bool soft_deprecatio if (soft_deprecation) { ENVOY_LOG_MISC(warn, "Deprecated field: {}", absl::StrCat(description, deprecation_error)); } else { - throw DeprecatedProtoFieldException(absl::StrCat(description, deprecation_error)); + throwExceptionOrPanic(DeprecatedProtoFieldException, + absl::StrCat(description, deprecation_error)); } } } // namespace @@ -75,8 +76,8 @@ void WarningValidationVisitorImpl::onWorkInProgress(absl::string_view descriptio } void StrictValidationVisitorImpl::onUnknownField(absl::string_view description) { - throw UnknownProtoFieldException( - absl::StrCat("Protobuf message (", description, ") has unknown fields")); + throwExceptionOrPanic(UnknownProtoFieldException, + absl::StrCat("Protobuf message (", description, ") has unknown fields")); } void StrictValidationVisitorImpl::onDeprecatedField(absl::string_view description, diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index f6c98b44e5ab..6a9cdb937234 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -10,6 +10,7 @@ #include "source/common/common/assert.h" #include "source/common/common/documentation_url.h" #include "source/common/common/fmt.h" +#include "source/common/protobuf/deterministic_hash.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/visitor.h" @@ -114,45 +115,37 @@ uint64_t fractionalPercentDenominatorToInt( } // namespace ProtobufPercentHelper -MissingFieldException::MissingFieldException(const std::string& field_name, - const Protobuf::Message& message) - : EnvoyException( - fmt::format("Field '{}' is missing in: {}", field_name, message.DebugString())) {} - -ProtoValidationException::ProtoValidationException(const std::string& validation_error, - const Protobuf::Message& message) - : EnvoyException(fmt::format("Proto constraint validation failed ({}): {}", validation_error, - message.DebugString())) { - ENVOY_LOG_MISC(debug, "Proto validation error; throwing {}", what()); -} - void ProtoExceptionUtil::throwMissingFieldException(const std::string& field_name, const Protobuf::Message& message) { - throw MissingFieldException(field_name, message); + std::string error = + fmt::format("Field '{}' is missing in: {}", field_name, message.DebugString()); + throwEnvoyExceptionOrPanic(error); } void ProtoExceptionUtil::throwProtoValidationException(const std::string& validation_error, const Protobuf::Message& message) { - throw ProtoValidationException(validation_error, message); + std::string error = fmt::format("Proto constraint validation failed ({}): {}", validation_error, + message.DebugString()); + throwEnvoyExceptionOrPanic(error); } size_t MessageUtil::hash(const Protobuf::Message& message) { - std::string text_format; - #if defined(ENVOY_ENABLE_FULL_PROTOS) - { + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.use_fast_protobuf_hash")) { + return DeterministicProtoHash::hash(message); + } else { + std::string text_format; Protobuf::TextFormat::Printer printer; printer.SetExpandAny(true); printer.SetUseFieldNumber(true); printer.SetSingleLineMode(true); printer.SetHideUnknownFields(true); printer.PrintToString(message, &text_format); + return HashUtil::xxHash64(text_format); } #else - absl::StrAppend(&text_format, message.SerializeAsString()); + return HashUtil::xxHash64(message.SerializeAsString()); #endif - - return HashUtil::xxHash64(text_format); } #if !defined(ENVOY_ENABLE_FULL_PROTOS) @@ -350,12 +343,12 @@ void MessageUtil::packFrom(ProtobufWkt::Any& any_message, const Protobuf::Messag void MessageUtil::unpackTo(const ProtobufWkt::Any& any_message, Protobuf::Message& message) { #if defined(ENVOY_ENABLE_FULL_PROTOS) if (!any_message.UnpackTo(&message)) { - throw EnvoyException(fmt::format("Unable to unpack as {}: {}", - message.GetDescriptor()->full_name(), - any_message.DebugString())); + throwEnvoyExceptionOrPanic(fmt::format("Unable to unpack as {}: {}", + message.GetDescriptor()->full_name(), + any_message.DebugString())); #else if (!message.ParseFromString(any_message.value())) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Unable to unpack as {}: {}", message.GetTypeName(), any_message.type_url())); #endif } @@ -378,6 +371,17 @@ absl::Status MessageUtil::unpackToNoThrow(const ProtobufWkt::Any& any_message, return absl::OkStatus(); } +std::string MessageUtil::convertToStringForLogs(const Protobuf::Message& message, bool pretty_print, + bool always_print_primitive_fields) { +#ifdef ENVOY_ENABLE_YAML + return getJsonStringFromMessageOrError(message, pretty_print, always_print_primitive_fields); +#else + UNREFERENCED_PARAMETER(pretty_print); + UNREFERENCED_PARAMETER(always_print_primitive_fields); + return message.DebugString(); +#endif +} + ProtobufWkt::Struct MessageUtil::keyValueStruct(const std::string& key, const std::string& value) { ProtobufWkt::Struct struct_obj; ProtobufWkt::Value val; @@ -605,7 +609,7 @@ void MessageUtil::wireCast(const Protobuf::Message& src, Protobuf::Message& dst) // This should should generally succeed, but if there are malformed UTF-8 strings in a message, // this can fail. if (!dst.ParseFromString(src.SerializeAsString())) { - throw EnvoyException("Unable to deserialize during wireCast()"); + throwEnvoyExceptionOrPanic("Unable to deserialize during wireCast()"); } } @@ -728,7 +732,7 @@ absl::Status validateDurationNoThrow(const ProtobufWkt::Duration& duration, void validateDuration(const ProtobufWkt::Duration& duration, int64_t max_seconds_value) { const auto result = validateDurationNoThrow(duration, max_seconds_value); if (!result.ok()) { - throw DurationUtil::OutOfRangeException(std::string(result.message())); + throwEnvoyExceptionOrPanic(std::string(result.message())); } } diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 2a016d4529c1..4d9c5bbdffcb 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -24,7 +24,7 @@ ((message).has_##field_name() ? (message).field_name().value() : (default_value)) // Obtain the value of a wrapped field (e.g. google.protobuf.UInt32Value) if set. Otherwise, throw -// a MissingFieldException. +// a EnvoyException. #define PROTOBUF_GET_WRAPPED_REQUIRED(message, field_name) \ ([](const auto& msg) { \ @@ -51,8 +51,8 @@ DurationUtil::durationToMilliseconds((message).field_name())) \ : absl::nullopt) -// Obtain the milliseconds value of a google.protobuf.Duration field if set. Otherwise, throw a -// MissingFieldException. +// Obtain the milliseconds value of a google.protobuf.Duration field if set. Otherwise, throw an +// EnvoyException. #define PROTOBUF_GET_MS_REQUIRED(message, field_name) \ ([](const auto& msg) { \ if (!msg.has_##field_name()) { \ @@ -61,8 +61,8 @@ return DurationUtil::durationToMilliseconds(msg.field_name()); \ }((message))) -// Obtain the seconds value of a google.protobuf.Duration field if set. Otherwise, throw a -// MissingFieldException. +// Obtain the seconds value of a google.protobuf.Duration field if set. Otherwise, throw an +// EnvoyException. #define PROTOBUF_GET_SECONDS_REQUIRED(message, field_name) \ ([](const auto& msg) { \ if (!msg.has_##field_name()) { \ @@ -133,11 +133,6 @@ uint64_t fractionalPercentDenominatorToInt( namespace Envoy { -class MissingFieldException : public EnvoyException { -public: - MissingFieldException(const std::string& field_name, const Protobuf::Message& message); -}; - class TypeUtil { public: static absl::string_view typeUrlToDescriptorFullName(absl::string_view type_url); @@ -203,10 +198,7 @@ class RepeatedPtrUtil { } }; -class ProtoValidationException : public EnvoyException { -public: - ProtoValidationException(const std::string& validation_error, const Protobuf::Message& message); -}; +using ProtoValidationException = EnvoyException; /** * utility functions to call when throwing exceptions in header files @@ -274,7 +266,7 @@ class MessageUtil { * @param message message to validate. * @param validation_visitor the validation visitor to use. * @param recurse_into_any whether to recurse into Any messages during unexpected checking. - * @throw ProtoValidationException if deprecated fields are used and listed + * @throw EnvoyException if deprecated fields are used and listed * in disallowed_features in runtime_features.h */ static void checkForUnexpectedFields(const Protobuf::Message& message, @@ -294,7 +286,7 @@ class MessageUtil { * @param message message to validate. * @param validation_visitor the validation visitor to use. * @param recurse_into_any whether to recurse into Any messages during unexpected checking. - * @throw ProtoValidationException if the message does not satisfy its type constraints. + * @throw EnvoyException if the message does not satisfy its type constraints. */ template static void validate(const MessageType& message, @@ -338,7 +330,7 @@ class MessageUtil { * of caller. * @param message const Protobuf::Message& to downcast and validate. * @return const MessageType& the concrete message type downcasted to on success. - * @throw ProtoValidationException if the message does not satisfy its type constraints. + * @throw EnvoyException if the message does not satisfy its type constraints. */ template static const MessageType& @@ -426,7 +418,7 @@ class MessageUtil { * @param message source google.protobuf.Any message. * * @return MessageType the typed message inside the Any. - * @throw ProtoValidationException if the message does not satisfy its type constraints. + * @throw EnvoyException if the message does not satisfy its type constraints. */ template static inline void anyConvertAndValidate(const ProtobufWkt::Any& message, @@ -521,6 +513,10 @@ class MessageUtil { bool always_print_primitive_fields = false); #endif + static std::string convertToStringForLogs(const Protobuf::Message& message, + bool pretty_print = false, + bool always_print_primitive_fields = false); + /** * Utility method to create a Struct containing the passed in key/value strings. * @@ -679,18 +675,13 @@ class HashedValue { class DurationUtil { public: - class OutOfRangeException : public EnvoyException { - public: - OutOfRangeException(const std::string& error) : EnvoyException(error) {} - }; - /** * Same as DurationUtil::durationToMilliseconds but with extra validation logic. * Same as Protobuf::util::TimeUtil::DurationToSeconds but with extra validation logic. * Specifically, we ensure that the duration is positive. * @param duration protobuf. * @return duration in milliseconds. - * @throw OutOfRangeException when duration is out-of-range. + * @throw EnvoyException when duration is out-of-range. */ static uint64_t durationToMilliseconds(const ProtobufWkt::Duration& duration); @@ -707,7 +698,7 @@ class DurationUtil { * Specifically, we ensure that the duration is positive. * @param duration protobuf. * @return duration in seconds. - * @throw OutOfRangeException when duration is out-of-range. + * @throw EnvoyException when duration is out-of-range. */ static uint64_t durationToSeconds(const ProtobufWkt::Duration& duration); }; diff --git a/source/common/protobuf/visitor.cc b/source/common/protobuf/visitor.cc index 38e0231dc7a7..1d544a693833 100644 --- a/source/common/protobuf/visitor.cc +++ b/source/common/protobuf/visitor.cc @@ -43,7 +43,8 @@ void traverseMessageWorker(ConstProtoVisitor& visitor, const Protobuf::Message& traverseMessageWorker(visitor, *inner_message, parents, true, recurse_into_any); return; } else if (!target_type_url.empty()) { - throw EnvoyException(fmt::format("Invalid type_url '{}' during traversal", target_type_url)); + throwEnvoyExceptionOrPanic( + fmt::format("Invalid type_url '{}' during traversal", target_type_url)); } } Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message); diff --git a/source/common/protobuf/visitor_helper.h b/source/common/protobuf/visitor_helper.h index 187648dcf970..c9e16c264491 100644 --- a/source/common/protobuf/visitor_helper.h +++ b/source/common/protobuf/visitor_helper.h @@ -24,7 +24,7 @@ convertTypedStruct(const Protobuf::Message& message) { MessageUtil::jsonConvert(typed_struct->value(), ProtobufMessage::getNullValidationVisitor(), *inner_message); #else - throw EnvoyException("JSON and YAML support compiled out."); + throwEnvoyExceptionOrPanic("JSON and YAML support compiled out."); #endif } return {std::move(inner_message), target_type_url}; diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index c22004dfa5cc..92d82da896b7 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -183,7 +183,9 @@ void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& messa void MessageUtil::loadFromFile(const std::string& path, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) { - const std::string contents = api.fileSystem().fileReadToEnd(path); + auto file_or_error = api.fileSystem().fileReadToEnd(path); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + const std::string contents = file_or_error.value(); // If the filename ends with .pb, attempt to parse it as a binary proto. if (absl::EndsWithIgnoreCase(path, FileExtensions::get().ProtoBinary)) { // Attempt to parse the binary format. @@ -345,24 +347,9 @@ std::string utf8CoerceToStructurallyValid(absl::string_view str, const char repl } // namespace std::string MessageUtil::sanitizeUtf8String(absl::string_view input) { - if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.service_sanitize_non_utf8_strings")) { - return std::string(input); - } - - // This returns the original string if it was already valid, and returns a pointer to - // `result.data()` if it needed to coerce. The coerced string is always - // the same length as the source string. - // - // Initializing `result` to `input` ensures that `result` is correctly sized to receive the - // modified string, or in the case where no modification is needed it already contains the correct - // value, so `result` can be returned in both cases. - // // The choice of '!' is somewhat arbitrary, but we wanted to avoid any character that has // special semantic meaning in URLs or similar. - std::string result = utf8CoerceToStructurallyValid(input, '!'); - - return result; + return utf8CoerceToStructurallyValid(input, '!'); } } // namespace Envoy diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index e1769f3919c4..c4511a02ee53 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -108,6 +108,7 @@ envoy_cc_library( ":quic_io_handle_wrapper_lib", ":quic_transport_socket_factory_lib", "//envoy/ssl:tls_certificate_config_interface", + "//source/common/quic:quic_server_transport_socket_factory_lib", "//source/common/stream_info:stream_info_lib", "//source/server:listener_stats", "@com_github_google_quiche//:quic_core_crypto_certificate_view_lib", @@ -142,16 +143,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "spdy_server_push_utils_for_envoy_lib", - srcs = ["spdy_server_push_utils_for_envoy.cc"], - tags = ["nofips"], - deps = [ - "//source/common/common:assert_lib", - "@com_github_google_quiche//:quic_core_http_spdy_server_push_utils_header", - ], -) - envoy_cc_library( name = "envoy_quic_stream_lib", hdrs = ["envoy_quic_stream.h"], @@ -336,7 +327,6 @@ envoy_cc_library( tags = ["nofips"], deps = [ "//envoy/network:connection_interface", - "//source/common/network:listen_socket_lib", ], ) @@ -349,6 +339,7 @@ envoy_cc_library( ":quic_io_handle_wrapper_lib", ":quic_network_connection_lib", "//source/common/network:generic_listener_filter_impl_base_lib", + "//source/common/network:listen_socket_lib", "//source/common/quic:envoy_quic_utils_lib", "@com_github_google_quiche//:quic_core_connection_lib", ], @@ -434,7 +425,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:header_utility_lib", "//source/common/network:address_lib", - "//source/common/network:listen_socket_lib", + "//source/common/network:connection_socket_lib", "//source/common/network:socket_option_factory_lib", "//source/common/quic:quic_io_handle_wrapper_lib", "@com_github_google_quiche//:quic_core_config_lib", @@ -446,8 +437,14 @@ envoy_cc_library( envoy_cc_library( name = "quic_transport_socket_factory_lib", - srcs = ["quic_transport_socket_factory.cc"], - hdrs = ["quic_transport_socket_factory.h"], + srcs = [ + "quic_client_transport_socket_factory.cc", + "quic_transport_socket_factory.cc", + ], + hdrs = [ + "quic_client_transport_socket_factory.h", + "quic_transport_socket_factory.h", + ], tags = ["nofips"], deps = [ ":envoy_quic_proof_verifier_lib", @@ -464,6 +461,31 @@ envoy_cc_library( alwayslink = LEGACY_ALWAYSLINK, ) +envoy_cc_library( + name = "quic_server_transport_socket_factory_lib", + srcs = [ + "quic_server_transport_socket_factory.cc", + ], + hdrs = [ + "quic_server_transport_socket_factory.h", + ], + tags = ["nofips"], + deps = [ + ":envoy_quic_proof_verifier_lib", + ":quic_transport_socket_factory_lib", + "//envoy/network:transport_socket_interface", + "//envoy/server:transport_socket_config_interface", + "//envoy/ssl:context_config_interface", + "//source/common/common:assert_lib", + "//source/common/network:transport_socket_options_lib", + "//source/extensions/transport_sockets/tls:context_config_lib", + "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "@com_github_google_quiche//:quic_core_crypto_crypto_handshake_lib", + "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto", + ], + alwayslink = LEGACY_ALWAYSLINK, +) + # Create a single target that contains all the libraries that register client factories. # All of these are needed for this extension to function. envoy_cc_library( @@ -542,8 +564,10 @@ envoy_cc_library( "//source/common/network:io_socket_error_lib", "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_lib", - "@com_github_google_quiche//:quic_core_batch_writer_gso_batch_writer_lib", - ], + ] + select({ + "//bazel:linux": ["@com_github_google_quiche//:quic_core_batch_writer_gso_batch_writer_lib"], + "//conditions:default": [], + }), ) envoy_cc_library( @@ -552,6 +576,7 @@ envoy_cc_library( hdrs = ["send_buffer_monitor.h"], tags = ["nofips"], deps = [ + "//source/common/common:assert_lib", "@com_github_google_quiche//:quic_core_session_lib", ], ) @@ -561,7 +586,9 @@ envoy_cc_library( hdrs = ["envoy_quic_client_crypto_stream_factory.h"], tags = ["nofips"], deps = [ + "//envoy/common:optref_lib", "//envoy/config:typed_config_interface", + "//envoy/network:transport_socket_interface", "@com_github_google_quiche//:quic_client_session_lib", "@com_github_google_quiche//:quic_core_http_spdy_session_lib", ], @@ -617,6 +644,7 @@ envoy_cc_library( tags = ["nofips"], deps = [ "//envoy/access_log:access_log_interface", + "//envoy/http:codec_interface", "//source/common/http:header_map_lib", "@com_github_google_quiche//:quic_core_ack_listener_interface_lib", ], @@ -628,6 +656,7 @@ envoy_cc_library( hdrs = ["http_datagram_handler.h"], deps = [ "//envoy/http:codec_interface", + "//source/common/buffer:buffer_lib", "//source/common/common:logger_lib", "//source/common/http:header_map_lib", "@com_github_google_quiche//:quic_core_http_spdy_session_lib", diff --git a/source/common/quic/active_quic_listener.cc b/source/common/quic/active_quic_listener.cc index c27f7653a6cf..ccdc9897e8d0 100644 --- a/source/common/quic/active_quic_listener.cc +++ b/source/common/quic/active_quic_listener.cc @@ -10,6 +10,7 @@ #include "source/common/config/utility.h" #include "source/common/http/utility.h" #include "source/common/network/socket_option_impl.h" +#include "source/common/network/udp_listener_impl.h" #include "source/common/quic/envoy_quic_alarm_factory.h" #include "source/common/quic/envoy_quic_connection_helper.h" #include "source/common/quic/envoy_quic_dispatcher.h" @@ -36,8 +37,8 @@ ActiveQuicListener::ActiveQuicListener( QuicConnectionIdGeneratorPtr&& cid_generator, QuicConnectionIdWorkerSelector worker_selector) : Server::ActiveUdpListenerBase( worker_index, concurrency, parent, *listen_socket, - dispatcher.createUdpListener( - listen_socket, *this, + std::make_unique( + dispatcher, listen_socket, *this, dispatcher.timeSource(), listener_config.udpListenerConfig()->config().downstream_socket_config()), &listener_config), dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedHttp3Versions()), diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc index 269dbbdf078f..d3efeba84e51 100644 --- a/source/common/quic/client_connection_factory_impl.cc +++ b/source/common/quic/client_connection_factory_impl.cc @@ -59,10 +59,9 @@ std::unique_ptr createQuicNetworkConnection( // QUICHE client session always use the 1st version to start handshake. return std::make_unique( - config, quic_versions, std::move(connection), server_id, std::move(crypto_config), - &info_impl->push_promise_index_, dispatcher, info_impl->buffer_limit_, - info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, scope, - transport_socket_options); + config, quic_versions, std::move(connection), server_id, std::move(crypto_config), dispatcher, + info_impl->buffer_limit_, info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, + scope, transport_socket_options); } } // namespace Quic diff --git a/source/common/quic/client_connection_factory_impl.h b/source/common/quic/client_connection_factory_impl.h index 4f21edb3038b..e2b8eede0e78 100644 --- a/source/common/quic/client_connection_factory_impl.h +++ b/source/common/quic/client_connection_factory_impl.h @@ -12,7 +12,6 @@ #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_client_stream.h" #include "source/extensions/transport_sockets/tls/ssl_socket.h" -#include "quiche/quic/core/http/quic_client_push_promise_index.h" #include "quiche/quic/core/quic_utils.h" namespace Envoy { @@ -29,9 +28,6 @@ struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo { quic::QuicConfig quic_config_; // The connection send buffer limits from cluster config. const uint32_t buffer_limit_; - // This arguably should not be shared across connections but as Envoy doesn't - // support push promise it's really moot point. - quic::QuicClientPushPromiseIndex push_promise_index_; // Hard code with the default crypto stream as there's no pluggable crypto for upstream Envoy. EnvoyQuicCryptoClientStreamFactoryImpl crypto_stream_factory_; }; diff --git a/source/common/quic/envoy_quic_client_connection.cc b/source/common/quic/envoy_quic_client_connection.cc index 9fd5f64bbe48..09a8745572bd 100644 --- a/source/common/quic/envoy_quic_client_connection.cc +++ b/source/common/quic/envoy_quic_client_connection.cc @@ -4,7 +4,6 @@ #include "envoy/config/core/v3/base.pb.h" -#include "source/common/network/listen_socket_impl.h" #include "source/common/network/socket_option_factory.h" #include "source/common/network/udp_packet_writer_handler_impl.h" #include "source/common/quic/envoy_quic_utils.h" @@ -107,6 +106,7 @@ void EnvoyQuicClientConnection::switchConnectionSocket( connection_socket->connectionInfoProvider().remoteAddress()->ip()); // The old socket is not closed in this call, because it could still receive useful packets. + num_socket_switches_++; setConnectionSocket(std::move(connection_socket)); setUpConnectionSocket(*connectionSocket(), delegate_); MigratePath(self_address, peer_address, writer.release(), true); @@ -118,7 +118,8 @@ void EnvoyQuicClientConnection::OnPathDegradingDetected() { } void EnvoyQuicClientConnection::maybeMigratePort() { - if (!IsHandshakeConfirmed() || HasPendingPathValidation() || !migrate_port_on_path_degrading_) { + if (!IsHandshakeConfirmed() || HasPendingPathValidation() || !migrate_port_on_path_degrading_ || + num_socket_switches_ >= kMaxNumSocketSwitches) { return; } @@ -173,6 +174,7 @@ void EnvoyQuicClientConnection::onPathValidationSuccess( peer_address() == envoy_context->peer_address()) { // probing_socket will be set as the new default socket. But old sockets are still able to // receive packets. + num_socket_switches_++; setConnectionSocket(std::move(probing_socket)); return; } diff --git a/source/common/quic/envoy_quic_client_connection.h b/source/common/quic/envoy_quic_client_connection.h index 5dc469049fd7..2ddd3e523e76 100644 --- a/source/common/quic/envoy_quic_client_connection.h +++ b/source/common/quic/envoy_quic_client_connection.h @@ -12,6 +12,9 @@ namespace Envoy { namespace Quic { +// Limits the max number of sockets created. +constexpr uint8_t kMaxNumSocketSwitches = 5; + class PacketsToReadDelegate { public: virtual ~PacketsToReadDelegate() = default; @@ -143,6 +146,7 @@ class EnvoyQuicClientConnection : public quic::QuicConnection, uint32_t packets_dropped_{0}; Event::Dispatcher& dispatcher_; bool migrate_port_on_path_degrading_{false}; + uint8_t num_socket_switches_{0}; }; } // namespace Quic diff --git a/source/common/quic/envoy_quic_client_session.cc b/source/common/quic/envoy_quic_client_session.cc index 152e2a484b93..d345a4fde0b9 100644 --- a/source/common/quic/envoy_quic_client_session.cc +++ b/source/common/quic/envoy_quic_client_session.cc @@ -46,8 +46,7 @@ class EnvoyQuicProofVerifyContextImpl : public EnvoyQuicProofVerifyContext { EnvoyQuicClientSession::EnvoyQuicClientSession( const quic::QuicConfig& config, const quic::ParsedQuicVersionVector& supported_versions, std::unique_ptr connection, const quic::QuicServerId& server_id, - std::shared_ptr crypto_config, - quic::QuicClientPushPromiseIndex* push_promise_index, Event::Dispatcher& dispatcher, + std::shared_ptr crypto_config, Event::Dispatcher& dispatcher, uint32_t send_buffer_limit, EnvoyQuicCryptoClientStreamFactoryInterface& crypto_stream_factory, QuicStatNames& quic_stat_names, OptRef rtt_cache, Stats::Scope& scope, @@ -59,7 +58,7 @@ EnvoyQuicClientSession::EnvoyQuicClientSession( dispatcher.timeSource(), connection->connectionSocket()->connectionInfoProviderSharedPtr())), quic::QuicSpdyClientSession(config, supported_versions, connection.release(), server_id, - crypto_config.get(), push_promise_index), + crypto_config.get()), crypto_config_(crypto_config), crypto_stream_factory_(crypto_stream_factory), quic_stat_names_(quic_stat_names), rtt_cache_(rtt_cache), scope_(scope), transport_socket_options_(transport_socket_options) { @@ -212,7 +211,7 @@ void EnvoyQuicClientSession::setHttp3Options( } static_cast(connection()) ->setNumPtosForPortMigration(PROTOBUF_GET_WRAPPED_OR_DEFAULT( - http3_options.quic_protocol_options(), num_timeouts_to_trigger_port_migration, 1)); + http3_options.quic_protocol_options(), num_timeouts_to_trigger_port_migration, 4)); if (http3_options_->quic_protocol_options().has_connection_keepalive()) { const uint64_t initial_interval = PROTOBUF_GET_MS_OR_DEFAULT( diff --git a/source/common/quic/envoy_quic_client_session.h b/source/common/quic/envoy_quic_client_session.h index a5dff63fb883..4e615fc5daf2 100644 --- a/source/common/quic/envoy_quic_client_session.h +++ b/source/common/quic/envoy_quic_client_session.h @@ -26,8 +26,7 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl, EnvoyQuicClientSession( const quic::QuicConfig& config, const quic::ParsedQuicVersionVector& supported_versions, std::unique_ptr connection, const quic::QuicServerId& server_id, - std::shared_ptr crypto_config, - quic::QuicClientPushPromiseIndex* push_promise_index, Event::Dispatcher& dispatcher, + std::shared_ptr crypto_config, Event::Dispatcher& dispatcher, uint32_t send_buffer_limit, EnvoyQuicCryptoClientStreamFactoryInterface& crypto_stream_factory, QuicStatNames& quic_stat_names, OptRef rtt_cache, diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc index d90c326b2da3..1c0211e083d4 100644 --- a/source/common/quic/envoy_quic_client_stream.cc +++ b/source/common/quic/envoy_quic_client_stream.cc @@ -91,9 +91,9 @@ Http::Status EnvoyQuicClientStream::encodeHeaders(const Http::RequestHeaderMap& #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") && (Http::HeaderUtility::isCapsuleProtocol(headers) || - Http::HeaderUtility::isConnectUdp(headers))) { + Http::HeaderUtility::isConnectUdpRequest(headers))) { useCapsuleProtocol(); - if (Http::HeaderUtility::isConnectUdp(headers)) { + if (Http::HeaderUtility::isConnectUdpRequest(headers)) { // HTTP/3 Datagrams sent over CONNECT-UDP are already congestion controlled, so make it // bypass the default Datagram queue. session()->SetForceFlushForDefaultQueue(true); @@ -147,7 +147,19 @@ void EnvoyQuicClientStream::encodeData(Buffer::Instance& data, bool end_stream) // TODO(danzh): investigate the cost of allocating one buffer per slice. // If it turns out to be expensive, add a new function to free data in the middle in buffer // interface and re-design QuicheMemSliceImpl. - quic_slices.emplace_back(quiche::QuicheMemSlice::InPlace(), data, slice.len_); + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.quiche_use_mem_slice_releasor_api")) { + quic_slices.emplace_back(quiche::QuicheMemSlice::InPlace(), data, slice.len_); + } else { + auto single_slice_buffer = std::make_unique(); + single_slice_buffer->move(data, slice.len_); + quic_slices.emplace_back( + reinterpret_cast(slice.mem_), slice.len_, + [single_slice_buffer = std::move(single_slice_buffer)](const char*) mutable { + // Free this memory explicitly when the callback is invoked. + single_slice_buffer = nullptr; + }); + } } quic::QuicConsumedData result{0, false}; absl::Span span(quic_slices); diff --git a/source/common/quic/envoy_quic_client_stream.h b/source/common/quic/envoy_quic_client_stream.h index e8a6272d07e1..edaba1b04ea5 100644 --- a/source/common/quic/envoy_quic_client_stream.h +++ b/source/common/quic/envoy_quic_client_stream.h @@ -3,8 +3,10 @@ #include "envoy/buffer/buffer.h" #include "source/common/quic/envoy_quic_stream.h" -#include "source/common/quic/http_datagram_handler.h" +#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS +#include "source/common/quic/http_datagram_handler.h" +#endif #include "quiche/common/simple_buffer_allocator.h" #include "quiche/quic/core/http/quic_spdy_client_stream.h" diff --git a/source/common/quic/envoy_quic_dispatcher.cc b/source/common/quic/envoy_quic_dispatcher.cc index 665b51ac4797..dd4f14183b0a 100644 --- a/source/common/quic/envoy_quic_dispatcher.cc +++ b/source/common/quic/envoy_quic_dispatcher.cc @@ -177,13 +177,26 @@ void EnvoyQuicDispatcher::closeConnectionsWithFilterChain( // Retain the number of connections in the list early because closing the connection will change // the size. const size_t num_connections = connections.size(); + bool delete_sessions_immediately = false; for (size_t i = 0; i < num_connections; ++i) { Network::Connection& connection = connections.front().get(); // This will remove the connection from the list. And the last removal will remove connections // from the map as well. connection.close(Network::ConnectionCloseType::NoFlush); + if (!delete_sessions_immediately && + dynamic_cast(connection).fix_quic_lifetime_issues()) { + // If `envoy.reloadable_features.quic_fix_filter_manager_uaf` is true, the closed sessions + // need to be deleted right away to consistently handle quic lifetimes. Because upon + // returning the filter chain configs will be destroyed, and no longer safe to be accessed. + // If any filters access those configs during destruction, it'll be use-after-free + delete_sessions_immediately = true; + } } ASSERT(connections_by_filter_chain_.find(filter_chain) == connections_by_filter_chain_.end()); + if (delete_sessions_immediately) { + // Explicitly destroy closed sessions in the current call stack. + DeleteSessions(); + } } } diff --git a/source/common/quic/envoy_quic_proof_source.h b/source/common/quic/envoy_quic_proof_source.h index bc3b3c127f32..d47ab9ba7bb3 100644 --- a/source/common/quic/envoy_quic_proof_source.h +++ b/source/common/quic/envoy_quic_proof_source.h @@ -1,7 +1,7 @@ #pragma once #include "source/common/quic/envoy_quic_proof_source_base.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_server_transport_socket_factory.h" #include "source/server/listener_stats.h" namespace Envoy { diff --git a/source/common/quic/envoy_quic_server_connection.h b/source/common/quic/envoy_quic_server_connection.h index 468257decfe8..5c74644c880d 100644 --- a/source/common/quic/envoy_quic_server_connection.h +++ b/source/common/quic/envoy_quic_server_connection.h @@ -78,8 +78,8 @@ class QuicListenerFilterManagerImpl : public Network::QuicListenerFilterManager, } bool shouldAdvertiseServerPreferredAddress( const quic::QuicSocketAddress& server_preferred_address) const override { - for (auto iter = accept_filters_.begin(); iter != accept_filters_.end(); iter++) { - if (!(*iter)->isCompatibleWithServerPreferredAddress(server_preferred_address)) { + for (const auto& accept_filter : accept_filters_) { + if (!accept_filter->isCompatibleWithServerPreferredAddress(server_preferred_address)) { return false; } } @@ -87,8 +87,8 @@ class QuicListenerFilterManagerImpl : public Network::QuicListenerFilterManager, } void onPeerAddressChanged(const quic::QuicSocketAddress& new_address, Network::Connection& connection) override { - for (auto iter = accept_filters_.begin(); iter != accept_filters_.end(); iter++) { - Network::FilterStatus status = (*iter)->onPeerAddressChanged(new_address, connection); + for (auto& accept_filter : accept_filters_) { + Network::FilterStatus status = accept_filter->onPeerAddressChanged(new_address, connection); if (status == Network::FilterStatus::StopIteration || connection.state() != Network::Connection::State::Open) { return; @@ -96,8 +96,8 @@ class QuicListenerFilterManagerImpl : public Network::QuicListenerFilterManager, } } void startFilterChain() { - for (auto iter = accept_filters_.begin(); iter != accept_filters_.end(); iter++) { - Network::FilterStatus status = (*iter)->onAccept(*this); + for (auto& accept_filter : accept_filters_) { + Network::FilterStatus status = accept_filter->onAccept(*this); if (status == Network::FilterStatus::StopIteration || !socket().ioHandle().isOpen()) { break; } diff --git a/source/common/quic/envoy_quic_server_session.cc b/source/common/quic/envoy_quic_server_session.cc index b0b016b7a898..66fdac04de41 100644 --- a/source/common/quic/envoy_quic_server_session.cc +++ b/source/common/quic/envoy_quic_server_session.cc @@ -207,6 +207,7 @@ void EnvoyQuicServerSession::ProcessUdpPacket(const quic::QuicSocketAddress& sel self_address == connection()->sent_server_preferred_address()) { connection_stats_.num_packets_rx_on_preferred_address_.inc(); } + maybeApplyDelayedClose(); } } // namespace Quic diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc index 47acf20f16d1..3ccc044d385b 100644 --- a/source/common/quic/envoy_quic_server_stream.cc +++ b/source/common/quic/envoy_quic_server_stream.cc @@ -121,7 +121,19 @@ void EnvoyQuicServerStream::encodeData(Buffer::Instance& data, bool end_stream) // TODO(danzh): investigate the cost of allocating one buffer per slice. // If it turns out to be expensive, add a new function to free data in the middle in buffer // interface and re-design QuicheMemSliceImpl. - quic_slices.emplace_back(quiche::QuicheMemSlice::InPlace(), data, slice.len_); + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.quiche_use_mem_slice_releasor_api")) { + quic_slices.emplace_back(quiche::QuicheMemSlice::InPlace(), data, slice.len_); + } else { + auto single_slice_buffer = std::make_unique(); + single_slice_buffer->move(data, slice.len_); + quic_slices.emplace_back( + reinterpret_cast(slice.mem_), slice.len_, + [single_slice_buffer = std::move(single_slice_buffer)](const char*) mutable { + // Free this memory explicitly when the callback is invoked. + single_slice_buffer = nullptr; + }); + } } quic::QuicConsumedData result{0, false}; absl::Span span(quic_slices); @@ -270,11 +282,11 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len, #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") && (Http::HeaderUtility::isCapsuleProtocol(*headers) || - Http::HeaderUtility::isConnectUdp(*headers))) { + Http::HeaderUtility::isConnectUdpRequest(*headers))) { useCapsuleProtocol(); // HTTP/3 Datagrams sent over CONNECT-UDP are already congestion controlled, so make it bypass // the default Datagram queue. - if (Http::HeaderUtility::isConnectUdp(*headers)) { + if (Http::HeaderUtility::isConnectUdpRequest(*headers)) { session()->SetForceFlushForDefaultQueue(true); } } diff --git a/source/common/quic/envoy_quic_server_stream.h b/source/common/quic/envoy_quic_server_stream.h index 35cef92dfe22..146e4f8b0f04 100644 --- a/source/common/quic/envoy_quic_server_stream.h +++ b/source/common/quic/envoy_quic_server_stream.h @@ -1,7 +1,10 @@ #pragma once #include "source/common/quic/envoy_quic_stream.h" + +#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS #include "source/common/quic/http_datagram_handler.h" +#endif #include "source/common/quic/quic_stats_gatherer.h" #include "quiche/common/platform/api/quiche_reference_counted.h" diff --git a/source/common/quic/envoy_quic_utils.cc b/source/common/quic/envoy_quic_utils.cc index 7dc7cb7c16f6..b036737992b9 100644 --- a/source/common/quic/envoy_quic_utils.cc +++ b/source/common/quic/envoy_quic_utils.cc @@ -285,9 +285,8 @@ void adjustNewConnectionIdForRouting(quic::QuicConnectionId& new_connection_id, const quic::QuicConnectionId& old_connection_id) { char* new_connection_id_data = new_connection_id.mutable_data(); const char* old_connection_id_ptr = old_connection_id.data(); - auto* first_four_bytes = reinterpret_cast(old_connection_id_ptr); // Override the first 4 bytes of the new CID to the original CID's first 4 bytes. - safeMemcpyUnsafeDst(new_connection_id_data, first_four_bytes); + memcpy(new_connection_id_data, old_connection_id_ptr, 4); // NOLINT(safe-memcpy) } } // namespace Quic diff --git a/source/common/quic/envoy_quic_utils.h b/source/common/quic/envoy_quic_utils.h index e5fcc2532a7b..ae63d9a78297 100644 --- a/source/common/quic/envoy_quic_utils.h +++ b/source/common/quic/envoy_quic_utils.h @@ -8,7 +8,7 @@ #include "source/common/http/header_map_impl.h" #include "source/common/http/header_utility.h" #include "source/common/network/address_impl.h" -#include "source/common/network/listen_socket_impl.h" +#include "source/common/network/connection_socket_impl.h" #include "source/common/quic/quic_io_handle_wrapper.h" #include "openssl/ssl.h" diff --git a/source/common/quic/http_datagram_handler.cc b/source/common/quic/http_datagram_handler.cc index 1af50e90a510..3ff34d2e524b 100644 --- a/source/common/quic/http_datagram_handler.cc +++ b/source/common/quic/http_datagram_handler.cc @@ -1,10 +1,12 @@ #include "source/common/quic/http_datagram_handler.h" +#include "source/common/buffer/buffer_impl.h" #include "source/common/common/logger.h" #include "source/common/http/header_map_impl.h" #include "absl/strings/string_view.h" -#include "quiche/common/simple_buffer_allocator.h" +#include "quiche/common/capsule.h" +#include "quiche/common/quiche_buffer_allocator.h" #include "quiche/quic/core/http/quic_spdy_stream.h" namespace Envoy { diff --git a/source/common/quic/platform/BUILD b/source/common/quic/platform/BUILD index f9eb06a247b5..ad64a7bc53e9 100644 --- a/source/common/quic/platform/BUILD +++ b/source/common/quic/platform/BUILD @@ -60,6 +60,7 @@ envoy_quiche_platform_impl_cc_library( hdrs = ["quiche_time_utils_impl.h"], external_deps = [ "abseil_base", + "abseil_optional", "abseil_time", ], ) @@ -106,15 +107,10 @@ envoy_quiche_platform_impl_cc_library( hdrs = [ "quiche_iovec_impl.h", ], - external_deps = [ - "abseil_base", - "abseil_hash", - "abseil_inlined_vector", - "abseil_memory", - "abseil_node_hash_map", - "abseil_node_hash_set", - ], tags = ["nofips"], + deps = [ + "//envoy/common:base_includes", + ], ) envoy_quiche_platform_impl_cc_library( @@ -146,6 +142,7 @@ envoy_quiche_platform_impl_cc_library( ":quiche_logging_impl_lib", "//source/common/buffer:buffer_lib", "@com_github_google_quiche//:quiche_common_buffer_allocator_lib", + "@com_github_google_quiche//:quiche_common_callbacks", ], ) diff --git a/source/common/quic/platform/mobile_impl/BUILD b/source/common/quic/platform/mobile_impl/BUILD new file mode 100644 index 000000000000..45b447c35c6b --- /dev/null +++ b/source/common/quic/platform/mobile_impl/BUILD @@ -0,0 +1,23 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//source/common/quic/platform:quiche.bzl", + "envoy_quiche_platform_impl_cc_library", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_quiche_platform_impl_cc_library( + name = "mobile_quiche_bug_tracker_impl_lib", + hdrs = [ + "quiche_bug_tracker_impl.h", + ], + tags = ["nofips"], + deps = [ + "@com_github_google_quiche//:quiche_common_platform_logging", + ], +) diff --git a/source/common/quic/platform/mobile_impl/quiche_bug_tracker_impl.h b/source/common/quic/platform/mobile_impl/quiche_bug_tracker_impl.h new file mode 100644 index 000000000000..8ed11ed422c2 --- /dev/null +++ b/source/common/quic/platform/mobile_impl/quiche_bug_tracker_impl.h @@ -0,0 +1,14 @@ +#pragma once + +// NOLINT(namespace-envoy) +// +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include "quiche/common/platform/api/quiche_logging.h" + +#define QUICHE_BUG_IMPL(b) QUICHE_DLOG(DFATAL) << #b ": " +#define QUICHE_BUG_IF_IMPL(b, condition) QUICHE_DLOG_IF(DFATAL, condition) << #b ": " +#define QUICHE_PEER_BUG_IMPL(b) QUICHE_DLOG(DFATAL) << #b ": " +#define QUICHE_PEER_BUG_IF_IMPL(b, condition) QUICHE_DLOG_IF(DFATAL, condition) << #b ": " diff --git a/source/common/quic/platform/quiche_flags_impl.cc b/source/common/quic/platform/quiche_flags_impl.cc index 187ce518720d..60dc6622a7a7 100644 --- a/source/common/quic/platform/quiche_flags_impl.cc +++ b/source/common/quic/platform/quiche_flags_impl.cc @@ -63,14 +63,6 @@ template <> constexpr int32_t maybeOverride(absl::string_view name, int // Flag definitions #define QUIC_FLAG(flag, value) ABSL_FLAG(bool, envoy_##flag, maybeOverride(#flag, value), ""); #include "quiche/quic/core/quic_flags_list.h" -QUIC_FLAG(quic_reloadable_flag_spdy_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_reloadable_flag_spdy_testonly_default_true, true) // NOLINT -QUIC_FLAG(quic_restart_flag_spdy_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_restart_flag_spdy_testonly_default_true, true) // NOLINT -QUIC_FLAG(quic_reloadable_flag_http2_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_reloadable_flag_http2_testonly_default_true, true) // NOLINT -QUIC_FLAG(quic_restart_flag_http2_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_restart_flag_http2_testonly_default_true, true) // NOLINT #undef QUIC_FLAG #define DEFINE_PROTOCOL_FLAG_IMPL(type, flag, value, help) \ diff --git a/source/common/quic/platform/quiche_flags_impl.h b/source/common/quic/platform/quiche_flags_impl.h index de425c9b05eb..e65277a3b9e4 100644 --- a/source/common/quic/platform/quiche_flags_impl.h +++ b/source/common/quic/platform/quiche_flags_impl.h @@ -47,14 +47,6 @@ class FlagRegistry { // Flag declarations #define QUIC_FLAG(flag, ...) ABSL_DECLARE_FLAG(bool, envoy_##flag); #include "quiche/quic/core/quic_flags_list.h" -QUIC_FLAG(quic_reloadable_flag_spdy_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_reloadable_flag_spdy_testonly_default_true, true) // NOLINT -QUIC_FLAG(quic_restart_flag_spdy_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_restart_flag_spdy_testonly_default_true, true) // NOLINT -QUIC_FLAG(quic_reloadable_flag_http2_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_reloadable_flag_http2_testonly_default_true, true) // NOLINT -QUIC_FLAG(quic_restart_flag_http2_testonly_default_false, false) // NOLINT -QUIC_FLAG(quic_restart_flag_http2_testonly_default_true, true) // NOLINT #undef QUIC_FLAG #define QUIC_PROTOCOL_FLAG(type, flag, ...) ABSL_DECLARE_FLAG(type, envoy_##flag); diff --git a/source/common/quic/platform/quiche_mem_slice_impl.cc b/source/common/quic/platform/quiche_mem_slice_impl.cc index 922959bc6ab8..44ffc80338a2 100644 --- a/source/common/quic/platform/quiche_mem_slice_impl.cc +++ b/source/common/quic/platform/quiche_mem_slice_impl.cc @@ -52,6 +52,18 @@ QuicheMemSliceImpl::QuicheMemSliceImpl(std::unique_ptr buffer, size_t le ASSERT(this->length() == length); } +QuicheMemSliceImpl::QuicheMemSliceImpl(char buffer[], size_t length, + SingleUseCallback deleter) + : fragment_(std::make_unique( + buffer, length, + [deleter = std::move(deleter)](const void* p, size_t, + const Envoy::Buffer::BufferFragmentImpl*) mutable { + std::move(deleter)(reinterpret_cast(p)); + })) { + single_slice_buffer_.addBufferFragment(*fragment_); + ASSERT(this->length() == length); +} + QuicheMemSliceImpl::~QuicheMemSliceImpl() { ASSERT(fragment_ == nullptr || (firstSliceLength(single_slice_buffer_) == fragment_->size() && data() == fragment_->data())); diff --git a/source/common/quic/platform/quiche_mem_slice_impl.h b/source/common/quic/platform/quiche_mem_slice_impl.h index 0521f1a45420..bc43f65d5947 100644 --- a/source/common/quic/platform/quiche_mem_slice_impl.h +++ b/source/common/quic/platform/quiche_mem_slice_impl.h @@ -13,6 +13,7 @@ #include "source/common/buffer/buffer_impl.h" #include "quiche/common/quiche_buffer_allocator.h" +#include "quiche/common/quiche_callbacks.h" namespace quiche { @@ -28,6 +29,7 @@ class QuicheMemSliceImpl { // Constructs a QuicheMemSliceImpl by taking ownership of the memory in `buffer`. QuicheMemSliceImpl(quiche::QuicheBuffer buffer); QuicheMemSliceImpl(std::unique_ptr buffer, size_t length); + QuicheMemSliceImpl(char buffer[], size_t length, SingleUseCallback); // Constructs a QuicheMemSliceImpl and moves the first slice of `buffer` into // it. Prerequisite: `buffer` must be non-empty, and its first slice must diff --git a/source/common/quic/quic_client_transport_socket_factory.cc b/source/common/quic/quic_client_transport_socket_factory.cc new file mode 100644 index 000000000000..1e0f289c319e --- /dev/null +++ b/source/common/quic/quic_client_transport_socket_factory.cc @@ -0,0 +1,66 @@ +#include "source/common/quic/quic_client_transport_socket_factory.h" + +#include + +#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.validate.h" + +#include "source/common/quic/envoy_quic_proof_verifier.h" +#include "source/common/runtime/runtime_features.h" +#include "source/extensions/transport_sockets/tls/context_config_impl.h" + +#include "quiche/quic/core/crypto/quic_client_session_cache.h" + +namespace Envoy { +namespace Quic { + +Network::UpstreamTransportSocketFactoryPtr +QuicClientTransportSocketConfigFactory::createTransportSocketFactory( + const Protobuf::Message& config, + Server::Configuration::TransportSocketFactoryContext& context) { + auto quic_transport = MessageUtil::downcastAndValidate< + const envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport&>( + config, context.messageValidationVisitor()); + auto client_config = std::make_unique( + quic_transport.upstream_tls_context(), context); + auto factory = + std::make_unique(std::move(client_config), context); + factory->initialize(); + return factory; +} + +QuicClientTransportSocketFactory::QuicClientTransportSocketFactory( + Ssl::ClientContextConfigPtr config, + Server::Configuration::TransportSocketFactoryContext& factory_context) + : QuicTransportSocketFactoryBase(factory_context.statsScope(), "client"), + fallback_factory_(std::make_unique( + std::move(config), factory_context.sslContextManager(), factory_context.statsScope())) {} + +ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +std::shared_ptr QuicClientTransportSocketFactory::getCryptoConfig() { + Envoy::Ssl::ClientContextSharedPtr context = sslCtx(); + // If the secrets haven't been loaded, there is no crypto config. + if (context == nullptr) { + ENVOY_LOG(warn, "SDS hasn't finished updating Ssl context config yet."); + stats_.upstream_context_secrets_not_ready_.inc(); + return nullptr; + } + + if (client_context_ != context) { + // If the context has been updated, update the crypto config. + client_context_ = context; + crypto_config_ = std::make_shared( + std::make_unique(std::move(context)), + std::make_unique()); + } + // Return the latest crypto config. + return crypto_config_; +} + +REGISTER_FACTORY(QuicClientTransportSocketConfigFactory, + Server::Configuration::UpstreamTransportSocketConfigFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/quic/quic_client_transport_socket_factory.h b/source/common/quic/quic_client_transport_socket_factory.h new file mode 100644 index 000000000000..3abc6e18be06 --- /dev/null +++ b/source/common/quic/quic_client_transport_socket_factory.h @@ -0,0 +1,74 @@ +#pragma once + +#include "source/common/quic/quic_transport_socket_factory.h" + +namespace Envoy { +namespace Quic { + +class QuicClientTransportSocketFactory : public Network::CommonUpstreamTransportSocketFactory, + public QuicTransportSocketFactoryBase { +public: + QuicClientTransportSocketFactory( + Ssl::ClientContextConfigPtr config, + Server::Configuration::TransportSocketFactoryContext& factory_context); + + void initialize() override {} + bool implementsSecureTransport() const override { return true; } + bool supportsAlpn() const override { return true; } + absl::string_view defaultServerNameIndication() const override { + return clientContextConfig()->serverNameIndication(); + } + + // As documented above for QuicTransportSocketFactoryBase, the actual HTTP/3 + // code does not create transport sockets. + // QuicClientTransportSocketFactory::createTransportSocket is called by the + // connection grid when upstream HTTP/3 fails over to TCP, and a raw SSL socket + // is needed. In this case the QuicClientTransportSocketFactory falls over to + // using the fallback factory. + Network::TransportSocketPtr + createTransportSocket(Network::TransportSocketOptionsConstSharedPtr options, + Upstream::HostDescriptionConstSharedPtr host) const override { + return fallback_factory_->createTransportSocket(options, host); + } + + Envoy::Ssl::ClientContextSharedPtr sslCtx() override { return fallback_factory_->sslCtx(); } + + OptRef clientContextConfig() const override { + return fallback_factory_->clientContextConfig(); + } + + // Returns a crypto config generated from the up-to-date client context config. Once the passed in + // context config gets updated, a new crypto config object will be returned by this method. + std::shared_ptr getCryptoConfig() override; + +protected: + // fallback_factory_ will update the context. + void onSecretUpdated() override {} + +private: + // The QUIC client transport socket can create TLS sockets for fallback to TCP. + std::unique_ptr fallback_factory_; + // Latch the latest client context, to determine if it has updated since last + // checked. + Envoy::Ssl::ClientContextSharedPtr client_context_; + // If client_context_ changes, client config will be updated as well. + std::shared_ptr crypto_config_; +}; + +class QuicClientTransportSocketConfigFactory + : public QuicTransportSocketConfigFactory, + public Server::Configuration::UpstreamTransportSocketConfigFactory { +public: + // Server::Configuration::UpstreamTransportSocketConfigFactory + Network::UpstreamTransportSocketFactoryPtr createTransportSocketFactory( + const Protobuf::Message& config, + Server::Configuration::TransportSocketFactoryContext& context) override; + + // Server::Configuration::TransportSocketConfigFactory + ProtobufTypes::MessagePtr createEmptyConfigProto() override; +}; + +DECLARE_FACTORY(QuicClientTransportSocketConfigFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/quic/quic_filter_manager_connection_impl.cc b/source/common/quic/quic_filter_manager_connection_impl.cc index a6731cd164fa..2f2068f3a58a 100644 --- a/source/common/quic/quic_filter_manager_connection_impl.cc +++ b/source/common/quic/quic_filter_manager_connection_impl.cc @@ -27,6 +27,8 @@ QuicFilterManagerConnectionImpl::QuicFilterManagerConnectionImpl( stream_info_->protocol(Http::Protocol::Http3); network_connection_->connectionSocket()->connectionInfoProvider().setSslConnection( Ssl::ConnectionInfoConstSharedPtr(quic_ssl_info_)); + fix_quic_lifetime_issues_ = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.quic_fix_filter_manager_uaf"); } void QuicFilterManagerConnectionImpl::addWriteFilter(Network::WriteFilterSharedPtr filter) { @@ -156,8 +158,9 @@ void QuicFilterManagerConnectionImpl::maybeUpdateDelayCloseTimer(bool has_sent_a } } -void QuicFilterManagerConnectionImpl::onWriteEventDone() { - // Apply delay close policy if there is any. +void QuicFilterManagerConnectionImpl::onWriteEventDone() { maybeApplyDelayedClose(); } + +void QuicFilterManagerConnectionImpl::maybeApplyDelayedClose() { if (!hasDataToWrite() && inDelayedClose() && delayed_close_state_ != DelayedCloseState::CloseAfterFlushAndWait) { closeConnectionImmediately(); @@ -178,7 +181,9 @@ void QuicFilterManagerConnectionImpl::onConnectionCloseEvent( network_connection_ = nullptr; } - filter_manager_ = nullptr; + if (!fix_quic_lifetime_issues_) { + filter_manager_ = nullptr; + } if (!codec_stats_.has_value()) { // The connection was closed before it could be used. Stats are not recorded. return; diff --git a/source/common/quic/quic_filter_manager_connection_impl.h b/source/common/quic/quic_filter_manager_connection_impl.h index b44515cea102..4a1a57a093d3 100644 --- a/source/common/quic/quic_filter_manager_connection_impl.h +++ b/source/common/quic/quic_filter_manager_connection_impl.h @@ -171,12 +171,16 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase, max_headers_count_ = max_headers_count; } + bool fix_quic_lifetime_issues() const { return fix_quic_lifetime_issues_; } + protected: // Propagate connection close to network_connection_callbacks_. void onConnectionCloseEvent(const quic::QuicConnectionCloseFrame& frame, quic::ConnectionCloseSource source, const quic::ParsedQuicVersion& version); + // Apply delay close policy if there is any. + void maybeApplyDelayedClose(); void closeConnectionImmediately() override; virtual bool hasDataToWrite() PURE; @@ -205,10 +209,10 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase, // Called when aggregated buffered bytes across all the streams declines to low watermark. void onSendBufferLowWatermark(); - // Currently ConnectionManagerImpl is the one and only filter. If more network - // filters are added, ConnectionManagerImpl should always be the last one. - // Its onRead() is only called once to trigger ReadFilter::onNewConnection() - // and the rest incoming data bypasses these filters. + // ConnectionManagerImpl should always be the last filter. Its onRead() is only called once to + // trigger ReadFilter::onNewConnection() and the rest incoming data bypasses these filters. + // It has the same life time as this connection, so do all the filters. If the connection gets + // defer-deleted, they will be defer-deleted together. std::unique_ptr filter_manager_; std::unique_ptr stream_info_; @@ -222,6 +226,7 @@ class QuicFilterManagerConnectionImpl : public Network::ConnectionImplBase, EnvoyQuicSimulatedWatermarkBuffer write_buffer_watermark_simulation_; Buffer::OwnedImpl empty_buffer_; absl::optional close_type_during_initialize_; + bool fix_quic_lifetime_issues_{false}; }; } // namespace Quic diff --git a/source/common/quic/quic_io_handle_wrapper.h b/source/common/quic/quic_io_handle_wrapper.h index 74ecbc1bc1ed..12dce060a2f2 100644 --- a/source/common/quic/quic_io_handle_wrapper.h +++ b/source/common/quic/quic_io_handle_wrapper.h @@ -27,30 +27,26 @@ class QuicIoHandleWrapper : public Network::IoHandle { Api::IoCallUint64Result readv(uint64_t max_length, Buffer::RawSlice* slices, uint64_t num_slice) override { if (closed_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.readv(max_length, slices, num_slice); } Api::IoCallUint64Result read(Buffer::Instance& buffer, absl::optional max_length) override { if (closed_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.read(buffer, max_length); } Api::IoCallUint64Result writev(const Buffer::RawSlice* slices, uint64_t num_slice) override { if (closed_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.writev(slices, num_slice); } Api::IoCallUint64Result write(Buffer::Instance& buffer) override { if (closed_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.write(buffer); } @@ -58,8 +54,7 @@ class QuicIoHandleWrapper : public Network::IoHandle { const Envoy::Network::Address::Ip* self_ip, const Network::Address::Instance& peer_address) override { if (closed_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.sendmsg(slices, num_slice, flags, self_ip, peer_address); } @@ -67,8 +62,7 @@ class QuicIoHandleWrapper : public Network::IoHandle { uint32_t self_port, RecvMsgOutput& output) override { if (closed_) { ASSERT(false, "recvmmsg is called after close."); - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.recvmsg(slices, num_slice, self_port, output); } @@ -76,16 +70,14 @@ class QuicIoHandleWrapper : public Network::IoHandle { RecvMsgOutput& output) override { if (closed_) { ASSERT(false, "recvmmsg is called after close."); - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.recvmmsg(slices, self_port, output); } Api::IoCallUint64Result recv(void* buffer, size_t length, int flags) override { if (closed_) { ASSERT(false, "recv called after close."); - return {0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } return io_handle_.recv(buffer, length, flags); } diff --git a/source/common/quic/quic_network_connection.h b/source/common/quic/quic_network_connection.h index 3a6783d75ae1..828f3eb1f761 100644 --- a/source/common/quic/quic_network_connection.h +++ b/source/common/quic/quic_network_connection.h @@ -61,7 +61,6 @@ class QuicNetworkConnection : protected Logger::Loggable // Hosts a list of active sockets, while only the last one is used for writing data. // Hosts a single default socket upon construction. New sockets can be pushed in later as a result // of QUIC connection migration. - // TODO(renjietang): Impose an upper limit. std::vector connection_sockets_; // Points to an instance of EnvoyQuicServerSession or EnvoyQuicClientSession. Network::Connection* envoy_connection_{nullptr}; diff --git a/source/common/quic/quic_server_transport_socket_factory.cc b/source/common/quic/quic_server_transport_socket_factory.cc new file mode 100644 index 000000000000..eaa139a0555f --- /dev/null +++ b/source/common/quic/quic_server_transport_socket_factory.cc @@ -0,0 +1,43 @@ +#include "source/common/quic/quic_server_transport_socket_factory.h" + +#include + +#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.validate.h" + +#include "source/common/runtime/runtime_features.h" +#include "source/extensions/transport_sockets/tls/context_config_impl.h" + +namespace Envoy { +namespace Quic { + +Network::DownstreamTransportSocketFactoryPtr +QuicServerTransportSocketConfigFactory::createTransportSocketFactory( + const Protobuf::Message& config, Server::Configuration::TransportSocketFactoryContext& context, + const std::vector& /*server_names*/) { + auto quic_transport = MessageUtil::downcastAndValidate< + const envoy::extensions::transport_sockets::quic::v3::QuicDownstreamTransport&>( + config, context.messageValidationVisitor()); + auto server_config = std::make_unique( + quic_transport.downstream_tls_context(), context); + // TODO(RyanTheOptimist): support TLS client authentication. + if (server_config->requireClientCertificate()) { + throwEnvoyExceptionOrPanic("TLS Client Authentication is not supported over QUIC"); + } + + auto factory = std::make_unique( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), + context.statsScope(), std::move(server_config)); + factory->initialize(); + return factory; +} + +ProtobufTypes::MessagePtr QuicServerTransportSocketConfigFactory::createEmptyConfigProto() { + return std::make_unique< + envoy::extensions::transport_sockets::quic::v3::QuicDownstreamTransport>(); +} + +REGISTER_FACTORY(QuicServerTransportSocketConfigFactory, + Server::Configuration::DownstreamTransportSocketConfigFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/quic/quic_server_transport_socket_factory.h b/source/common/quic/quic_server_transport_socket_factory.h new file mode 100644 index 000000000000..8e1439d526d6 --- /dev/null +++ b/source/common/quic/quic_server_transport_socket_factory.h @@ -0,0 +1,77 @@ +#pragma once + +#include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" +#include "envoy/network/transport_socket.h" +#include "envoy/server/transport_socket_config.h" +#include "envoy/ssl/context_config.h" + +#include "source/common/common/assert.h" +#include "source/common/network/transport_socket_options_impl.h" +#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/extensions/transport_sockets/tls/ssl_socket.h" + +namespace Envoy { +namespace Quic { + +// TODO(danzh): when implement ProofSource, examine of it's necessary to +// differentiate server and client side context config. +class QuicServerTransportSocketFactory : public Network::DownstreamTransportSocketFactory, + public QuicTransportSocketFactoryBase { +public: + QuicServerTransportSocketFactory(bool enable_early_data, Stats::Scope& store, + Ssl::ServerContextConfigPtr config) + : QuicTransportSocketFactoryBase(store, "server"), config_(std::move(config)), + enable_early_data_(enable_early_data) {} + + // Network::DownstreamTransportSocketFactory + Network::TransportSocketPtr createDownstreamTransportSocket() const override { + PANIC("not implemented"); + } + bool implementsSecureTransport() const override { return true; } + + void initialize() override { + config_->setSecretUpdateCallback([this]() { + // The callback also updates config_ with the new secret. + onSecretUpdated(); + }); + } + + // Return TLS certificates if the context config is ready. + std::vector> + getTlsCertificates() const { + if (!config_->isReady()) { + ENVOY_LOG(warn, "SDS hasn't finished updating Ssl context config yet."); + stats_.downstream_context_secrets_not_ready_.inc(); + return {}; + } + return config_->tlsCertificates(); + } + + bool earlyDataEnabled() const { return enable_early_data_; } + +protected: + void onSecretUpdated() override { stats_.context_config_update_by_sds_.inc(); } + +private: + Ssl::ServerContextConfigPtr config_; + bool enable_early_data_; +}; + +class QuicServerTransportSocketConfigFactory + : public QuicTransportSocketConfigFactory, + public Server::Configuration::DownstreamTransportSocketConfigFactory { +public: + // Server::Configuration::DownstreamTransportSocketConfigFactory + Network::DownstreamTransportSocketFactoryPtr + createTransportSocketFactory(const Protobuf::Message& config, + Server::Configuration::TransportSocketFactoryContext& context, + const std::vector& server_names) override; + + // Server::Configuration::TransportSocketConfigFactory + ProtobufTypes::MessagePtr createEmptyConfigProto() override; +}; + +DECLARE_FACTORY(QuicServerTransportSocketConfigFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/quic/quic_stats_gatherer.cc b/source/common/quic/quic_stats_gatherer.cc index d7acb9ffc0e6..cd663aa3cb15 100644 --- a/source/common/quic/quic_stats_gatherer.cc +++ b/source/common/quic/quic_stats_gatherer.cc @@ -28,12 +28,15 @@ void QuicStatsGatherer::maybeDoDeferredLog(bool record_ack_timing) { } stream_info_->addBytesRetransmitted(retransmitted_bytes_); stream_info_->addPacketsRetransmitted(retransmitted_packets_); - const Http::RequestHeaderMap* request_headers = request_header_map_.get(); - const Http::ResponseHeaderMap* response_headers = response_header_map_.get(); - const Http::ResponseTrailerMap* response_trailers = response_trailer_map_.get(); + + const Formatter::HttpFormatterContext log_context{request_header_map_.get(), + response_header_map_.get(), + response_trailer_map_.get(), + {}, + AccessLog::AccessLogType::DownstreamEnd}; + for (const AccessLog::InstanceSharedPtr& log_handler : access_log_handlers_) { - log_handler->log(request_headers, response_headers, response_trailers, *stream_info_, - AccessLog::AccessLogType::DownstreamEnd); + log_handler->log(log_context, *stream_info_); } } diff --git a/source/common/quic/quic_transport_socket_factory.cc b/source/common/quic/quic_transport_socket_factory.cc index b038c0eb2b0d..3893dde4b6e7 100644 --- a/source/common/quic/quic_transport_socket_factory.cc +++ b/source/common/quic/quic_transport_socket_factory.cc @@ -10,86 +10,4 @@ #include "quiche/quic/core/crypto/quic_client_session_cache.h" -namespace Envoy { -namespace Quic { - -Network::DownstreamTransportSocketFactoryPtr -QuicServerTransportSocketConfigFactory::createTransportSocketFactory( - const Protobuf::Message& config, Server::Configuration::TransportSocketFactoryContext& context, - const std::vector& /*server_names*/) { - auto quic_transport = MessageUtil::downcastAndValidate< - const envoy::extensions::transport_sockets::quic::v3::QuicDownstreamTransport&>( - config, context.messageValidationVisitor()); - auto server_config = std::make_unique( - quic_transport.downstream_tls_context(), context); - // TODO(RyanTheOptimist): support TLS client authentication. - if (server_config->requireClientCertificate()) { - throw EnvoyException("TLS Client Authentication is not supported over QUIC"); - } - - auto factory = std::make_unique( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_transport, enable_early_data, true), - context.statsScope(), std::move(server_config)); - factory->initialize(); - return factory; -} - -ProtobufTypes::MessagePtr QuicServerTransportSocketConfigFactory::createEmptyConfigProto() { - return std::make_unique< - envoy::extensions::transport_sockets::quic::v3::QuicDownstreamTransport>(); -} - -Network::UpstreamTransportSocketFactoryPtr -QuicClientTransportSocketConfigFactory::createTransportSocketFactory( - const Protobuf::Message& config, - Server::Configuration::TransportSocketFactoryContext& context) { - auto quic_transport = MessageUtil::downcastAndValidate< - const envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport&>( - config, context.messageValidationVisitor()); - auto client_config = std::make_unique( - quic_transport.upstream_tls_context(), context); - auto factory = - std::make_unique(std::move(client_config), context); - factory->initialize(); - return factory; -} - -QuicClientTransportSocketFactory::QuicClientTransportSocketFactory( - Ssl::ClientContextConfigPtr config, - Server::Configuration::TransportSocketFactoryContext& factory_context) - : QuicTransportSocketFactoryBase(factory_context.statsScope(), "client"), - fallback_factory_(std::make_unique( - std::move(config), factory_context.sslContextManager(), factory_context.statsScope())) {} - -ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyConfigProto() { - return std::make_unique(); -} - -std::shared_ptr QuicClientTransportSocketFactory::getCryptoConfig() { - Envoy::Ssl::ClientContextSharedPtr context = sslCtx(); - // If the secrets haven't been loaded, there is no crypto config. - if (context == nullptr) { - ENVOY_LOG(warn, "SDS hasn't finished updating Ssl context config yet."); - stats_.upstream_context_secrets_not_ready_.inc(); - return nullptr; - } - - if (client_context_ != context) { - // If the context has been updated, update the crypto config. - client_context_ = context; - crypto_config_ = std::make_shared( - std::make_unique(std::move(context)), - std::make_unique()); - } - // Return the latest crypto config. - return crypto_config_; -} - -REGISTER_FACTORY(QuicServerTransportSocketConfigFactory, - Server::Configuration::DownstreamTransportSocketConfigFactory); - -REGISTER_FACTORY(QuicClientTransportSocketConfigFactory, - Server::Configuration::UpstreamTransportSocketConfigFactory); - -} // namespace Quic -} // namespace Envoy +namespace Envoy {} // namespace Envoy diff --git a/source/common/quic/quic_transport_socket_factory.h b/source/common/quic/quic_transport_socket_factory.h index e9ae0288e311..ce6dbe312b48 100644 --- a/source/common/quic/quic_transport_socket_factory.h +++ b/source/common/quic/quic_transport_socket_factory.h @@ -52,100 +52,6 @@ class QuicTransportSocketFactoryBase : protected Logger::LoggablesetSecretUpdateCallback([this]() { - // The callback also updates config_ with the new secret. - onSecretUpdated(); - }); - } - - // Return TLS certificates if the context config is ready. - std::vector> - getTlsCertificates() const { - if (!config_->isReady()) { - ENVOY_LOG(warn, "SDS hasn't finished updating Ssl context config yet."); - stats_.downstream_context_secrets_not_ready_.inc(); - return {}; - } - return config_->tlsCertificates(); - } - - bool earlyDataEnabled() const { return enable_early_data_; } - -protected: - void onSecretUpdated() override { stats_.context_config_update_by_sds_.inc(); } - -private: - Ssl::ServerContextConfigPtr config_; - bool enable_early_data_; -}; - -class QuicClientTransportSocketFactory : public Network::CommonUpstreamTransportSocketFactory, - public QuicTransportSocketFactoryBase { -public: - QuicClientTransportSocketFactory( - Ssl::ClientContextConfigPtr config, - Server::Configuration::TransportSocketFactoryContext& factory_context); - - void initialize() override {} - bool implementsSecureTransport() const override { return true; } - bool supportsAlpn() const override { return true; } - absl::string_view defaultServerNameIndication() const override { - return clientContextConfig()->serverNameIndication(); - } - - // As documented above for QuicTransportSocketFactoryBase, the actual HTTP/3 - // code does not create transport sockets. - // QuicClientTransportSocketFactory::createTransportSocket is called by the - // connection grid when upstream HTTP/3 fails over to TCP, and a raw SSL socket - // is needed. In this case the QuicClientTransportSocketFactory falls over to - // using the fallback factory. - Network::TransportSocketPtr - createTransportSocket(Network::TransportSocketOptionsConstSharedPtr options, - Upstream::HostDescriptionConstSharedPtr host) const override { - return fallback_factory_->createTransportSocket(options, host); - } - - Envoy::Ssl::ClientContextSharedPtr sslCtx() override { return fallback_factory_->sslCtx(); } - - OptRef clientContextConfig() const override { - return fallback_factory_->clientContextConfig(); - } - - // Returns a crypto config generated from the up-to-date client context config. Once the passed in - // context config gets updated, a new crypto config object will be returned by this method. - std::shared_ptr getCryptoConfig() override; - -protected: - // fallback_factory_ will update the context. - void onSecretUpdated() override {} - -private: - // The QUIC client transport socket can create TLS sockets for fallback to TCP. - std::unique_ptr fallback_factory_; - // Latch the latest client context, to determine if it has updated since last - // checked. - Envoy::Ssl::ClientContextSharedPtr client_context_; - // If client_context_ changes, client config will be updated as well. - std::shared_ptr crypto_config_; -}; - // Base class to create above QuicTransportSocketFactory for server and client // side. class QuicTransportSocketConfigFactory @@ -155,36 +61,5 @@ class QuicTransportSocketConfigFactory std::string name() const override { return "envoy.transport_sockets.quic"; } }; -class QuicServerTransportSocketConfigFactory - : public QuicTransportSocketConfigFactory, - public Server::Configuration::DownstreamTransportSocketConfigFactory { -public: - // Server::Configuration::DownstreamTransportSocketConfigFactory - Network::DownstreamTransportSocketFactoryPtr - createTransportSocketFactory(const Protobuf::Message& config, - Server::Configuration::TransportSocketFactoryContext& context, - const std::vector& server_names) override; - - // Server::Configuration::TransportSocketConfigFactory - ProtobufTypes::MessagePtr createEmptyConfigProto() override; -}; - -DECLARE_FACTORY(QuicServerTransportSocketConfigFactory); - -class QuicClientTransportSocketConfigFactory - : public QuicTransportSocketConfigFactory, - public Server::Configuration::UpstreamTransportSocketConfigFactory { -public: - // Server::Configuration::UpstreamTransportSocketConfigFactory - Network::UpstreamTransportSocketFactoryPtr createTransportSocketFactory( - const Protobuf::Message& config, - Server::Configuration::TransportSocketFactoryContext& context) override; - - // Server::Configuration::TransportSocketConfigFactory - ProtobufTypes::MessagePtr createEmptyConfigProto() override; -}; - -DECLARE_FACTORY(QuicClientTransportSocketConfigFactory); - } // namespace Quic } // namespace Envoy diff --git a/source/common/quic/send_buffer_monitor.cc b/source/common/quic/send_buffer_monitor.cc index b949f0a89782..0eb32a561647 100644 --- a/source/common/quic/send_buffer_monitor.cc +++ b/source/common/quic/send_buffer_monitor.cc @@ -1,5 +1,7 @@ #include "source/common/quic/send_buffer_monitor.h" +#include "source/common/common/assert.h" + namespace Envoy { namespace Quic { diff --git a/source/common/quic/spdy_server_push_utils_for_envoy.cc b/source/common/quic/spdy_server_push_utils_for_envoy.cc deleted file mode 100644 index 70029bdef076..000000000000 --- a/source/common/quic/spdy_server_push_utils_for_envoy.cc +++ /dev/null @@ -1,39 +0,0 @@ -#include "quiche/quic/core/http/spdy_server_push_utils.h" - -// NOLINT(namespace-envoy) - -// This file has a substitute definition for -// quiche/quic/core/http/spdy_server_push_utils.cc which depends on GURL. -// Since Envoy doesn't support server push, these functions shouldn't be -// executed at all. - -using spdy::Http2HeaderBlock; - -namespace quic { - -// static -// NOLINTNEXTLINE(readability-identifier-naming) -std::string SpdyServerPushUtils::GetPromisedUrlFromHeaders(const Http2HeaderBlock& /*headers*/) { - return ""; -} - -// static -std::string -// NOLINTNEXTLINE(readability-identifier-naming) -SpdyServerPushUtils::GetPromisedHostNameFromHeaders(const Http2HeaderBlock& /*headers*/) { - return ""; -} - -// static -// NOLINTNEXTLINE(readability-identifier-naming) -bool SpdyServerPushUtils::PromisedUrlIsValid(const Http2HeaderBlock& /*headers*/) { return false; } - -// static -// NOLINTNEXTLINE(readability-identifier-naming) -std::string SpdyServerPushUtils::GetPushPromiseUrl(absl::string_view /*scheme*/, - absl::string_view /*authority*/, - absl::string_view /*path*/) { - return ""; -} - -} // namespace quic diff --git a/source/common/quic/udp_gso_batch_writer.cc b/source/common/quic/udp_gso_batch_writer.cc index 7171806d7640..e31f6ef6cc14 100644 --- a/source/common/quic/udp_gso_batch_writer.cc +++ b/source/common/quic/udp_gso_batch_writer.cc @@ -16,25 +16,23 @@ Api::IoCallUint64Result convertQuicWriteResult(quic::WriteResult quic_result, si } // Return payload_len as rc & nullptr as error on success return {/*rc=*/payload_len, - /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + /*err=*/Api::IoError::none()}; case quic::WRITE_STATUS_BLOCKED_DATA_BUFFERED: // Data was buffered, Return payload_len as rc & nullptr as error ENVOY_LOG_MISC(trace, "sendmsg blocked, message buffered to send"); return {/*rc=*/payload_len, - /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + /*err=*/Api::IoError::none()}; case quic::WRITE_STATUS_BLOCKED: // Writer blocked, return error ENVOY_LOG_MISC(trace, "sendmsg blocked, message not buffered"); return {/*rc=*/0, - /*err=*/Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)}; + /*err=*/Network::IoSocketError::getIoSocketEagainError()}; default: // Write Failed, return {0 and error_code} ENVOY_LOG_MISC(trace, "sendmsg failed with error code {}", static_cast(quic_result.error_code)); return {/*rc=*/0, - /*err=*/Api::IoErrorPtr(new Network::IoSocketError(quic_result.error_code), - Network::IoSocketError::deleteIoError)}; + /*err=*/Network::IoSocketError::create(quic_result.error_code)}; } } diff --git a/source/common/rds/rds_route_config_subscription.cc b/source/common/rds/rds_route_config_subscription.cc index 5a155af697f7..44d2e15d59a6 100644 --- a/source/common/rds/rds_route_config_subscription.cc +++ b/source/common/rds/rds_route_config_subscription.cc @@ -28,8 +28,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( stat_prefix_(stat_prefix), rds_type_(rds_type), stats_({ALL_RDS_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}), route_config_provider_manager_(route_config_provider_manager), - manager_identifier_(manager_identifier), route_config_provider_(nullptr), - config_update_info_(std::move(config_update)), + manager_identifier_(manager_identifier), config_update_info_(std::move(config_update)), resource_decoder_(std::move(resource_decoder)) { const auto resource_type = route_config_provider_manager_.protoTraits().resourceType(); subscription_ = @@ -53,21 +52,30 @@ RdsRouteConfigSubscription::~RdsRouteConfigSubscription() { absl::Status RdsRouteConfigSubscription::onConfigUpdate( const std::vector& resources, const std::string& version_info) { - if (!validateUpdateSize(resources.size())) { + if (resources.empty()) { + ENVOY_LOG(debug, "Missing {} RouteConfiguration for {} in onConfigUpdate()", rds_type_, + route_config_name_); + stats_.update_empty_.inc(); + local_init_target_.ready(); return absl::OkStatus(); } + if (resources.size() != 1) { + return absl::InvalidArgumentError( + fmt::format("Unexpected {} resource length: {}", rds_type_, resources.size())); + } + const auto& route_config = resources[0].get().resource(); Protobuf::ReflectableMessage reflectable_config = createReflectableMessage(route_config); if (reflectable_config->GetDescriptor()->full_name() != route_config_provider_manager_.protoTraits().resourceType()) { - throw EnvoyException(fmt::format("Unexpected {} configuration type (expecting {}): {}", - rds_type_, - route_config_provider_manager_.protoTraits().resourceType(), - reflectable_config->GetDescriptor()->full_name())); + return absl::InvalidArgumentError( + fmt::format("Unexpected {} configuration type (expecting {}): {}", rds_type_, + route_config_provider_manager_.protoTraits().resourceType(), + reflectable_config->GetDescriptor()->full_name())); } if (resourceName(route_config_provider_manager_.protoTraits(), route_config) != route_config_name_) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("Unexpected {} configuration (expecting {}): {}", rds_type_, route_config_name_, resourceName(route_config_provider_manager_.protoTraits(), route_config))); } @@ -117,21 +125,5 @@ void RdsRouteConfigSubscription::onConfigUpdateFailed( local_init_target_.ready(); } -bool RdsRouteConfigSubscription::validateUpdateSize(int num_resources) { - if (num_resources == 0) { - ENVOY_LOG(debug, "Missing {} RouteConfiguration for {} in onConfigUpdate()", rds_type_, - route_config_name_); - stats_.update_empty_.inc(); - local_init_target_.ready(); - return false; - } - if (num_resources != 1) { - throw EnvoyException( - fmt::format("Unexpected {} resource length: {}", rds_type_, num_resources)); - // (would be a return false here) - } - return true; -} - } // namespace Rds } // namespace Envoy diff --git a/source/common/rds/rds_route_config_subscription.h b/source/common/rds/rds_route_config_subscription.h index 6c31198748df..205375cef14f 100644 --- a/source/common/rds/rds_route_config_subscription.h +++ b/source/common/rds/rds_route_config_subscription.h @@ -72,8 +72,6 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException*) override; - bool validateUpdateSize(int num_resources); - virtual void beforeProviderUpdate(std::unique_ptr&, std::unique_ptr&) {} virtual void afterProviderUpdate() {} @@ -98,7 +96,7 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, RdsStats stats_; RouteConfigProviderManager& route_config_provider_manager_; const uint64_t manager_identifier_; - RouteConfigProvider* route_config_provider_; + RouteConfigProvider* route_config_provider_{nullptr}; RouteConfigUpdatePtr config_update_info_; Envoy::Config::OpaqueResourceDecoderSharedPtr resource_decoder_; }; diff --git a/source/common/rds/route_config_update_receiver_impl.cc b/source/common/rds/route_config_update_receiver_impl.cc index 3d0122616b97..3551b3d7b1b6 100644 --- a/source/common/rds/route_config_update_receiver_impl.cc +++ b/source/common/rds/route_config_update_receiver_impl.cc @@ -10,7 +10,7 @@ RouteConfigUpdateReceiverImpl::RouteConfigUpdateReceiverImpl( Server::Configuration::ServerFactoryContext& factory_context) : config_traits_(config_traits), proto_traits_(proto_traits), factory_context_(factory_context), time_source_(factory_context.timeSource()), - route_config_proto_(proto_traits_.createEmptyProto()), last_config_hash_(0ull), + route_config_proto_(proto_traits_.createEmptyProto()), config_(config_traits_.createNullConfig()) {} void RouteConfigUpdateReceiverImpl::updateConfig( diff --git a/source/common/rds/route_config_update_receiver_impl.h b/source/common/rds/route_config_update_receiver_impl.h index 153dab4d491b..b230dc57ac1f 100644 --- a/source/common/rds/route_config_update_receiver_impl.h +++ b/source/common/rds/route_config_update_receiver_impl.h @@ -35,7 +35,7 @@ class RouteConfigUpdateReceiverImpl : public RouteConfigUpdateReceiver { Server::Configuration::ServerFactoryContext& factory_context_; TimeSource& time_source_; ProtobufTypes::MessagePtr route_config_proto_; - uint64_t last_config_hash_; + uint64_t last_config_hash_{0ull}; SystemTime last_updated_; absl::optional config_info_; ConfigConstSharedPtr config_; diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 1a0d3c11c5c4..81e062ef480e 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -39,7 +39,6 @@ envoy_cc_library( deps = [ ":config_utility_lib", ":context_lib", - ":header_formatter_lib", ":header_parser_lib", ":metadatamatchcriteria_lib", ":reset_header_parser_lib", @@ -96,6 +95,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:empty_string", "//source/common/common:matchers_lib", + "//source/common/config:datasource_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", "//source/common/protobuf:utility_lib", @@ -298,6 +298,7 @@ envoy_cc_library( "//envoy/http:filter_interface", "//envoy/http:stateful_session_interface", "//envoy/local_info:local_info_interface", + "//envoy/router:router_filter_interface", "//envoy/router:shadow_writer_interface", "//envoy/runtime:runtime_interface", "//envoy/server:factory_context_interface", @@ -419,33 +420,17 @@ envoy_cc_library( ], hdrs = ["header_parser.h"], deps = [ - ":header_formatter_lib", "//envoy/http:header_evaluator", "//envoy/http:header_map_interface", + "//source/common/formatter:substitution_formatter_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", + "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) -envoy_cc_library( - name = "header_formatter_lib", - hdrs = ["header_formatter.h"], - external_deps = ["abseil_optional"], - deps = [ - "//envoy/router:string_accessor_interface", - "//envoy/stream_info:filter_state_interface", - "//envoy/stream_info:stream_info_interface", - "//source/common/common:minimal_logger_lib", - "//source/common/common:utility_lib", - "//source/common/config:metadata_lib", - "//source/common/formatter:substitution_formatter_lib", - "//source/common/http:header_map_lib", - "//source/common/json:json_loader_lib", - ], -) - envoy_cc_library( name = "reset_header_parser_lib", srcs = ["reset_header_parser.cc"], diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index d760155d7c51..71735d340da3 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -75,7 +75,6 @@ void mergeTransforms(Http::HeaderTransforms& dest, const Http::HeaderTransforms& RouteEntryImplBaseConstSharedPtr createAndValidateRoute( const envoy::config::route::v3::Route& route_config, const CommonVirtualHostSharedPtr& vhost, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, const absl::optional& validation_clusters) { @@ -83,34 +82,31 @@ RouteEntryImplBaseConstSharedPtr createAndValidateRoute( RouteEntryImplBaseConstSharedPtr route; switch (route_config.match().path_specifier_case()) { case envoy::config::route::v3::RouteMatch::PathSpecifierCase::kPrefix: - route = std::make_shared(vhost, route_config, optional_http_filters, - factory_context, validator); + route = std::make_shared(vhost, route_config, factory_context, validator); break; case envoy::config::route::v3::RouteMatch::PathSpecifierCase::kPath: - route = std::make_shared(vhost, route_config, optional_http_filters, - factory_context, validator); + route = std::make_shared(vhost, route_config, factory_context, validator); break; case envoy::config::route::v3::RouteMatch::PathSpecifierCase::kSafeRegex: - route = std::make_shared(vhost, route_config, optional_http_filters, - factory_context, validator); + route = std::make_shared(vhost, route_config, factory_context, validator); break; case envoy::config::route::v3::RouteMatch::PathSpecifierCase::kConnectMatcher: - route = std::make_shared(vhost, route_config, optional_http_filters, - factory_context, validator); + route = + std::make_shared(vhost, route_config, factory_context, validator); break; case envoy::config::route::v3::RouteMatch::PathSpecifierCase::kPathSeparatedPrefix: - route = std::make_shared( - vhost, route_config, optional_http_filters, factory_context, validator); + route = std::make_shared(vhost, route_config, + factory_context, validator); break; case envoy::config::route::v3::RouteMatch::PathSpecifierCase::kPathMatchPolicy: - route = std::make_shared( - vhost, route_config, optional_http_filters, factory_context, validator); + route = std::make_shared(vhost, route_config, factory_context, + validator); break; case envoy::config::route::v3::RouteMatch::PathSpecifierCase::PATH_SPECIFIER_NOT_SET: break; // throw the error below. } if (!route) { - throw EnvoyException("Invalid route config"); + throwEnvoyExceptionOrPanic("Invalid route config"); } if (validation_clusters.has_value()) { @@ -119,7 +115,7 @@ RouteEntryImplBaseConstSharedPtr createAndValidateRoute( if (!shadow_policy->cluster().empty()) { ASSERT(shadow_policy->clusterHeader().get().empty()); if (!validation_clusters->hasCluster(shadow_policy->cluster())) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("route: unknown shadow cluster '{}'", shadow_policy->cluster())); } } @@ -151,9 +147,9 @@ class RouteActionValidationVisitor const envoy::config::route::v3::WeightedCluster::ClusterWeight& validateWeightedClusterSpecifier( const envoy::config::route::v3::WeightedCluster::ClusterWeight& cluster) { if (!cluster.name().empty() && !cluster.cluster_header().empty()) { - throw EnvoyException("Only one of name or cluster_header can be specified"); + throwEnvoyExceptionOrPanic("Only one of name or cluster_header can be specified"); } else if (cluster.name().empty() && cluster.cluster_header().empty()) { - throw EnvoyException("At least one of name or cluster_header need to be specified"); + throwEnvoyExceptionOrPanic("At least one of name or cluster_header need to be specified"); } return cluster; } @@ -195,7 +191,7 @@ getClusterSpecifierPluginByTheProto(const envoy::config::route::v3::ClusterSpeci if (plugin.is_optional()) { return std::make_shared(); } - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Didn't find a registered implementation for '{}' with type URL: '{}'", plugin.extension().name(), Envoy::Config::Utility::getFactoryType(plugin.extension().typed_config()))); @@ -312,7 +308,7 @@ RetryPolicyImpl::RetryPolicyImpl(const envoy::config::route::v3::RetryPolicy& re } if ((*max_interval_).count() < (*base_interval_).count()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "retry_policy.max_interval must greater than or equal to the base_interval"); } } @@ -368,6 +364,12 @@ InternalRedirectPolicyImpl::InternalRedirectPolicyImpl( Envoy::Config::Utility::translateOpaqueConfig(predicate.typed_config(), validator, *config); predicate_factories_.emplace_back(&factory, std::move(config)); } + for (const auto& header : policy_config.response_headers_to_copy()) { + if (!Http::HeaderUtility::isModifiableHeader(header)) { + throwEnvoyExceptionOrPanic(":-prefixed headers or Hosts may not be specified here."); + } + response_headers_to_copy_.emplace_back(header); + } } std::vector InternalRedirectPolicyImpl::predicates() const { @@ -380,6 +382,11 @@ std::vector InternalRedirectPolicyImpl::pred return predicates; } +const std::vector& +InternalRedirectPolicyImpl::responseHeadersToCopy() const { + return response_headers_to_copy_; +} + absl::flat_hash_set InternalRedirectPolicyImpl::buildRedirectResponseCodes( const envoy::config::route::v3::InternalRedirectPolicy& policy_config) const { if (policy_config.redirect_response_codes_size() == 0) { @@ -400,13 +407,13 @@ absl::flat_hash_set InternalRedirectPolicyImpl::buildRedirectRespons void validateMirrorClusterSpecifier( const envoy::config::route::v3::RouteAction::RequestMirrorPolicy& config) { if (!config.cluster().empty() && !config.cluster_header().empty()) { - throw EnvoyException(fmt::format("Only one of cluster '{}' or cluster_header '{}' " - "in request mirror policy can be specified", - config.cluster(), config.cluster_header())); + throwEnvoyExceptionOrPanic(fmt::format("Only one of cluster '{}' or cluster_header '{}' " + "in request mirror policy can be specified", + config.cluster(), config.cluster_header())); } else if (config.cluster().empty() && config.cluster_header().empty()) { // For shadow policies with `cluster_header_`, we only verify that this field is not // empty because the cluster name is not set yet at config time. - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Exactly one of cluster or cluster_header in request mirror policy need to be specified"); } } @@ -480,7 +487,6 @@ const Tracing::CustomTagMap& RouteTracingImpl::getCustomTags() const { return cu RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) : prefix_rewrite_(route.route().prefix_rewrite()), @@ -520,8 +526,7 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, direct_response_body_(ConfigUtility::parseDirectResponseBody( route, factory_context.api(), vhost_->globalRouteConfig().maxDirectResponseBodySizeBytes())), - per_filter_configs_(route.typed_per_filter_config(), optional_http_filters, factory_context, - validator), + per_filter_configs_(route.typed_per_filter_config(), factory_context, validator), route_name_(route.name()), time_source_(factory_context.mainThreadDispatcher().timeSource()), retry_shadow_buffer_limit_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( route, per_request_buffer_limit_bytes, vhost->retryShadowBufferLimit())), @@ -543,8 +548,7 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, route.response_headers_to_remove()); } if (route.has_metadata()) { - metadata_ = std::make_unique(route.metadata()); - typed_metadata_ = std::make_unique(route.metadata()); + metadata_ = std::make_unique(route.metadata()); } if (route.route().has_metadata_match()) { const auto filter_it = route.route().metadata_match().filter_metadata().find( @@ -578,12 +582,11 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, weighted_clusters.reserve(route.route().weighted_clusters().clusters().size()); for (const auto& cluster : route.route().weighted_clusters().clusters()) { auto cluster_entry = std::make_unique( - this, runtime_key_prefix + "." + cluster.name(), factory_context, validator, cluster, - optional_http_filters); + this, runtime_key_prefix + "." + cluster.name(), factory_context, validator, cluster); weighted_clusters.emplace_back(std::move(cluster_entry)); total_weight += weighted_clusters.back()->clusterWeight(); if (total_weight > std::numeric_limits::max()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("The sum of weights of all weighted clusters of route {} exceeds {}", route_name_, std::numeric_limits::max())); } @@ -591,7 +594,7 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, // Reject the config if the total_weight of all clusters is 0. if (total_weight == 0) { - throw EnvoyException("Sum of weights in the weighted_cluster must be greater than 0."); + throwEnvoyExceptionOrPanic("Sum of weights in the weighted_cluster must be greater than 0."); } weighted_clusters_config_ = @@ -645,7 +648,7 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, Envoy::Http::LowerCaseString(upgrade_config.upgrade_type()).get(), enabled)) .second; if (!success) { - throw EnvoyException(absl::StrCat("Duplicate upgrade ", upgrade_config.upgrade_type())); + throwEnvoyExceptionOrPanic(absl::StrCat("Duplicate upgrade ", upgrade_config.upgrade_type())); } if (absl::EqualsIgnoreCase(upgrade_config.upgrade_type(), Http::Headers::get().MethodValues.Connect) || @@ -654,8 +657,8 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, Http::Headers::get().UpgradeValues.ConnectUdp))) { connect_config_ = std::make_unique(upgrade_config.connect_config()); } else if (upgrade_config.has_connect_config()) { - throw EnvoyException(absl::StrCat("Non-CONNECT upgrade type ", upgrade_config.upgrade_type(), - " has ConnectConfig")); + throwEnvoyExceptionOrPanic(absl::StrCat("Non-CONNECT upgrade type ", + upgrade_config.upgrade_type(), " has ConnectConfig")); } } @@ -673,12 +676,12 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, } if (num_rewrite_polices > 1) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Specify only one of prefix_rewrite, regex_rewrite or path_rewrite_policy"); } if (!prefix_rewrite_.empty() && path_matcher_ != nullptr) { - throw EnvoyException("Cannot use prefix_rewrite with matcher extension"); + throwEnvoyExceptionOrPanic("Cannot use prefix_rewrite with matcher extension"); } if (route.route().has_regex_rewrite()) { @@ -690,7 +693,7 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, if (path_rewriter_ != nullptr) { absl::Status compatible_status = path_rewriter_->isCompatiblePathMatcher(path_matcher_); if (!compatible_status.ok()) { - throw EnvoyException(std::string(compatible_status.message())); + throwEnvoyExceptionOrPanic(std::string(compatible_status.message())); } } @@ -787,8 +790,8 @@ bool RouteEntryImplBase::matchRoute(const Http::RequestHeaderMap& headers, return false; } if (!config_query_parameters_.empty()) { - Http::Utility::QueryParams query_parameters = - Http::Utility::parseQueryString(headers.getPathValue()); + auto query_parameters = + Http::Utility::QueryParamsMulti::parseQueryString(headers.getPathValue()); matches &= ConfigUtility::matchQueryParams(query_parameters, config_query_parameters_); if (!matches) { return false; @@ -864,11 +867,8 @@ void RouteEntryImplBase::finalizeResponseHeaders(Http::ResponseHeaderMap& header for (const HeaderParser* header_parser : getResponseHeaderParsers( /*specificity_ascend=*/vhost_->globalRouteConfig().mostSpecificHeaderMutationsWins())) { // Later evaluated header parser wins. - header_parser->evaluateHeaders(headers, - stream_info.getRequestHeaders() == nullptr - ? *Http::StaticEmptyHeaders::get().request_headers - : *stream_info.getRequestHeaders(), - headers, stream_info); + header_parser->evaluateHeaders(headers, {stream_info.getRequestHeaders(), &headers}, + stream_info); } } @@ -1011,13 +1011,7 @@ absl::optional RouteEntryImplBase::currentUrlPathAfterRewriteWithMa if (!new_path.ok()) { return std::string(headers.getPathValue()); } - - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.append_query_parameters_path_rewriter")) { - return path.replace(0, just_path.size(), new_path.value()); - } else { - return *std::move(new_path); - } + return path.replace(0, just_path.size(), new_path.value()); } // There are no rewrites configured. @@ -1169,7 +1163,7 @@ RouteEntryImplBase::buildPathRewriter(envoy::config::route::v3::Route route, absl::StatusOr rewriter = factory.createPathRewriter(*config); if (!rewriter.ok()) { - throw EnvoyException(std::string(rewriter.status().message())); + throwEnvoyExceptionOrPanic(std::string(rewriter.status().message())); } return rewriter.value(); @@ -1190,7 +1184,7 @@ RouteEntryImplBase::buildPathMatcher(envoy::config::route::v3::Route route, absl::StatusOr matcher = factory.createPathMatcher(*config); if (!matcher.ok()) { - throw EnvoyException(std::string(matcher.status().message())); + throwEnvoyExceptionOrPanic(std::string(matcher.status().message())); } return matcher.value(); @@ -1352,14 +1346,14 @@ void RouteEntryImplBase::validateClusters( // route tables. This would enable the all CDS with static route table case. if (!cluster_name_.empty()) { if (!cluster_info_maps.hasCluster(cluster_name_)) { - throw EnvoyException(fmt::format("route: unknown cluster '{}'", cluster_name_)); + throwEnvoyExceptionOrPanic(fmt::format("route: unknown cluster '{}'", cluster_name_)); } } else if (weighted_clusters_config_ != nullptr) { for (const WeightedClusterEntrySharedPtr& cluster : weighted_clusters_config_->weighted_clusters_) { if (!cluster->clusterName().empty()) { if (!cluster_info_maps.hasCluster(cluster->clusterName())) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("route: unknown weighted cluster '{}'", cluster->clusterName())); } } @@ -1367,13 +1361,13 @@ void RouteEntryImplBase::validateClusters( // not empty because the cluster name is not set yet at config time (hence the validation // here). else if (cluster->clusterHeaderName().get().empty()) { - throw EnvoyException("route: unknown weighted cluster with no cluster_header field"); + throwEnvoyExceptionOrPanic("route: unknown weighted cluster with no cluster_header field"); } } } } -bool RouteEntryImplBase::filterDisabled(absl::string_view config_name) const { +absl::optional RouteEntryImplBase::filterDisabled(absl::string_view config_name) const { absl::optional result = per_filter_configs_.disabled(config_name); if (result.has_value()) { return result.value(); @@ -1392,17 +1386,24 @@ void RouteEntryImplBase::traversePerFilterConfig( } } +const envoy::config::core::v3::Metadata& RouteEntryImplBase::metadata() const { + return metadata_ != nullptr ? metadata_->proto_metadata_ + : DefaultRouteMetadataPack::get().proto_metadata_; +} +const Envoy::Config::TypedMetadata& RouteEntryImplBase::typedMetadata() const { + return metadata_ != nullptr ? metadata_->typed_metadata_ + : DefaultRouteMetadataPack::get().typed_metadata_; +} + RouteEntryImplBase::WeightedClusterEntry::WeightedClusterEntry( const RouteEntryImplBase* parent, const std::string& runtime_key, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, - const envoy::config::route::v3::WeightedCluster::ClusterWeight& cluster, - const OptionalHttpFilters& optional_http_filters) + const envoy::config::route::v3::WeightedCluster::ClusterWeight& cluster) : DynamicRouteEntry(parent, nullptr, validateWeightedClusterSpecifier(cluster).name()), runtime_key_(runtime_key), loader_(factory_context.runtime()), cluster_weight_(PROTOBUF_GET_WRAPPED_REQUIRED(cluster, weight)), - per_filter_configs_(cluster.typed_per_filter_config(), optional_http_filters, factory_context, - validator), + per_filter_configs_(cluster.typed_per_filter_config(), factory_context, validator), host_rewrite_(cluster.host_rewrite_literal()), cluster_header_name_(cluster.cluster_header()) { if (!cluster.request_headers_to_add().empty() || !cluster.request_headers_to_remove().empty()) { @@ -1458,10 +1459,9 @@ void RouteEntryImplBase::WeightedClusterEntry::traversePerFilterConfig( UriTemplateMatcherRouteEntryImpl::UriTemplateMatcherRouteEntryImpl( const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) - : RouteEntryImplBase(vhost, route, optional_http_filters, factory_context, validator), + : RouteEntryImplBase(vhost, route, factory_context, validator), uri_template_(path_matcher_->uriTemplate()){}; void UriTemplateMatcherRouteEntryImpl::rewritePathHeader(Http::RequestHeaderMap& headers, @@ -1487,10 +1487,9 @@ UriTemplateMatcherRouteEntryImpl::matches(const Http::RequestHeaderMap& headers, PrefixRouteEntryImpl::PrefixRouteEntryImpl( const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) - : RouteEntryImplBase(vhost, route, optional_http_filters, factory_context, validator), + : RouteEntryImplBase(vhost, route, factory_context, validator), path_matcher_( Matchers::PathMatcher::createPrefix(route.match().prefix(), !case_sensitive())) {} @@ -1516,10 +1515,9 @@ RouteConstSharedPtr PrefixRouteEntryImpl::matches(const Http::RequestHeaderMap& PathRouteEntryImpl::PathRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) - : RouteEntryImplBase(vhost, route, optional_http_filters, factory_context, validator), + : RouteEntryImplBase(vhost, route, factory_context, validator), path_matcher_(Matchers::PathMatcher::createExact(route.match().path(), !case_sensitive())) {} void PathRouteEntryImpl::rewritePathHeader(Http::RequestHeaderMap& headers, @@ -1545,10 +1543,9 @@ RouteConstSharedPtr PathRouteEntryImpl::matches(const Http::RequestHeaderMap& he RegexRouteEntryImpl::RegexRouteEntryImpl( const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) - : RouteEntryImplBase(vhost, route, optional_http_filters, factory_context, validator), + : RouteEntryImplBase(vhost, route, factory_context, validator), path_matcher_(Matchers::PathMatcher::createSafeRegex(route.match().safe_regex())) { ASSERT(route.match().path_specifier_case() == envoy::config::route::v3::RouteMatch::PathSpecifierCase::kSafeRegex); @@ -1582,10 +1579,9 @@ RouteConstSharedPtr RegexRouteEntryImpl::matches(const Http::RequestHeaderMap& h ConnectRouteEntryImpl::ConnectRouteEntryImpl( const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) - : RouteEntryImplBase(vhost, route, optional_http_filters, factory_context, validator) {} + : RouteEntryImplBase(vhost, route, factory_context, validator) {} void ConnectRouteEntryImpl::rewritePathHeader(Http::RequestHeaderMap& headers, bool insert_envoy_original_path) const { @@ -1604,7 +1600,7 @@ RouteConstSharedPtr ConnectRouteEntryImpl::matches(const Http::RequestHeaderMap& uint64_t random_value) const { if ((Http::HeaderUtility::isConnect(headers) || (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") && - Http::HeaderUtility::isConnectUdp(headers))) && + Http::HeaderUtility::isConnectUdpRequest(headers))) && RouteEntryImplBase::matchRoute(headers, stream_info, random_value)) { return clusterEntry(headers, random_value); } @@ -1613,10 +1609,9 @@ RouteConstSharedPtr ConnectRouteEntryImpl::matches(const Http::RequestHeaderMap& PathSeparatedPrefixRouteEntryImpl::PathSeparatedPrefixRouteEntryImpl( const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) - : RouteEntryImplBase(vhost, route, optional_http_filters, factory_context, validator), + : RouteEntryImplBase(vhost, route, factory_context, validator), path_matcher_(Matchers::PathMatcher::createPrefix(route.match().path_separated_prefix(), !case_sensitive())) {} @@ -1650,14 +1645,12 @@ PathSeparatedPrefixRouteEntryImpl::matches(const Http::RequestHeaderMap& headers CommonVirtualHostImpl::CommonVirtualHostImpl( const envoy::config::route::v3::VirtualHost& virtual_host, - const OptionalHttpFilters& optional_http_filters, const CommonConfigSharedPtr& global_route_config, Server::Configuration::ServerFactoryContext& factory_context, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validator) : stat_name_storage_(virtual_host.name(), factory_context.scope().symbolTable()), global_route_config_(global_route_config), - per_filter_configs_(virtual_host.typed_per_filter_config(), optional_http_filters, - factory_context, validator), + per_filter_configs_(virtual_host.typed_per_filter_config(), factory_context, validator), retry_shadow_buffer_limit_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( virtual_host, per_request_buffer_limit_bytes, std::numeric_limits::max())), include_attempt_count_in_request_(virtual_host.include_request_attempt_count()), @@ -1700,7 +1693,7 @@ CommonVirtualHostImpl::CommonVirtualHostImpl( } if (virtual_host.has_matcher() && !virtual_host.routes().empty()) { - throw EnvoyException("cannot set both matcher and routes on virtual host"); + throwEnvoyExceptionOrPanic("cannot set both matcher and routes on virtual host"); } if (!virtual_host.virtual_clusters().empty()) { @@ -1719,6 +1712,10 @@ CommonVirtualHostImpl::CommonVirtualHostImpl( if (virtual_host.has_cors()) { cors_policy_ = std::make_unique(virtual_host.cors(), factory_context.runtime()); } + + if (virtual_host.has_metadata()) { + metadata_ = std::make_unique(virtual_host.metadata()); + } } CommonVirtualHostImpl::VirtualClusterEntry::VirtualClusterEntry( @@ -1728,7 +1725,7 @@ CommonVirtualHostImpl::VirtualClusterEntry::VirtualClusterEntry( VirtualClusterBase(virtual_cluster.name(), stat_name_storage_.statName(), scope.scopeFromStatName(stat_name_storage_.statName()), stat_names) { if (virtual_cluster.headers().empty()) { - throw EnvoyException("virtual clusters must define 'headers'"); + throwEnvoyExceptionOrPanic("virtual clusters must define 'headers'"); } ASSERT(!virtual_cluster.headers().empty()); @@ -1737,7 +1734,7 @@ CommonVirtualHostImpl::VirtualClusterEntry::VirtualClusterEntry( const CommonConfig& CommonVirtualHostImpl::routeConfig() const { return *global_route_config_; } -bool CommonVirtualHostImpl::filterDisabled(absl::string_view config_name) const { +absl::optional CommonVirtualHostImpl::filterDisabled(absl::string_view config_name) const { absl::optional result = per_filter_configs_.disabled(config_name); if (result.has_value()) { return result.value(); @@ -1765,16 +1762,24 @@ void CommonVirtualHostImpl::traversePerFilterConfig( } } +const envoy::config::core::v3::Metadata& CommonVirtualHostImpl::metadata() const { + return metadata_ != nullptr ? metadata_->proto_metadata_ + : DefaultRouteMetadataPack::get().proto_metadata_; +} +const Envoy::Config::TypedMetadata& CommonVirtualHostImpl::typedMetadata() const { + return metadata_ != nullptr ? metadata_->typed_metadata_ + : DefaultRouteMetadataPack::get().typed_metadata_; +} + VirtualHostImpl::VirtualHostImpl( const envoy::config::route::v3::VirtualHost& virtual_host, - const OptionalHttpFilters& optional_http_filters, const CommonConfigSharedPtr& global_route_config, Server::Configuration::ServerFactoryContext& factory_context, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validator, const absl::optional& validation_clusters) { - shared_virtual_host_ = std::make_shared( - virtual_host, optional_http_filters, global_route_config, factory_context, scope, validator); + shared_virtual_host_ = std::make_shared(virtual_host, global_route_config, + factory_context, scope, validator); switch (virtual_host.require_tls()) { PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; @@ -1790,7 +1795,7 @@ VirtualHostImpl::VirtualHostImpl( } if (virtual_host.has_matcher()) { - RouteActionContext context{shared_virtual_host_, optional_http_filters, factory_context}; + RouteActionContext context{shared_virtual_host_, factory_context}; RouteActionValidationVisitor validation_visitor; Matcher::MatchTreeFactory factory( context, factory_context, validation_visitor); @@ -1799,14 +1804,14 @@ VirtualHostImpl::VirtualHostImpl( if (!validation_visitor.errors().empty()) { // TODO(snowp): Output all violations. - throw EnvoyException(fmt::format("requirement violation while creating route match tree: {}", - validation_visitor.errors()[0])); + throwEnvoyExceptionOrPanic( + fmt::format("requirement violation while creating route match tree: {}", + validation_visitor.errors()[0])); } } else { for (const auto& route : virtual_host.routes()) { - routes_.emplace_back(createAndValidateRoute(route, shared_virtual_host_, - optional_http_filters, factory_context, validator, - validation_clusters)); + routes_.emplace_back(createAndValidateRoute(route, shared_virtual_host_, factory_context, + validator, validation_clusters)); } } } @@ -1926,7 +1931,6 @@ const VirtualHostImpl* RouteMatcher::findWildcardVirtualHost( } RouteMatcher::RouteMatcher(const envoy::config::route::v3::RouteConfiguration& route_config, - const OptionalHttpFilters& optional_http_filters, const CommonConfigSharedPtr& global_route_config, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, bool validate_clusters) @@ -1938,17 +1942,17 @@ RouteMatcher::RouteMatcher(const envoy::config::route::v3::RouteConfiguration& r validation_clusters = factory_context.clusterManager().clusters(); } for (const auto& virtual_host_config : route_config.virtual_hosts()) { - VirtualHostSharedPtr virtual_host = std::make_shared( - virtual_host_config, optional_http_filters, global_route_config, factory_context, - *vhost_scope_, validator, validation_clusters); + VirtualHostSharedPtr virtual_host = + std::make_shared(virtual_host_config, global_route_config, factory_context, + *vhost_scope_, validator, validation_clusters); for (const std::string& domain_name : virtual_host_config.domains()) { const Http::LowerCaseString lower_case_domain_name(domain_name); absl::string_view domain = lower_case_domain_name; bool duplicate_found = false; if ("*" == domain) { if (default_virtual_host_) { - throw EnvoyException(fmt::format("Only a single wildcard domain is permitted in route {}", - route_config.name())); + throwEnvoyExceptionOrPanic(fmt::format( + "Only a single wildcard domain is permitted in route {}", route_config.name())); } default_virtual_host_ = virtual_host; } else if (!domain.empty() && '*' == domain[0]) { @@ -1963,9 +1967,10 @@ RouteMatcher::RouteMatcher(const envoy::config::route::v3::RouteConfiguration& r duplicate_found = !virtual_hosts_.emplace(domain, virtual_host).second; } if (duplicate_found) { - throw EnvoyException(fmt::format("Only unique values for domains are permitted. Duplicate " - "entry of domain {} in route {}", - domain, route_config.name())); + throwEnvoyExceptionOrPanic( + fmt::format("Only unique values for domains are permitted. Duplicate " + "entry of domain {} in route {}", + domain, route_config.name())); } } } @@ -2052,12 +2057,10 @@ CommonVirtualHostImpl::virtualClusterFromEntries(const Http::HeaderMap& headers) } CommonConfigImpl::CommonConfigImpl(const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) : name_(config.name()), symbol_table_(factory_context.scope().symbolTable()), - per_filter_configs_(config.typed_per_filter_config(), optional_http_filters, factory_context, - validator), + per_filter_configs_(config.typed_per_filter_config(), factory_context, validator), max_direct_response_body_size_bytes_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_direct_response_body_size_bytes, DEFAULT_MAX_DIRECT_RESPONSE_BODY_SIZE_BYTES)), @@ -2090,29 +2093,40 @@ CommonConfigImpl::CommonConfigImpl(const envoy::config::route::v3::RouteConfigur response_headers_parser_ = HeaderParser::configure(config.response_headers_to_add(), config.response_headers_to_remove()); } + + if (config.has_metadata()) { + metadata_ = std::make_unique(config.metadata()); + } } ClusterSpecifierPluginSharedPtr CommonConfigImpl::clusterSpecifierPlugin(absl::string_view provider) const { auto iter = cluster_specifier_plugins_.find(provider); if (iter == cluster_specifier_plugins_.end() || iter->second == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Unknown cluster specifier plugin name: {} is used in the route", provider)); } return iter->second; } +const envoy::config::core::v3::Metadata& CommonConfigImpl::metadata() const { + return metadata_ != nullptr ? metadata_->proto_metadata_ + : DefaultRouteMetadataPack::get().proto_metadata_; +} +const Envoy::Config::TypedMetadata& CommonConfigImpl::typedMetadata() const { + return metadata_ != nullptr ? metadata_->typed_metadata_ + : DefaultRouteMetadataPack::get().typed_metadata_; +} + ConfigImpl::ConfigImpl(const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, bool validate_clusters_default) { - shared_config_ = - std::make_shared(config, optional_http_filters, factory_context, validator); + shared_config_ = std::make_shared(config, factory_context, validator); route_matcher_ = std::make_unique( - config, optional_http_filters, shared_config_, factory_context, validator, + config, shared_config_, factory_context, validator, PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, validate_clusters, validate_clusters_default)); } @@ -2123,6 +2137,13 @@ RouteConstSharedPtr ConfigImpl::route(const RouteCallback& cb, return route_matcher_->route(cb, headers, stream_info, random_value); } +const envoy::config::core::v3::Metadata& NullConfigImpl::metadata() const { + return DefaultRouteMetadataPack::get().proto_metadata_; +} +const Envoy::Config::TypedMetadata& NullConfigImpl::typedMetadata() const { + return DefaultRouteMetadataPack::get().typed_metadata_; +} + RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFilterConfig( const std::string& name, const ProtobufWkt::Any& typed_config, bool is_optional, Server::Configuration::ServerFactoryContext& factory_context, @@ -2137,7 +2158,7 @@ RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFil name, Envoy::Config::Utility::getFactoryType(typed_config)); return nullptr; } else { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Didn't find a registered implementation for '{}' with type URL: '{}'", name, Envoy::Config::Utility::getFactoryType(typed_config))); } @@ -2154,7 +2175,7 @@ RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFil "optional, so ignore it.", name); } else { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "The filter {} doesn't support virtual host or route specific configurations", name)); } } @@ -2163,13 +2184,9 @@ RouteSpecificFilterConfigConstSharedPtr PerFilterConfigs::createRouteSpecificFil PerFilterConfigs::PerFilterConfigs( const Protobuf::Map& typed_configs, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) { - const bool ignore_optional_option_from_hcm_for_route_config(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config")); - std::string filter_config_type = envoy::config::route::v3::FilterConfig::default_instance().GetTypeName(); @@ -2177,20 +2194,6 @@ PerFilterConfigs::PerFilterConfigs( const std::string& name = per_filter_config.first; RouteSpecificFilterConfigConstSharedPtr config; - // There are two ways to mark a route/virtual host per filter configuration as optional: - // 1. Mark it as optional in the HTTP filter of HCM. This way is deprecated but still works - // when the runtime flag - // `envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config` - // is explicitly set to false. - // 2. Mark it as optional in the route/virtual host per filter configuration. This way is - // recommended. - // - // We check the first way first to ensure if this filter configuration is marked as optional - // or not. This will be true if the runtime flag is explicitly reverted to false and the - // config name is in the optional http filter list. - bool is_optional_by_hcm = !ignore_optional_option_from_hcm_for_route_config && - (optional_http_filters.find(name) != optional_http_filters.end()); - if (TypeUtil::typeUrlToDescriptorFullName(per_filter_config.second.type_url()) == filter_config_type) { envoy::config::route::v3::FilterConfig filter_config; @@ -2205,22 +2208,21 @@ PerFilterConfigs::PerFilterConfigs( // If the field `config` is not configured, we treat it as configuration error. if (!filter_config.has_config()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Empty route/virtual host per filter configuration for {} filter", name)); } - // If the field `config` is configured but is empty, we treat the filter as disabled + // If the field `config` is configured but is empty, we treat the filter is enabled // explicitly. if (filter_config.config().type_url().empty()) { configs_.emplace(name, FilterConfig{nullptr, false}); continue; } - config = createRouteSpecificFilterConfig(name, filter_config.config(), - is_optional_by_hcm || filter_config.is_optional(), - factory_context, validator); + config = createRouteSpecificFilterConfig( + name, filter_config.config(), filter_config.is_optional(), factory_context, validator); } else { - config = createRouteSpecificFilterConfig(name, per_filter_config.second, is_optional_by_hcm, + config = createRouteSpecificFilterConfig(name, per_filter_config.second, false, factory_context, validator); } @@ -2251,8 +2253,8 @@ Matcher::ActionFactoryCb RouteMatchActionFactory::createActionFactoryCb( const auto& route_config = MessageUtil::downcastAndValidate(config, validation_visitor); - auto route = createAndValidateRoute(route_config, context.vhost, context.optional_http_filters, - context.factory_context, validation_visitor, absl::nullopt); + auto route = createAndValidateRoute(route_config, context.vhost, context.factory_context, + validation_visitor, absl::nullopt); return [route]() { return std::make_unique(route); }; } @@ -2267,9 +2269,8 @@ Matcher::ActionFactoryCb RouteListMatchActionFactory::createActionFactoryCb( std::vector routes; for (const auto& route : route_config.routes()) { - routes.emplace_back(createAndValidateRoute(route, context.vhost, context.optional_http_filters, - context.factory_context, validation_visitor, - absl::nullopt)); + routes.emplace_back(createAndValidateRoute(route, context.vhost, context.factory_context, + validation_visitor, absl::nullopt)); } return [routes]() { return std::make_unique(routes); }; } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index ed9c0cd0acdd..88588139786c 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -29,7 +29,6 @@ #include "source/common/http/header_utility.h" #include "source/common/matcher/matcher.h" #include "source/common/router/config_utility.h" -#include "source/common/router/header_formatter.h" #include "source/common/router/header_parser.h" #include "source/common/router/metadatamatchcriteria_impl.h" #include "source/common/router/router_ratelimit.h" @@ -42,6 +41,10 @@ namespace Envoy { namespace Router { +using RouteMetadataPack = Envoy::Config::MetadataPack; +using RouteMetadataPackPtr = Envoy::Config::MetadataPackPtr; +using DefaultRouteMetadataPack = ConstSingleton; + /** * Original port from the authority header. */ @@ -77,8 +80,6 @@ class Matchable { virtual bool supportsPathlessHeaders() const { return false; } }; -using OptionalHttpFilters = absl::flat_hash_set; - class PerFilterConfigs : public Logger::Loggable { public: struct FilterConfig { @@ -87,7 +88,6 @@ class PerFilterConfigs : public Logger::Loggable { }; PerFilterConfigs(const Protobuf::Map& typed_configs, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -140,7 +140,7 @@ class SslRedirectRoute : public Route { const RouteSpecificFilterConfig* mostSpecificPerFilterConfig(const std::string&) const override { return nullptr; } - bool filterDisabled(absl::string_view) const override { return false; } + absl::optional filterDisabled(absl::string_view) const override { return {}; } void traversePerFilterConfig( const std::string&, std::function) const override {} @@ -235,7 +235,6 @@ using CommonConfigSharedPtr = std::shared_ptr; class CommonVirtualHostImpl : public VirtualHost, Logger::Loggable { public: CommonVirtualHostImpl(const envoy::config::route::v3::VirtualHost& virtual_host, - const OptionalHttpFilters& optional_http_filters, const CommonConfigSharedPtr& global_route_config, Server::Configuration::ServerFactoryContext& factory_context, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validator); @@ -254,7 +253,7 @@ class CommonVirtualHostImpl : public VirtualHost, Logger::Loggable filterDisabled(absl::string_view config_name) const; // Router::VirtualHost const CorsPolicy* corsPolicy() const override { return cors_policy_.get(); } @@ -288,6 +287,8 @@ class CommonVirtualHostImpl : public VirtualHost, Logger::Loggable cb) const override; + const envoy::config::core::v3::Metadata& metadata() const override; + const Envoy::Config::TypedMetadata& typedMetadata() const override; private: struct StatNameProvider { @@ -344,6 +345,7 @@ class CommonVirtualHostImpl : public VirtualHost, Logger::Loggable retry_policy_; std::unique_ptr hedge_policy_; std::unique_ptr virtual_cluster_catch_all_; + RouteMetadataPackPtr metadata_; // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. uint32_t retry_shadow_buffer_limit_{std::numeric_limits::max()}; const bool include_attempt_count_in_request_ : 1; @@ -360,7 +362,6 @@ class VirtualHostImpl : Logger::Loggable { public: VirtualHostImpl( const envoy::config::route::v3::VirtualHost& virtual_host, - const OptionalHttpFilters& optional_http_filters, const CommonConfigSharedPtr& global_route_config, Server::Configuration::ServerFactoryContext& factory_context, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validator, @@ -572,6 +573,7 @@ class InternalRedirectPolicyImpl : public InternalRedirectPolicy { } std::vector predicates() const override; + const std::vector& responseHeadersToCopy() const override; uint32_t maxInternalRedirects() const override { return max_internal_redirects_; } @@ -585,6 +587,10 @@ class InternalRedirectPolicyImpl : public InternalRedirectPolicy { const absl::flat_hash_set redirect_response_codes_; std::vector> predicate_factories_; + // Vector of header names (as a lower case string to simplify use + // later on), to copy from the response that triggers the redirect + // into the following request. + std::vector response_headers_to_copy_; // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. const uint32_t max_internal_redirects_{1}; const bool enabled_{false}; @@ -607,7 +613,6 @@ class RouteEntryImplBase : public RouteEntryAndRoute, */ RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -748,22 +753,8 @@ class RouteEntryImplBase : public RouteEntryAndRoute, return opaque_config_; } bool includeVirtualHostRateLimits() const override { return include_vh_rate_limits_; } - using DefaultMetadata = ConstSingleton; - const envoy::config::core::v3::Metadata& metadata() const override { - if (metadata_ != nullptr) { - return *metadata_; - } - return DefaultMetadata::get(); - } - using RouteTypedMetadata = Envoy::Config::TypedMetadataImpl; - const Envoy::Config::TypedMetadata& typedMetadata() const override { - if (typed_metadata_ != nullptr) { - return *typed_metadata_; - } - static const RouteTypedMetadata* defaultTypedMetadata = - new RouteTypedMetadata(DefaultMetadata::get()); - return *defaultTypedMetadata; - } + const envoy::config::core::v3::Metadata& metadata() const override; + const Envoy::Config::TypedMetadata& typedMetadata() const override; const PathMatchCriterion& pathMatchCriterion() const override { return *this; } bool includeAttemptCountInRequest() const override { return vhost_->includeAttemptCountInRequest(); @@ -791,7 +782,7 @@ class RouteEntryImplBase : public RouteEntryAndRoute, const RouteEntry* routeEntry() const override; const Decorator* decorator() const override { return decorator_.get(); } const RouteTracing* tracingConfig() const override { return route_tracing_.get(); } - bool filterDisabled(absl::string_view config_name) const override; + absl::optional filterDisabled(absl::string_view config_name) const override; const RouteSpecificFilterConfig* mostSpecificPerFilterConfig(const std::string& name) const override { auto* config = per_filter_configs_.get(name); @@ -924,7 +915,7 @@ class RouteEntryImplBase : public RouteEntryAndRoute, const RouteEntry* routeEntry() const override { return this; } const Decorator* decorator() const override { return parent_->decorator(); } const RouteTracing* tracingConfig() const override { return parent_->tracingConfig(); } - bool filterDisabled(absl::string_view config_name) const override { + absl::optional filterDisabled(absl::string_view config_name) const override { return parent_->filterDisabled(config_name); } const RouteSpecificFilterConfig* @@ -963,8 +954,7 @@ class RouteEntryImplBase : public RouteEntryAndRoute, WeightedClusterEntry(const RouteEntryImplBase* parent, const std::string& rutime_key, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, - const envoy::config::route::v3::WeightedCluster::ClusterWeight& cluster, - const OptionalHttpFilters& optional_http_filters); + const envoy::config::route::v3::WeightedCluster::ClusterWeight& cluster); uint64_t clusterWeight() const { return loader_.snapshot().getInteger(runtime_key_, cluster_weight_); @@ -1007,13 +997,13 @@ class RouteEntryImplBase : public RouteEntryAndRoute, stream_info.getRequestHeaders() == nullptr ? *Http::StaticEmptyHeaders::get().request_headers : *stream_info.getRequestHeaders(); - responseHeaderParser().evaluateHeaders(headers, request_headers, headers, stream_info); + responseHeaderParser().evaluateHeaders(headers, {&request_headers, &headers}, stream_info); DynamicRouteEntry::finalizeResponseHeaders(headers, stream_info); } Http::HeaderTransforms responseHeaderTransforms(const StreamInfo::StreamInfo& stream_info, bool do_formatting = true) const override; - bool filterDisabled(absl::string_view config_name) const override { + absl::optional filterDisabled(absl::string_view config_name) const override { absl::optional result = per_filter_configs_.disabled(config_name); if (result.has_value()) { return result.value(); @@ -1204,8 +1194,7 @@ class RouteEntryImplBase : public RouteEntryAndRoute, TlsContextMatchCriteriaConstPtr tls_context_match_criteria_; HeaderParserPtr request_headers_parser_; HeaderParserPtr response_headers_parser_; - std::unique_ptr metadata_; - std::unique_ptr typed_metadata_; + RouteMetadataPackPtr metadata_; const std::vector dynamic_metadata_; // TODO(danielhochman): refactor multimap into unordered_map since JSON is unordered map. @@ -1239,7 +1228,6 @@ class UriTemplateMatcherRouteEntryImpl : public RouteEntryImplBase { public: UriTemplateMatcherRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1271,7 +1259,6 @@ class PrefixRouteEntryImpl : public RouteEntryImplBase { public: PrefixRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1305,7 +1292,6 @@ class PathRouteEntryImpl : public RouteEntryImplBase { public: PathRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1339,7 +1325,6 @@ class RegexRouteEntryImpl : public RouteEntryImplBase { public: RegexRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1374,7 +1359,6 @@ class ConnectRouteEntryImpl : public RouteEntryImplBase { public: ConnectRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1404,7 +1388,6 @@ class PathSeparatedPrefixRouteEntryImpl : public RouteEntryImplBase { public: PathSeparatedPrefixRouteEntryImpl(const CommonVirtualHostSharedPtr& vhost, const envoy::config::route::v3::Route& route, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1434,7 +1417,6 @@ class PathSeparatedPrefixRouteEntryImpl : public RouteEntryImplBase { // Contextual information used to construct the route actions for a match tree. struct RouteActionContext { const CommonVirtualHostSharedPtr& vhost; - const OptionalHttpFilters& optional_http_filters; Server::Configuration::ServerFactoryContext& factory_context; }; @@ -1496,7 +1478,6 @@ DECLARE_FACTORY(RouteListMatchActionFactory); class RouteMatcher { public: RouteMatcher(const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& optional_http_filters, const CommonConfigSharedPtr& global_route_config, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, bool validate_clusters); @@ -1539,7 +1520,6 @@ class RouteMatcher { class CommonConfigImpl : public CommonConfig { public: CommonConfigImpl(const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator); @@ -1559,8 +1539,8 @@ class CommonConfigImpl : public CommonConfig { const RouteSpecificFilterConfig* perFilterConfig(const std::string& name) const { return per_filter_configs_.get(name); } - bool filterDisabled(absl::string_view config_name) const { - return per_filter_configs_.disabled(config_name).value_or(false); + absl::optional filterDisabled(absl::string_view config_name) const { + return per_filter_configs_.disabled(config_name); } // Router::CommonConfig @@ -1580,6 +1560,8 @@ class CommonConfigImpl : public CommonConfig { bool ignorePathParametersInPathMatching() const { return ignore_path_parameters_in_path_matching_; } + const envoy::config::core::v3::Metadata& metadata() const override; + const Envoy::Config::TypedMetadata& typedMetadata() const override; private: std::list internal_only_headers_; @@ -1591,6 +1573,7 @@ class CommonConfigImpl : public CommonConfig { // Cluster specifier plugins/providers. absl::flat_hash_map cluster_specifier_plugins_; PerFilterConfigs per_filter_configs_; + RouteMetadataPackPtr metadata_; // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. const uint32_t max_direct_response_body_size_bytes_; const bool uses_vhds_ : 1; @@ -1604,7 +1587,6 @@ class CommonConfigImpl : public CommonConfig { class ConfigImpl : public Config { public: ConfigImpl(const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, bool validate_clusters_default); @@ -1638,6 +1620,12 @@ class ConfigImpl : public Config { bool ignorePathParametersInPathMatching() const { return shared_config_->ignorePathParametersInPathMatching(); } + const envoy::config::core::v3::Metadata& metadata() const override { + return shared_config_->metadata(); + } + const Envoy::Config::TypedMetadata& typedMetadata() const override { + return shared_config_->typedMetadata(); + } private: CommonConfigSharedPtr shared_config_; @@ -1668,6 +1656,8 @@ class NullConfigImpl : public Config { bool usesVhds() const override { return false; } bool mostSpecificHeaderMutationsWins() const override { return false; } uint32_t maxDirectResponseBodySizeBytes() const override { return 0; } + const envoy::config::core::v3::Metadata& metadata() const override; + const Envoy::Config::TypedMetadata& typedMetadata() const override; private: std::list internal_only_headers_; diff --git a/source/common/router/config_utility.cc b/source/common/router/config_utility.cc index 70362166294b..2493bab40e91 100644 --- a/source/common/router/config_utility.cc +++ b/source/common/router/config_utility.cc @@ -9,6 +9,7 @@ #include "source/common/common/assert.h" #include "source/common/common/regex.h" +#include "source/common/config/datasource.h" namespace Envoy { namespace Router { @@ -38,16 +39,19 @@ ConfigUtility::QueryParameterMatcher::QueryParameterMatcher( : name_(config.name()), matcher_(maybeCreateStringMatcher(config)) {} bool ConfigUtility::QueryParameterMatcher::matches( - const Http::Utility::QueryParams& request_query_params) const { - auto query_param = request_query_params.find(name_); - if (query_param == request_query_params.end()) { + const Http::Utility::QueryParamsMulti& request_query_params) const { + // This preserves the legacy behavior of ignoring all but the first value for a given key + auto data = request_query_params.getFirstValue(name_); + if (!data.has_value()) { return false; - } else if (!matcher_.has_value()) { - // Present match. + } + + if (!matcher_.has_value()) { + // Present check return true; - } else { - return matcher_.value().match(query_param->second); } + + return matcher_.value().match(data.value()); } Upstream::ResourcePriority @@ -63,7 +67,7 @@ ConfigUtility::parsePriority(const envoy::config::core::v3::RoutingPriority& pri } bool ConfigUtility::matchQueryParams( - const Http::Utility::QueryParams& query_params, + const Http::Utility::QueryParamsMulti& query_params, const std::vector& config_query_params) { for (const auto& config_query_param : config_query_params) { if (!config_query_param->matches(query_params)) { @@ -108,28 +112,14 @@ std::string ConfigUtility::parseDirectResponseBody(const envoy::config::route::v return EMPTY_STRING; } const auto& body = route.direct_response().body(); - const std::string& filename = body.filename(); - if (!filename.empty()) { - if (!api.fileSystem().fileExists(filename)) { - throw EnvoyException(fmt::format("response body file {} does not exist", filename)); - } - const ssize_t size = api.fileSystem().fileSize(filename); - if (size < 0) { - throw EnvoyException(absl::StrCat("cannot determine size of response body file ", filename)); - } - if (static_cast(size) > max_body_size_bytes) { - throw EnvoyException(fmt::format("response body file {} size is {} bytes; maximum is {}", - filename, size, max_body_size_bytes)); - } - return api.fileSystem().fileReadToEnd(filename); - } - const std::string inline_body(body.inline_bytes().empty() ? body.inline_string() - : body.inline_bytes()); - if (inline_body.length() > max_body_size_bytes) { - throw EnvoyException(fmt::format("response body size is {} bytes; maximum is {}", - inline_body.length(), max_body_size_bytes)); + + const std::string string_body = + Envoy::Config::DataSource::read(body, true, api, max_body_size_bytes); + if (string_body.length() > max_body_size_bytes) { + throwEnvoyExceptionOrPanic(fmt::format("response body size is {} bytes; maximum is {}", + string_body.length(), max_body_size_bytes)); } - return inline_body; + return string_body; } Http::Code ConfigUtility::parseClusterNotFoundResponseCode( diff --git a/source/common/router/config_utility.h b/source/common/router/config_utility.h index 4895f103de3a..d5773941eebe 100644 --- a/source/common/router/config_utility.h +++ b/source/common/router/config_utility.h @@ -39,7 +39,7 @@ class ConfigUtility { * @param request_query_params supplies the parsed query parameters from a request. * @return bool true if a match for this QueryParameterMatcher exists in request_query_params. */ - bool matches(const Http::Utility::QueryParams& request_query_params) const; + bool matches(const Http::Utility::QueryParamsMulti& request_query_params) const; private: const std::string name_; @@ -62,7 +62,7 @@ class ConfigUtility { * @return bool true if all the query params (and values) in the config_params are found in the * query_params */ - static bool matchQueryParams(const Http::Utility::QueryParams& query_params, + static bool matchQueryParams(const Http::Utility::QueryParamsMulti& query_params, const std::vector& config_query_params); /** diff --git a/source/common/router/delegating_route_impl.h b/source/common/router/delegating_route_impl.h index 1c82de40bf59..2212615fcaaa 100644 --- a/source/common/router/delegating_route_impl.h +++ b/source/common/router/delegating_route_impl.h @@ -43,7 +43,7 @@ class DelegatingRoute : public Router::Route { const Envoy::Config::TypedMetadata& typedMetadata() const override { return base_route_->typedMetadata(); } - bool filterDisabled(absl::string_view name) const override { + absl::optional filterDisabled(absl::string_view name) const override { return base_route_->filterDisabled(name); } const std::string& routeName() const override { return base_route_->routeName(); } diff --git a/source/common/router/header_formatter.h b/source/common/router/header_formatter.h deleted file mode 100644 index 825b70514447..000000000000 --- a/source/common/router/header_formatter.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "envoy/formatter/substitution_formatter.h" - -#include "source/common/http/header_map_impl.h" - -#include "absl/container/node_hash_map.h" -#include "absl/strings/string_view.h" - -namespace Envoy { -namespace Router { - -/** - * HttpHeaderFormatter is used by HTTP headers manipulators. - **/ -class HttpHeaderFormatter { -public: - virtual ~HttpHeaderFormatter() = default; - - virtual const std::string format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Envoy::StreamInfo::StreamInfo& stream_info) const PURE; -}; - -using HttpHeaderFormatterPtr = std::unique_ptr; - -/** - * Implementation of HttpHeaderFormatter. - * Actual formatting is done via substitution formatters. - */ -class HttpHeaderFormatterImpl : public HttpHeaderFormatter { -public: - HttpHeaderFormatterImpl(Formatter::FormatterPtr&& formatter) : formatter_(std::move(formatter)) {} - - // HttpHeaderFormatter::format - // Trailers are not available when HTTP headers are manipulated. - const std::string format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Envoy::StreamInfo::StreamInfo& stream_info) const override { - std::string buf; - buf = formatter_->formatWithContext({&request_headers, &response_headers}, stream_info); - return buf; - }; - -private: - const Formatter::FormatterPtr formatter_; -}; - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index bc48fe67f506..b983c4074df7 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -22,7 +22,7 @@ namespace Router { namespace { -HttpHeaderFormatterPtr +Formatter::FormatterPtr parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_value) { const std::string& key = header_value.key(); // PGV constraints provide this guarantee. @@ -35,7 +35,7 @@ parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_valu // Host is disallowed as it created confusing and inconsistent behaviors for // HTTP/1 and HTTP/2. It could arguably be allowed on the response path. if (!Http::HeaderUtility::isModifiableHeader(key)) { - throw EnvoyException(":-prefixed or host headers may not be modified"); + throwEnvoyExceptionOrPanic(":-prefixed or host headers may not be modified"); } // UPSTREAM_METADATA and DYNAMIC_METADATA must be translated from JSON ["a", "b"] format to colon @@ -45,8 +45,7 @@ parseHttpHeaderFormatter(const envoy::config::core::v3::HeaderValue& header_valu final_header_value = HeaderParser::translatePerRequestState(final_header_value); // Let the substitution formatter parse the final_header_value. - return std::make_unique( - std::make_unique(final_header_value, true)); + return std::make_unique(final_header_value, true); } } // namespace @@ -58,7 +57,7 @@ HeadersToAddEntry::HeadersToAddEntry(const HeaderValueOption& header_value_optio if (header_value_option.has_append()) { // 'append' is set and ensure the 'append_action' value is equal to the default value. if (header_value_option.append_action() != HeaderValueOption::APPEND_IF_EXISTS_OR_ADD) { - throw EnvoyException("Both append and append_action are set and it's not allowed"); + throwEnvoyExceptionOrPanic("Both append and append_action are set and it's not allowed"); } append_action_ = header_value_option.append().value() @@ -112,7 +111,7 @@ HeaderParser::configure(const Protobuf::RepeatedPtrField& hea // request finalization assume their existence and they are needed for well-formedness in most // cases. if (!Http::HeaderUtility::isRemovableHeader(header)) { - throw EnvoyException(":-prefixed or host headers may not be removed"); + throwEnvoyExceptionOrPanic(":-prefixed or host headers may not be removed"); } header_parser->headers_to_remove_.emplace_back(header); } @@ -121,15 +120,13 @@ HeaderParser::configure(const Protobuf::RepeatedPtrField& hea } void HeaderParser::evaluateHeaders(Http::HeaderMap& headers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const { - evaluateHeaders(headers, request_headers, response_headers, &stream_info); + evaluateHeaders(headers, context, &stream_info); } void HeaderParser::evaluateHeaders(Http::HeaderMap& headers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo* stream_info) const { // Removing headers in the headers_to_remove_ list first makes // remove-before-add the default behavior as expected by users. @@ -159,7 +156,7 @@ void HeaderParser::evaluateHeaders(Http::HeaderMap& headers, for (const auto& [key, entry] : headers_to_add_) { absl::string_view value; if (stream_info != nullptr) { - value_buffer = entry.formatter_->format(request_headers, response_headers, *stream_info); + value_buffer = entry.formatter_->formatWithContext(context, *stream_info); value = value_buffer; } else { value = entry.original_value_; @@ -204,9 +201,7 @@ Http::HeaderTransforms HeaderParser::getHeaderTransforms(const StreamInfo::Strea for (const auto& [key, entry] : headers_to_add_) { if (do_formatting) { - const std::string value = - entry.formatter_->format(*Http::StaticEmptyHeaders::get().request_headers, - *Http::StaticEmptyHeaders::get().response_headers, stream_info); + const std::string value = entry.formatter_->formatWithContext({}, stream_info); if (!value.empty() || entry.add_if_empty_) { switch (entry.append_action_) { case HeaderValueOption::APPEND_IF_EXISTS_OR_ADD: diff --git a/source/common/router/header_parser.h b/source/common/router/header_parser.h index 5e7069c3f922..c2785924c58e 100644 --- a/source/common/router/header_parser.h +++ b/source/common/router/header_parser.h @@ -10,7 +10,6 @@ #include "source/common/http/header_map_impl.h" #include "source/common/protobuf/protobuf.h" -#include "source/common/router/header_formatter.h" namespace Envoy { namespace Router { @@ -29,7 +28,7 @@ struct HeadersToAddEntry { std::string original_value_; bool add_if_empty_ = false; - HttpHeaderFormatterPtr formatter_; + Formatter::FormatterPtr formatter_; HeaderAppendAction append_action_; }; @@ -71,11 +70,10 @@ class HeaderParser : public Http::HeaderEvaluator { return *instance; } - void evaluateHeaders(Http::HeaderMap& headers, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + void evaluateHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const override; - void evaluateHeaders(Http::HeaderMap& headers, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, + + void evaluateHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo* stream_info) const; /** @@ -84,20 +82,13 @@ class HeaderParser : public Http::HeaderEvaluator { * empty. */ void evaluateHeaders(Http::HeaderMap& headers, const StreamInfo::StreamInfo& stream_info) const { - evaluateHeaders(headers, - stream_info.getRequestHeaders() != nullptr - ? *stream_info.getRequestHeaders() - : *Http::StaticEmptyHeaders::get().request_headers, - *Http::StaticEmptyHeaders::get().response_headers.get(), stream_info); + evaluateHeaders(headers, {stream_info.getRequestHeaders()}, &stream_info); } void evaluateHeaders(Http::HeaderMap& headers, const StreamInfo::StreamInfo* stream_info) const { evaluateHeaders(headers, - stream_info == nullptr - ? *Http::StaticEmptyHeaders::get().request_headers - : (stream_info->getRequestHeaders() != nullptr - ? *stream_info->getRequestHeaders() - : *Http::StaticEmptyHeaders::get().request_headers), - *Http::StaticEmptyHeaders::get().response_headers.get(), stream_info); + Formatter::HttpFormatterContext{ + stream_info == nullptr ? nullptr : stream_info->getRequestHeaders()}, + stream_info); } /* diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index fe4cc2ba5bae..18475ae48d06 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -28,24 +28,18 @@ RouteConfigProviderSharedPtr RouteConfigProviderUtil::create( Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator, Init::Manager& init_manager, const std::string& stat_prefix, RouteConfigProviderManager& route_config_provider_manager) { - OptionalHttpFilters optional_http_filters; - auto& filters = config.http_filters(); - for (const auto& filter : filters) { - if (filter.is_optional()) { - optional_http_filters.insert(filter.name()); - } - } + switch (config.route_specifier_case()) { case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::kRouteConfig: return route_config_provider_manager.createStaticRouteConfigProvider( - config.route_config(), optional_http_filters, factory_context, validator); + config.route_config(), factory_context, validator); case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::kRds: return route_config_provider_manager.createRdsRouteConfigProvider( // At the creation of a RDS route config provider, the factory_context's initManager is // always valid, though the init manager may go away later when the listener goes away. - config.rds(), optional_http_filters, factory_context, stat_prefix, init_manager); + config.rds(), factory_context, stat_prefix, init_manager); case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::kScopedRoutes: FALLTHRU; // PANIC @@ -228,15 +222,13 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(OptRef( - proto_traits_, factory_context, optional_http_filters); + [&factory_context, &rds, &stat_prefix, this](uint64_t manager_identifier) { + auto config_update = + std::make_unique(proto_traits_, factory_context); auto resource_decoder = std::make_shared< Envoy::Config::OpaqueResourceDecoderImpl>( factory_context.messageValidationContext().dynamicValidationVisitor(), "name"); @@ -253,15 +245,13 @@ Router::RouteConfigProviderSharedPtr RouteConfigProviderManagerImpl::createRdsRo RouteConfigProviderPtr RouteConfigProviderManagerImpl::createStaticRouteConfigProvider( const envoy::config::route::v3::RouteConfiguration& route_config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) { - auto provider = manager_.addStaticProvider( - [&optional_http_filters, &factory_context, &validator, &route_config, this]() { - ConfigTraitsImpl config_traits(optional_http_filters, validator); - return std::make_unique(route_config, config_traits, - factory_context, manager_); - }); + auto provider = manager_.addStaticProvider([&factory_context, &validator, &route_config, this]() { + ConfigTraitsImpl config_traits(validator); + return std::make_unique(route_config, config_traits, + factory_context, manager_); + }); ASSERT(dynamic_cast(provider.get())); return RouteConfigProviderPtr(static_cast(provider.release())); } diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 643c06ff720e..5d5eb826d5c7 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -189,13 +189,11 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, // RouteConfigProviderManager RouteConfigProviderSharedPtr createRdsRouteConfigProvider( const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, Init::Manager& init_manager) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::config::route::v3::RouteConfiguration& route_config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) override; diff --git a/source/common/router/route_config_update_receiver_impl.cc b/source/common/router/route_config_update_receiver_impl.cc index 62cafc189531..8a929aec5eb0 100644 --- a/source/common/router/route_config_update_receiver_impl.cc +++ b/source/common/router/route_config_update_receiver_impl.cc @@ -45,8 +45,8 @@ ConfigTraitsImpl::createConfig(const Protobuf::Message& rc, bool validate_clusters_default) const { ASSERT(dynamic_cast(&rc)); return std::make_shared( - static_cast(rc), optional_http_filters_, - factory_context, validator_, validate_clusters_default); + static_cast(rc), factory_context, + validator_, validate_clusters_default); } bool RouteConfigUpdateReceiverImpl::onRdsUpdate(const Protobuf::Message& rc, diff --git a/source/common/router/route_config_update_receiver_impl.h b/source/common/router/route_config_update_receiver_impl.h index 33d82c833f8e..ae0403ada20f 100644 --- a/source/common/router/route_config_update_receiver_impl.h +++ b/source/common/router/route_config_update_receiver_impl.h @@ -20,9 +20,7 @@ namespace Router { class ConfigTraitsImpl : public Rds::ConfigTraits { public: - ConfigTraitsImpl(const OptionalHttpFilters& optional_http_filters, - ProtobufMessage::ValidationVisitor& validator) - : optional_http_filters_(optional_http_filters), validator_(validator) {} + ConfigTraitsImpl(ProtobufMessage::ValidationVisitor& validator) : validator_(validator) {} Rds::ConfigConstSharedPtr createNullConfig() const override; Rds::ConfigConstSharedPtr createConfig(const Protobuf::Message& rc, @@ -30,17 +28,14 @@ class ConfigTraitsImpl : public Rds::ConfigTraits { bool validate_clusters_default) const override; private: - const OptionalHttpFilters optional_http_filters_; ProtobufMessage::ValidationVisitor& validator_; }; class RouteConfigUpdateReceiverImpl : public RouteConfigUpdateReceiver { public: RouteConfigUpdateReceiverImpl(Rds::ProtoTraits& proto_traits, - Server::Configuration::ServerFactoryContext& factory_context, - const OptionalHttpFilters& optional_http_filters) - : config_traits_(optional_http_filters, - factory_context.messageValidationContext().dynamicValidationVisitor()), + Server::Configuration::ServerFactoryContext& factory_context) + : config_traits_(factory_context.messageValidationContext().dynamicValidationVisitor()), base_(config_traits_, proto_traits, factory_context) {} using VirtualHostMap = std::map; diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 89a688cead20..7391c00bacf4 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -68,17 +68,19 @@ FilterConfig::FilterConfig(Stats::StatName stat_prefix, Server::Configuration::FactoryContext& context, ShadowWriterPtr&& shadow_writer, const envoy::extensions::filters::http::router::v3::Router& config) - : FilterConfig(stat_prefix, context.localInfo(), context.scope(), context.clusterManager(), - context.runtime(), context.api().randomGenerator(), std::move(shadow_writer), - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, dynamic_stats, true), - config.start_child_span(), config.suppress_envoy_headers(), - config.respect_expected_rq_timeout(), - config.suppress_grpc_request_failure_code_stats(), - config.has_upstream_log_options() - ? config.upstream_log_options().flush_upstream_log_on_upstream_stream() - : false, - config.strict_check_headers(), context.api().timeSource(), context.httpContext(), - context.routerContext()) { + : FilterConfig( + stat_prefix, context.serverFactoryContext().localInfo(), context.scope(), + context.serverFactoryContext().clusterManager(), context.serverFactoryContext().runtime(), + context.serverFactoryContext().api().randomGenerator(), std::move(shadow_writer), + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, dynamic_stats, true), config.start_child_span(), + config.suppress_envoy_headers(), config.respect_expected_rq_timeout(), + config.suppress_grpc_request_failure_code_stats(), + config.has_upstream_log_options() + ? config.upstream_log_options().flush_upstream_log_on_upstream_stream() + : false, + config.strict_check_headers(), context.serverFactoryContext().api().timeSource(), + context.serverFactoryContext().httpContext(), + context.serverFactoryContext().routerContext()) { for (const auto& upstream_log : config.upstream_log()) { upstream_logs_.push_back(AccessLog::AccessLogFactory::fromProto(upstream_log, context)); } @@ -90,7 +92,7 @@ FilterConfig::FilterConfig(Stats::StatName stat_prefix, } if (config.upstream_http_filters_size() > 0) { - auto& server_factory_ctx = context.getServerFactoryContext(); + auto& server_factory_ctx = context.serverFactoryContext(); const Http::FilterChainUtility::FiltersList& upstream_http_filters = config.upstream_http_filters(); std::shared_ptr filter_config_provider_manager = @@ -101,8 +103,8 @@ FilterConfig::FilterConfig(Stats::StatName stat_prefix, server_factory_ctx, context.initManager(), context.scope()); Http::FilterChainHelper - helper(*filter_config_provider_manager, server_factory_ctx, context.clusterManager(), - *upstream_ctx_, prefix); + helper(*filter_config_provider_manager, server_factory_ctx, + context.serverFactoryContext().clusterManager(), *upstream_ctx_, prefix); THROW_IF_NOT_OK(helper.processFilters(upstream_http_filters, "router upstream http", "router upstream http", upstream_http_filter_factories_)); } @@ -151,11 +153,11 @@ bool FilterUtility::shouldShadow(const ShadowPolicy& policy, Runtime::Loader& ru stable_random); } -FilterUtility::TimeoutData -FilterUtility::finalTimeout(const RouteEntry& route, Http::RequestHeaderMap& request_headers, - bool insert_envoy_expected_request_timeout_ms, bool grpc_request, - bool per_try_timeout_hedging_enabled, - bool respect_expected_rq_timeout) { +TimeoutData FilterUtility::finalTimeout(const RouteEntry& route, + Http::RequestHeaderMap& request_headers, + bool insert_envoy_expected_request_timeout_ms, + bool grpc_request, bool per_try_timeout_hedging_enabled, + bool respect_expected_rq_timeout) { // See if there is a user supplied timeout in a request header. If there is we take that. // Otherwise if the request is gRPC and a maximum gRPC timeout is configured we use the timeout // in the gRPC headers (or infinity when gRPC headers have no timeout), but cap that timeout to @@ -239,8 +241,7 @@ FilterUtility::finalTimeout(const RouteEntry& route, Http::RequestHeaderMap& req return timeout; } -void FilterUtility::setTimeoutHeaders(uint64_t elapsed_time, - const FilterUtility::TimeoutData& timeout, +void FilterUtility::setTimeoutHeaders(uint64_t elapsed_time, const TimeoutData& timeout, const RouteEntry& route, Http::RequestHeaderMap& request_headers, bool insert_envoy_expected_request_timeout_ms, @@ -327,13 +328,6 @@ Filter::~Filter() { // Upstream resources should already have been cleaned. ASSERT(upstream_requests_.empty()); ASSERT(!retry_state_); - - // Unregister from shadow stream notifications and cancel active streams. - for (auto* shadow_stream : shadow_streams_) { - shadow_stream->removeDestructorCallback(); - shadow_stream->removeWatermarkCallbacks(); - shadow_stream->cancel(); - } } const FilterUtility::StrictHeaderChecker::HeaderCheckResult @@ -563,6 +557,11 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, return Http::FilterHeadersStatus::StopIteration; } + // Support DROP_OVERLOAD config from control plane to drop certain percentage of traffic. + if (checkDropOverload(*cluster, modify_headers)) { + return Http::FilterHeadersStatus::StopIteration; + } + // Fetch a connection pool for the upstream cluster. const auto& upstream_http_protocol_options = cluster_->upstreamHttpProtocolOptions(); @@ -816,7 +815,7 @@ Filter::createConnPool(Upstream::ThreadLocalCluster& thread_local_cluster) { if (route_entry_->connectConfig().has_value()) { auto method = downstream_headers_->getMethodValue(); if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") && - Http::HeaderUtility::isConnectUdp(*downstream_headers_)) { + Http::HeaderUtility::isConnectUdpRequest(*downstream_headers_)) { upstream_protocol = UpstreamProtocol::UDP; } else if (method == Http::Headers::get().MethodValues.Connect || (route_entry_->connectConfig()->allow_post() && @@ -825,7 +824,8 @@ Filter::createConnPool(Upstream::ThreadLocalCluster& thread_local_cluster) { upstream_protocol = UpstreamProtocol::TCP; } } - return factory->createGenericConnPool(thread_local_cluster, upstream_protocol, *route_entry_, + return factory->createGenericConnPool(thread_local_cluster, upstream_protocol, + route_entry_->priority(), callbacks_->streamInfo().protocol(), this); } @@ -1055,6 +1055,14 @@ void Filter::onRequestComplete() { void Filter::onDestroy() { // Reset any in-flight upstream requests. resetAll(); + + // Unregister from shadow stream notifications and cancel active streams. + for (auto* shadow_stream : shadow_streams_) { + shadow_stream->removeDestructorCallback(); + shadow_stream->removeWatermarkCallbacks(); + shadow_stream->cancel(); + } + cleanup(); } @@ -1373,10 +1381,28 @@ void Filter::onUpstreamReset(Http::StreamResetReason reset_reason, onUpstreamAbort(error_code, response_flags, body, dropped, details); } -void Filter::onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host) { +void Filter::onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host, + bool pool_success) { if (retry_state_ && host) { retry_state_->onHostAttempted(host); } + + if (!pool_success) { + return; + } + + if (request_vcluster_) { + // The cluster increases its upstream_rq_total_ counter right before firing this onPoolReady + // callback. Hence, the upstream request increases the virtual cluster's upstream_rq_total_ stat + // here. + request_vcluster_->stats().upstream_rq_total_.inc(); + } + if (route_stats_context_.has_value()) { + // The cluster increases its upstream_rq_total_ counter right before firing this onPoolReady + // callback. Hence, the upstream request increases the route level upstream_rq_total_ stat + // here. + route_stats_context_->stats().upstream_rq_total_.inc(); + } } StreamInfo::ResponseFlag @@ -1740,7 +1766,8 @@ bool Filter::setupRedirect(const Http::ResponseHeaderMap& headers) { // Redirects are not supported for streaming requests yet. if (downstream_end_stream_ && (!request_buffer_overflowed_ || !callbacks_->decodingBuffer()) && location != nullptr && - convertRequestHeadersForInternalRedirect(*downstream_headers_, *location, status_code) && + convertRequestHeadersForInternalRedirect(*downstream_headers_, headers, *location, + status_code) && callbacks_->recreateStream(&headers)) { ENVOY_STREAM_LOG(debug, "Internal redirect succeeded", *callbacks_); cluster_->trafficStats()->upstream_internal_redirect_succeeded_total_.inc(); @@ -1760,9 +1787,9 @@ bool Filter::setupRedirect(const Http::ResponseHeaderMap& headers) { return false; } -bool Filter::convertRequestHeadersForInternalRedirect(Http::RequestHeaderMap& downstream_headers, - const Http::HeaderEntry& internal_redirect, - uint64_t status_code) { +bool Filter::convertRequestHeadersForInternalRedirect( + Http::RequestHeaderMap& downstream_headers, const Http::ResponseHeaderMap& upstream_headers, + const Http::HeaderEntry& internal_redirect, uint64_t status_code) { if (!downstream_headers.Path()) { ENVOY_STREAM_LOG(trace, "Internal redirect failed: no path in downstream_headers", *callbacks_); return false; @@ -1816,17 +1843,41 @@ bool Filter::convertRequestHeadersForInternalRedirect(Http::RequestHeaderMap& do return false; } // Copy the old values, so they can be restored if the redirect fails. - const std::string original_host(downstream_headers.getHostValue()); - const std::string original_path(downstream_headers.getPathValue()); const bool scheme_is_set = (downstream_headers.Scheme() != nullptr); + + std::unique_ptr saved_headers = Http::RequestHeaderMapImpl::create(); + Http::RequestHeaderMapImpl::copyFrom(*saved_headers, downstream_headers); + + for (const Http::LowerCaseString& header : + route_entry_->internalRedirectPolicy().responseHeadersToCopy()) { + Http::HeaderMap::GetResult result = upstream_headers.get(header); + Http::HeaderMap::GetResult downstream_result = downstream_headers.get(header); + if (result.empty()) { + // Clear headers if present, else do nothing: + if (downstream_result.empty()) { + continue; + } + downstream_headers.remove(header); + } else { + // The header exists in the response, copy into the downstream headers + if (!downstream_result.empty()) { + downstream_headers.remove(header); + } + for (size_t idx = 0; idx < result.size(); idx++) { + downstream_headers.addCopy(header, result[idx]->value().getStringView()); + } + } + } + Cleanup restore_original_headers( - [&downstream_headers, original_host, original_path, scheme_is_set, scheme_is_http]() { - downstream_headers.setHost(original_host); - downstream_headers.setPath(original_path); + [&downstream_headers, scheme_is_set, scheme_is_http, &saved_headers]() { + downstream_headers.clear(); if (scheme_is_set) { downstream_headers.setScheme(scheme_is_http ? Http::Headers::get().SchemeValues.Http : Http::Headers::get().SchemeValues.Https); } + + Http::RequestHeaderMapImpl::copyFrom(downstream_headers, *saved_headers); }); // Replace the original host, scheme and path. @@ -1881,10 +1932,10 @@ bool Filter::convertRequestHeadersForInternalRedirect(Http::RequestHeaderMap& do num_internal_redirect->increment(); restore_original_headers.cancel(); // Preserve the original request URL for the second pass. - downstream_headers.setEnvoyOriginalUrl(absl::StrCat(scheme_is_http - ? Http::Headers::get().SchemeValues.Http - : Http::Headers::get().SchemeValues.Https, - "://", original_host, original_path)); + downstream_headers.setEnvoyOriginalUrl( + absl::StrCat(scheme_is_http ? Http::Headers::get().SchemeValues.Http + : Http::Headers::get().SchemeValues.Https, + "://", saved_headers->getHostValue(), saved_headers->getPathValue())); return true; } @@ -1973,6 +2024,33 @@ uint32_t Filter::numRequestsAwaitingHeaders() { [](const auto& req) -> bool { return req->awaitingHeaders(); }); } +bool Filter::checkDropOverload(Upstream::ThreadLocalCluster& cluster, + std::function& modify_headers) { + if (cluster.dropOverload().value()) { + ENVOY_STREAM_LOG(debug, "Router filter: cluster DROP_OVERLOAD configuration: {}", *callbacks_, + cluster.dropOverload().value()); + if (config_.random_.bernoulli(cluster.dropOverload())) { + ENVOY_STREAM_LOG(debug, "The request is dropped by DROP_OVERLOAD", *callbacks_); + callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::DropOverLoad); + chargeUpstreamCode(Http::Code::ServiceUnavailable, nullptr, true); + callbacks_->sendLocalReply( + Http::Code::ServiceUnavailable, "drop overload", + [modify_headers, this](Http::ResponseHeaderMap& headers) { + if (!config_.suppress_envoy_headers_) { + headers.addReference(Http::Headers::get().EnvoyDropOverload, + Http::Headers::get().EnvoyDropOverloadValues.True); + } + modify_headers(headers); + }, + absl::nullopt, StreamInfo::ResponseCodeDetails::get().DropOverload); + + cluster.info()->loadReportStats().upstream_rq_drop_overload_.inc(); + return true; + } + } + return false; +} + RetryStatePtr ProdFilter::createRetryState(const RetryPolicy& policy, Http::RequestHeaderMap& request_headers, const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, diff --git a/source/common/router/router.h b/source/common/router/router.h index af4bbe8c66d2..cd6f6c29d948 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -16,6 +16,7 @@ #include "envoy/http/hash_policy.h" #include "envoy/http/stateful_session.h" #include "envoy/local_info/local_info.h" +#include "envoy/router/router_filter_interface.h" #include "envoy/router/shadow_writer.h" #include "envoy/runtime/runtime.h" #include "envoy/server/factory_context.h" @@ -57,12 +58,6 @@ MAKE_STATS_STRUCT(FilterStats, StatNames, ALL_ROUTER_STATS); */ class FilterUtility { public: - struct TimeoutData { - std::chrono::milliseconds global_timeout_{0}; - std::chrono::milliseconds per_try_timeout_{0}; - std::chrono::milliseconds per_try_idle_timeout_{0}; - }; - struct HedgingParams { bool hedge_on_per_try_timeout_ : 1; }; @@ -165,7 +160,7 @@ class FilterUtility { * @param grpc_request tells if the request is a gRPC request. * @param per_try_timeout_headging_enabled is request hedging enabled? */ - static void setTimeoutHeaders(uint64_t elapsed_time, const FilterUtility::TimeoutData& timeout, + static void setTimeoutHeaders(uint64_t elapsed_time, const TimeoutData& timeout, const RouteEntry& route, Http::RequestHeaderMap& request_headers, bool insert_envoy_expected_request_timeout_ms, bool grpc_request, bool per_try_timeout_hedging_enabled); @@ -250,7 +245,7 @@ class FilterConfig : Http::FilterChainFactory { bool createUpgradeFilterChain(absl::string_view, const UpgradeMap*, Http::FilterChainManager&) const override { - // Upgrade filter chains not yet supported for upstream filters. + // Upgrade filter chains not yet supported for upstream HTTP filters. return false; } @@ -294,47 +289,6 @@ using FilterConfigSharedPtr = std::shared_ptr; class UpstreamRequest; using UpstreamRequestPtr = std::unique_ptr; -// The interface the UpstreamRequest has to interact with the router filter. -// Split out primarily for unit test mocks. -class RouterFilterInterface { -public: - virtual ~RouterFilterInterface() = default; - - virtual void onUpstream1xxHeaders(Http::ResponseHeaderMapPtr&& headers, - UpstreamRequest& upstream_request) PURE; - virtual void onUpstreamHeaders(uint64_t response_code, Http::ResponseHeaderMapPtr&& headers, - UpstreamRequest& upstream_request, bool end_stream) PURE; - virtual void onUpstreamData(Buffer::Instance& data, UpstreamRequest& upstream_request, - bool end_stream) PURE; - virtual void onUpstreamTrailers(Http::ResponseTrailerMapPtr&& trailers, - UpstreamRequest& upstream_request) PURE; - virtual void onUpstreamMetadata(Http::MetadataMapPtr&& metadata_map) PURE; - virtual void onUpstreamReset(Http::StreamResetReason reset_reason, - absl::string_view transport_failure, - UpstreamRequest& upstream_request) PURE; - virtual void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host) PURE; - virtual void onPerTryTimeout(UpstreamRequest& upstream_request) PURE; - virtual void onPerTryIdleTimeout(UpstreamRequest& upstream_request) PURE; - virtual void onStreamMaxDurationReached(UpstreamRequest& upstream_request) PURE; - - virtual Http::StreamDecoderFilterCallbacks* callbacks() PURE; - virtual Upstream::ClusterInfoConstSharedPtr cluster() PURE; - virtual FilterConfig& config() PURE; - virtual FilterUtility::TimeoutData timeout() PURE; - virtual absl::optional dynamicMaxStreamDuration() const PURE; - virtual Http::RequestHeaderMap* downstreamHeaders() PURE; - virtual Http::RequestTrailerMap* downstreamTrailers() PURE; - virtual bool downstreamResponseStarted() const PURE; - virtual bool downstreamEndStream() const PURE; - virtual uint32_t attemptCount() const PURE; - virtual const VirtualCluster* requestVcluster() const PURE; - virtual const RouteStatsContextOptRef routeStatsContext() const PURE; - virtual const Route* route() const PURE; - virtual const std::list& upstreamRequests() const PURE; - virtual const UpstreamRequest* finalUpstreamRequest() const PURE; - virtual TimeSource& timeSource() PURE; -}; - /** * Service routing filter. */ @@ -499,14 +453,15 @@ class Filter : Logger::Loggable, void onUpstreamMetadata(Http::MetadataMapPtr&& metadata_map) override; void onUpstreamReset(Http::StreamResetReason reset_reason, absl::string_view transport_failure, UpstreamRequest& upstream_request) override; - void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host) override; + void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host, + bool pool_success) override; void onPerTryTimeout(UpstreamRequest& upstream_request) override; void onPerTryIdleTimeout(UpstreamRequest& upstream_request) override; void onStreamMaxDurationReached(UpstreamRequest& upstream_request) override; Http::StreamDecoderFilterCallbacks* callbacks() override { return callbacks_; } Upstream::ClusterInfoConstSharedPtr cluster() override { return cluster_; } FilterConfig& config() override { return config_; } - FilterUtility::TimeoutData timeout() override { return timeout_; } + TimeoutData timeout() override { return timeout_; } absl::optional dynamicMaxStreamDuration() const override { return dynamic_max_stream_duration_; } @@ -515,15 +470,10 @@ class Filter : Logger::Loggable, bool downstreamResponseStarted() const override { return downstream_response_started_; } bool downstreamEndStream() const override { return downstream_end_stream_; } uint32_t attemptCount() const override { return attempt_count_; } - const VirtualCluster* requestVcluster() const override { return request_vcluster_; } - const RouteStatsContextOptRef routeStatsContext() const override { return route_stats_context_; } - const Route* route() const override { return route_.get(); } - const std::list& upstreamRequests() const override { - return upstream_requests_; - } - const UpstreamRequest* finalUpstreamRequest() const override { return final_upstream_request_; } - TimeSource& timeSource() override { return config_.timeSource(); } + const std::list& upstreamRequests() const { return upstream_requests_; } + TimeSource& timeSource() { return config_.timeSource(); } + const Route* route() const { return route_.get(); } const FilterStats& stats() { return stats_; } protected: @@ -586,6 +536,7 @@ class Filter : Logger::Loggable, void sendNoHealthyUpstreamResponse(); bool setupRedirect(const Http::ResponseHeaderMap& headers); bool convertRequestHeadersForInternalRedirect(Http::RequestHeaderMap& downstream_headers, + const Http::ResponseHeaderMap& upstream_headers, const Http::HeaderEntry& internal_redirect, uint64_t status_code); void updateOutlierDetection(Upstream::Outlier::Result result, UpstreamRequest& upstream_request, @@ -598,6 +549,8 @@ class Filter : Logger::Loggable, UpstreamRequest& upstream_request, bool end_stream, uint64_t grpc_to_http_status); Http::Context& httpContext() { return config_.http_context_; } + bool checkDropOverload(Upstream::ThreadLocalCluster& cluster, + std::function& modify_headers); RetryStatePtr retry_state_; FilterConfig& config_; @@ -609,7 +562,7 @@ class Filter : Logger::Loggable, const VirtualCluster* request_vcluster_{}; RouteStatsContextOptRef route_stats_context_; Event::TimerPtr response_timeout_; - FilterUtility::TimeoutData timeout_; + TimeoutData timeout_; std::list upstream_requests_; FilterStats stats_; // Tracks which upstream request "wins" and will have the corresponding diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index 0d75d283fe9d..b53943de89f4 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -257,8 +257,8 @@ QueryParameterValueMatchAction::QueryParameterValueMatchAction( bool QueryParameterValueMatchAction::populateDescriptor( RateLimit::DescriptorEntry& descriptor_entry, const std::string&, const Http::RequestHeaderMap& headers, const StreamInfo::StreamInfo&) const { - Http::Utility::QueryParams query_parameters = - Http::Utility::parseAndDecodeQueryString(headers.getPathValue()); + Http::Utility::QueryParamsMulti query_parameters = + Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(headers.getPathValue()); if (expect_match_ == ConfigUtility::matchQueryParams(query_parameters, action_query_parameters_)) { descriptor_entry = {descriptor_key_, descriptor_value_}; @@ -334,7 +334,7 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl( if (producer) { actions_.emplace_back(std::move(producer)); } else { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Rate limit descriptor extension failed: ", action.extension().name())); } break; @@ -348,7 +348,7 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl( new QueryParameterValueMatchAction(action.query_parameter_value_match())); break; case envoy::config::route::v3::RateLimit::Action::ActionSpecifierCase::ACTION_SPECIFIER_NOT_SET: - throw EnvoyException("invalid config"); + throwEnvoyExceptionOrPanic("invalid config"); } } if (config.has_limit()) { @@ -359,7 +359,7 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl( break; case envoy::config::route::v3::RateLimit_Override::OverrideSpecifierCase:: OVERRIDE_SPECIFIER_NOT_SET: - throw EnvoyException("invalid config"); + throwEnvoyExceptionOrPanic("invalid config"); } } } diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 45a16d9c4c46..e6566a2f872c 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -29,14 +29,14 @@ HeaderValueExtractorImpl::HeaderValueExtractorImpl( ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex) { if (header_value_extractor_config_.index() != 0 && header_value_extractor_config_.element_separator().empty()) { - throw ProtoValidationException("Index > 0 for empty string element separator.", - header_value_extractor_config_); + ProtoExceptionUtil::throwProtoValidationException( + "Index > 0 for empty string element separator.", config_); } } if (header_value_extractor_config_.extract_type_case() == ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::EXTRACT_TYPE_NOT_SET) { - throw ProtoValidationException("HeaderValueExtractor extract_type not set.", - header_value_extractor_config_); + ProtoExceptionUtil::throwProtoValidationException("HeaderValueExtractor extract_type not set.", + config_); } } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index d241be70a9b1..3e4c6526df49 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -42,13 +42,6 @@ ConfigProviderPtr create( ASSERT(config.route_specifier_case() == envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::kScopedRoutes); - OptionalHttpFilters optional_http_filters; - auto& filters = config.http_filters(); - for (const auto& filter : filters) { - if (filter.is_optional()) { - optional_http_filters.insert(filter.name()); - } - } switch (config.scoped_routes().config_specifier_case()) { case envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: ConfigSpecifierCase::kScopedRouteConfigurationsList: { @@ -61,16 +54,14 @@ ConfigProviderPtr create( ProtobufTypes::ConstMessagePtrVector>(scoped_route_list.scoped_route_configurations()), factory_context, ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), - config.scoped_routes().rds_config_source(), - optional_http_filters)); + config.scoped_routes().rds_config_source())); } case envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: ConfigSpecifierCase::kScopedRds: return scoped_routes_config_provider_manager.createXdsConfigProvider( config.scoped_routes().scoped_rds(), factory_context, init_manager, stat_prefix, ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), - config.scoped_routes().rds_config_source(), - optional_http_filters)); + config.scoped_routes().rds_config_source())); case envoy::extensions::filters::network::http_connection_manager::v3::ScopedRoutes:: ConfigSpecifierCase::CONFIG_SPECIFIER_NOT_SET: PANIC("not implemented"); @@ -95,23 +86,24 @@ namespace { std::vector makeScopedRouteInfos(ProtobufTypes::ConstMessagePtrVector&& config_protos, Server::Configuration::ServerFactoryContext& factory_context, - ScopedRoutesConfigProviderManager& config_provider_manager, - const OptionalHttpFilters& optional_http_filters) { + ScopedRoutesConfigProviderManager& config_provider_manager) { std::vector scopes; for (std::unique_ptr& config_proto : config_protos) { auto scoped_route_config = MessageUtil::downcastAndValidate( *config_proto, factory_context.messageValidationContext().staticValidationVisitor()); if (!scoped_route_config.route_configuration_name().empty()) { - throw EnvoyException("Fetching routes via RDS (route_configuration_name) is not supported " - "with inline scoped routes."); + throwEnvoyExceptionOrPanic( + "Fetching routes via RDS (route_configuration_name) is not supported " + "with inline scoped routes."); } if (!scoped_route_config.has_route_configuration()) { - throw EnvoyException("You must specify a route_configuration with inline scoped routes."); + throwEnvoyExceptionOrPanic( + "You must specify a route_configuration with inline scoped routes."); } RouteConfigProviderPtr route_config_provider = config_provider_manager.routeConfigProviderManager().createStaticRouteConfigProvider( - scoped_route_config.route_configuration(), optional_http_filters, factory_context, + scoped_route_config.route_configuration(), factory_context, factory_context.messageValidationContext().staticValidationVisitor()); scopes.push_back(std::make_shared(scoped_route_config, route_config_provider->configCast())); @@ -126,22 +118,21 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ProtobufTypes::ConstMessagePtrVector&& config_protos, std::string name, Server::Configuration::ServerFactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, - envoy::config::core::v3::ConfigSource rds_config_source, - const OptionalHttpFilters& optional_http_filters) + envoy::config::core::v3::ConfigSource rds_config_source) : Envoy::Config::ImmutableConfigProviderBase(factory_context, config_provider_manager, ConfigProviderInstanceType::Inline, ConfigProvider::ApiType::Delta), name_(std::move(name)), - scopes_(makeScopedRouteInfos(std::move(config_protos), factory_context, - config_provider_manager, optional_http_filters)), + scopes_( + makeScopedRouteInfos(std::move(config_protos), factory_context, config_provider_manager)), config_(std::make_shared(scopes_)), rds_config_source_(std::move(rds_config_source)) {} ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRds& scoped_rds, - const OptionalHttpFilters& optional_http_filters, const uint64_t manager_identifier, - const std::string& name, Server::Configuration::ServerFactoryContext& factory_context, - const std::string& stat_prefix, envoy::config::core::v3::ConfigSource rds_config_source, + const uint64_t manager_identifier, const std::string& name, + Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, + envoy::config::core::v3::ConfigSource rds_config_source, RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, @@ -152,8 +143,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}), rds_config_source_(std::move(rds_config_source)), stat_prefix_(stat_prefix), - route_config_provider_manager_(route_config_provider_manager), - optional_http_filters_(optional_http_filters) { + route_config_provider_manager_(route_config_provider_manager) { const auto resource_name = getResourceName(); if (scoped_rds.srds_resources_locator().empty()) { subscription_ = @@ -230,8 +220,7 @@ void ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::initRdsConfigPro Init::Manager& init_manager) { route_provider_ = std::dynamic_pointer_cast( parent_.route_config_provider_manager_.createRdsRouteConfigProvider( - rds, parent_.optional_http_filters_, parent_.factory_context_, parent_.stat_prefix_, - init_manager)); + rds, parent_.factory_context_, parent_.stat_prefix_, init_manager)); rds_update_callback_handle_ = route_provider_->subscription().addUpdateCallback([this]() { // Subscribe to RDS update. @@ -430,8 +419,8 @@ absl::Status ScopedRdsConfigSubscription::onConfigUpdate( if (!status_or_applied.status().ok()) { return status_or_applied.status(); } - bool any_applied = status_or_applied.value(); - auto status = ConfigSubscriptionCommonBase::onConfigUpdate(); + const bool any_applied = status_or_applied.value(); + const auto status = ConfigSubscriptionCommonBase::onConfigUpdate(); if (!status.ok()) { return status; } @@ -627,9 +616,8 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRds&>( config_source_proto); return std::make_shared( - scoped_rds_config_source, typed_optarg.optional_http_filters_, manager_identifier, - typed_optarg.scoped_routes_name_, factory_context, stat_prefix, - typed_optarg.rds_config_source_, + scoped_rds_config_source, manager_identifier, typed_optarg.scoped_routes_name_, + factory_context, stat_prefix, typed_optarg.rds_config_source_, static_cast(config_provider_manager) .routeConfigProviderManager(), static_cast(config_provider_manager)); @@ -645,7 +633,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( const auto& typed_optarg = static_cast(optarg); return std::make_unique( std::move(config_protos), typed_optarg.scoped_routes_name_, factory_context, *this, - typed_optarg.rds_config_source_, typed_optarg.optional_http_filters_); + typed_optarg.rds_config_source_); } } // namespace Router diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 3dc1b7e615ae..b10b518493ba 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -57,8 +57,7 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr std::string name, Server::Configuration::ServerFactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, - envoy::config::core::v3::ConfigSource rds_config_source, - const OptionalHttpFilters& optional_http_filters); + envoy::config::core::v3::ConfigSource rds_config_source); ~InlineScopedRoutesConfigProvider() override = default; @@ -117,9 +116,9 @@ class ScopedRdsConfigSubscription ScopedRdsConfigSubscription( const envoy::extensions::filters::network::http_connection_manager::v3::ScopedRds& scoped_rds, - const OptionalHttpFilters& optional_http_filters, const uint64_t manager_identifier, - const std::string& name, Server::Configuration::ServerFactoryContext& factory_context, - const std::string& stat_prefix, envoy::config::core::v3::ConfigSource rds_config_source, + const uint64_t manager_identifier, const std::string& name, + Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, + envoy::config::core::v3::ConfigSource rds_config_source, RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); @@ -242,7 +241,6 @@ class ScopedRdsConfigSubscription absl::flat_hash_map route_provider_by_scope_; // A map of (hash, scope-name), used to detect the key conflict between scopes. absl::flat_hash_map scope_name_by_hash_; - const OptionalHttpFilters optional_http_filters_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; @@ -315,14 +313,11 @@ class ScopedRoutesConfigProviderManagerOptArg public: ScopedRoutesConfigProviderManagerOptArg( std::string scoped_routes_name, - const envoy::config::core::v3::ConfigSource& rds_config_source, - const OptionalHttpFilters& optional_http_filters) - : scoped_routes_name_(std::move(scoped_routes_name)), rds_config_source_(rds_config_source), - optional_http_filters_(optional_http_filters) {} + const envoy::config::core::v3::ConfigSource& rds_config_source) + : scoped_routes_name_(std::move(scoped_routes_name)), rds_config_source_(rds_config_source) {} const std::string scoped_routes_name_; const envoy::config::core::v3::ConfigSource& rds_config_source_; - const OptionalHttpFilters& optional_http_filters_; }; } // namespace Router diff --git a/source/common/router/upstream_codec_filter.h b/source/common/router/upstream_codec_filter.h index 657a59f99db6..3e090387b79d 100644 --- a/source/common/router/upstream_codec_filter.h +++ b/source/common/router/upstream_codec_filter.h @@ -18,7 +18,7 @@ namespace Envoy { namespace Router { -// This is the last filter in the upstream filter chain. +// This is the last filter in the upstream HTTP filter chain. // It takes request headers/body/data from the filter manager and encodes them to the upstream // codec. It also registers the CodecBridge with the upstream stream, and takes response // headers/body/data from the upstream stream and sends them to the filter manager. @@ -113,7 +113,7 @@ class UpstreamCodecFilterFactory UpstreamCodecFilterFactory() : CommonFactoryBase("envoy.filters.http.upstream_codec") {} std::string category() const override { return "envoy.filters.http.upstream"; } - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, Server::Configuration::UpstreamFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index c4beb31cdb33..5a27efd3e11c 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -41,7 +41,7 @@ namespace Envoy { namespace Router { -// The upstream filter manager class. +// The upstream HTTP filter manager class. class UpstreamFilterManager : public Http::FilterManager { public: UpstreamFilterManager(Http::FilterManagerCallbacks& filter_manager_callbacks, @@ -60,8 +60,8 @@ class UpstreamFilterManager : public Http::FilterManager { const StreamInfo::StreamInfo& streamInfo() const override { return upstream_request_.parent_.callbacks()->streamInfo(); } - // Send local replies via the downstream filter manager. - // Local replies will not be seen by upstream filters. + // Send local replies via the downstream HTTP filter manager. + // Local replies will not be seen by upstream HTTP filters. void sendLocalReply(Http::Code code, absl::string_view body, const std::function& modify_headers, const absl::optional grpc_status, @@ -94,12 +94,12 @@ UpstreamRequest::UpstreamRequest(RouterFilterInterface& parent, stream_options_({can_send_early_data, can_use_http3}), grpc_rq_success_deferred_(false), upstream_wait_for_response_headers_before_disabling_read_(Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.upstream_wait_for_response_headers_before_disabling_read")) { - if (parent_.config().start_child_span_) { - if (auto tracing_config = parent_.callbacks()->tracingConfig(); tracing_config.has_value()) { + if (auto tracing_config = parent_.callbacks()->tracingConfig(); tracing_config.has_value()) { + if (tracing_config->spawnUpstreamSpan() || parent_.config().start_child_span_) { span_ = parent_.callbacks()->activeSpan().spawnChild( tracing_config.value().get(), absl::StrCat("router ", parent.cluster()->observabilityName(), " egress"), - parent.timeSource().systemTime()); + parent_.callbacks()->dispatcher().timeSource().systemTime()); if (parent.attemptCount() != 1) { // This is a retry request, add this metadata to span. span_->setTag(Tracing::Tags::get().RetryCount, std::to_string(parent.attemptCount() - 1)); @@ -109,33 +109,34 @@ UpstreamRequest::UpstreamRequest(RouterFilterInterface& parent, // The router checks that the connection pool is non-null before creating the upstream request. auto upstream_host = conn_pool_->host(); + Tracing::HttpTraceContext trace_context(*parent_.downstreamHeaders()); if (span_ != nullptr) { - span_->injectContext(*parent_.downstreamHeaders(), upstream_host); + span_->injectContext(trace_context, upstream_host); } else { // No independent child span for current upstream request then inject the parent span's tracing // context into the request headers. // The injectContext() of the parent span may be called repeatedly when the request is retried. - parent_.callbacks()->activeSpan().injectContext(*parent_.downstreamHeaders(), upstream_host); + parent_.callbacks()->activeSpan().injectContext(trace_context, upstream_host); } stream_info_.setUpstreamInfo(std::make_shared()); - stream_info_.route_ = parent.callbacks()->route(); + stream_info_.route_ = parent_.callbacks()->route(); parent_.callbacks()->streamInfo().setUpstreamInfo(stream_info_.upstreamInfo()); stream_info_.healthCheck(parent_.callbacks()->streamInfo().healthCheck()); + stream_info_.setIsShadow(parent_.callbacks()->streamInfo().isShadow()); absl::optional cluster_info = parent_.callbacks()->streamInfo().upstreamClusterInfo(); if (cluster_info.has_value()) { stream_info_.setUpstreamClusterInfo(*cluster_info); } - // Set up the upstream filter manager. + // Set up the upstream HTTP filter manager. filter_manager_callbacks_ = std::make_unique(*this); filter_manager_ = std::make_unique( - *filter_manager_callbacks_, parent_.callbacks()->dispatcher(), - parent_.callbacks()->connection(), parent_.callbacks()->streamId(), - parent_.callbacks()->account(), true, parent_.callbacks()->decoderBufferLimit(), - *parent_.cluster(), *this); + *filter_manager_callbacks_, parent_.callbacks()->dispatcher(), connection(), + parent_.callbacks()->streamId(), parent_.callbacks()->account(), true, + parent_.callbacks()->decoderBufferLimit(), *parent_.cluster(), *this); // Attempt to create custom cluster-specified filter chain bool created = parent_.cluster()->createFilterChain(*filter_manager_, /*only_create_if_configured=*/true); @@ -224,16 +225,21 @@ void UpstreamRequest::cleanUp() { parent_.cluster()->trafficStats()->upstream_flow_control_drained_total_.inc(); --downstream_data_disabled_; } - // The upstream filter chain callbacks own headers/trailers while they are traversing the filter - // chain. Make sure to not delete them immediately when the stream ends, as the stream often - // ends during filter chain processing and it causes use-after-free violations. + // The upstream HTTP filter chain callbacks own headers/trailers while they are traversing the + // filter chain. Make sure to not delete them immediately when the stream ends, as the stream + // often ends during filter chain processing and it causes use-after-free violations. parent_.callbacks()->dispatcher().deferredDelete(std::move(filter_manager_callbacks_)); } void UpstreamRequest::upstreamLog(AccessLog::AccessLogType access_log_type) { + const Formatter::HttpFormatterContext log_context{parent_.downstreamHeaders(), + upstream_headers_.get(), + upstream_trailers_.get(), + {}, + access_log_type}; + for (const auto& upstream_log : parent_.config().upstream_logs_) { - upstream_log->log(parent_.downstreamHeaders(), upstream_headers_.get(), - upstream_trailers_.get(), stream_info_, access_log_type); + upstream_log->log(log_context, stream_info_); } } @@ -338,7 +344,7 @@ void UpstreamRequest::dumpState(std::ostream& os, int indent_level) const { } } -const Route& UpstreamRequest::route() const { return *parent_.route(); } +const Route& UpstreamRequest::route() const { return *parent_.callbacks()->route(); } OptRef UpstreamRequest::connection() const { return parent_.callbacks()->connection(); @@ -354,11 +360,12 @@ void UpstreamRequest::maybeEndDecode(bool end_stream) { } } -void UpstreamRequest::onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host) { +void UpstreamRequest::onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host, + bool pool_success) { StreamInfo::UpstreamInfo& upstream_info = *streamInfo().upstreamInfo(); upstream_info.setUpstreamHost(host); upstream_host_ = host; - parent_.onUpstreamHostSelected(host); + parent_.onUpstreamHostSelected(host, pool_success); } void UpstreamRequest::acceptHeadersFromRouter(bool end_stream) { @@ -375,7 +382,7 @@ void UpstreamRequest::acceptHeadersFromRouter(bool end_stream) { } // Kick off creation of the upstream connection immediately upon receiving headers. - // In future it may be possible for upstream filters to delay this, or influence connection + // In future it may be possible for upstream HTTP filters to delay this, or influence connection // creation but for now optimize for minimal latency and fetch the connection // as soon as possible. conn_pool_->newStream(this); @@ -548,7 +555,7 @@ void UpstreamRequest::onPoolFailure(ConnectionPool::PoolFailureReason reason, stream_info_.upstreamInfo()->setUpstreamTransportFailureReason(transport_failure_reason); // Mimic an upstream reset. - onUpstreamHostSelected(host); + onUpstreamHostSelected(host, false); onResetStream(reset_reason, transport_failure_reason); } @@ -566,22 +573,9 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, // Have the upstream use the account of the downstream. upstream_->setAccount(parent_.callbacks()->account()); - if (parent_.requestVcluster()) { - // The cluster increases its upstream_rq_total_ counter right before firing this onPoolReady - // callback. Hence, the upstream request increases the virtual cluster's upstream_rq_total_ stat - // here. - parent_.requestVcluster()->stats().upstream_rq_total_.inc(); - } - if (parent_.routeStatsContext().has_value()) { - // The cluster increases its upstream_rq_total_ counter right before firing this onPoolReady - // callback. Hence, the upstream request increases the route level upstream_rq_total_ stat - // here. - parent_.routeStatsContext()->stats().upstream_rq_total_.inc(); - } - host->outlierDetector().putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess); - onUpstreamHostSelected(host); + onUpstreamHostSelected(host, true); if (protocol) { stream_info_.protocol(protocol.value()); @@ -599,7 +593,7 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, upstream_info.setUpstreamNumStreams(info.upstreamInfo()->upstreamNumStreams()); } - // Upstream filters might have already created/set a filter state. + // Upstream HTTP filters might have already created/set a filter state. const StreamInfo::FilterStateSharedPtr& filter_state = info.filterState(); if (!filter_state) { upstream_info.setUpstreamFilterState( @@ -651,7 +645,7 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, max_stream_duration_timer_->enableTimer(*max_stream_duration); } - const auto* route_entry = parent_.route()->routeEntry(); + const auto* route_entry = route().routeEntry(); if (route_entry->autoHostRewrite() && !host->hostname().empty()) { Http::Utility::updateAuthority(*parent_.downstreamHeaders(), host->hostname(), route_entry->appendXfh()); @@ -733,14 +727,6 @@ void UpstreamRequest::readDisableOrDefer(bool disable) { void UpstreamRequest::DownstreamWatermarkManager::onAboveWriteBufferHighWatermark() { ASSERT(parent_.upstream_); - - // There are two states we should get this callback in: 1) the watermark was - // hit due to writes from a different filter instance over a shared - // downstream connection, or 2) the watermark was hit due to THIS filter - // instance writing back the "winning" upstream request. In either case we - // can disable reads from upstream. - ASSERT(!parent_.parent_.finalUpstreamRequest() || - &parent_ == parent_.parent_.finalUpstreamRequest()); parent_.readDisableOrDefer(true); } @@ -750,32 +736,12 @@ void UpstreamRequest::DownstreamWatermarkManager::onBelowWriteBufferLowWatermark } void UpstreamRequest::disableDataFromDownstreamForFlowControl() { - // If there is only one upstream request, we can be assured that - // disabling reads will not slow down other upstream requests. If we've - // already seen the full downstream request (downstream_end_stream_) then - // disabling reads is a noop. - // This assert condition must be true because - // parent_.upstreamRequests().size() can only be greater than 1 in the - // case of a per-try-timeout with hedge_on_per_try_timeout enabled, and - // the per try timeout timer is started only after downstream_end_stream_ - // is true. - ASSERT(parent_.upstreamRequests().size() == 1 || parent_.downstreamEndStream()); parent_.cluster()->trafficStats()->upstream_flow_control_backed_up_total_.inc(); parent_.callbacks()->onDecoderFilterAboveWriteBufferHighWatermark(); ++downstream_data_disabled_; } void UpstreamRequest::enableDataFromDownstreamForFlowControl() { - // If there is only one upstream request, we can be assured that - // disabling reads will not overflow any write buffers in other upstream - // requests. If we've already seen the full downstream request - // (downstream_end_stream_) then enabling reads is a noop. - // This assert condition must be true because - // parent_.upstreamRequests().size() can only be greater than 1 in the - // case of a per-try-timeout with hedge_on_per_try_timeout enabled, and - // the per try timeout timer is started only after downstream_end_stream_ - // is true. - ASSERT(parent_.upstreamRequests().size() == 1 || parent_.downstreamEndStream()); parent_.cluster()->trafficStats()->upstream_flow_control_drained_total_.inc(); parent_.callbacks()->onDecoderFilterBelowWriteBufferLowWatermark(); ASSERT(downstream_data_disabled_ != 0); diff --git a/source/common/router/upstream_request.h b/source/common/router/upstream_request.h index a91b75c833f2..b2369c8cb125 100644 --- a/source/common/router/upstream_request.h +++ b/source/common/router/upstream_request.h @@ -84,7 +84,7 @@ class UpstreamRequest : public Logger::Loggable, void resetStream(); void setupPerTryTimeout(); void maybeEndDecode(bool end_stream); - void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host); + void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host, bool pool_success); // Http::StreamDecoder void decodeData(Buffer::Instance& data, bool end_stream) override; @@ -342,10 +342,10 @@ class UpstreamRequestFilterManagerCallbacks : public Http::FilterManagerCallback // Unsupported functions. void recreateStream(StreamInfo::FilterStateSharedPtr) override { - IS_ENVOY_BUG("recreateStream called from upstream filter"); + IS_ENVOY_BUG("recreateStream called from upstream HTTP filter"); } void upgradeFilterChainCreated() override { - IS_ENVOY_BUG("upgradeFilterChainCreated called from upstream filter"); + IS_ENVOY_BUG("upgradeFilterChainCreated called from upstream HTTP filter"); } OptRef upstreamCallbacks() override { return {*this}; } diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 939852a50fc6..7f0139907337 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -43,7 +43,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, .api_config_source() .api_type(); if (config_source != envoy::config::core::v3::ApiConfigSource::DELTA_GRPC) { - throw EnvoyException("vhds: only 'DELTA_GRPC' is supported as an api_type."); + throwEnvoyExceptionOrPanic("vhds: only 'DELTA_GRPC' is supported as an api_type."); } const auto resource_name = getResourceName(); Envoy::Config::SubscriptionOptions options; diff --git a/source/common/runtime/BUILD b/source/common/runtime/BUILD index 97a43f3b4964..b5b1dc888386 100644 --- a/source/common/runtime/BUILD +++ b/source/common/runtime/BUILD @@ -9,6 +9,19 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "runtime_keys_lib", + srcs = [ + "runtime_keys.cc", + ], + hdrs = [ + "runtime_keys.h", + ], + deps = [ + "@com_google_absl//absl/strings", + ], +) + envoy_cc_library( name = "runtime_features_lib", srcs = [ diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index ceaf22acc2a4..716b83ea8a16 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -30,55 +30,58 @@ // ASAP by filing a bug on github. Overriding non-buggy code is strongly discouraged to avoid the // problem of the bugs being found after the old code path has been removed. RUNTIME_GUARD(envoy_reloadable_features_allow_absolute_url_with_mixed_scheme); -RUNTIME_GUARD(envoy_reloadable_features_allow_compact_maglev); -RUNTIME_GUARD(envoy_reloadable_features_append_query_parameters_path_rewriter); RUNTIME_GUARD(envoy_reloadable_features_append_xfh_idempotent); RUNTIME_GUARD(envoy_reloadable_features_check_mep_on_first_eject); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); +RUNTIME_GUARD(envoy_reloadable_features_convert_legacy_lb_config); RUNTIME_GUARD(envoy_reloadable_features_copy_response_code_to_downstream_stream_info); RUNTIME_GUARD(envoy_reloadable_features_count_unused_mapped_pages_as_free); +RUNTIME_GUARD(envoy_reloadable_features_defer_processing_backedup_streams); RUNTIME_GUARD(envoy_reloadable_features_detect_and_raise_rst_tcp_connection); RUNTIME_GUARD(envoy_reloadable_features_dfp_mixed_scheme); +RUNTIME_GUARD(envoy_reloadable_features_dns_cache_set_first_resolve_complete); RUNTIME_GUARD(envoy_reloadable_features_enable_aws_credentials_file); RUNTIME_GUARD(envoy_reloadable_features_enable_compression_bomb_protection); RUNTIME_GUARD(envoy_reloadable_features_enable_connect_udp_support); RUNTIME_GUARD(envoy_reloadable_features_enable_intermediate_ca); RUNTIME_GUARD(envoy_reloadable_features_enable_zone_routing_different_zone_counts); -RUNTIME_GUARD(envoy_reloadable_features_expand_agnostic_stream_lifetime); RUNTIME_GUARD(envoy_reloadable_features_ext_authz_http_send_original_xff); -RUNTIME_GUARD(envoy_reloadable_features_format_ports_as_numbers); +RUNTIME_GUARD(envoy_reloadable_features_grpc_http1_reverse_bridge_handle_empty_response); RUNTIME_GUARD(envoy_reloadable_features_handle_uppercase_scheme); RUNTIME_GUARD(envoy_reloadable_features_hmac_base64_encoding_only); RUNTIME_GUARD(envoy_reloadable_features_http1_allow_codec_error_response_after_1xx_headers); RUNTIME_GUARD(envoy_reloadable_features_http1_connection_close_header_in_redirect); RUNTIME_GUARD(envoy_reloadable_features_http1_use_balsa_parser); RUNTIME_GUARD(envoy_reloadable_features_http2_decode_metadata_with_quiche); +RUNTIME_GUARD(envoy_reloadable_features_http2_discard_host_header); +RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2); RUNTIME_GUARD(envoy_reloadable_features_http2_validate_authority_with_quiche); RUNTIME_GUARD(envoy_reloadable_features_http_allow_partial_urls_in_referer); -RUNTIME_GUARD(envoy_reloadable_features_http_ext_auth_failure_mode_allow_header_add); RUNTIME_GUARD(envoy_reloadable_features_http_filter_avoid_reentrant_local_reply); // Delay deprecation and decommission until UHV is enabled. RUNTIME_GUARD(envoy_reloadable_features_http_reject_path_with_fragment); -RUNTIME_GUARD(envoy_reloadable_features_ignore_optional_option_from_hcm_for_route_config); RUNTIME_GUARD(envoy_reloadable_features_immediate_response_use_filter_mutation_rule); RUNTIME_GUARD(envoy_reloadable_features_initialize_upstream_filters); RUNTIME_GUARD(envoy_reloadable_features_keep_endpoint_active_hc_status_on_locality_update); RUNTIME_GUARD(envoy_reloadable_features_locality_routing_use_new_routing_logic); RUNTIME_GUARD(envoy_reloadable_features_lowercase_scheme); +RUNTIME_GUARD(envoy_reloadable_features_no_downgrade_to_canonical_name); RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name); RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch); +RUNTIME_GUARD(envoy_reloadable_features_normalize_host_for_preresolve_dfp_dns); RUNTIME_GUARD(envoy_reloadable_features_oauth_make_token_cookie_httponly); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_standard_max_age_value); RUNTIME_GUARD(envoy_reloadable_features_oauth_use_url_encoding); RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout); RUNTIME_GUARD(envoy_reloadable_features_overload_manager_error_unknown_action); -RUNTIME_GUARD(envoy_reloadable_features_prohibit_route_refresh_after_response_headers_sent); -RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener); -RUNTIME_GUARD(envoy_reloadable_features_sanitize_original_path); +RUNTIME_GUARD(envoy_reloadable_features_proxy_status_upstream_request_timeout); +RUNTIME_GUARD(envoy_reloadable_features_quic_fix_filter_manager_uaf); +RUNTIME_GUARD(envoy_reloadable_features_sanitize_te); RUNTIME_GUARD(envoy_reloadable_features_send_header_raw_value); -RUNTIME_GUARD(envoy_reloadable_features_service_sanitize_non_utf8_strings); RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); +RUNTIME_GUARD(envoy_reloadable_features_ssl_transport_failure_reason_format); RUNTIME_GUARD(envoy_reloadable_features_stateful_session_encode_ttl_in_cookie); +RUNTIME_GUARD(envoy_reloadable_features_stop_decode_metadata_on_local_reply); RUNTIME_GUARD(envoy_reloadable_features_test_feature_true); RUNTIME_GUARD(envoy_reloadable_features_thrift_allow_negative_field_ids); RUNTIME_GUARD(envoy_reloadable_features_thrift_connection_draining); @@ -89,9 +92,9 @@ RUNTIME_GUARD(envoy_reloadable_features_upstream_wait_for_response_headers_befor RUNTIME_GUARD(envoy_reloadable_features_use_cluster_cache_for_alt_protocols_filter); RUNTIME_GUARD(envoy_reloadable_features_use_http3_header_normalisation); RUNTIME_GUARD(envoy_reloadable_features_validate_connect); -RUNTIME_GUARD(envoy_reloadable_features_validate_detailed_override_host_statuses); RUNTIME_GUARD(envoy_reloadable_features_validate_grpc_header_before_log_grpc_status); RUNTIME_GUARD(envoy_reloadable_features_validate_upstream_headers); +RUNTIME_GUARD(envoy_restart_features_send_goaway_for_premature_rst_streams); RUNTIME_GUARD(envoy_restart_features_udp_read_normalize_addresses); // Begin false flags. These should come with a TODO to flip true. @@ -101,11 +104,6 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_test_feature_false); FALSE_RUNTIME_GUARD(envoy_reloadable_features_streaming_shadow); // TODO(adisuissa) reset to true to enable unified mux by default FALSE_RUNTIME_GUARD(envoy_reloadable_features_unified_mux); -// TODO(kbaichoo): Make this enabled by default when fairness and chunking -// are implemented, and we've had more cpu time. -FALSE_RUNTIME_GUARD(envoy_reloadable_features_defer_processing_backedup_streams); -// TODO(birenroy) flip after a burn-in period -FALSE_RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2); // Used to track if runtime is initialized. FALSE_RUNTIME_GUARD(envoy_reloadable_features_runtime_initialized); // TODO(mattklein123): Flip this to true and/or remove completely once verified by Envoy Mobile. @@ -118,14 +116,23 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_enable_include_histograms); FALSE_RUNTIME_GUARD(envoy_reloadable_features_refresh_rtt_after_request); // TODO(danzh) false deprecate it once QUICHE has its own enable/disable flag. FALSE_RUNTIME_GUARD(envoy_reloadable_features_quic_reject_all); +// TODO(steveWang) flip this to true after this is verified in prod. +FALSE_RUNTIME_GUARD(envoy_reloadable_features_quiche_use_mem_slice_releasor_api); +// TODO(suniltheta): Once the newly added http async technique is stabilized move it under +// RUNTIME_GUARD so that this option becomes default enabled. Once this option proves effective +// remove the feature flag and remove code path that relies on old technique to fetch credentials +// via libcurl and remove the bazel steps to pull and test the curl dependency. +FALSE_RUNTIME_GUARD(envoy_reloadable_features_use_http_client_to_fetch_aws_credentials); // TODO(adisuissa): enable by default once this is tested in prod. FALSE_RUNTIME_GUARD(envoy_restart_features_use_eds_cache_for_ads); // TODO(#10646) change to true when UHV is sufficiently tested // For more information about Universal Header Validation, please see // https://github.com/envoyproxy/envoy/issues/10646 FALSE_RUNTIME_GUARD(envoy_reloadable_features_enable_universal_header_validator); -// TODO(wbpcode): enable by default after a complete deprecation period. -FALSE_RUNTIME_GUARD(envoy_reloadable_features_no_downgrade_to_canonical_name); +// TODO(pksohn): enable after fixing https://github.com/envoyproxy/envoy/issues/29930 +FALSE_RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener); +// TODO(#31276): flip this to true after some test time. +FALSE_RUNTIME_GUARD(envoy_restart_features_use_fast_protobuf_hash); // Block of non-boolean flags. Use of int flags is deprecated. Do not add more. ABSL_FLAG(uint64_t, re2_max_program_size_error_level, 100, ""); // NOLINT diff --git a/source/common/runtime/runtime_features.h b/source/common/runtime/runtime_features.h index 0f89b781c352..0c2fb2d9b9e7 100644 --- a/source/common/runtime/runtime_features.h +++ b/source/common/runtime/runtime_features.h @@ -26,8 +26,6 @@ void maybeSetRuntimeGuard(absl::string_view name, bool value); void maybeSetDeprecatedInts(absl::string_view name, uint32_t value); constexpr absl::string_view defer_processing_backedup_streams = "envoy.reloadable_features.defer_processing_backedup_streams"; -constexpr absl::string_view expand_agnostic_stream_lifetime = - "envoy.reloadable_features.expand_agnostic_stream_lifetime"; } // namespace Runtime } // namespace Envoy diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 8368c3fa11c6..5e5ca22fe27e 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -417,15 +417,19 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix ENVOY_LOG(debug, "walking directory: {}", path); if (depth > MaxWalkDepth) { - throw EnvoyException(absl::StrCat("Walk recursion depth exceeded ", MaxWalkDepth)); + throwEnvoyExceptionOrPanic(absl::StrCat("Walk recursion depth exceeded ", MaxWalkDepth)); } // Check if this is an obviously bad path. if (api.fileSystem().illegalPath(path)) { - throw EnvoyException(absl::StrCat("Invalid path: ", path)); + throwEnvoyExceptionOrPanic(absl::StrCat("Invalid path: ", path)); } Filesystem::Directory directory(path); - for (const Filesystem::DirectoryEntry& entry : directory) { + Filesystem::DirectoryIteratorImpl it = directory.begin(); + THROW_IF_NOT_OK_REF(it.status()); + for (; it != directory.end(); ++it) { + THROW_IF_NOT_OK_REF(it.status()); + Filesystem::DirectoryEntry entry = *it; std::string full_path = path + "/" + entry.name_; std::string full_prefix; if (prefix.empty()) { @@ -446,7 +450,9 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix // Read the file and remove any comments. A comment is a line starting with a '#' character. // Comments are useful for placeholder files with no value. - const std::string text_file{api.fileSystem().fileReadToEnd(full_path)}; + auto file_or_error = api.fileSystem().fileReadToEnd(full_path); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + const std::string text_file{file_or_error.value()}; const auto lines = StringUtil::splitToken(text_file, "\n"); for (const auto& line : lines) { @@ -472,6 +478,7 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix #endif } } + THROW_IF_NOT_OK_REF(it.status()); } ProtoLayer::ProtoLayer(absl::string_view name, const ProtobufWkt::Struct& proto) @@ -486,7 +493,7 @@ void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string& case ProtobufWkt::Value::KIND_NOT_SET: case ProtobufWkt::Value::kListValue: case ProtobufWkt::Value::kNullValue: - throw EnvoyException(absl::StrCat("Invalid runtime entry value for ", prefix)); + throwEnvoyExceptionOrPanic(absl::StrCat("Invalid runtime entry value for ", prefix)); break; case ProtobufWkt::Value::kStringValue: SnapshotImpl::addEntry(values_, prefix, v, ""); @@ -527,7 +534,7 @@ LoaderImpl::LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator for (const auto& layer : config_.layers()) { auto ret = layer_names.insert(layer.name()); if (!ret.second) { - throw EnvoyException(absl::StrCat("Duplicate layer name: ", layer.name())); + throwEnvoyExceptionOrPanic(absl::StrCat("Duplicate layer name: ", layer.name())); } switch (layer.layer_specifier_case()) { case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kStaticLayer: @@ -535,7 +542,7 @@ LoaderImpl::LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator break; case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kAdminLayer: if (admin_layer_ != nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Too many admin layers specified in LayeredRuntime, at most one may be specified"); } admin_layer_ = std::make_unique(layer.name(), stats_); @@ -553,7 +560,7 @@ LoaderImpl::LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator init_manager_.add(subscriptions_.back()->init_target_); break; case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::LAYER_SPECIFIER_NOT_SET: - throw EnvoyException("layer specifier not set"); + throwEnvoyExceptionOrPanic("layer specifier not set"); } } @@ -701,7 +708,7 @@ SnapshotConstSharedPtr LoaderImpl::threadsafeSnapshot() { void LoaderImpl::mergeValues(const absl::node_hash_map& values) { if (admin_layer_ == nullptr) { - throw EnvoyException("No admin layer specified"); + throwEnvoyExceptionOrPanic("No admin layer specified"); } admin_layer_->mergeValues(values); loadNewSnapshot(); diff --git a/source/common/runtime/runtime_keys.cc b/source/common/runtime/runtime_keys.cc new file mode 100644 index 000000000000..621003372536 --- /dev/null +++ b/source/common/runtime/runtime_keys.cc @@ -0,0 +1,9 @@ +#include "source/common/runtime/runtime_keys.h" + +namespace Envoy { +namespace Runtime { + +const absl::string_view Keys::GlobalMaxCxRuntimeKey = "overload.global_downstream_max_connections"; + +} // namespace Runtime +} // namespace Envoy diff --git a/source/common/runtime/runtime_keys.h b/source/common/runtime/runtime_keys.h new file mode 100644 index 000000000000..5631b97c255c --- /dev/null +++ b/source/common/runtime/runtime_keys.h @@ -0,0 +1,16 @@ +#pragma once + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Runtime { + +// This class provides a place to register runtime keys which must be accessed by +// generic classes. +class Keys { +public: + static const absl::string_view GlobalMaxCxRuntimeKey; +}; + +} // namespace Runtime +} // namespace Envoy diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index ebb6ed1c3d05..8a0c3f43d2e1 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -25,7 +25,7 @@ SdsApi::SdsApi(envoy::config::core::v3::ConfigSource sds_config, absl::string_vi dispatcher_(dispatcher), api_(api), scope_(stats.createScope(absl::StrCat("sds.", sds_config_name, "."))), sds_api_stats_(generateStats(*scope_)), sds_config_(std::move(sds_config)), - sds_config_name_(sds_config_name), secret_hash_(0), clean_up_(std::move(destructor_cb)), + sds_config_name_(sds_config_name), clean_up_(std::move(destructor_cb)), subscription_factory_(subscription_factory), time_source_(time_source), secret_data_{sds_config_name_, "uninitialized", time_source_.systemTime()} { @@ -81,12 +81,15 @@ void SdsApi::onWatchUpdate() { absl::Status SdsApi::onConfigUpdate(const std::vector& resources, const std::string& version_info) { - validateUpdateSize(resources.size()); + const absl::Status status = validateUpdateSize(resources.size()); + if (!status.ok()) { + return status; + } const auto& secret = dynamic_cast( resources[0].get().resource()); if (secret.name() != sds_config_name_) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("Unexpected SDS secret (expecting {}): {}", sds_config_name_, secret.name())); } @@ -116,8 +119,9 @@ absl::Status SdsApi::onConfigUpdate(const std::vectoraddWatch(absl::StrCat(result.directory_, "/"), + const auto result_or_error = api_.fileSystem().splitPathFromFilename(filename); + THROW_IF_STATUS_NOT_OK(result_or_error, throw); + watcher_->addWatch(absl::StrCat(result_or_error.value().directory_, "/"), Filesystem::Watcher::Events::MovedTo, [this](uint32_t) { onWatchUpdate(); }); } @@ -135,7 +139,10 @@ absl::Status SdsApi::onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField&, const std::string&) { - validateUpdateSize(added_resources.size()); + const absl::Status status = validateUpdateSize(added_resources.size()); + if (!status.ok()) { + return status; + } return onConfigUpdate(added_resources, added_resources[0].get().version()); } @@ -146,14 +153,16 @@ void SdsApi::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reaso init_target_.ready(); } -void SdsApi::validateUpdateSize(int num_resources) { +absl::Status SdsApi::validateUpdateSize(int num_resources) { if (num_resources == 0) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("Missing SDS resources for {} in onConfigUpdate()", sds_config_name_)); } if (num_resources != 1) { - throw EnvoyException(fmt::format("Unexpected SDS secrets length: {}", num_resources)); + return absl::InvalidArgumentError( + fmt::format("Unexpected SDS secrets length: {}", num_resources)); } + return absl::OkStatus(); } void SdsApi::initialize() { @@ -167,7 +176,9 @@ SdsApi::SecretData SdsApi::secretData() { return secret_data_; } SdsApi::FileContentMap SdsApi::loadFiles() { FileContentMap files; for (auto const& filename : getDataSourceFilenames()) { - files[filename] = api_.fileSystem().fileReadToEnd(filename); + auto file_or_error = api_.fileSystem().fileReadToEnd(filename); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + files[filename] = file_or_error.value(); } return files; } diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index dbe410ae177e..1aee5f4a0b78 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -92,7 +92,7 @@ class SdsApi : public Envoy::Config::SubscriptionBase< Api::Api& api_; private: - void validateUpdateSize(int num_resources); + absl::Status validateUpdateSize(int num_resources); void initialize(); FileContentMap loadFiles(); uint64_t getHashForFiles(const FileContentMap& files); @@ -107,7 +107,7 @@ class SdsApi : public Envoy::Config::SubscriptionBase< Config::SubscriptionPtr subscription_; const std::string sds_config_name_; - uint64_t secret_hash_; + uint64_t secret_hash_{0}; uint64_t files_hash_; Cleanup clean_up_; Config::SubscriptionFactory& subscription_factory_; diff --git a/source/common/secret/secret_manager_impl.cc b/source/common/secret/secret_manager_impl.cc index ac7624e5d901..caa5619bc720 100644 --- a/source/common/secret/secret_manager_impl.cc +++ b/source/common/secret/secret_manager_impl.cc @@ -26,7 +26,7 @@ SecretManagerImpl::SecretManagerImpl(OptRef config_tracke } } -void SecretManagerImpl::addStaticSecret( +absl::Status SecretManagerImpl::addStaticSecret( const envoy::extensions::transport_sockets::tls::v3::Secret& secret) { switch (secret.type_case()) { case envoy::extensions::transport_sockets::tls::v3::Secret::TypeCase::kTlsCertificate: { @@ -34,7 +34,7 @@ void SecretManagerImpl::addStaticSecret( std::make_shared(secret.tls_certificate()); if (!static_tls_certificate_providers_.insert(std::make_pair(secret.name(), secret_provider)) .second) { - throw EnvoyException( + return absl::InvalidArgumentError( absl::StrCat("Duplicate static TlsCertificate secret name ", secret.name())); } break; @@ -45,7 +45,7 @@ void SecretManagerImpl::addStaticSecret( if (!static_certificate_validation_context_providers_ .insert(std::make_pair(secret.name(), secret_provider)) .second) { - throw EnvoyException(absl::StrCat( + return absl::InvalidArgumentError(absl::StrCat( "Duplicate static CertificateValidationContext secret name ", secret.name())); } break; @@ -56,7 +56,7 @@ void SecretManagerImpl::addStaticSecret( if (!static_session_ticket_keys_providers_ .insert(std::make_pair(secret.name(), secret_provider)) .second) { - throw EnvoyException( + return absl::InvalidArgumentError( absl::StrCat("Duplicate static TlsSessionTicketKeys secret name ", secret.name())); } break; @@ -66,14 +66,15 @@ void SecretManagerImpl::addStaticSecret( std::make_shared(secret.generic_secret()); if (!static_generic_secret_providers_.insert(std::make_pair(secret.name(), secret_provider)) .second) { - throw EnvoyException( + return absl::InvalidArgumentError( absl::StrCat("Duplicate static GenericSecret secret name ", secret.name())); } break; } default: - throw EnvoyException("Secret type not implemented"); + return absl::InvalidArgumentError("Secret type not implemented"); } + return absl::OkStatus(); } TlsCertificateConfigProviderSharedPtr diff --git a/source/common/secret/secret_manager_impl.h b/source/common/secret/secret_manager_impl.h index 952206f106b7..f4f780ac8d1b 100644 --- a/source/common/secret/secret_manager_impl.h +++ b/source/common/secret/secret_manager_impl.h @@ -19,7 +19,7 @@ namespace Secret { class SecretManagerImpl : public SecretManager { public: SecretManagerImpl(OptRef config_tracker); - void + absl::Status addStaticSecret(const envoy::extensions::transport_sockets::tls::v3::Secret& secret) override; TlsCertificateConfigProviderSharedPtr diff --git a/source/common/singleton/manager_impl.cc b/source/common/singleton/manager_impl.cc index 0e3fca61b870..a71fd7f3a9f6 100644 --- a/source/common/singleton/manager_impl.cc +++ b/source/common/singleton/manager_impl.cc @@ -8,18 +8,23 @@ namespace Envoy { namespace Singleton { -InstanceSharedPtr ManagerImpl::get(const std::string& name, SingletonFactoryCb cb) { +InstanceSharedPtr ManagerImpl::get(const std::string& name, SingletonFactoryCb cb, bool pin) { ASSERT(run_tid_ == thread_factory_.currentThreadId()); ENVOY_BUG(Registry::FactoryRegistry::getFactory(name) != nullptr, "invalid singleton name '" + name + "'. Make sure it is registered."); - if (nullptr == singletons_[name].lock()) { + auto existing_singleton = singletons_[name].lock(); + + if (existing_singleton == nullptr) { InstanceSharedPtr singleton = cb(); singletons_[name] = singleton; + if (pin && singleton != nullptr) { + pinned_singletons_.push_back(singleton); + } return singleton; } else { - return singletons_[name].lock(); + return existing_singleton; } } diff --git a/source/common/singleton/manager_impl.h b/source/common/singleton/manager_impl.h index 32de2d112750..cc715f55c1b8 100644 --- a/source/common/singleton/manager_impl.h +++ b/source/common/singleton/manager_impl.h @@ -21,10 +21,12 @@ class ManagerImpl : public Manager, NonCopyable { : thread_factory_(thread_factory), run_tid_(thread_factory.currentThreadId()) {} // Singleton::Manager - InstanceSharedPtr get(const std::string& name, SingletonFactoryCb cb) override; + InstanceSharedPtr get(const std::string& name, SingletonFactoryCb cb, bool pin) override; private: absl::node_hash_map> singletons_; + std::vector pinned_singletons_; + Thread::ThreadFactory& thread_factory_; const Thread::ThreadId run_tid_; }; diff --git a/source/common/ssl/tls_certificate_config_impl.cc b/source/common/ssl/tls_certificate_config_impl.cc index 7e96344d7f9a..0b131494b5da 100644 --- a/source/common/ssl/tls_certificate_config_impl.cc +++ b/source/common/ssl/tls_certificate_config_impl.cc @@ -17,7 +17,7 @@ std::vector readOcspStaple(const envoy::config::core::v3::DataSource& s std::string staple = Config::DataSource::read(source, true, api); if (source.specifier_case() == envoy::config::core::v3::DataSource::SpecifierCase::kInlineString) { - throw EnvoyException("OCSP staple cannot be provided via inline_string"); + throwEnvoyExceptionOrPanic("OCSP staple cannot be provided via inline_string"); } return {staple.begin(), staple.end()}; @@ -48,15 +48,15 @@ TlsCertificateConfigImpl::TlsCertificateConfigImpl( private_key_method_(nullptr) { if (config.has_pkcs12()) { if (config.has_private_key()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Certificate configuration can't have both pkcs12 and private_key")); } if (config.has_certificate_chain()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Certificate configuration can't have both pkcs12 and certificate_chain")); } if (config.has_private_key_provider()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Certificate configuration can't have both pkcs12 and private_key_provider")); } } else { @@ -67,8 +67,8 @@ TlsCertificateConfigImpl::TlsCertificateConfigImpl( .createPrivateKeyMethodProvider(config.private_key_provider(), factory_context); if (private_key_method_ == nullptr || (!private_key_method_->isAvailable() && !config.private_key_provider().fallback())) { - throw EnvoyException(fmt::format("Failed to load private key provider: {}", - config.private_key_provider().provider_name())); + throwEnvoyExceptionOrPanic(fmt::format("Failed to load private key provider: {}", + config.private_key_provider().provider_name())); } if (!private_key_method_->isAvailable()) { @@ -76,13 +76,13 @@ TlsCertificateConfigImpl::TlsCertificateConfigImpl( } } if (certificate_chain_.empty()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Failed to load incomplete certificate from {}: certificate chain not set", certificate_chain_path_)); } if (private_key_.empty() && private_key_method_ == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Failed to load incomplete private key from path: {}", private_key_path_)); } } diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index b795283f7718..5c35a29df60f 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( deps = [ ":metric_impl_lib", ":stat_merger_lib", + "//envoy/stats:sink_interface", "//source/common/common:assert_lib", "//source/common/common:hash_lib", "//source/common/common:thread_annotations", @@ -28,6 +29,7 @@ envoy_cc_library( name = "custom_stat_namespaces_lib", srcs = ["custom_stat_namespaces_impl.cc"], hdrs = ["custom_stat_namespaces_impl.h"], + external_deps = ["abseil_flat_hash_set"], deps = [ "//envoy/stats:custom_stat_namespaces_interface", "//source/common/common:assert_lib", diff --git a/source/common/stats/deferred_creation.h b/source/common/stats/deferred_creation.h index b2999666e7a0..511eb73ac454 100644 --- a/source/common/stats/deferred_creation.h +++ b/source/common/stats/deferred_creation.h @@ -49,7 +49,7 @@ class DeferredStats : public DeferredCreationCompatibleInterface& computedBuckets() const override { return computed_buckets_; } std::vector computeDisjointBuckets() const override; uint64_t sampleCount() const override { return sample_count_; } + uint64_t outOfBoundCount() const override { return out_of_bound_count_; } double sampleSum() const override { return sample_sum_; } private: @@ -67,6 +68,7 @@ class HistogramStatisticsImpl final : public HistogramStatistics, NonCopyable { std::vector computed_quantiles_; std::vector computed_buckets_; uint64_t sample_count_{0}; + uint64_t out_of_bound_count_{0}; double sample_sum_{0}; const Histogram::Unit unit_{Histogram::Unit::Unspecified}; }; diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 1d29b90b08e9..84ea935102b9 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -236,6 +236,7 @@ class IsolatedStoreImpl : public Store { void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override {} void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override {} + const TagVector& fixedTags() override { CONSTRUCT_ON_FIRST_USE(TagVector); } protected: /** diff --git a/source/common/stats/symbol_table.h b/source/common/stats/symbol_table.h index f7ad027eeeca..5f79985ae4f2 100644 --- a/source/common/stats/symbol_table.h +++ b/source/common/stats/symbol_table.h @@ -702,7 +702,7 @@ class StatNameManagedStorage : public StatNameStorage { StatNameManagedStorage(StatName src, SymbolTable& table) noexcept : StatNameStorage(src, table), symbol_table_(table) {} - ~StatNameManagedStorage() { free(symbol_table_); } + ~StatNameManagedStorage() { free(symbol_table_); } // NOLINT(clang-analyzer-unix.Malloc) private: SymbolTable& symbol_table_; diff --git a/source/common/stats/tag_extractor_impl.cc b/source/common/stats/tag_extractor_impl.cc index 78b220eefa0c..c48370880519 100644 --- a/source/common/stats/tag_extractor_impl.cc +++ b/source/common/stats/tag_extractor_impl.cc @@ -72,11 +72,11 @@ TagExtractorPtr TagExtractorImplBase::createTagExtractor(absl::string_view name, absl::string_view negative_match, Regex::Type re_type) { if (name.empty()) { - throw EnvoyException("tag_name cannot be empty"); + throwEnvoyExceptionOrPanic("tag_name cannot be empty"); } if (regex.empty()) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "No regex specified for tag specifier and no default regex for name: '{}'", name)); } switch (re_type) { diff --git a/source/common/stats/tag_producer_impl.cc b/source/common/stats/tag_producer_impl.cc index 4d8f5a518f88..44e6c862e3ae 100644 --- a/source/common/stats/tag_producer_impl.cc +++ b/source/common/stats/tag_producer_impl.cc @@ -21,6 +21,7 @@ TagProducerImpl::TagProducerImpl(const envoy::config::metrics::v3::StatsConfig& for (const auto& cli_tag : cli_tags) { addExtractor(std::make_unique(cli_tag.name_, cli_tag.value_)); + fixed_tags_.push_back(cli_tag); } for (const auto& tag_specifier : config.stats_tags()) { @@ -34,7 +35,7 @@ TagProducerImpl::TagProducerImpl(const envoy::config::metrics::v3::StatsConfig& if (tag_specifier.regex().empty()) { if (addExtractorsMatching(name) == 0) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "No regex specified for tag specifier and no default regex for name: '{}'", name)); } } else { @@ -43,6 +44,7 @@ TagProducerImpl::TagProducerImpl(const envoy::config::metrics::v3::StatsConfig& } else if (tag_specifier.tag_value_case() == envoy::config::metrics::v3::TagSpecifier::TagValueCase::kFixedValue) { addExtractor(std::make_unique(name, tag_specifier.fixed_value())); + fixed_tags_.push_back(Tag{name, tag_specifier.fixed_value()}); } } } diff --git a/source/common/stats/tag_producer_impl.h b/source/common/stats/tag_producer_impl.h index db87e2a31f0e..e253dd002dd5 100644 --- a/source/common/stats/tag_producer_impl.h +++ b/source/common/stats/tag_producer_impl.h @@ -43,6 +43,8 @@ class TagProducerImpl : public TagProducer { */ std::string produceTags(absl::string_view metric_name, TagVector& tags) const override; + const TagVector& fixedTags() const override { return fixed_tags_; } + private: friend class DefaultTagRegexTester; @@ -104,6 +106,8 @@ class TagProducerImpl : public TagProducer { // we need do elide duplicate extractors during extraction. It is not valid to // send duplicate tag names to Prometheus so this needs to be filtered out. absl::flat_hash_map> extractor_map_; + + TagVector fixed_tags_; }; } // namespace Stats diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index faa2e771d1aa..09e235e6ac4c 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -258,7 +258,7 @@ ThreadLocalStoreImpl::CentralCacheEntry::~CentralCacheEntry() { // is because many tests will not populate rejected_stats_. ASSERT(symbol_table_.toString(StatNameManagedStorage("Hello.world", symbol_table_).statName()) == "Hello.world"); - rejected_stats_.free(symbol_table_); + rejected_stats_.free(symbol_table_); // NOLINT(clang-analyzer-unix.Malloc) } void ThreadLocalStoreImpl::releaseScopeCrossThread(ScopeImpl* scope) { @@ -803,8 +803,7 @@ ThreadLocalHistogramImpl::ThreadLocalHistogramImpl(StatName name, Histogram::Uni const StatNameTagVector& stat_name_tags, SymbolTable& symbol_table) : HistogramImplHelper(name, tag_extracted_name, stat_name_tags, symbol_table), unit_(unit), - current_active_(0), used_(false), created_thread_id_(std::this_thread::get_id()), - symbol_table_(symbol_table) { + used_(false), created_thread_id_(std::this_thread::get_id()), symbol_table_(symbol_table) { histograms_[0] = hist_alloc(); histograms_[1] = hist_alloc(); } @@ -836,8 +835,7 @@ ParentHistogramImpl::ParentHistogramImpl(StatName name, Histogram::Unit unit, unit_(unit), thread_local_store_(thread_local_store), interval_histogram_(hist_alloc()), cumulative_histogram_(hist_alloc()), interval_statistics_(interval_histogram_, unit, supported_buckets), - cumulative_statistics_(cumulative_histogram_, unit, supported_buckets), merged_(false), - id_(id) {} + cumulative_statistics_(cumulative_histogram_, unit, supported_buckets), id_(id) {} ParentHistogramImpl::~ParentHistogramImpl() { thread_local_store_.releaseHistogramCrossThread(id_); diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 081aee90fbe4..f8f27558b400 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -65,7 +65,7 @@ class ThreadLocalHistogramImpl : public HistogramImplHelper { private: Histogram::Unit unit_; uint64_t otherHistogramIndex() const { return 1 - current_active_; } - uint64_t current_active_; + uint64_t current_active_{0}; histogram_t* histograms_[2]; std::atomic used_; std::thread::id created_thread_id_; @@ -140,7 +140,7 @@ class ParentHistogramImpl : public MetricImpl { HistogramStatisticsImpl cumulative_statistics_; mutable Thread::MutexBasicLockable merge_lock_; std::list tls_histograms_ ABSL_GUARDED_BY(merge_lock_); - bool merged_; + bool merged_{false}; std::atomic shutting_down_{false}; std::atomic ref_count_{0}; const uint64_t id_; // Index into TlsCache::histogram_cache_. @@ -224,6 +224,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo void extractAndAppendTags(StatName name, StatNamePool& pool, StatNameTagVector& tags) override; void extractAndAppendTags(absl::string_view name, StatNamePool& pool, StatNameTagVector& tags) override; + const TagVector& fixedTags() override { return tag_producer_->fixedTags(); }; private: friend class ThreadLocalStoreTestingPeer; diff --git a/source/common/stream_info/BUILD b/source/common/stream_info/BUILD index 0f470829c667..9acfe214baaf 100644 --- a/source/common/stream_info/BUILD +++ b/source/common/stream_info/BUILD @@ -43,6 +43,7 @@ envoy_cc_library( "//envoy/http:codes_interface", "//envoy/stream_info:stream_info_interface", "//source/common/http:default_server_string_lib", + "//source/common/runtime:runtime_features_lib", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", ], ) diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index ca72643ba82c..5d1494640824 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -134,6 +134,8 @@ struct StreamInfoImpl : public StreamInfo { MonotonicTime startTimeMonotonic() const override { return start_time_monotonic_; } + TimeSource& timeSource() const override { return time_source_; } + absl::optional duration(absl::optional time) const { if (!time) { return {}; diff --git a/source/common/stream_info/utility.cc b/source/common/stream_info/utility.cc index ab720c88eda2..61304c9ad94c 100644 --- a/source/common/stream_info/utility.cc +++ b/source/common/stream_info/utility.cc @@ -5,6 +5,7 @@ #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "source/common/http/default_server_string.h" +#include "source/common/runtime/runtime_features.h" #include "absl/strings/str_format.h" @@ -36,7 +37,7 @@ const std::string ResponseFlagUtils::toString(const StreamInfo& stream_info, boo } absl::flat_hash_map ResponseFlagUtils::getFlagMap() { - static_assert(ResponseFlag::LastFlag == 0x4000000, + static_assert(ResponseFlag::LastFlag == 0x8000000, "A flag has been added. Add the new flag to ALL_RESPONSE_STRINGS_FLAGS."); absl::flat_hash_map res; for (auto [flag_strings, flag] : ResponseFlagUtils::ALL_RESPONSE_STRINGS_FLAGS) { @@ -313,7 +314,7 @@ ProxyStatusUtils::proxyStatusErrorToString(const ProxyStatusError proxy_status) case ProxyStatusError::TlsProtocolError: return TLS_PROTOCOL_ERROR; case ProxyStatusError::TlsCertificateError: - return TLS_CERTIFICATE_ERORR; + return TLS_CERTIFICATE_ERROR; case ProxyStatusError::TlsAlertReceived: return TLS_ALERT_RECEIVED; case ProxyStatusError::HttpRequestError: @@ -366,7 +367,11 @@ ProxyStatusUtils::fromStreamInfo(const StreamInfo& stream_info) { } else if (stream_info.hasResponseFlag(ResponseFlag::NoHealthyUpstream)) { return ProxyStatusError::DestinationUnavailable; } else if (stream_info.hasResponseFlag(ResponseFlag::UpstreamRequestTimeout)) { - return ProxyStatusError::ConnectionTimeout; + if (!Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.proxy_status_upstream_request_timeout")) { + return ProxyStatusError::ConnectionTimeout; + } + return ProxyStatusError::HttpResponseTimeout; } else if (stream_info.hasResponseFlag(ResponseFlag::LocalReset)) { return ProxyStatusError::ConnectionTimeout; } else if (stream_info.hasResponseFlag(ResponseFlag::UpstreamRemoteReset)) { diff --git a/source/common/stream_info/utility.h b/source/common/stream_info/utility.h index 5f8c84a1bc36..e7794165b500 100644 --- a/source/common/stream_info/utility.h +++ b/source/common/stream_info/utility.h @@ -26,6 +26,8 @@ class ResponseFlagUtils { using FlagStringsAndEnum = std::pair; + // When adding a new flag, it's required to update the access log docs and the string + // mapping below - ``ALL_RESPONSE_STRINGS_FLAGS``. constexpr static absl::string_view NONE = "-"; constexpr static absl::string_view DOWNSTREAM_CONNECTION_TERMINATION = "DC"; constexpr static absl::string_view FAILED_LOCAL_HEALTH_CHECK = "LH"; @@ -54,6 +56,7 @@ class ResponseFlagUtils { constexpr static absl::string_view NO_CLUSTER_FOUND = "NC"; constexpr static absl::string_view OVERLOAD_MANAGER = "OM"; constexpr static absl::string_view DNS_FAIL = "DF"; + constexpr static absl::string_view DROP_OVERLOAD = "DO"; constexpr static absl::string_view DOWNSTREAM_CONNECTION_TERMINATION_LONG = "DownstreamConnectionTermination"; @@ -87,6 +90,8 @@ class ResponseFlagUtils { constexpr static absl::string_view UPSTREAM_PROTOCOL_ERROR_LONG = "UpstreamProtocolError"; constexpr static absl::string_view NO_CLUSTER_FOUND_LONG = "NoClusterFound"; constexpr static absl::string_view OVERLOAD_MANAGER_LONG = "OverloadManagerTerminated"; + constexpr static absl::string_view DNS_FAIL_LONG = "DnsResolutionFailed"; + constexpr static absl::string_view DROP_OVERLOAD_LONG = "DropOverload"; static constexpr std::array ALL_RESPONSE_STRINGS_FLAGS{ FlagStringsAndEnum{{FAILED_LOCAL_HEALTH_CHECK, FAILED_LOCAL_HEALTH_CHECK_LONG}, @@ -135,6 +140,8 @@ class ResponseFlagUtils { ResponseFlag::UpstreamProtocolError}, FlagStringsAndEnum{{NO_CLUSTER_FOUND, NO_CLUSTER_FOUND_LONG}, ResponseFlag::NoClusterFound}, FlagStringsAndEnum{{OVERLOAD_MANAGER, OVERLOAD_MANAGER_LONG}, ResponseFlag::OverloadManager}, + FlagStringsAndEnum{{DNS_FAIL, DNS_FAIL_LONG}, ResponseFlag::DnsResolutionFailed}, + FlagStringsAndEnum{{DROP_OVERLOAD, DROP_OVERLOAD_LONG}, ResponseFlag::DropOverLoad}, }; private: @@ -249,7 +256,7 @@ class ProxyStatusUtils { constexpr static absl::string_view CONNECTION_WRITE_TIMEOUT = "connection_write_timeout"; constexpr static absl::string_view CONNECTION_LIMIT_REACHED = "connection_limit_reached"; constexpr static absl::string_view TLS_PROTOCOL_ERROR = "tls_protocol_error"; - constexpr static absl::string_view TLS_CERTIFICATE_ERORR = "tls_certificate_error"; + constexpr static absl::string_view TLS_CERTIFICATE_ERROR = "tls_certificate_error"; constexpr static absl::string_view TLS_ALERT_RECEIVED = "tls_alert_received"; constexpr static absl::string_view HTTP_REQUEST_ERROR = "http_request_error"; constexpr static absl::string_view HTTP_REQUEST_DENIED = "http_request_denied"; diff --git a/source/common/tcp/async_tcp_client_impl.h b/source/common/tcp/async_tcp_client_impl.h index 1786cfaaf589..6250fb574d6d 100644 --- a/source/common/tcp/async_tcp_client_impl.h +++ b/source/common/tcp/async_tcp_client_impl.h @@ -4,6 +4,7 @@ #include #include +#include "envoy/common/optref.h" #include "envoy/event/dispatcher.h" #include "envoy/network/connection.h" #include "envoy/network/filter.h" @@ -15,6 +16,7 @@ #include "source/common/network/filter_impl.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" namespace Envoy { namespace Tcp { @@ -58,6 +60,14 @@ class AsyncTcpClientImpl : public AsyncTcpClient, Event::Dispatcher& dispatcher() override { return dispatcher_; } + OptRef getStreamInfo() override { + if (connection_) { + return connection_->streamInfo(); + } else { + return absl::nullopt; + } + } + private: struct NetworkReadFilter : public Network::ReadFilterBaseImpl { NetworkReadFilter(AsyncTcpClientImpl& parent) : parent_(parent) {} diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index 0b07d56cd641..15f24fe8d743 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -139,9 +139,9 @@ Config::SharedConfig::SharedConfig( Config::Config(const envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy& config, Server::Configuration::FactoryContext& context) : max_connect_attempts_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_connect_attempts, 1)), - upstream_drain_manager_slot_(context.threadLocal().allocateSlot()), + upstream_drain_manager_slot_(context.serverFactoryContext().threadLocal().allocateSlot()), shared_config_(std::make_shared(config, context)), - random_generator_(context.api().randomGenerator()) { + random_generator_(context.serverFactoryContext().api().randomGenerator()) { upstream_drain_manager_slot_->set([](Event::Dispatcher&) { ThreadLocal::ThreadLocalObjectSharedPtr drain_manager = std::make_shared(); @@ -224,10 +224,7 @@ Filter::~Filter() { disableAccessLogFlushTimer(); // Flush the final end stream access log entry. - for (const auto& access_log : config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, getStreamInfo(), - AccessLog::AccessLogType::TcpConnectionEnd); - } + flushAccessLog(AccessLog::AccessLogType::TcpConnectionEnd); ASSERT(generic_conn_pool_ == nullptr); ASSERT(upstream_ == nullptr); @@ -271,6 +268,15 @@ void Filter::initialize(Network::ReadFilterCallbacks& callbacks, bool set_connec } void Filter::onInitFailure(UpstreamFailureReason reason) { + // If ODCDS fails, the filter will not attempt to create a connection to + // upstream, as it does not have an assigned upstream. As such the filter will + // not have started attempting to connect to an upstream and there is no + // connection pool callback latency to record. + if (initial_upstream_connection_start_time_.has_value()) { + getStreamInfo().upstreamInfo()->upstreamTiming().recordConnectionPoolCallbackLatency( + initial_upstream_connection_start_time_.value(), + read_callbacks_->connection().dispatcher().timeSource()); + } read_callbacks_->connection().close( Network::ConnectionCloseType::NoFlush, absl::StrCat(StreamInfo::LocalCloseReasons::get().TcpProxyInitializationFailure, @@ -428,6 +434,10 @@ Network::FilterStatus Filter::establishUpstreamConnection() { ENVOY_CONN_LOG(debug, "Creating connection to cluster {}", read_callbacks_->connection(), cluster_name); + if (!initial_upstream_connection_start_time_.has_value()) { + initial_upstream_connection_start_time_.emplace( + read_callbacks_->connection().dispatcher().timeSource().monotonicTime()); + } const Upstream::ClusterInfoConstSharedPtr& cluster = thread_local_cluster->info(); getStreamInfo().setUpstreamClusterInfo(cluster); @@ -572,6 +582,9 @@ void Filter::onGenericPoolReady(StreamInfo::StreamInfo* info, generic_conn_pool_.reset(); read_callbacks_->upstreamHost(host); StreamInfo::UpstreamInfo& upstream_info = *getStreamInfo().upstreamInfo(); + upstream_info.upstreamTiming().recordConnectionPoolCallbackLatency( + initial_upstream_connection_start_time_.value(), + read_callbacks_->connection().dispatcher().timeSource()); upstream_info.setUpstreamHost(host); upstream_info.setUpstreamLocalAddress(address_provider.localAddress()); upstream_info.setUpstreamRemoteAddress(address_provider.remoteAddress()); @@ -836,10 +849,7 @@ void Filter::onUpstreamConnection() { } if (config_->flushAccessLogOnConnected()) { - for (const auto& access_log : config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, getStreamInfo(), - AccessLog::AccessLogType::TcpUpstreamConnected); - } + flushAccessLog(AccessLog::AccessLogType::TcpUpstreamConnected); } } @@ -862,10 +872,7 @@ void Filter::onMaxDownstreamConnectionDuration() { } void Filter::onAccessLogFlushInterval() { - for (const auto& access_log : config_->accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, getStreamInfo(), - AccessLog::AccessLogType::TcpPeriodic); - } + flushAccessLog(AccessLog::AccessLogType::TcpPeriodic); const SystemTime now = read_callbacks_->connection().dispatcher().timeSource().systemTime(); getStreamInfo().getDownstreamBytesMeter()->takeDownstreamPeriodicLoggingSnapshot(now); if (getStreamInfo().getUpstreamBytesMeter()) { @@ -874,6 +881,14 @@ void Filter::onAccessLogFlushInterval() { resetAccessLogFlushTimer(); } +void Filter::flushAccessLog(AccessLog::AccessLogType access_log_type) { + const Formatter::HttpFormatterContext log_context{nullptr, nullptr, nullptr, {}, access_log_type}; + + for (const auto& access_log : config_->accessLogs()) { + access_log->log(log_context, getStreamInfo()); + } +} + void Filter::resetAccessLogFlushTimer() { if (access_log_flush_timer_ != nullptr) { ASSERT(config_->accessLogFlushInterval().has_value()); diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index ae360352de07..82ebcb8fb9d9 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -185,9 +185,9 @@ class OnDemandConfig { OnDemandConfig(const envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy_OnDemand& on_demand_message, Server::Configuration::FactoryContext& context, Stats::Scope& scope) - : odcds_(context.clusterManager().allocateOdCdsApi(on_demand_message.odcds_config(), - OptRef(), - context.messageValidationVisitor())), + : odcds_(context.serverFactoryContext().clusterManager().allocateOdCdsApi( + on_demand_message.odcds_config(), OptRef(), + context.messageValidationVisitor())), lookup_timeout_(std::chrono::milliseconds( PROTOBUF_GET_MS_OR_DEFAULT(on_demand_message, timeout, 60000))), stats_(generateStats(scope)) {} @@ -515,6 +515,7 @@ class Filter : public Network::ReadFilter, void onMaxDownstreamConnectionDuration(); void onAccessLogFlushInterval(); void resetAccessLogFlushTimer(); + void flushAccessLog(AccessLog::AccessLogType access_log_type); void disableAccessLogFlushTimer(); const ConfigSharedPtr config_; @@ -538,6 +539,10 @@ class Filter : public Network::ReadFilter, // This will be non-null from when an upstream connection is attempted until // it either succeeds or fails. std::unique_ptr generic_conn_pool_; + // Time the filter first attempted to connect to the upstream after the + // cluster is discovered. Capture the first time as the filter may try multiple times to connect + // to the upstream. + absl::optional initial_upstream_connection_start_time_; RouteConstSharedPtr route_; Router::MetadataMatchCriteriaConstPtr metadata_match_criteria_; Network::TransportSocketOptionsConstSharedPtr transport_socket_options_; diff --git a/source/common/tcp_proxy/upstream.cc b/source/common/tcp_proxy/upstream.cc index 82c389d6faf0..5e4eaa35338d 100644 --- a/source/common/tcp_proxy/upstream.cc +++ b/source/common/tcp_proxy/upstream.cc @@ -77,12 +77,53 @@ TcpUpstream::onDownstreamEvent(Network::ConnectionEvent event) { HttpUpstream::HttpUpstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, const TunnelingConfigHelper& config, - StreamInfo::StreamInfo& downstream_info) + StreamInfo::StreamInfo& downstream_info, Http::CodecType type) : config_(config), downstream_info_(downstream_info), response_decoder_(*this), - upstream_callbacks_(callbacks) {} + upstream_callbacks_(callbacks), type_(type) {} HttpUpstream::~HttpUpstream() { resetEncoder(Network::ConnectionEvent::LocalClose); } +bool HttpUpstream::isValidResponse(const Http::ResponseHeaderMap& headers) { + if (type_ == Http::CodecType::HTTP1) { + // According to RFC7231 any 2xx response indicates that the connection is + // established. + // Any 'Content-Length' or 'Transfer-Encoding' header fields MUST be ignored. + // https://tools.ietf.org/html/rfc7231#section-4.3.6 + return Http::CodeUtility::is2xx(Http::Utility::getResponseStatus(headers)); + } + return Http::Utility::getResponseStatus(headers) == 200; +} + +void HttpUpstream::setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl) { + request_encoder_ = &request_encoder; + request_encoder_->getStream().addCallbacks(*this); + auto headers = Http::createHeaderMap({ + {Http::Headers::get().Method, config_.usePost() ? "POST" : "CONNECT"}, + {Http::Headers::get().Host, config_.host(downstream_info_)}, + }); + if (config_.usePost()) { + headers->addReference(Http::Headers::get().Path, config_.postPath()); + } + + if (type_ == Http::CodecType::HTTP1) { + request_encoder_->enableTcpTunneling(); + ASSERT(request_encoder_->http1StreamEncoderOptions() != absl::nullopt); + } else { + const std::string& scheme = + is_ssl ? Http::Headers::get().SchemeValues.Https : Http::Headers::get().SchemeValues.Http; + + if (config_.usePost()) { + headers->addReference(Http::Headers::get().Scheme, scheme); + } + } + + config_.headerEvaluator().evaluateHeaders(*headers, {downstream_info_.getRequestHeaders()}, + downstream_info_); + const auto status = request_encoder_->encodeHeaders(*headers, false); + // Encoding can only fail on missing required request headers. + ASSERT(status.ok()); +} + bool HttpUpstream::readDisable(bool disable) { if (!request_encoder_) { return false; @@ -95,8 +136,15 @@ void HttpUpstream::encodeData(Buffer::Instance& data, bool end_stream) { if (!request_encoder_) { return; } + auto codec = type_; request_encoder_->encodeData(data, end_stream); - if (end_stream) { + + // doneWriting() is being skipped for H1 codec to avoid resetEncoder() call. + // This is because H1 codec does not support half-closed stream. Calling resetEncoder() + // will fully close the upstream connection without flushing any pending data, rather than a http + // stream reset. + // More details can be found on https://github.com/envoyproxy/envoy/pull/13293 + if ((codec != Http::CodecType::HTTP1) && (end_stream)) { doneWriting(); } } @@ -245,11 +293,7 @@ HttpConnPool::~HttpConnPool() { void HttpConnPool::newStream(GenericConnectionPoolCallbacks& callbacks) { callbacks_ = &callbacks; - if (type_ == Http::CodecType::HTTP1) { - upstream_ = std::make_unique(upstream_callbacks_, config_, downstream_info_); - } else { - upstream_ = std::make_unique(upstream_callbacks_, config_, downstream_info_); - } + upstream_ = std::make_unique(upstream_callbacks_, config_, downstream_info_, type_); Tcp::ConnectionPool::Cancellable* handle = conn_pool_data_.value().newStream(upstream_->responseDecoder(), *this, {/*can_send_early_data_=*/false, @@ -291,91 +335,5 @@ void HttpConnPool::onGenericPoolReady(Upstream::HostDescriptionConstSharedPtr& h callbacks_->onGenericPoolReady(nullptr, std::move(upstream_), host, address_provider, ssl_info); } -Http2Upstream::Http2Upstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, - const TunnelingConfigHelper& config, - StreamInfo::StreamInfo& downstream_info) - : HttpUpstream(callbacks, config, downstream_info) {} - -bool Http2Upstream::isValidResponse(const Http::ResponseHeaderMap& headers) { - if (Http::Utility::getResponseStatus(headers) != 200) { - return false; - } - return true; -} - -void Http2Upstream::setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl) { - request_encoder_ = &request_encoder; - request_encoder_->getStream().addCallbacks(*this); - - const std::string& scheme = - is_ssl ? Http::Headers::get().SchemeValues.Https : Http::Headers::get().SchemeValues.Http; - auto headers = Http::createHeaderMap({ - {Http::Headers::get().Method, config_.usePost() ? "POST" : "CONNECT"}, - {Http::Headers::get().Host, config_.host(downstream_info_)}, - }); - - if (config_.usePost()) { - headers->addReference(Http::Headers::get().Path, config_.postPath()); - headers->addReference(Http::Headers::get().Scheme, scheme); - } - - config_.headerEvaluator().evaluateHeaders(*headers, - downstream_info_.getRequestHeaders() == nullptr - ? *Http::StaticEmptyHeaders::get().request_headers - : *downstream_info_.getRequestHeaders(), - *Http::StaticEmptyHeaders::get().response_headers, - downstream_info_); - const auto status = request_encoder_->encodeHeaders(*headers, false); - // Encoding can only fail on missing required request headers. - ASSERT(status.ok()); -} - -Http1Upstream::Http1Upstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, - const TunnelingConfigHelper& config, - StreamInfo::StreamInfo& downstream_info) - : HttpUpstream(callbacks, config, downstream_info) {} - -void Http1Upstream::setRequestEncoder(Http::RequestEncoder& request_encoder, bool) { - request_encoder_ = &request_encoder; - request_encoder_->getStream().addCallbacks(*this); - request_encoder_->enableTcpTunneling(); - ASSERT(request_encoder_->http1StreamEncoderOptions() != absl::nullopt); - - auto headers = Http::createHeaderMap({ - {Http::Headers::get().Method, config_.usePost() ? "POST" : "CONNECT"}, - {Http::Headers::get().Host, config_.host(downstream_info_)}, - }); - - if (config_.usePost()) { - // Path is required for POST requests. - headers->addReference(Http::Headers::get().Path, config_.postPath()); - } - - config_.headerEvaluator().evaluateHeaders(*headers, - downstream_info_.getRequestHeaders() == nullptr - ? *Http::StaticEmptyHeaders::get().request_headers - : *downstream_info_.getRequestHeaders(), - *Http::StaticEmptyHeaders::get().response_headers, - downstream_info_); - const auto status = request_encoder_->encodeHeaders(*headers, false); - // Encoding can only fail on missing required request headers. - ASSERT(status.ok()); -} - -bool Http1Upstream::isValidResponse(const Http::ResponseHeaderMap& headers) { - // According to RFC7231 any 2xx response indicates that the connection is - // established. - // Any 'Content-Length' or 'Transfer-Encoding' header fields MUST be ignored. - // https://tools.ietf.org/html/rfc7231#section-4.3.6 - return Http::CodeUtility::is2xx(Http::Utility::getResponseStatus(headers)); -} - -void Http1Upstream::encodeData(Buffer::Instance& data, bool end_stream) { - if (!request_encoder_) { - return; - } - request_encoder_->encodeData(data, end_stream); -} - } // namespace TcpProxy } // namespace Envoy diff --git a/source/common/tcp_proxy/upstream.h b/source/common/tcp_proxy/upstream.h index 5532f8148016..d115bc440cc2 100644 --- a/source/common/tcp_proxy/upstream.h +++ b/source/common/tcp_proxy/upstream.h @@ -129,9 +129,11 @@ class HttpUpstream : public GenericUpstream, protected Http::StreamCallbacks { public: using TunnelingConfig = envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy_TunnelingConfig; - + HttpUpstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, + const TunnelingConfigHelper& config, StreamInfo::StreamInfo& downstream_info, + Http::CodecType type); ~HttpUpstream() override; - virtual bool isValidResponse(const Http::ResponseHeaderMap&) PURE; + bool isValidResponse(const Http::ResponseHeaderMap&); void doneReading(); void doneWriting(); @@ -152,15 +154,13 @@ class HttpUpstream : public GenericUpstream, protected Http::StreamCallbacks { void onAboveWriteBufferHighWatermark() override; void onBelowWriteBufferLowWatermark() override; - virtual void setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl) PURE; + void setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl); void setConnPoolCallbacks(std::unique_ptr&& callbacks) { conn_pool_callbacks_ = std::move(callbacks); } Ssl::ConnectionInfoConstSharedPtr getUpstreamConnectionSslInfo() override { return nullptr; } protected: - HttpUpstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, - const TunnelingConfigHelper& config, StreamInfo::StreamInfo& downstream_info); void resetEncoder(Network::ConnectionEvent event, bool inform_downstream = true); // The encoder offered by the upstream http client. @@ -208,6 +208,7 @@ class HttpUpstream : public GenericUpstream, protected Http::StreamCallbacks { }; DecoderShim response_decoder_; Tcp::ConnectionPool::UpstreamCallbacks& upstream_callbacks_; + const Http::CodecType type_; bool read_half_closed_{}; bool write_half_closed_{}; @@ -216,24 +217,5 @@ class HttpUpstream : public GenericUpstream, protected Http::StreamCallbacks { std::unique_ptr conn_pool_callbacks_; }; -class Http1Upstream : public HttpUpstream { -public: - Http1Upstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, - const TunnelingConfigHelper& config, StreamInfo::StreamInfo& downstream_info); - - void encodeData(Buffer::Instance& data, bool end_stream) override; - void setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl) override; - bool isValidResponse(const Http::ResponseHeaderMap& headers) override; -}; - -class Http2Upstream : public HttpUpstream { -public: - Http2Upstream(Tcp::ConnectionPool::UpstreamCallbacks& callbacks, - const TunnelingConfigHelper& config, StreamInfo::StreamInfo& downstream_info); - - void setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl) override; - bool isValidResponse(const Http::ResponseHeaderMap& headers) override; -}; - } // namespace TcpProxy } // namespace Envoy diff --git a/source/common/tracing/BUILD b/source/common/tracing/BUILD index b59a5612b590..b0c1132467b6 100644 --- a/source/common/tracing/BUILD +++ b/source/common/tracing/BUILD @@ -38,6 +38,7 @@ envoy_cc_library( "http_tracer_impl.h", ], deps = [ + ":trace_context_lib", ":tracer_lib", "//envoy/tracing:tracer_interface", "//source/common/formatter:substitution_formatter_lib", @@ -100,6 +101,21 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "trace_context_lib", + srcs = [ + "trace_context_impl.cc", + ], + hdrs = [ + "trace_context_impl.h", + ], + deps = [ + "//envoy/http:header_map_interface", + "//envoy/tracing:trace_context_interface", + "//source/common/http:header_map_lib", + ], +) + envoy_cc_library( name = "custom_tag_lib", srcs = [ @@ -109,6 +125,7 @@ envoy_cc_library( "custom_tag_impl.h", ], deps = [ + ":trace_context_lib", "//envoy/router:router_interface", "//envoy/tracing:custom_tag_interface", "//source/common/config:metadata_lib", diff --git a/source/common/tracing/custom_tag_impl.cc b/source/common/tracing/custom_tag_impl.cc index 5d34282c6041..3b9c8e5fc0a6 100644 --- a/source/common/tracing/custom_tag_impl.cc +++ b/source/common/tracing/custom_tag_impl.cc @@ -49,11 +49,8 @@ RequestHeaderCustomTag::RequestHeaderCustomTag( default_value_(request_header.default_value()) {} absl::string_view RequestHeaderCustomTag::value(const CustomTagContext& ctx) const { - if (ctx.trace_context == nullptr) { - return default_value_; - } // TODO(https://github.com/envoyproxy/envoy/issues/13454): Potentially populate all header values. - const auto entry = ctx.trace_context->getByKey(name_); + const auto entry = name_.get(ctx.trace_context); return entry.value_or(default_value_); } diff --git a/source/common/tracing/custom_tag_impl.h b/source/common/tracing/custom_tag_impl.h index ec141da2778b..f17ffc3ab16c 100644 --- a/source/common/tracing/custom_tag_impl.h +++ b/source/common/tracing/custom_tag_impl.h @@ -4,6 +4,7 @@ #include "envoy/type/tracing/v3/custom_tag.pb.h" #include "source/common/config/metadata.h" +#include "source/common/tracing/trace_context_impl.h" namespace Envoy { namespace Tracing { @@ -51,7 +52,7 @@ class RequestHeaderCustomTag : public CustomTagBase { absl::string_view value(const CustomTagContext& ctx) const override; private: - const Http::LowerCaseString name_; + const Tracing::TraceContextHandler name_; const std::string default_value_; }; diff --git a/source/common/tracing/http_tracer_impl.cc b/source/common/tracing/http_tracer_impl.cc index e55153de6613..cd2cd817cfc8 100644 --- a/source/common/tracing/http_tracer_impl.cc +++ b/source/common/tracing/http_tracer_impl.cc @@ -239,7 +239,10 @@ void HttpTracerUtility::setCommonTags(Span& span, const StreamInfo::StreamInfo& span.setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); } - CustomTagContext ctx{stream_info.getRequestHeaders(), stream_info}; + ReadOnlyHttpTraceContext trace_context{stream_info.getRequestHeaders() != nullptr + ? *stream_info.getRequestHeaders() + : *Http::StaticEmptyHeaders::get().request_headers}; + CustomTagContext ctx{trace_context, stream_info}; if (const CustomTagMap* custom_tag_map = tracing_config.customTags(); custom_tag_map) { for (const auto& it : *custom_tag_map) { it.second->applySpan(span, ctx); diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index 47b3eb9ab7e2..25ce607f2e0c 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -17,6 +17,60 @@ namespace Envoy { namespace Tracing { +template class HttpTraceContextBase : public TraceContext { +public: + static_assert(std::is_same::type, Http::RequestHeaderMap>::value, + "T must be Http::RequestHeaderMap or const Http::RequestHeaderMap"); + + HttpTraceContextBase(T& request_headers) : request_headers_(request_headers) {} + + absl::string_view protocol() const override { return request_headers_.getProtocolValue(); } + absl::string_view host() const override { return request_headers_.getHostValue(); } + absl::string_view path() const override { return request_headers_.getPathValue(); } + absl::string_view method() const override { return request_headers_.getMethodValue(); } + void forEach(IterateCallback callback) const override { + request_headers_.iterate([cb = std::move(callback)](const Http::HeaderEntry& entry) { + if (cb(entry.key().getStringView(), entry.value().getStringView())) { + return Http::HeaderMap::Iterate::Continue; + } + return Http::HeaderMap::Iterate::Break; + }); + } + absl::optional get(absl::string_view key) const override { + Http::LowerCaseString lower_key{std::string(key)}; + const auto entry = request_headers_.get(lower_key); + if (!entry.empty()) { + return entry[0]->value().getStringView(); + } + return absl::nullopt; + } + void set(absl::string_view, absl::string_view) override {} + void remove(absl::string_view) override {} + OptRef requestHeaders() const override { return request_headers_; }; + OptRef requestHeaders() override { return {}; }; + +protected: + T& request_headers_; +}; + +// Read only http trace context that could be constructed from const Http::RequestHeaderMap. +// This is mainly used for custom tag extraction. +using ReadOnlyHttpTraceContext = HttpTraceContextBase; + +class HttpTraceContext : public HttpTraceContextBase { +public: + using HttpTraceContextBase::HttpTraceContextBase; + + void set(absl::string_view key, absl::string_view value) override { + request_headers_.setCopy(Http::LowerCaseString(std::string(key)), value); + } + void remove(absl::string_view key) override { + request_headers_.remove(Http::LowerCaseString(std::string(key))); + } + OptRef requestHeaders() const override { return request_headers_; }; + OptRef requestHeaders() override { return request_headers_; }; +}; + class HttpTracerUtility { public: /** diff --git a/source/common/tracing/trace_context_impl.cc b/source/common/tracing/trace_context_impl.cc new file mode 100644 index 000000000000..5eb4075dac59 --- /dev/null +++ b/source/common/tracing/trace_context_impl.cc @@ -0,0 +1,97 @@ +#include "source/common/tracing/trace_context_impl.h" + +#include "source/common/http/header_map_impl.h" + +namespace Envoy { +namespace Tracing { + +TraceContextHandler::TraceContextHandler(absl::string_view key) : key_(key) { + // This will force the header map to be finalized in unit tests and do nothing in prod ( + // where the header map is already finalized when the server is initializing). + Http::TypedHeaderMapImpl::inlineHeadersSize(); + + handle_ = Http::CustomInlineHeaderRegistry::getInlineHeader< + Http::CustomInlineHeaderRegistry::Type::RequestHeaders>(key_); +} + +void TraceContextHandler::set(TraceContext& trace_context, absl::string_view value) const { + // Will dynamic_cast be better? + auto header_map = trace_context.requestHeaders(); + if (!header_map.has_value()) { + trace_context.set(key_, value); + return; + } + + if (handle_.has_value()) { + header_map->setInline(handle_.value(), value); + } else { + header_map->setCopy(key_, value); + } +} + +void TraceContextHandler::setRefKey(TraceContext& trace_context, absl::string_view value) const { + auto header_map = trace_context.requestHeaders(); + if (!header_map.has_value()) { + trace_context.set(key_, value); + return; + } + + if (handle_.has_value()) { + header_map->setInline(handle_.value(), value); + } else { + header_map->setReferenceKey(key_, value); + } +} + +void TraceContextHandler::setRef(TraceContext& trace_context, absl::string_view value) const { + auto header_map = trace_context.requestHeaders(); + if (!header_map.has_value()) { + trace_context.set(key_, value); + return; + } + + if (handle_.has_value()) { + header_map->setReferenceInline(handle_.value(), value); + } else { + header_map->setReference(key_, value); + } +} + +absl::optional +TraceContextHandler::get(const TraceContext& trace_context) const { + auto header_map = trace_context.requestHeaders(); + if (!header_map.has_value()) { + return trace_context.get(key_); + } + + if (handle_.has_value()) { + auto* entry = header_map->getInline(handle_.value()); + if (entry == nullptr) { + return absl::nullopt; + } + return entry->value().getStringView(); + } else { + auto results = header_map->get(key_); + if (results.empty()) { + return absl::nullopt; + } + return results[0]->value().getStringView(); + } +} + +void TraceContextHandler::remove(TraceContext& trace_context) const { + auto header_map = trace_context.requestHeaders(); + if (!header_map.has_value()) { + trace_context.remove(key_); + return; + } + + if (handle_.has_value()) { + header_map->removeInline(handle_.value()); + } else { + header_map->remove(key_); + } +} + +} // namespace Tracing +} // namespace Envoy diff --git a/source/common/tracing/trace_context_impl.h b/source/common/tracing/trace_context_impl.h new file mode 100644 index 000000000000..8ee2c4aeb83f --- /dev/null +++ b/source/common/tracing/trace_context_impl.h @@ -0,0 +1,75 @@ +#pragma once + +#include "envoy/http/header_map.h" +#include "envoy/tracing/trace_context.h" + +namespace Envoy { +namespace Tracing { + +using InlineHandle = Http::CustomInlineHeaderRegistry::Handle< + Http::CustomInlineHeaderRegistry::Type::RequestHeaders>; + +/** + * A handler class for trace context. This class could be used to set, get and erase the key/value + * pair in the trace context. This handler is optimized for HTTP to support reference semantics and + * inline header. If HTTP header is used as the trace context and the key is registered in the + * custom inline header registry, then inline handle will be used to improve the performance. + */ +class TraceContextHandler { +public: + /** + * Construct a handler with the given key. Note that the key will be lowercase and copied. + * @param key the key of the handler. + */ + TraceContextHandler(absl::string_view key); + + /** + * Get the key of the handler. + * @return absl::string_view the key of the handler. Note that the key is lowercase. + */ + const Http::LowerCaseString& key() const { return key_; } + + /** + * Erase the key/value pair from the trace context. + * @param trace_context the trace context to erase the key/value pair. + */ + void remove(TraceContext& trace_context) const; + + /** + * Get the value from the trace context by the key. + * @param trace_context the trace context to get the value. + * @return absl::optional the value of the key. If the key is not found, then + * absl::nullopt will be returned. + */ + absl::optional get(const TraceContext& trace_context) const; + + /* + * Set the key/value pair in the trace context. + * @param trace_context the trace context to set the key/value pair. + * @param value the value to set. + */ + void set(TraceContext& trace_context, absl::string_view value) const; + + /** + * Set the key/value pair in the trace context. This method should only be used when the handler + * has longer lifetime than current stream. + * @param trace_context the trace context to set the key/value pair. + * @param value the value to set. + */ + void setRefKey(TraceContext& trace_context, absl::string_view value) const; + + /** + * Set the key/value pair in the trace context. This method should only be used when both the + * handler and the value have longer lifetime than current stream. + * @param trace_context the trace context to set the key/value pair. + * @param value the value to set. + */ + void setRef(TraceContext& trace_context, absl::string_view value) const; + +private: + const Http::LowerCaseString key_; + absl::optional handle_; +}; + +} // namespace Tracing +} // namespace Envoy diff --git a/source/common/tracing/tracer_config_impl.h b/source/common/tracing/tracer_config_impl.h index 16cad4dc53b3..9ea8e1fe58b9 100644 --- a/source/common/tracing/tracer_config_impl.h +++ b/source/common/tracing/tracer_config_impl.h @@ -50,6 +50,9 @@ class ConnectionManagerTracingConfigImpl : public ConnectionManagerTracingConfig break; } + spawn_upstream_span_ = + PROTOBUF_GET_WRAPPED_OR_DEFAULT(tracing_config, spawn_upstream_span, false); + for (const auto& tag : tracing_config.custom_tags()) { custom_tags_.emplace(tag.tag(), Tracing::CustomTagUtility::createCustomTag(tag)); } @@ -101,6 +104,7 @@ class ConnectionManagerTracingConfigImpl : public ConnectionManagerTracingConfig Tracing::OperationName operationName() const override { return operation_name_; } bool verbose() const override { return verbose_; } uint32_t maxPathTagLength() const override { return max_path_tag_length_; } + bool spawnUpstreamSpan() const override { return spawn_upstream_span_; } // TODO(wbpcode): keep this field be public for compatibility. Then the HCM needn't change much // code to use this config. @@ -111,6 +115,7 @@ class ConnectionManagerTracingConfigImpl : public ConnectionManagerTracingConfig envoy::type::v3::FractionalPercent overall_sampling_; bool verbose_{}; uint32_t max_path_tag_length_{}; + bool spawn_upstream_span_{}; }; } // namespace Tracing diff --git a/source/common/tracing/tracer_impl.cc b/source/common/tracing/tracer_impl.cc index 01ee929bbd35..8205d0d2f8cd 100644 --- a/source/common/tracing/tracer_impl.cc +++ b/source/common/tracing/tracer_impl.cc @@ -125,7 +125,7 @@ void TracerUtility::finalizeSpan(Span& span, const TraceContext& trace_context, } // Custom tag from configuration. - CustomTagContext ctx{&trace_context, stream_info}; + CustomTagContext ctx{trace_context, stream_info}; if (const CustomTagMap* custom_tag_map = tracing_config.customTags(); custom_tag_map) { for (const auto& it : *custom_tag_map) { it.second->applySpan(span, ctx); diff --git a/source/common/tracing/tracer_impl.h b/source/common/tracing/tracer_impl.h index 5050cb05ac13..70354ef4ba8c 100644 --- a/source/common/tracing/tracer_impl.h +++ b/source/common/tracing/tracer_impl.h @@ -55,6 +55,8 @@ class EgressConfigImpl : public Config { const CustomTagMap* customTags() const override { return nullptr; } bool verbose() const override { return false; } uint32_t maxPathTagLength() const override { return Tracing::DefaultMaxPathTagLength; } + // This EgressConfigImpl is only used for async client tracing. Return false here is OK. + bool spawnUpstreamSpan() const override { return false; } }; using EgressConfig = ConstSingleton; @@ -63,7 +65,7 @@ class NullTracer : public Tracer { public: // Tracing::Tracer SpanPtr startSpan(const Config&, TraceContext&, const StreamInfo::StreamInfo&, - const Tracing::Decision) override { + Tracing::Decision) override { return SpanPtr{new NullSpan()}; } }; @@ -75,7 +77,7 @@ class TracerImpl : public Tracer { // Tracing::Tracer SpanPtr startSpan(const Config& config, TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, - const Tracing::Decision tracing_decision) override; + Tracing::Decision tracing_decision) override; DriverSharedPtr driverForTest() const { return driver_; } diff --git a/source/common/tracing/tracer_manager_impl.cc b/source/common/tracing/tracer_manager_impl.cc index 0e7ea0fa7b17..e80f2380d78c 100644 --- a/source/common/tracing/tracer_manager_impl.cc +++ b/source/common/tracing/tracer_manager_impl.cc @@ -62,11 +62,11 @@ void TracerManagerImpl::removeExpiredCacheEntries() { std::shared_ptr TracerManagerImpl::singleton(Server::Configuration::FactoryContext& context) { - return context.singletonManager().getTyped( + return context.serverFactoryContext().singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(tracer_manager), [&context] { return std::make_shared( std::make_unique( - context.getServerFactoryContext(), context.messageValidationVisitor())); + context.serverFactoryContext(), context.messageValidationVisitor())); }); } diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index f74052b5f907..efe386867b8b 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -122,7 +122,6 @@ envoy_cc_library( "//source/common/upstream:priority_conn_pool_map_impl_lib", "//source/common/upstream:upstream_lib", "//source/extensions/filters/network/http_connection_manager:config", - "//source/server:factory_context_base_impl_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", @@ -250,18 +249,35 @@ envoy_cc_library( srcs = ["host_utility.cc"], hdrs = ["host_utility.h"], deps = [ + "//envoy/stats:primitive_stats_interface", "//envoy/upstream:load_balancer_interface", "//envoy/upstream:upstream_interface", + "//source/common/config:well_known_names", "//source/common/runtime:runtime_lib", ], ) +envoy_cc_library( + name = "subset_lb_config_lib", + srcs = ["subset_lb_config.cc"], + hdrs = ["subset_lb_config.h"], + deps = [ + "//envoy/upstream:load_balancer_interface", + "//envoy/upstream:upstream_interface", + "//source/common/config:utility_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "load_balancer_lib", srcs = ["load_balancer_impl.cc"], hdrs = ["load_balancer_impl.h"], deps = [ ":scheduler_lib", + ":subset_lb_config_lib", "//envoy/common:random_generator_interface", "//envoy/runtime:runtime_interface", "//envoy/stats:stats_interface", @@ -272,7 +288,6 @@ envoy_cc_library( "//source/common/runtime:runtime_protos_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/least_request/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/round_robin/v3:pkg_cc_proto", @@ -603,4 +618,6 @@ envoy_cc_library( "//source/common/network:resolver_lib", "@envoy_api//envoy/config/upstream/local_address_selector/v3:pkg_cc_proto", ], + # Ensure this factory in the source is always linked in. + alwayslink = 1, ) diff --git a/source/common/upstream/cds_api_helper.cc b/source/common/upstream/cds_api_helper.cc index 306d6c0fe1e9..26b18892af17 100644 --- a/source/common/upstream/cds_api_helper.cc +++ b/source/common/upstream/cds_api_helper.cc @@ -42,7 +42,9 @@ CdsApiHelper::onConfigUpdate(const std::vector& adde cluster = dynamic_cast(resource.get().resource()); if (!cluster_names.insert(cluster.name()).second) { // NOTE: at this point, the first of these duplicates has already been successfully applied. - throw EnvoyException(fmt::format("duplicate cluster {} found", cluster.name())); + exception_msgs.push_back( + fmt::format("{}: duplicate cluster {} found", cluster.name(), cluster.name())); + continue; } if (cm_.addOrUpdateCluster(cluster, resource.get().version())) { any_applied = true; diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index b9a43923f30f..cfb0ee3e8b21 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -64,7 +64,7 @@ CdsApiImpl::onConfigUpdate(const std::vector& added_ helper_.onConfigUpdate(added_resources, removed_resources, system_version_info); runInitializeCallbackIfAny(); if (!exception_msgs.empty()) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("Error adding/updating cluster(s) {}", absl::StrJoin(exception_msgs, ", "))); } return absl::OkStatus(); diff --git a/source/common/upstream/cluster_factory_impl.cc b/source/common/upstream/cluster_factory_impl.cc index 48c8779608a2..0cb89487aca2 100644 --- a/source/common/upstream/cluster_factory_impl.cc +++ b/source/common/upstream/cluster_factory_impl.cc @@ -115,10 +115,12 @@ ClusterFactoryImplBase::create(const envoy::config::cluster::v3::Cluster& cluste } } - new_cluster_pair.first->setOutlierDetector(Outlier::DetectorImplFactory::createForCluster( + auto detector_or_error = Outlier::DetectorImplFactory::createForCluster( *new_cluster_pair.first, cluster, server_context.mainThreadDispatcher(), server_context.runtime(), context.outlierEventLogger(), - server_context.api().randomGenerator())); + server_context.api().randomGenerator()); + RETURN_IF_STATUS_NOT_OK(detector_or_error); + new_cluster_pair.first->setOutlierDetector(detector_or_error.value()); return status_or_cluster; } diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 5fb1a6b2dc2d..d5cf2c62dc7c 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -86,6 +86,20 @@ getOrigin(const Network::TransportSocketOptionsConstSharedPtr& options, HostCons return {{"https", sni, host->address()->ip()->port()}}; } +bool isBlockingAdsCluster(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + absl::string_view cluster_name) { + if (bootstrap.dynamic_resources().has_ads_config()) { + const auto& ads_config_source = bootstrap.dynamic_resources().ads_config(); + // We only care about EnvoyGrpc, not GoogleGrpc, because we only need to delay ADS mux + // initialization if it uses an Envoy cluster that needs to be initialized first. We don't + // depend on the same cluster initialization when opening a gRPC stream for GoogleGrpc. + return (ads_config_source.grpc_services_size() > 0 && + ads_config_source.grpc_services(0).has_envoy_grpc() && + ads_config_source.grpc_services(0).envoy_grpc().cluster_name() == cluster_name); + } + return false; +} + } // namespace void ClusterManagerInitHelper::addCluster(ClusterManagerCluster& cm_cluster) { @@ -287,7 +301,7 @@ ClusterManagerImpl::ClusterManagerImpl( ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context, Grpc::Context& grpc_context, Router::Context& router_context, const Server::Instance& server) - : factory_(factory), runtime_(runtime), stats_(stats), tls_(tls), + : server_(server), factory_(factory), runtime_(runtime), stats_(stats), tls_(tls), random_(api.randomGenerator()), deferred_cluster_creation_(bootstrap.cluster_manager().enable_deferred_cluster_creation()), bind_config_(bootstrap.cluster_manager().has_upstream_bind_config() @@ -296,8 +310,8 @@ ClusterManagerImpl::ClusterManagerImpl( local_info_(local_info), cm_stats_(generateStats(*stats.rootScope())), init_helper_(*this, [this](ClusterManagerCluster& cluster) { onClusterInit(cluster); }), time_source_(main_thread_dispatcher.timeSource()), dispatcher_(main_thread_dispatcher), - http_context_(http_context), router_context_(router_context), - cluster_stat_names_(stats.symbolTable()), + http_context_(http_context), validation_context_(validation_context), + router_context_(router_context), cluster_stat_names_(stats.symbolTable()), cluster_config_update_stat_names_(stats.symbolTable()), cluster_lb_stat_names_(stats.symbolTable()), cluster_endpoint_stat_names_(stats.symbolTable()), @@ -317,7 +331,8 @@ ClusterManagerImpl::ClusterManagerImpl( }); } async_client_manager_ = std::make_unique( - *this, tls, time_source_, api, grpc_context.statNames()); + *this, tls, time_source_, api, grpc_context.statNames(), + bootstrap.grpc_async_client_manager_config()); const auto& cm_config = bootstrap.cluster_manager(); if (cm_config.has_outlier_detection()) { const std::string event_log_file_path = cm_config.outlier_detection().event_log_path(); @@ -354,8 +369,11 @@ ClusterManagerImpl::ClusterManagerImpl( local_info, main_thread_dispatcher, *this, validation_context.dynamicValidationVisitor(), api, server, makeOptRefFromPtr(xds_resources_delegate_.get()), makeOptRefFromPtr(xds_config_tracker_.get())); +} - const auto& dyn_resources = bootstrap.dynamic_resources(); +absl::Status ClusterManagerImpl::init(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + ASSERT(!initialized_); + initialized_ = true; // Cluster loading happens in two phases: first all the primary clusters are loaded, and then all // the secondary clusters are loaded. As it currently stands all non-EDS clusters and EDS which @@ -377,13 +395,25 @@ ClusterManagerImpl::ClusterManagerImpl( primary_clusters_.insert(cluster.name()); } } + + bool has_ads_cluster = false; // Load all the primary clusters. for (const auto& cluster : bootstrap.static_resources().clusters()) { if (is_primary_cluster(cluster)) { - loadCluster(cluster, MessageUtil::hash(cluster), "", false, active_clusters_); + const bool required_for_ads = isBlockingAdsCluster(bootstrap, cluster.name()); + has_ads_cluster |= required_for_ads; + // TODO(abeyad): Consider passing a lambda for a "post-cluster-init" callback, which would + // include a conditional ads_mux_->start() call, if other uses cases for "post-cluster-init" + // functionality pops up. + auto status_or_cluster = + loadCluster(cluster, MessageUtil::hash(cluster), "", /*added_via_api=*/false, + required_for_ads, active_clusters_); + RETURN_IF_STATUS_NOT_OK(status_or_cluster); } } + const auto& dyn_resources = bootstrap.dynamic_resources(); + // Now setup ADS if needed, this might rely on a primary cluster. // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. // After here, we just have a GrpcMux interface held in ads_mux_, which hides @@ -391,7 +421,7 @@ ClusterManagerImpl::ClusterManagerImpl( if (dyn_resources.has_ads_config()) { Config::CustomConfigValidatorsPtr custom_config_validators = std::make_unique( - validation_context.dynamicValidationVisitor(), server, + validation_context_.dynamicValidationVisitor(), server_, dyn_resources.ads_config().config_validators()); JitteredExponentialBackOffStrategyPtr backoff_strategy = @@ -404,7 +434,8 @@ ClusterManagerImpl::ClusterManagerImpl( Runtime::runtimeFeatureEnabled("envoy.restart_features.use_eds_cache_for_ads"); if (dyn_resources.ads_config().api_type() == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC) { - THROW_IF_NOT_OK(Config::Utility::checkTransportVersion(dyn_resources.ads_config())); + absl::Status status = Config::Utility::checkTransportVersion(dyn_resources.ads_config()); + RETURN_IF_NOT_OK(status); std::string name; if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { name = "envoy.config_mux.delta_grpc_mux_factory"; @@ -413,17 +444,18 @@ ClusterManagerImpl::ClusterManagerImpl( } auto* factory = Config::Utility::getFactoryByName(name); if (!factory) { - throw EnvoyException(fmt::format("{} not found", name)); + return absl::InvalidArgumentError(fmt::format("{} not found", name)); } ads_mux_ = factory->create( Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) + *async_client_manager_, dyn_resources.ads_config(), *stats_.rootScope(), false) ->createUncachedRawAsyncClient(), - main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), - local_info, std::move(custom_config_validators), std::move(backoff_strategy), + dispatcher_, random_, *stats_.rootScope(), dyn_resources.ads_config(), local_info_, + std::move(custom_config_validators), std::move(backoff_strategy), makeOptRefFromPtr(xds_config_tracker_.get()), {}, use_eds_cache); } else { - THROW_IF_NOT_OK(Config::Utility::checkTransportVersion(dyn_resources.ads_config())); + absl::Status status = Config::Utility::checkTransportVersion(dyn_resources.ads_config()); + RETURN_IF_NOT_OK(status); auto xds_delegate_opt_ref = makeOptRefFromPtr(xds_resources_delegate_.get()); std::string name; if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { @@ -434,14 +466,14 @@ ClusterManagerImpl::ClusterManagerImpl( auto* factory = Config::Utility::getFactoryByName(name); if (!factory) { - throw EnvoyException(fmt::format("{} not found", name)); + return absl::InvalidArgumentError(fmt::format("{} not found", name)); } ads_mux_ = factory->create( Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, dyn_resources.ads_config(), *stats.rootScope(), false) + *async_client_manager_, dyn_resources.ads_config(), *stats_.rootScope(), false) ->createUncachedRawAsyncClient(), - main_thread_dispatcher, random_, *stats_.rootScope(), dyn_resources.ads_config(), - local_info, std::move(custom_config_validators), std::move(backoff_strategy), + dispatcher_, random_, *stats_.rootScope(), dyn_resources.ads_config(), local_info_, + std::move(custom_config_validators), std::move(backoff_strategy), makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref, use_eds_cache); } } else { @@ -454,7 +486,14 @@ ClusterManagerImpl::ClusterManagerImpl( if (cluster.type() == envoy::config::cluster::v3::Cluster::EDS && !Config::SubscriptionFactory::isPathBasedConfigSource( cluster.eds_cluster_config().eds_config().config_source_specifier_case())) { - loadCluster(cluster, MessageUtil::hash(cluster), "", false, active_clusters_); + const bool required_for_ads = isBlockingAdsCluster(bootstrap, cluster.name()); + has_ads_cluster |= required_for_ads; + auto status_or_cluster = + loadCluster(cluster, MessageUtil::hash(cluster), "", /*added_via_api=*/false, + required_for_ads, active_clusters_); + if (!status_or_cluster.status().ok()) { + return status_or_cluster.status(); + } } } @@ -465,7 +504,7 @@ ClusterManagerImpl::ClusterManagerImpl( if (local_cluster_name_) { auto local_cluster = active_clusters_.find(local_cluster_name_.value()); if (local_cluster == active_clusters_.end()) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("local cluster '{}' must be defined", local_cluster_name_.value())); } local_cluster_params.emplace(); @@ -504,10 +543,15 @@ ClusterManagerImpl::ClusterManagerImpl( // clusters have already initialized. (E.g., if all static). init_helper_.onStaticLoadComplete(); - ads_mux_->start(); + if (!has_ads_cluster) { + // There is no ADS cluster, so we won't be starting the ADS mux after a cluster has finished + // initializing, so we must start ADS here. + ads_mux_->start(); + } + return absl::OkStatus(); } -void ClusterManagerImpl::initializeSecondaryClusters( +absl::Status ClusterManagerImpl::initializeSecondaryClusters( const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { init_helper_.startInitializingSecondaryClusters(); @@ -515,7 +559,8 @@ void ClusterManagerImpl::initializeSecondaryClusters( if (cm_config.has_load_stats_config()) { const auto& load_stats_config = cm_config.load_stats_config(); - THROW_IF_NOT_OK(Config::Utility::checkTransportVersion(load_stats_config)); + absl::Status status = Config::Utility::checkTransportVersion(load_stats_config); + RETURN_IF_NOT_OK(status); load_stats_reporter_ = std::make_unique( local_info_, *this, *stats_.rootScope(), Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, load_stats_config, @@ -523,6 +568,7 @@ void ClusterManagerImpl::initializeSecondaryClusters( ->createUncachedRawAsyncClient(), dispatcher_); } + return absl::OkStatus(); } ClusterManagerStats ClusterManagerImpl::generateStats(Stats::Scope& scope) { @@ -763,8 +809,10 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::config::cluster::v3::Cl init_helper_.state() == ClusterManagerInitHelper::State::AllClustersInitialized; // Preserve the previous cluster data to avoid early destroy. The same cluster should be added // before destroy to avoid early initialization complete. - const auto previous_cluster = - loadCluster(cluster, new_hash, version_info, true, warming_clusters_); + auto status_or_cluster = loadCluster(cluster, new_hash, version_info, /*added_via_api=*/true, + /*required_for_ads=*/false, warming_clusters_); + THROW_IF_STATUS_NOT_OK(status_or_cluster, throw); + const ClusterDataPtr previous_cluster = std::move(status_or_cluster.value()); auto& cluster_entry = warming_clusters_.at(cluster_name); cluster_entry->cluster_->info()->configUpdateStats().warming_state_.set(1); if (!all_clusters_initialized) { @@ -845,16 +893,17 @@ bool ClusterManagerImpl::removeCluster(const std::string& cluster_name) { return removed; } -ClusterManagerImpl::ClusterDataPtr +absl::StatusOr ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& cluster, const uint64_t cluster_hash, const std::string& version_info, - bool added_via_api, ClusterMap& cluster_map) { + bool added_via_api, const bool required_for_ads, + ClusterMap& cluster_map) { absl::StatusOr> new_cluster_pair_or_error = factory_.clusterFromProto(cluster, *this, outlier_event_logger_, added_via_api); if (!new_cluster_pair_or_error.ok()) { - throw EnvoyException(std::string(new_cluster_pair_or_error.status().message())); + return absl::InvalidArgumentError(std::string(new_cluster_pair_or_error.status().message())); } auto& new_cluster = new_cluster_pair_or_error->first; auto& lb = new_cluster_pair_or_error->second; @@ -864,7 +913,7 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust if (!added_via_api) { if (cluster_map.find(cluster_info->name()) != cluster_map.end()) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("cluster manager: duplicate cluster '{}'", cluster_info->name())); } } @@ -880,12 +929,13 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } if (cluster_provided_lb && lb == nullptr) { - throw EnvoyException(fmt::format("cluster manager: cluster provided LB specified but cluster " - "'{}' did not provide one. Check cluster documentation.", - cluster_info->name())); + return absl::InvalidArgumentError( + fmt::format("cluster manager: cluster provided LB specified but cluster " + "'{}' did not provide one. Check cluster documentation.", + cluster_info->name())); } if (!cluster_provided_lb && lb != nullptr) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("cluster manager: cluster provided LB not specified but cluster " "'{}' provided one. Check cluster documentation.", cluster_info->name())); @@ -916,14 +966,14 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust if (cluster_entry_it != cluster_map.end()) { result = std::exchange(cluster_entry_it->second, std::make_unique(cluster, cluster_hash, version_info, - added_via_api, std::move(new_cluster), - time_source_)); + added_via_api, required_for_ads, + std::move(new_cluster), time_source_)); } else { bool inserted = false; std::tie(cluster_entry_it, inserted) = cluster_map.emplace( cluster_info->name(), std::make_unique(cluster, cluster_hash, version_info, added_via_api, - std::move(new_cluster), time_source_)); + required_for_ads, std::move(new_cluster), time_source_)); ASSERT(inserted); } // If an LB is thread aware, create it here. The LB is not initialized until cluster pre-init @@ -973,16 +1023,16 @@ void ClusterManagerImpl::updateClusterCounts() { init_helper_.state() == ClusterManagerInitHelper::State::AllClustersInitialized; if (all_clusters_initialized && ads_mux_) { const auto type_url = Config::getTypeUrl(); - const uint64_t previous_warming = cm_stats_.warming_clusters_.value(); - if (previous_warming == 0 && !warming_clusters_.empty()) { + if (last_recorded_warming_clusters_count_ == 0 && !warming_clusters_.empty()) { resume_cds_ = ads_mux_->pause(type_url); - } else if (previous_warming > 0 && warming_clusters_.empty()) { + } else if (last_recorded_warming_clusters_count_ > 0 && warming_clusters_.empty()) { ASSERT(resume_cds_ != nullptr); resume_cds_.reset(); } } cm_stats_.active_clusters_.set(active_clusters_.size()); cm_stats_.warming_clusters_.set(warming_clusters_.size()); + last_recorded_warming_clusters_count_ = warming_clusters_.size(); } ThreadLocalCluster* ClusterManagerImpl::getThreadLocalCluster(absl::string_view cluster) { @@ -1093,14 +1143,16 @@ void ClusterManagerImpl::drainConnections(DrainConnectionsHostPredicate predicat }); } -void ClusterManagerImpl::checkActiveStaticCluster(const std::string& cluster) { +absl::Status ClusterManagerImpl::checkActiveStaticCluster(const std::string& cluster) { const auto& it = active_clusters_.find(cluster); if (it == active_clusters_.end()) { - throw EnvoyException(fmt::format("Unknown gRPC client cluster '{}'", cluster)); + return absl::InvalidArgumentError(fmt::format("Unknown gRPC client cluster '{}'", cluster)); } if (it->second->added_via_api_) { - throw EnvoyException(fmt::format("gRPC client cluster '{}' is not static", cluster)); + return absl::InvalidArgumentError( + fmt::format("gRPC client cluster '{}' is not static", cluster)); } + return absl::OkStatus(); } void ClusterManagerImpl::postThreadLocalRemoveHosts(const Cluster& cluster, @@ -1174,15 +1226,16 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ pending_cluster_creations_.erase(cm_cluster.cluster().info()->name()); + const UnitFloat drop_overload = cm_cluster.cluster().dropOverload(); // Populate the cluster initialization object based on this update. ClusterInitializationObjectConstSharedPtr cluster_initialization_object = - addOrUpdateClusterInitializationObjectIfSupported(params, cm_cluster.cluster().info(), - load_balancer_factory, host_map); + addOrUpdateClusterInitializationObjectIfSupported( + params, cm_cluster.cluster().info(), load_balancer_factory, host_map, drop_overload); tls_.runOnAllThreads([info = cm_cluster.cluster().info(), params = std::move(params), add_or_update_cluster, load_balancer_factory, map = std::move(host_map), - cluster_initialization_object = std::move(cluster_initialization_object)]( - OptRef cluster_manager) { + cluster_initialization_object = std::move(cluster_initialization_object), + drop_overload](OptRef cluster_manager) { ASSERT(cluster_manager.has_value(), "Expected the ThreadLocalClusterManager to be set during ClusterManagerImpl creation."); @@ -1238,6 +1291,9 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ cluster_manager->thread_local_clusters_.size()); } + if (cluster_manager->thread_local_clusters_[info->name()]) { + cluster_manager->thread_local_clusters_[info->name()]->setDropOverload(drop_overload); + } for (const auto& per_priority : params.per_priority_update_params_) { cluster_manager->updateClusterMembership( info->name(), per_priority.priority_, per_priority.update_hosts_params_, @@ -1260,12 +1316,20 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ } } }); + + // By this time, the main thread has received the cluster initialization update, so we can start + // the ADS mux if the ADS mux is dependent on this cluster's initialization. + if (cm_cluster.requiredForAds() && !ads_mux_initialized_) { + ads_mux_->start(); + ads_mux_initialized_ = true; + } } ClusterManagerImpl::ClusterInitializationObjectConstSharedPtr ClusterManagerImpl::addOrUpdateClusterInitializationObjectIfSupported( const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, - LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map) { + LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, + UnitFloat drop_overload) { if (!deferralIsSupportedForCluster(cluster_info)) { return nullptr; } @@ -1274,15 +1338,20 @@ ClusterManagerImpl::addOrUpdateClusterInitializationObjectIfSupported( auto entry = cluster_initialization_map_.find(cluster_name); // TODO(kbaichoo): if EDS can be configured via cluster_type() then modify the // merging logic below. - // We should only merge if the cluster type is the same as before and this is - // an EDS cluster. This is due to the fact that EDS clusters get - // ClusterLoadAssignment from the configuration server but pass per priority - // deltas in updates to the ClusterManager. In the future we may decide to - // change how the updates propagate among those components. + // + // This method may be called multiple times to create multiple ClusterInitializationObject + // instances for the same cluster. And before the thread local clusters are actually initialized, + // the new instances will override the old instances in the work threads. But part of data is be + // created only once, such as load balancer factory. So we should always to merge the new instance + // with the old one to keep the latest instance have all necessary data. + // + // More specifically, this will happen in the following scenarios for now: + // 1. EDS clusters: the ClusterLoadAssignment of EDS cluster may be updated multiples before + // the thread local cluster is initialized. + // 2. Clusters in the unit tests: the cluster in the unit test may be updated multiples before + // the thread local cluster is initialized by calling 'updateHosts' manually. const bool should_merge_with_prior_cluster = - entry != cluster_initialization_map_.end() && - entry->second->cluster_info_->type() == cluster_info->type() && - cluster_info->type() == envoy::config::cluster::v3::Cluster::EDS; + entry != cluster_initialization_map_.end() && entry->second->cluster_info_ == cluster_info; if (should_merge_with_prior_cluster) { // We need to copy from an existing Cluster Initialization Object. In @@ -1291,13 +1360,13 @@ ClusterManagerImpl::addOrUpdateClusterInitializationObjectIfSupported( entry->second->per_priority_state_, params, std::move(cluster_info), load_balancer_factory == nullptr ? entry->second->load_balancer_factory_ : load_balancer_factory, - map); + map, drop_overload); cluster_initialization_map_[cluster_name] = new_initialization_object; return new_initialization_object; } else { // We need to create a fresh Cluster Initialization Object. auto new_initialization_object = std::make_shared( - params, std::move(cluster_info), load_balancer_factory, map); + params, std::move(cluster_info), load_balancer_factory, map, drop_overload); cluster_initialization_map_[cluster_name] = new_initialization_object; return new_initialization_object; } @@ -1330,6 +1399,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::initializeClusterInlineIfExis per_priority.overprovisioning_factor_, initialization_object->cross_priority_host_map_); } + thread_local_clusters_[cluster]->setDropOverload(initialization_object->drop_overload_); // Remove the CIO as we've initialized the cluster. thread_local_deferred_clusters_.erase(entry); @@ -1339,9 +1409,10 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::initializeClusterInlineIfExis ClusterManagerImpl::ClusterInitializationObject::ClusterInitializationObject( const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, - LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map) + LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, + UnitFloat drop_overload) : cluster_info_(std::move(cluster_info)), load_balancer_factory_(load_balancer_factory), - cross_priority_host_map_(map) { + cross_priority_host_map_(map), drop_overload_(drop_overload) { // Copy the update since the map is empty. for (const auto& update : params.per_priority_update_params_) { per_priority_state_.emplace(update.priority_, update); @@ -1351,13 +1422,12 @@ ClusterManagerImpl::ClusterInitializationObject::ClusterInitializationObject( ClusterManagerImpl::ClusterInitializationObject::ClusterInitializationObject( const absl::flat_hash_map& per_priority_state, const ThreadLocalClusterUpdateParams& update_params, ClusterInfoConstSharedPtr cluster_info, - LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map) + LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, + UnitFloat drop_overload) : per_priority_state_(per_priority_state), cluster_info_(std::move(cluster_info)), - load_balancer_factory_(load_balancer_factory), cross_priority_host_map_(map) { + load_balancer_factory_(load_balancer_factory), cross_priority_host_map_(map), + drop_overload_(drop_overload) { - ASSERT(cluster_info_->type() == envoy::config::cluster::v3::Cluster::EDS, - fmt::format("Using merge constructor on possible non-mergable cluster of type {}", - cluster_info_->type())); // Because EDS Clusters receive the entire ClusterLoadAssignment but only // provides the delta we must process the hosts_added and hosts_removed and // not simply overwrite with hosts added. @@ -2046,6 +2116,9 @@ HostConstSharedPtr ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEnt if (host != nullptr) { return host; } + if (!HostUtility::allowLBChooseHost(context)) { + return nullptr; + } return lb_->chooseHost(context); } @@ -2147,11 +2220,13 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::tcpConnPoolIsIdle( ClusterManagerPtr ProdClusterManagerFactory::clusterManagerFromProto( const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - return ClusterManagerPtr{new ClusterManagerImpl( + auto cluster_manager_impl = std::unique_ptr{new ClusterManagerImpl( bootstrap, *this, stats_, tls_, context_.runtime(), context_.localInfo(), context_.accessLogManager(), context_.mainThreadDispatcher(), context_.admin(), context_.messageValidationContext(), context_.api(), http_context_, context_.grpcContext(), context_.routerContext(), server_)}; + THROW_IF_NOT_OK(cluster_manager_impl->init(bootstrap)); + return cluster_manager_impl; } Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 294af652d761..3bac52e9aa9c 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -41,7 +41,6 @@ #include "source/common/upstream/od_cds_api_impl.h" #include "source/common/upstream/priority_conn_pool_map.h" #include "source/common/upstream/upstream_impl.h" -#include "source/server/factory_context_base_impl.h" namespace Envoy { namespace Upstream { @@ -62,7 +61,8 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { : context_(context), stats_(stats), tls_(tls), http_context_(http_context), dns_resolver_fn_(dns_resolver_fn), ssl_context_manager_(ssl_context_manager), secret_manager_(secret_manager), quic_stat_names_(quic_stat_names), - alternate_protocols_cache_manager_factory_(context.singletonManager(), tls, {context}), + alternate_protocols_cache_manager_factory_(context.singletonManager(), tls, + {context, context.messageValidationVisitor()}), alternate_protocols_cache_manager_(alternate_protocols_cache_manager_factory_.get()), server_(server) {} @@ -131,6 +131,10 @@ class ClusterManagerCluster { // Set when a cluster has been added or updated. This is only called a single time for a cluster. virtual void setAddedOrUpdated() PURE; + + // Return true if the cluster must be ready-for-use before ADS (Aggregated Discovery Service) can + // be initialized; will only occur if ADS is configured to use the cluster via EnvoyGrpc. + virtual bool requiredForAds() const PURE; }; /** @@ -242,15 +246,17 @@ class ClusterManagerImpl : public ClusterManager, public MissingClusterNotifier, Logger::Loggable { public: - ClusterManagerImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, - ClusterManagerFactory& factory, Stats::Store& stats, - ThreadLocal::Instance& tls, Runtime::Loader& runtime, - const LocalInfo::LocalInfo& local_info, - AccessLog::AccessLogManager& log_manager, - Event::Dispatcher& main_thread_dispatcher, OptRef admin, - ProtobufMessage::ValidationContext& validation_context, Api::Api& api, - Http::Context& http_context, Grpc::Context& grpc_context, - Router::Context& router_context, const Server::Instance& server); + // Initializes the ClusterManagerImpl instance based on the given Bootstrap config. + // + // This method *must* be called prior to invoking any other methods on the class and *must* only + // be called once. This method should be called immediately after ClusterManagerImpl construction + // and from the same thread in which the ClusterManagerImpl was constructed. + // + // The initialization is separated from the constructor because lots of work, including ADS + // initialization, is done in this method. If the contents of this method are invoked during + // construction, a derived class cannot override any of the virtual methods and have them invoked + // instead, since the base class's methods are used when in a base class constructor. + absl::Status init(const envoy::config::bootstrap::v3::Bootstrap& bootstrap); std::size_t warmingClusterCount() const { return warming_clusters_.size(); } @@ -329,7 +335,7 @@ class ClusterManagerImpl : public ClusterManager, Config::SubscriptionFactory& subscriptionFactory() override { return *subscription_factory_; } - void + absl::Status initializeSecondaryClusters(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) override; const ClusterTrafficStatNames& clusterStatNames() const override { return cluster_stat_names_; } @@ -358,7 +364,7 @@ class ClusterManagerImpl : public ClusterManager, void drainConnections(DrainConnectionsHostPredicate predicate) override; - void checkActiveStaticCluster(const std::string& cluster) override; + absl::Status checkActiveStaticCluster(const std::string& cluster) override; // Upstream::MissingClusterNotifier void notifyMissingCluster(absl::string_view name) override; @@ -377,6 +383,18 @@ class ClusterManagerImpl : public ClusterManager, Config::EdsResourcesCacheOptRef edsResourcesCache() override; protected: + // ClusterManagerImpl's constructor should not be invoked directly; create instances from the + // clusterManagerFromProto() static method. The init() method must be called after construction. + ClusterManagerImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + ClusterManagerFactory& factory, Stats::Store& stats, + ThreadLocal::Instance& tls, Runtime::Loader& runtime, + const LocalInfo::LocalInfo& local_info, + AccessLog::AccessLogManager& log_manager, + Event::Dispatcher& main_thread_dispatcher, OptRef admin, + ProtobufMessage::ValidationContext& validation_context, Api::Api& api, + Http::Context& http_context, Grpc::Context& grpc_context, + Router::Context& router_context, const Server::Instance& server); + virtual void postThreadLocalRemoveHosts(const Cluster& cluster, const HostVector& hosts_removed); // Parameters for calling postThreadLocalClusterUpdate() @@ -414,18 +432,20 @@ class ClusterManagerImpl : public ClusterManager, ClusterInitializationObject(const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, - HostMapConstSharedPtr map); + HostMapConstSharedPtr map, UnitFloat drop_overload); ClusterInitializationObject( const absl::flat_hash_map& per_priority_state, const ThreadLocalClusterUpdateParams& update_params, ClusterInfoConstSharedPtr cluster_info, - LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map); + LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, + UnitFloat drop_overload); absl::flat_hash_map per_priority_state_; const ClusterInfoConstSharedPtr cluster_info_; const LoadBalancerFactorySharedPtr load_balancer_factory_; const HostMapConstSharedPtr cross_priority_host_map_; + UnitFloat drop_overload_{0}; }; using ClusterInitializationObjectConstSharedPtr = @@ -485,6 +505,9 @@ class ClusterManagerImpl : public ClusterManager, ClusterDiscoveryManager createAndSwapClusterDiscoveryManager(std::string thread_name); private: + // To enable access to the protected constructor. + friend ProdClusterManagerFactory; + /** * Thread local cached cluster data. Each thread local cluster gets updates from the parent * central dynamic cluster (if applicable). It maintains load balancer state and any created @@ -591,6 +614,8 @@ class ClusterManagerImpl : public ClusterManager, // Drain any connection pools associated with the hosts filtered by the predicate. void drainConnPools(DrainConnectionsHostPredicate predicate, ConnectionPool::DrainBehavior behavior); + UnitFloat dropOverload() const override { return drop_overload_; } + void setDropOverload(UnitFloat drop_overload) override { drop_overload_ = drop_overload; } private: Http::ConnectionPool::Instance* @@ -606,6 +631,7 @@ class ClusterManagerImpl : public ClusterManager, ThreadLocalClusterManagerImpl& parent_; PrioritySetImpl priority_set_; + UnitFloat drop_overload_{0}; // Don't change the order of cluster_info_ and lb_factory_/lb_ as the the lb_factory_/lb_ // may keep a reference to the cluster_info_. @@ -631,14 +657,6 @@ class ClusterManagerImpl : public ClusterManager, // * 0b010000: envoy::config::core::v3::HealthStatus::TIMEOUT // * 0b100000: envoy::config::core::v3::HealthStatus::DEGRADED // - // If runtime flag `envoy.reloadable_features.validate_detailed_override_host_statuses` is - // disabled, the old coarse health status Host::Health will be used. The specific - // correspondence is shown below: - // - // * 0b001: Host::Health::Unhealthy - // * 0b010: Host::Health::Degraded - // * 0b100: Host::Health::Healthy - // // If multiple bit fields are set, it is acceptable as long as the status of override host is // in any of these statuses. const HostUtility::HostStatusSet override_host_statuses_{}; @@ -718,11 +736,12 @@ class ClusterManagerImpl : public ClusterManager, struct ClusterData : public ClusterManagerCluster { ClusterData(const envoy::config::cluster::v3::Cluster& cluster_config, const uint64_t cluster_config_hash, const std::string& version_info, - bool added_via_api, ClusterSharedPtr&& cluster, TimeSource& time_source) + bool added_via_api, bool required_for_ads, ClusterSharedPtr&& cluster, + TimeSource& time_source) : cluster_config_(cluster_config), config_hash_(cluster_config_hash), version_info_(version_info), cluster_(std::move(cluster)), last_updated_(time_source.systemTime()), - added_via_api_(added_via_api), added_or_updated_{} {} + added_via_api_(added_via_api), added_or_updated_{}, required_for_ads_(required_for_ads) {} bool blockUpdate(uint64_t hash) { return !added_via_api_ || config_hash_ == hash; } @@ -740,6 +759,7 @@ class ClusterManagerImpl : public ClusterManager, ASSERT(!added_or_updated_); added_or_updated_ = true; } + bool requiredForAds() const override { return required_for_ads_; } const envoy::config::cluster::v3::Cluster cluster_config_; const uint64_t config_hash_; @@ -755,6 +775,7 @@ class ClusterManagerImpl : public ClusterManager, // Keep smaller fields near the end to reduce padding const bool added_via_api_ : 1; bool added_or_updated_ : 1; + const bool required_for_ads_ : 1; }; struct ClusterUpdateCallbacksHandleImpl : public ClusterUpdateCallbacksHandle, @@ -824,11 +845,13 @@ class ClusterManagerImpl : public ClusterManager, /** * @return ClusterDataPtr contains the previous cluster in the cluster_map, or - * nullptr if cluster_map did not contain the same cluster. + * nullptr if cluster_map did not contain the same cluster or an error if + * cluster load fails. */ - ClusterDataPtr loadCluster(const envoy::config::cluster::v3::Cluster& cluster, - const uint64_t cluster_hash, const std::string& version_info, - bool added_via_api, ClusterMap& cluster_map); + absl::StatusOr loadCluster(const envoy::config::cluster::v3::Cluster& cluster, + const uint64_t cluster_hash, + const std::string& version_info, bool added_via_api, + bool required_for_ads, ClusterMap& cluster_map); void onClusterInit(ClusterManagerCluster& cluster); void postThreadLocalHealthFailure(const HostSharedPtr& host); void updateClusterCounts(); @@ -844,6 +867,10 @@ class ClusterManagerImpl : public ClusterManager, void notifyClusterDiscoveryStatus(absl::string_view name, ClusterDiscoveryStatus status); +protected: + ClusterMap active_clusters_; + ClusterInitializationMap cluster_initialization_map_; + private: /** * Builds the cluster initialization object for this given cluster. @@ -853,10 +880,12 @@ class ClusterManagerImpl : public ClusterManager, */ ClusterInitializationObjectConstSharedPtr addOrUpdateClusterInitializationObjectIfSupported( const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, - LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map); + LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, + UnitFloat drop_overload); bool deferralIsSupportedForCluster(const ClusterInfoConstSharedPtr& info) const; + const Server::Instance& server_; ClusterManagerFactory& factory_; Runtime::Loader& runtime_; Stats::Store& stats_; @@ -864,14 +893,8 @@ class ClusterManagerImpl : public ClusterManager, // Contains information about ongoing on-demand cluster discoveries. ClusterCreationsMap pending_cluster_creations_; Random::RandomGenerator& random_; - -protected: - ClusterMap active_clusters_; - -private: ClusterMap warming_clusters_; const bool deferred_cluster_creation_; - ClusterInitializationMap cluster_initialization_map_; absl::optional bind_config_; Outlier::EventLoggerSharedPtr outlier_event_logger_; const LocalInfo::LocalInfo& local_info_; @@ -890,6 +913,7 @@ class ClusterManagerImpl : public ClusterManager, ClusterUpdatesMap updates_map_; Event::Dispatcher& dispatcher_; Http::Context& http_context_; + ProtobufMessage::ValidationContext& validation_context_; Router::Context& router_context_; ClusterTrafficStatNames cluster_stat_names_; ClusterConfigUpdateStatNames cluster_config_update_stat_names_; @@ -909,7 +933,18 @@ class ClusterManagerImpl : public ClusterManager, std::unique_ptr xds_resources_delegate_; std::unique_ptr xds_config_tracker_; + bool initialized_{}; + bool ads_mux_initialized_{}; std::atomic shutdown_{}; + + // Records the last `warming_clusters_` map size from updateClusterCounts(). This variable is + // used for bookkeeping to run the `resume_cds_` cleanup that decrements the pause count and + // enables the resumption of DiscoveryRequests for the Cluster type url. + // + // The `warming_clusters` gauge is not suitable for this purpose, because different environments + // (e.g. mobile) may have different stats enabled, leading to the gauge not having a reliable + // previous warming clusters size value. + std::size_t last_recorded_warming_clusters_count_{0}; }; } // namespace Upstream diff --git a/source/common/upstream/default_local_address_selector_factory.cc b/source/common/upstream/default_local_address_selector_factory.cc index f10dd91b4451..ea0726cb2a4c 100644 --- a/source/common/upstream/default_local_address_selector_factory.cc +++ b/source/common/upstream/default_local_address_selector_factory.cc @@ -9,18 +9,19 @@ namespace { constexpr absl::string_view kDefaultLocalAddressSelectorName = "envoy.upstream.local_address_selector.default_local_address_selector"; -void validate(const std::vector<::Envoy::Upstream::UpstreamLocalAddress>& upstream_local_addresses, - absl::optional cluster_name) { +absl::Status +validate(const std::vector<::Envoy::Upstream::UpstreamLocalAddress>& upstream_local_addresses, + absl::optional cluster_name) { if (upstream_local_addresses.empty()) { - throw EnvoyException(fmt::format("{}'s upstream binding config has no valid source address.", - !(cluster_name.has_value()) - ? "Bootstrap" - : fmt::format("Cluster {}", cluster_name.value()))); + return absl::InvalidArgumentError( + fmt::format("{}'s upstream binding config has no valid source address.", + !(cluster_name.has_value()) ? "Bootstrap" + : fmt::format("Cluster {}", cluster_name.value()))); } if (upstream_local_addresses.size() > 2) { - throw EnvoyException(fmt::format( + return absl::InvalidArgumentError(fmt::format( "{}'s upstream binding config has more than one extra/additional source addresses. Only " "one extra/additional source can be supported in BindConfig's " "extra_source_addresses/additional_source_addresses field", @@ -42,7 +43,7 @@ void validate(const std::vector<::Envoy::Upstream::UpstreamLocalAddress>& upstre if (upstream_local_addresses[0].address_->ip()->version() == upstream_local_addresses[1].address_->ip()->version()) { - throw EnvoyException(fmt::format( + return absl::InvalidArgumentError(fmt::format( "{}'s upstream binding config has two same IP version source addresses. Only two " "different IP version source addresses can be supported in BindConfig's source_address " "and extra_source_addresses/additional_source_addresses fields", @@ -50,6 +51,7 @@ void validate(const std::vector<::Envoy::Upstream::UpstreamLocalAddress>& upstre : fmt::format("Cluster {}", cluster_name.value()))); } } + return absl::OkStatus(); } } // namespace @@ -58,11 +60,14 @@ std::string DefaultUpstreamLocalAddressSelectorFactory::name() const { return std::string(kDefaultLocalAddressSelectorName); } -UpstreamLocalAddressSelectorConstSharedPtr +absl::StatusOr DefaultUpstreamLocalAddressSelectorFactory::createLocalAddressSelector( std::vector<::Envoy::Upstream::UpstreamLocalAddress> upstream_local_addresses, absl::optional cluster_name) const { - validate(upstream_local_addresses, cluster_name); + absl::Status status = validate(upstream_local_addresses, cluster_name); + if (!status.ok()) { + return status; + } return std::make_shared(std::move(upstream_local_addresses)); } diff --git a/source/common/upstream/default_local_address_selector_factory.h b/source/common/upstream/default_local_address_selector_factory.h index 7f8baa0fa3d6..6dfa875fc430 100644 --- a/source/common/upstream/default_local_address_selector_factory.h +++ b/source/common/upstream/default_local_address_selector_factory.h @@ -18,7 +18,7 @@ class DefaultUpstreamLocalAddressSelectorFactory : public UpstreamLocalAddressSe public: std::string name() const override; - UpstreamLocalAddressSelectorConstSharedPtr createLocalAddressSelector( + absl::StatusOr createLocalAddressSelector( std::vector<::Envoy::Upstream::UpstreamLocalAddress> upstream_local_addresses, absl::optional cluster_name) const override; diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 659a701b18b7..b1c659a5529f 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -37,7 +37,7 @@ HdsDelegate::HdsDelegate(Server::Configuration::ServerFactoryContext& server_con async_client_(std::move(async_client)), dispatcher_(server_context.mainThreadDispatcher()), server_context_(server_context), store_stats_(stats), ssl_context_manager_(ssl_context_manager), info_factory_(info_factory), - tls_(server_context_.threadLocal()), specifier_hash_(0) { + tls_(server_context_.threadLocal()) { health_check_request_.mutable_health_check_request()->mutable_node()->MergeFrom( server_context.localInfo().node()); backoff_strategy_ = std::make_unique( diff --git a/source/common/upstream/health_discovery_service.h b/source/common/upstream/health_discovery_service.h index 7880649b0319..2cd21b06da79 100644 --- a/source/common/upstream/health_discovery_service.h +++ b/source/common/upstream/health_discovery_service.h @@ -72,6 +72,8 @@ class HdsCluster : public Cluster, Logger::Loggable { std::vector healthCheckers() { return health_checkers_; }; std::vector hosts() { return *hosts_; }; + UnitFloat dropOverload() const override { return UnitFloat(0); } + void setDropOverload(UnitFloat) override {} protected: PrioritySetImpl priority_set_; @@ -182,7 +184,7 @@ class HdsDelegate : Grpc::AsyncStreamCallbacks clusters_; std::vector hds_clusters_; diff --git a/source/common/upstream/host_utility.cc b/source/common/upstream/host_utility.cc index 5efe759dfb36..341b0674d815 100644 --- a/source/common/upstream/host_utility.cc +++ b/source/common/upstream/host_utility.cc @@ -2,6 +2,7 @@ #include +#include "source/common/config/well_known_names.h" #include "source/common/runtime/runtime_features.h" namespace Envoy { @@ -97,38 +98,6 @@ HostUtility::HostStatusSet HostUtility::createOverrideHostStatus( const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) { HostStatusSet override_host_status; - if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.validate_detailed_override_host_statuses")) { - // Old code path that should be removed once the runtime flag is removed directly. - // Coarse health status is used here. - - if (!common_config.has_override_host_status()) { - // No override host status and 'Healthy' and 'Degraded' will be applied by default. - override_host_status.set(static_cast(Host::Health::Healthy)); - override_host_status.set(static_cast(Host::Health::Degraded)); - return override_host_status; - } - - for (auto single_status : common_config.override_host_status().statuses()) { - switch (static_cast(single_status)) { - PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; - case envoy::config::core::v3::HealthStatus::UNKNOWN: - case envoy::config::core::v3::HealthStatus::HEALTHY: - override_host_status.set(static_cast(Host::Health::Healthy)); - break; - case envoy::config::core::v3::HealthStatus::UNHEALTHY: - case envoy::config::core::v3::HealthStatus::DRAINING: - case envoy::config::core::v3::HealthStatus::TIMEOUT: - override_host_status.set(static_cast(Host::Health::Unhealthy)); - break; - case envoy::config::core::v3::HealthStatus::DEGRADED: - override_host_status.set(static_cast(Host::Health::Degraded)); - break; - } - } - return override_host_status; - } - if (!common_config.has_override_host_status()) { // No override host status and [UNKNOWN, HEALTHY, DEGRADED] will be applied by default. override_host_status.set(static_cast(envoy::config::core::v3::HealthStatus::UNKNOWN)); @@ -169,7 +138,7 @@ HostConstSharedPtr HostUtility::selectOverrideHost(const HostMap* host_map, Host return nullptr; } - auto host_iter = host_map->find(override_host.value()); + auto host_iter = host_map->find(override_host.value().first); // The override host cannot be found in the host map. if (host_iter == host_map->end()) { @@ -179,22 +148,91 @@ HostConstSharedPtr HostUtility::selectOverrideHost(const HostMap* host_map, Host HostConstSharedPtr host = host_iter->second; ASSERT(host != nullptr); - if (!Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.validate_detailed_override_host_statuses")) { - // Old code path that should be removed once the runtime flag is removed directly. - // Coarse health status is used here. - - if (status[static_cast(host->coarseHealth())]) { - return host; - } - return nullptr; - } - if (status[static_cast(host->healthStatus())]) { return host; } return nullptr; } +bool HostUtility::allowLBChooseHost(LoadBalancerContext* context) { + if (context == nullptr) { + return true; + } + + auto override_host = context->overrideHostToSelect(); + if (!override_host.has_value()) { + return true; + } + + // Return opposite value to "strict" setting. + return !override_host.value().second; +} + +void HostUtility::forEachHostMetric( + const ClusterManager& cluster_manager, + const std::function& counter_cb, + const std::function& gauge_cb) { + for (const auto& [unused_name, cluster_ref] : cluster_manager.clusters().active_clusters_) { + Upstream::ClusterInfoConstSharedPtr cluster_info = cluster_ref.get().info(); + if (cluster_info->perEndpointStatsEnabled()) { + const std::string cluster_name = + Stats::Utility::sanitizeStatsName(cluster_info->observabilityName()); + + const Stats::TagVector& fixed_tags = cluster_info->statsScope().store().fixedTags(); + + for (auto& host_set : cluster_ref.get().prioritySet().hostSetsPerPriority()) { + for (auto& host : host_set->hosts()) { + + Stats::TagVector tags; + tags.reserve(fixed_tags.size() + 3); + tags.insert(tags.end(), fixed_tags.begin(), fixed_tags.end()); + tags.emplace_back(Stats::Tag{Envoy::Config::TagNames::get().CLUSTER_NAME, cluster_name}); + tags.emplace_back(Stats::Tag{"envoy.endpoint_address", host->address()->asString()}); + + const auto& hostname = host->hostname(); + if (!hostname.empty()) { + tags.push_back({"envoy.endpoint_hostname", hostname}); + } + + auto set_metric_metadata = [&](absl::string_view metric_name, + Stats::PrimitiveMetricMetadata& metric) { + metric.setName( + absl::StrCat("cluster.", cluster_name, ".endpoint.", + Stats::Utility::sanitizeStatsName(host->address()->asStringView()), + ".", metric_name)); + metric.setTagExtractedName(absl::StrCat("cluster.endpoint.", metric_name)); + metric.setTags(tags); + + // Validate that all components were sanitized. + ASSERT(metric.name() == Stats::Utility::sanitizeStatsName(metric.name())); + ASSERT(metric.tagExtractedName() == + Stats::Utility::sanitizeStatsName(metric.tagExtractedName())); + }; + + for (auto& [metric_name, primitive] : host->counters()) { + Stats::PrimitiveCounterSnapshot metric(primitive.get()); + set_metric_metadata(metric_name, metric); + + counter_cb(std::move(metric)); + } + + auto gauges = host->gauges(); + + // Add synthetic "healthy" gauge. + Stats::PrimitiveGauge healthy_gauge; + healthy_gauge.set((host->coarseHealth() == Host::Health::Healthy) ? 1 : 0); + gauges.emplace_back(absl::string_view("healthy"), healthy_gauge); + + for (auto& [metric_name, primitive] : gauges) { + Stats::PrimitiveGaugeSnapshot metric(primitive.get()); + set_metric_metadata(metric_name, metric); + gauge_cb(std::move(metric)); + } + } + } + } + } +} + } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/host_utility.h b/source/common/upstream/host_utility.h index 637b24cff255..49c5749562bf 100644 --- a/source/common/upstream/host_utility.h +++ b/source/common/upstream/host_utility.h @@ -2,6 +2,7 @@ #include +#include "envoy/stats/primitive_stats.h" #include "envoy/upstream/load_balancer.h" #include "envoy/upstream/upstream.h" @@ -28,6 +29,14 @@ class HostUtility { // A utility function to select override host from host map according to load balancer context. static HostConstSharedPtr selectOverrideHost(const HostMap* host_map, HostStatusSet status, LoadBalancerContext* context); + + // Iterate over all per-endpoint metrics, for clusters with `per_endpoint_stats` enabled. + static void + forEachHostMetric(const ClusterManager& cluster_manager, + const std::function& counter_cb, + const std::function& gauge_cb); + + static bool allowLBChooseHost(LoadBalancerContext* context); }; } // namespace Upstream diff --git a/source/common/upstream/load_balancer_factory_base.h b/source/common/upstream/load_balancer_factory_base.h index ae9c066a550b..eb5cd70e63d7 100644 --- a/source/common/upstream/load_balancer_factory_base.h +++ b/source/common/upstream/load_balancer_factory_base.h @@ -5,19 +5,6 @@ namespace Envoy { namespace Upstream { -class LoadBalancerConfigWrapper : public LoadBalancerConfig { -public: - LoadBalancerConfigWrapper(ProtobufTypes::MessagePtr config) : config_(std::move(config)) {} - - template OptRef typedProtoConfig() const { - auto* typed_config = dynamic_cast(config_.get()); - return makeOptRefFromPtr(typed_config); - } - -private: - ProtobufTypes::MessagePtr config_; -}; - /** * Base class for cluster provided load balancers and load balancers specified by load balancing * policy config. This class should be extended directly if the load balancing policy specifies a @@ -35,11 +22,6 @@ template class TypedLoadBalancerFactoryBase : public TypedLoadBala return ProtobufTypes::MessagePtr{new Proto()}; } - LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config, - ProtobufMessage::ValidationVisitor&) override { - return std::make_unique(std::move(config)); - } - protected: TypedLoadBalancerFactoryBase(const std::string& name) : name_(name) {} diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 72a76e7df0d8..c85565bfc6fc 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -232,8 +232,10 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // all hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which // results in 100%. If 80% of hosts are healthy, that priority's health is still 100% // (80%*1.4=112% and capped at 100%). - per_priority_health.get()[priority] = std::min( - 100, (host_set.overprovisioningFactor() * healthy_weight / total_weight)); + per_priority_health.get()[priority] = + std::min(100, + // NOLINTNEXTLINE(clang-analyzer-core.DivideZero) + (host_set.overprovisioningFactor() * healthy_weight / total_weight)); // We perform the same computation for degraded hosts. per_priority_degraded.get()[priority] = std::min( @@ -1344,47 +1346,5 @@ HostConstSharedPtr RandomLoadBalancer::peekOrChoose(LoadBalancerContext* context return hosts_to_use[random_hash % hosts_to_use.size()]; } -SubsetSelectorImpl::SubsetSelectorImpl( - const Protobuf::RepeatedPtrField& selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy, - const Protobuf::RepeatedPtrField& fallback_keys_subset, - bool single_host_per_subset) - : selector_keys_(selector_keys.begin(), selector_keys.end()), - fallback_keys_subset_(fallback_keys_subset.begin(), fallback_keys_subset.end()), - fallback_policy_(fallback_policy), single_host_per_subset_(single_host_per_subset) { - - if (fallback_policy_ != - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET) { - // defining fallback_keys_subset_ for a fallback policy other than KEYS_SUBSET doesn't have - // any effect and it is probably a user mistake. We should let the user know about it. - if (!fallback_keys_subset_.empty()) { - throw EnvoyException("fallback_keys_subset can be set only for KEYS_SUBSET fallback_policy"); - } - return; - } - - // if KEYS_SUBSET fallback policy is selected, fallback_keys_subset must not be empty, because - // it would be the same as not defining fallback policy at all (global fallback policy would be - // used) - if (fallback_keys_subset_.empty()) { - throw EnvoyException("fallback_keys_subset cannot be empty"); - } - - // We allow only for a fallback to a subset of the selector keys because this is probably the - // only use case that makes sense (fallback from more specific selector to less specific - // selector). Potentially we can relax this constraint in the future if there will be a use case - // for this. - if (!std::includes(selector_keys_.begin(), selector_keys_.end(), fallback_keys_subset_.begin(), - fallback_keys_subset_.end())) { - throw EnvoyException("fallback_keys_subset must be a subset of selector keys"); - } - - // Enforce that the fallback_keys_subset_ set is smaller than the selector_keys_ set. Otherwise - // we could end up with a infinite recursion of SubsetLoadBalancer::chooseHost(). - if (selector_keys_.size() == fallback_keys_subset_.size()) { - throw EnvoyException("fallback_keys_subset cannot be equal to keys"); - } -} } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 8338da065da4..614541057798 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -11,13 +11,9 @@ #include "envoy/common/callback.h" #include "envoy/common/random_generator.h" #include "envoy/config/cluster/v3/cluster.pb.h" -#include "envoy/extensions/load_balancing_policies/common/v3/common.pb.h" #include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.h" -#include "envoy/extensions/load_balancing_policies/least_request/v3/least_request.pb.validate.h" #include "envoy/extensions/load_balancing_policies/random/v3/random.pb.h" -#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.validate.h" #include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" -#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h" #include "envoy/runtime/runtime.h" #include "envoy/stream_info/stream_info.h" #include "envoy/upstream/load_balancer.h" @@ -26,6 +22,7 @@ #include "source/common/protobuf/utility.h" #include "source/common/runtime/runtime_protos.h" #include "source/common/upstream/edf_scheduler.h" +#include "source/common/upstream/subset_lb_config.h" namespace Envoy { namespace Upstream { @@ -782,97 +779,5 @@ class RandomLoadBalancer : public ZoneAwareLoadBalancerBase { HostConstSharedPtr peekOrChoose(LoadBalancerContext* context, bool peek); }; -/** - * Implementation of SubsetSelector - */ -class SubsetSelectorImpl : public SubsetSelector { -public: - SubsetSelectorImpl(const Protobuf::RepeatedPtrField& selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy, - const Protobuf::RepeatedPtrField& fallback_keys_subset, - bool single_host_per_subset); - - // SubsetSelector - const std::set& selectorKeys() const override { return selector_keys_; } - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy - fallbackPolicy() const override { - return fallback_policy_; - } - const std::set& fallbackKeysSubset() const override { return fallback_keys_subset_; } - bool singleHostPerSubset() const override { return single_host_per_subset_; } - -private: - const std::set selector_keys_; - const std::set fallback_keys_subset_; - // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. - const envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy_; - const bool single_host_per_subset_ : 1; -}; - -/** - * Implementation of LoadBalancerSubsetInfo. - */ -class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { -public: - LoadBalancerSubsetInfoImpl( - const envoy::config::cluster::v3::Cluster::LbSubsetConfig& subset_config) - : default_subset_(subset_config.default_subset()), - fallback_policy_(subset_config.fallback_policy()), - metadata_fallback_policy_(subset_config.metadata_fallback_policy()), - enabled_(!subset_config.subset_selectors().empty()), - locality_weight_aware_(subset_config.locality_weight_aware()), - scale_locality_weight_(subset_config.scale_locality_weight()), - panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()) { - for (const auto& subset : subset_config.subset_selectors()) { - if (!subset.keys().empty()) { - subset_selectors_.emplace_back(std::make_shared( - subset.keys(), subset.fallback_policy(), subset.fallback_keys_subset(), - subset.single_host_per_subset())); - } - } - } - LoadBalancerSubsetInfoImpl() - : LoadBalancerSubsetInfoImpl( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance()) {} - - // Upstream::LoadBalancerSubsetInfo - bool isEnabled() const override { return enabled_; } - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy - fallbackPolicy() const override { - return fallback_policy_; - } - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy - metadataFallbackPolicy() const override { - return metadata_fallback_policy_; - } - const ProtobufWkt::Struct& defaultSubset() const override { return default_subset_; } - const std::vector& subsetSelectors() const override { - return subset_selectors_; - } - bool localityWeightAware() const override { return locality_weight_aware_; } - bool scaleLocalityWeight() const override { return scale_locality_weight_; } - bool panicModeAny() const override { return panic_mode_any_; } - bool listAsAny() const override { return list_as_any_; } - bool allowRedundantKeys() const override { return false; } - -private: - const ProtobufWkt::Struct default_subset_; - std::vector subset_selectors_; - // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. - const envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy - fallback_policy_; - const envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy - metadata_fallback_policy_; - const bool enabled_ : 1; - const bool locality_weight_aware_ : 1; - const bool scale_locality_weight_ : 1; - const bool panic_mode_any_ : 1; - const bool list_as_any_ : 1; -}; -using DefaultLoadBalancerSubsetInfoImpl = ConstSingleton; - } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/load_stats_reporter.cc b/source/common/upstream/load_stats_reporter.cc index 386f79b6be99..a06d85c40ded 100644 --- a/source/common/upstream/load_stats_reporter.cc +++ b/source/common/upstream/load_stats_reporter.cc @@ -123,6 +123,14 @@ void LoadStatsReporter::sendLoadStatsRequest() { } cluster_stats->set_total_dropped_requests( cluster.info()->loadReportStats().upstream_rq_dropped_.latch()); + const uint64_t drop_overload_count = + cluster.info()->loadReportStats().upstream_rq_drop_overload_.latch(); + if (drop_overload_count > 0) { + auto* dropped_request = cluster_stats->add_dropped_requests(); + dropped_request->set_category("drop_overload"); + dropped_request->set_dropped_count(drop_overload_count); + } + const auto now = time_source_.monotonicTime().time_since_epoch(); const auto measured_interval = now - cluster_name_and_timestamp.second; cluster_stats->mutable_load_report_interval()->MergeFrom( @@ -214,6 +222,7 @@ void LoadStatsReporter::startLoadReportPeriod() { } } cluster.info()->loadReportStats().upstream_rq_dropped_.latch(); + cluster.info()->loadReportStats().upstream_rq_drop_overload_.latch(); }; if (message_->send_all_clusters()) { for (const auto& p : all_clusters.active_clusters_) { diff --git a/source/common/upstream/od_cds_api_impl.cc b/source/common/upstream/od_cds_api_impl.cc index c976b1000f5e..a9ec3c349693 100644 --- a/source/common/upstream/od_cds_api_impl.cc +++ b/source/common/upstream/od_cds_api_impl.cc @@ -25,7 +25,7 @@ OdCdsApiImpl::OdCdsApiImpl(const envoy::config::core::v3::ConfigSource& odcds_co : Envoy::Config::SubscriptionBase(validation_visitor, "name"), helper_(cm, "odcds"), cm_(cm), notifier_(notifier), - scope_(scope.createScope("cluster_manager.odcds.")), status_(StartStatus::NotStarted) { + scope_(scope.createScope("cluster_manager.odcds.")) { // TODO(krnowak): Move the subscription setup to CdsApiHelper. Maybe make CdsApiHelper a base // class for CDS and ODCDS. const auto resource_name = getResourceName(); @@ -63,7 +63,7 @@ OdCdsApiImpl::onConfigUpdate(const std::vector& adde notifier_.notifyMissingCluster(resource_name); } if (!exception_msgs.empty()) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("Error adding/updating cluster(s) {}", absl::StrJoin(exception_msgs, ", "))); } return absl::OkStatus(); diff --git a/source/common/upstream/od_cds_api_impl.h b/source/common/upstream/od_cds_api_impl.h index 11686be76465..87c4d8c334f4 100644 --- a/source/common/upstream/od_cds_api_impl.h +++ b/source/common/upstream/od_cds_api_impl.h @@ -88,7 +88,7 @@ class OdCdsApiImpl : public OdCdsApi, ClusterManager& cm_; MissingClusterNotifier& notifier_; Stats::ScopeSharedPtr scope_; - StartStatus status_; + StartStatus status_{StartStatus::NotStarted}; absl::flat_hash_set awaiting_names_; Config::SubscriptionPtr subscription_; }; diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc index 6377abbeaf68..79a9e304e699 100644 --- a/source/common/upstream/outlier_detection_impl.cc +++ b/source/common/upstream/outlier_detection_impl.cc @@ -23,7 +23,7 @@ namespace Envoy { namespace Upstream { namespace Outlier { -DetectorSharedPtr DetectorImplFactory::createForCluster( +absl::StatusOr DetectorImplFactory::createForCluster( Cluster& cluster, const envoy::config::cluster::v3::Cluster& cluster_config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, EventLoggerSharedPtr event_logger, Random::RandomGenerator& random) { @@ -284,7 +284,7 @@ DetectorImpl::~DetectorImpl() { } } -std::shared_ptr +absl::StatusOr> DetectorImpl::create(Cluster& cluster, const envoy::config::cluster::v3::OutlierDetection& config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, TimeSource& time_source, EventLoggerSharedPtr event_logger, @@ -293,7 +293,7 @@ DetectorImpl::create(Cluster& cluster, const envoy::config::cluster::v3::Outlier new DetectorImpl(cluster, config, dispatcher, runtime, time_source, event_logger, random)); if (detector->config().maxEjectionTimeMs() < detector->config().baseEjectionTimeMs()) { - throw EnvoyException( + return absl::InvalidArgumentError( "outlier detector's max_ejection_time cannot be smaller than base_ejection_time"); } detector->initialize(cluster); @@ -354,10 +354,6 @@ void DetectorImpl::armIntervalTimer() { void DetectorImpl::checkHostForUneject(HostSharedPtr host, DetectorHostMonitorImpl* monitor, MonotonicTime now) { if (!host->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)) { - // Node seems to be healthy and was not ejected since the last check. - if (monitor->ejectTimeBackoff() != 0) { - monitor->ejectTimeBackoff()--; - } return; } @@ -760,6 +756,22 @@ void DetectorImpl::onIntervalTimer() { processSuccessRateEjections(DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin); processSuccessRateEjections(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin); + // Decrement time backoff for all hosts which have not been ejected. + for (auto host : host_monitors_) { + if (!host.first->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)) { + auto& monitor = host.second; + // Node is healthy and was not ejected since the last check. + if (monitor->lastUnejectionTime().has_value() && + ((now - monitor->lastUnejectionTime().value()) >= + std::chrono::milliseconds( + runtime_.snapshot().getInteger(IntervalMsRuntime, config_.intervalMs())))) { + if (monitor->ejectTimeBackoff() != 0) { + monitor->ejectTimeBackoff()--; + } + } + } + } + armIntervalTimer(); } diff --git a/source/common/upstream/outlier_detection_impl.h b/source/common/upstream/outlier_detection_impl.h index 4044cd0d1ecf..1a70d3c75129 100644 --- a/source/common/upstream/outlier_detection_impl.h +++ b/source/common/upstream/outlier_detection_impl.h @@ -34,7 +34,7 @@ namespace Outlier { */ class DetectorImplFactory { public: - static DetectorSharedPtr + static absl::StatusOr createForCluster(Cluster& cluster, const envoy::config::cluster::v3::Cluster& cluster_config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, EventLoggerSharedPtr event_logger, Random::RandomGenerator& random); @@ -372,7 +372,7 @@ class DetectorConfig { */ class DetectorImpl : public Detector, public std::enable_shared_from_this { public: - static std::shared_ptr + static absl::StatusOr> create(Cluster& cluster, const envoy::config::cluster::v3::OutlierDetection& config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, TimeSource& time_source, EventLoggerSharedPtr event_logger, Random::RandomGenerator& random); diff --git a/source/common/upstream/subset_lb_config.cc b/source/common/upstream/subset_lb_config.cc new file mode 100644 index 000000000000..ef3cf228ebd7 --- /dev/null +++ b/source/common/upstream/subset_lb_config.cc @@ -0,0 +1,53 @@ +#include "source/common/upstream/subset_lb_config.h" + +#include "source/common/config/utility.h" + +namespace Envoy { +namespace Upstream { + +SubsetSelectorImpl::SubsetSelectorImpl( + const Protobuf::RepeatedPtrField& selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy fallback_policy, + const Protobuf::RepeatedPtrField& fallback_keys_subset, + bool single_host_per_subset) + : selector_keys_(selector_keys.begin(), selector_keys.end()), + fallback_keys_subset_(fallback_keys_subset.begin(), fallback_keys_subset.end()), + fallback_policy_(fallback_policy), single_host_per_subset_(single_host_per_subset) { + + if (fallback_policy_ != + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET) { + // defining fallback_keys_subset_ for a fallback policy other than KEYS_SUBSET doesn't have + // any effect and it is probably a user mistake. We should let the user know about it. + if (!fallback_keys_subset_.empty()) { + throwEnvoyExceptionOrPanic( + "fallback_keys_subset can be set only for KEYS_SUBSET fallback_policy"); + } + return; + } + + // if KEYS_SUBSET fallback policy is selected, fallback_keys_subset must not be empty, because + // it would be the same as not defining fallback policy at all (global fallback policy would be + // used) + if (fallback_keys_subset_.empty()) { + throwEnvoyExceptionOrPanic("fallback_keys_subset cannot be empty"); + } + + // We allow only for a fallback to a subset of the selector keys because this is probably the + // only use case that makes sense (fallback from more specific selector to less specific + // selector). Potentially we can relax this constraint in the future if there will be a use case + // for this. + if (!std::includes(selector_keys_.begin(), selector_keys_.end(), fallback_keys_subset_.begin(), + fallback_keys_subset_.end())) { + throwEnvoyExceptionOrPanic("fallback_keys_subset must be a subset of selector keys"); + } + + // Enforce that the fallback_keys_subset_ set is smaller than the selector_keys_ set. Otherwise + // we could end up with a infinite recursion of SubsetLoadBalancer::chooseHost(). + if (selector_keys_.size() == fallback_keys_subset_.size()) { + throwEnvoyExceptionOrPanic("fallback_keys_subset cannot be equal to keys"); + } +} + +} // namespace Upstream +} // namespace Envoy diff --git a/source/common/upstream/subset_lb_config.h b/source/common/upstream/subset_lb_config.h new file mode 100644 index 000000000000..58b97111ad4f --- /dev/null +++ b/source/common/upstream/subset_lb_config.h @@ -0,0 +1,141 @@ +#pragma once + +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/load_balancing_policies/common/v3/common.pb.h" +#include "envoy/extensions/load_balancing_policies/common/v3/common.pb.validate.h" +#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.h" +#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.validate.h" +#include "envoy/upstream/load_balancer.h" + +namespace Envoy { +namespace Upstream { + +/** + * Implementation of SubsetSelector. This is part of subset load balancer config and is used to + * store config of single selector. + */ +class SubsetSelectorImpl : public SubsetSelector { +public: + SubsetSelectorImpl(const Protobuf::RepeatedPtrField& selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy fallback_policy, + const Protobuf::RepeatedPtrField& fallback_keys_subset, + bool single_host_per_subset); + + // SubsetSelector + const std::set& selectorKeys() const override { return selector_keys_; } + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy + fallbackPolicy() const override { + return fallback_policy_; + } + const std::set& fallbackKeysSubset() const override { return fallback_keys_subset_; } + bool singleHostPerSubset() const override { return single_host_per_subset_; } + +private: + const std::set selector_keys_; + const std::set fallback_keys_subset_; + // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. + const envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy fallback_policy_; + const bool single_host_per_subset_ : 1; +}; + +using SubsetLoadbalancingPolicyProto = + envoy::extensions::load_balancing_policies::subset::v3::Subset; +using LegacySubsetLoadbalancingPolicyProto = envoy::config::cluster::v3::Cluster::LbSubsetConfig; + +/** + * Implementation of LoadBalancerSubsetInfo. Both the legacy and extension subset proto configs + * are converted to this class. + */ +class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { +public: + using FallbackPolicy = + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy; + using MetadataFallbackPolicy = + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy; + using SubsetFallbackPolicy = envoy::config::cluster::v3::Cluster::LbSubsetConfig:: + LbSubsetSelector::LbSubsetSelectorFallbackPolicy; + + LoadBalancerSubsetInfoImpl(const SubsetLoadbalancingPolicyProto& subset_config) + : default_subset_(subset_config.default_subset()), + fallback_policy_(static_cast(subset_config.fallback_policy())), + metadata_fallback_policy_( + static_cast(subset_config.metadata_fallback_policy())), + enabled_(!subset_config.subset_selectors().empty()), + locality_weight_aware_(subset_config.locality_weight_aware()), + scale_locality_weight_(subset_config.scale_locality_weight()), + panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()), + allow_redundant_keys_(subset_config.allow_redundant_keys()) { + for (const auto& subset : subset_config.subset_selectors()) { + if (!subset.keys().empty()) { + subset_selectors_.emplace_back(std::make_shared( + subset.keys(), static_cast(subset.fallback_policy()), + subset.fallback_keys_subset(), subset.single_host_per_subset())); + } + } + + if (allow_redundant_keys_) { + // Sort subset selectors by number of keys, descending. This will ensure that the longest + // matching subset selector will be at the beginning of the list. + std::stable_sort(subset_selectors_.begin(), subset_selectors_.end(), + [](const SubsetSelectorPtr& a, const SubsetSelectorPtr& b) -> bool { + return a->selectorKeys().size() > b->selectorKeys().size(); + }); + } + } + + LoadBalancerSubsetInfoImpl(const LegacySubsetLoadbalancingPolicyProto& subset_config) + : default_subset_(subset_config.default_subset()), + fallback_policy_(subset_config.fallback_policy()), + metadata_fallback_policy_(subset_config.metadata_fallback_policy()), + enabled_(!subset_config.subset_selectors().empty()), + locality_weight_aware_(subset_config.locality_weight_aware()), + scale_locality_weight_(subset_config.scale_locality_weight()), + panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()) { + for (const auto& subset : subset_config.subset_selectors()) { + if (!subset.keys().empty()) { + subset_selectors_.emplace_back(std::make_shared( + subset.keys(), subset.fallback_policy(), subset.fallback_keys_subset(), + subset.single_host_per_subset())); + } + } + } + LoadBalancerSubsetInfoImpl() + : LoadBalancerSubsetInfoImpl( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance()) {} + + // Upstream::LoadBalancerSubsetInfo + bool isEnabled() const override { return enabled_; } + FallbackPolicy fallbackPolicy() const override { return fallback_policy_; } + MetadataFallbackPolicy metadataFallbackPolicy() const override { + return metadata_fallback_policy_; + } + const ProtobufWkt::Struct& defaultSubset() const override { return default_subset_; } + const std::vector& subsetSelectors() const override { + return subset_selectors_; + } + bool localityWeightAware() const override { return locality_weight_aware_; } + bool scaleLocalityWeight() const override { return scale_locality_weight_; } + bool panicModeAny() const override { return panic_mode_any_; } + bool listAsAny() const override { return list_as_any_; } + bool allowRedundantKeys() const override { return allow_redundant_keys_; } + +private: + const ProtobufWkt::Struct default_subset_; + std::vector subset_selectors_; + // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. + const FallbackPolicy fallback_policy_; + const MetadataFallbackPolicy metadata_fallback_policy_; + const bool enabled_ : 1; + const bool locality_weight_aware_ : 1; + const bool scale_locality_weight_ : 1; + const bool panic_mode_any_ : 1; + const bool list_as_any_ : 1; + const bool allow_redundant_keys_{}; +}; +using DefaultLoadBalancerSubsetInfoImpl = ConstSingleton; + +} // namespace Upstream +} // namespace Envoy diff --git a/source/common/upstream/thread_aware_lb_impl.cc b/source/common/upstream/thread_aware_lb_impl.cc index a787c466a533..7c16473d9c1c 100644 --- a/source/common/upstream/thread_aware_lb_impl.cc +++ b/source/common/upstream/thread_aware_lb_impl.cc @@ -19,7 +19,7 @@ void normalizeHostWeights(const HostVector& hosts, double normalized_locality_we for (const auto& host : hosts) { sum += host->weight(); if (sum > std::numeric_limits::max()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("The sum of weights of all upstream hosts in a locality exceeds {}", std::numeric_limits::max())); } @@ -45,7 +45,7 @@ void normalizeLocalityWeights(const HostsPerLocality& hosts_per_locality, for (const auto weight : locality_weights) { sum += weight; if (sum > std::numeric_limits::max()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("The sum of weights of all localities at the same priority exceeds {}", std::numeric_limits::max())); } diff --git a/source/common/upstream/upstream_factory_context_impl.h b/source/common/upstream/upstream_factory_context_impl.h index 6ad4b7e4d564..5b51672f992a 100644 --- a/source/common/upstream/upstream_factory_context_impl.h +++ b/source/common/upstream/upstream_factory_context_impl.h @@ -9,7 +9,7 @@ namespace Upstream { /* * Upstream Factory Context used by both Clusters and Routers to configure - * upstream filters. + * upstream HTTP filters. */ class UpstreamFactoryContextImpl : public Server::Configuration::UpstreamFactoryContext { public: @@ -17,7 +17,7 @@ class UpstreamFactoryContextImpl : public Server::Configuration::UpstreamFactory Init::Manager& init_manager, Stats::Scope& scope) : server_context_(context), init_manager_(init_manager), scope_(scope) {} - Server::Configuration::ServerFactoryContext& getServerFactoryContext() const override { + Server::Configuration::ServerFactoryContext& serverFactoryContext() const override { return server_context_; } diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a21ae634b57f..aa4e8fc849f9 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -97,15 +97,16 @@ createProtocolOptionsConfig(const std::string& name, const ProtobufWkt::Any& typ } if (factory == nullptr) { - throw EnvoyException(fmt::format("Didn't find a registered network or http filter or protocol " - "options implementation for name: '{}'", - name)); + throwEnvoyExceptionOrPanic( + fmt::format("Didn't find a registered network or http filter or protocol " + "options implementation for name: '{}'", + name)); } ProtobufTypes::MessagePtr proto_config = factory->createEmptyProtocolOptionsProto(); if (proto_config == nullptr) { - throw EnvoyException(fmt::format("filter {} does not support protocol options", name)); + throwEnvoyExceptionOrPanic(fmt::format("filter {} does not support protocol options", name)); } Envoy::Config::Utility::translateOpaqueConfig( @@ -287,10 +288,10 @@ parseBindConfig(::Envoy::OptRef bind_ if (upstream_local_addresses.size() > 1) { for (auto const& upstream_local_address : upstream_local_addresses) { if (upstream_local_address.address_ == nullptr) { - throw EnvoyException(fmt::format("{}'s upstream binding config has invalid IP addresses.", - !(cluster_name.has_value()) - ? "Bootstrap" - : fmt::format("Cluster {}", cluster_name.value()))); + throwEnvoyExceptionOrPanic(fmt::format( + "{}'s upstream binding config has invalid IP addresses.", + !(cluster_name.has_value()) ? "Bootstrap" + : fmt::format("Cluster {}", cluster_name.value()))); } } } @@ -317,7 +318,7 @@ Envoy::Upstream::UpstreamLocalAddressSelectorConstSharedPtr createUpstreamLocalA if (bind_config.has_value()) { if (bind_config->additional_source_addresses_size() > 0 && bind_config->extra_source_addresses_size() > 0) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "Can't specify both `extra_source_addresses` and `additional_source_addresses` " "in the {}'s upstream binding config", !(cluster_name.has_value()) ? "Bootstrap" @@ -327,7 +328,7 @@ Envoy::Upstream::UpstreamLocalAddressSelectorConstSharedPtr createUpstreamLocalA if (!bind_config->has_source_address() && (bind_config->extra_source_addresses_size() > 0 || bind_config->additional_source_addresses_size() > 0)) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "{}'s upstream binding config has extra/additional source addresses but no " "source_address. Extra/additional addresses cannot be specified if " "source_address is not set.", @@ -353,7 +354,7 @@ Envoy::Upstream::UpstreamLocalAddressSelectorConstSharedPtr createUpstreamLocalA Config::Utility::getAndCheckFactory(typed_extension, false); } - return local_address_selector_factory->createLocalAddressSelector( + auto selector_or_error = local_address_selector_factory->createLocalAddressSelector( parseBindConfig( bind_config, cluster_name, buildBaseSocketOptions(cluster_config, bootstrap_bind_config.value_or( @@ -361,6 +362,8 @@ Envoy::Upstream::UpstreamLocalAddressSelectorConstSharedPtr createUpstreamLocalA buildClusterSocketOptions(cluster_config, bootstrap_bind_config.value_or( envoy::config::core::v3::BindConfig{}))), cluster_name); + THROW_IF_STATUS_NOT_OK(selector_or_error, throw); + return selector_or_error.value(); } } // namespace @@ -411,7 +414,7 @@ HostDescriptionImpl::HostDescriptionImpl( if (health_check_config.port_value() != 0 && dest_address->type() != Network::Address::Type::Ip) { // Setting the health check port to non-0 only works for IP-type addresses. Setting the port // for a pipe address is a misconfiguration. Throw an exception. - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Invalid host configuration: non-zero port for non-IP address")); } health_check_address_ = resolveHealthCheckAddress(health_check_config, dest_address); @@ -863,74 +866,6 @@ ClusterInfoImpl::generateTimeoutBudgetStats(Stats::Scope& scope, return {stat_names, scope}; } -// Implements the FactoryContext interface required by network filters. -class FactoryContextImpl : public Server::Configuration::CommonFactoryContext { -public: - // Create from a TransportSocketFactoryContext using parent stats_scope and runtime - // other contexts taken from TransportSocketFactoryContext. - FactoryContextImpl(Stats::Scope& stats_scope, Envoy::Runtime::Loader& runtime, - Server::Configuration::TransportSocketFactoryContext& c) - : admin_(c.serverFactoryContext().admin()), - server_scope_(c.serverFactoryContext().serverScope()), stats_scope_(stats_scope), - cluster_manager_(c.clusterManager()), local_info_(c.serverFactoryContext().localInfo()), - dispatcher_(c.serverFactoryContext().mainThreadDispatcher()), runtime_(runtime), - singleton_manager_(c.serverFactoryContext().singletonManager()), - tls_(c.serverFactoryContext().threadLocal()), api_(c.serverFactoryContext().api()), - options_(c.serverFactoryContext().options()), - message_validation_visitor_(c.messageValidationVisitor()) {} - - Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } - Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; } - const Server::Options& options() override { return options_; } - const LocalInfo::LocalInfo& localInfo() const override { return local_info_; } - Envoy::Runtime::Loader& runtime() override { return runtime_; } - Stats::Scope& scope() override { return stats_scope_; } - Stats::Scope& serverScope() override { return server_scope_; } - Singleton::Manager& singletonManager() override { return singleton_manager_; } - ThreadLocal::SlotAllocator& threadLocal() override { return tls_; } - OptRef admin() override { return admin_; } - TimeSource& timeSource() override { return api().timeSource(); } - ProtobufMessage::ValidationContext& messageValidationContext() override { - // TODO(davinci26): Needs an implementation for this context. Currently not used. - PANIC("unimplemented"); - } - - AccessLog::AccessLogManager& accessLogManager() override { - // TODO(davinci26): Needs an implementation for this context. Currently not used. - PANIC("unimplemented"); - } - - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return message_validation_visitor_; - } - - Server::ServerLifecycleNotifier& lifecycleNotifier() override { - // TODO(davinci26): Needs an implementation for this context. Currently not used. - PANIC("unimplemented"); - } - - Init::Manager& initManager() override { - // TODO(davinci26): Needs an implementation for this context. Currently not used. - PANIC("unimplemented"); - } - - Api::Api& api() override { return api_; } - -private: - OptRef admin_; - Stats::Scope& server_scope_; - Stats::Scope& stats_scope_; - Upstream::ClusterManager& cluster_manager_; - const LocalInfo::LocalInfo& local_info_; - Event::Dispatcher& dispatcher_; - Envoy::Runtime::Loader& runtime_; - Singleton::Manager& singleton_manager_; - ThreadLocal::SlotAllocator& tls_; - Api::Api& api_; - const Server::Options& options_; - ProtobufMessage::ValidationVisitor& message_validation_visitor_; -}; - std::shared_ptr createOptions(const envoy::config::cluster::v3::Cluster& config, std::shared_ptr&& options, @@ -942,8 +877,9 @@ createOptions(const envoy::config::cluster::v3::Cluster& config, if (config.protocol_selection() == envoy::config::cluster::v3::Cluster::USE_CONFIGURED_PROTOCOL) { // Make sure multiple protocol configurations are not present if (config.has_http_protocol_options() && config.has_http2_protocol_options()) { - throw EnvoyException(fmt::format("cluster: Both HTTP1 and HTTP2 options may only be " - "configured with non-default 'protocol_selection' values")); + throwEnvoyExceptionOrPanic( + fmt::format("cluster: Both HTTP1 and HTTP2 options may only be " + "configured with non-default 'protocol_selection' values")); } } @@ -958,6 +894,75 @@ createOptions(const envoy::config::cluster::v3::Cluster& config, config.has_http2_protocol_options(), validation_visitor); } +absl::StatusOr +LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProtoWithoutSubset( + const ClusterProto& cluster, ProtobufMessage::ValidationVisitor& visitor) { + + LoadBalancerConfigPtr lb_config; + TypedLoadBalancerFactory* lb_factory = nullptr; + + switch (cluster.lb_policy()) { + PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; + case ClusterProto::ROUND_ROBIN: + lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.round_robin"); + break; + case ClusterProto::LEAST_REQUEST: + lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.least_request"); + break; + case ClusterProto::RANDOM: + lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.random"); + break; + case ClusterProto::RING_HASH: + lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.ring_hash"); + break; + case ClusterProto::MAGLEV: + lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.maglev"); + break; + case ClusterProto::CLUSTER_PROVIDED: + lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.cluster_provided"); + break; + case ClusterProto::LOAD_BALANCING_POLICY_CONFIG: + // 'LOAD_BALANCING_POLICY_CONFIG' should be handled by the 'configureLbPolicies' + // function and should not reach here. + PANIC("getTypedLbConfigFromLegacyProtoWithoutSubset: should not reach here"); + break; + } + + if (lb_factory == nullptr) { + return absl::InvalidArgumentError( + fmt::format("No load balancer factory found for LB type: {}", + ClusterProto::LbPolicy_Name(cluster.lb_policy()))); + } + + ASSERT(lb_factory != nullptr); + return Result{lb_factory, lb_factory->loadConfig(cluster, visitor)}; +} + +absl::StatusOr +LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProto( + const ClusterProto& cluster, ProtobufMessage::ValidationVisitor& visitor) { + + // Handle the lb subset config case first. + // Note it is possible to have a lb_subset_config without actually having any subset selectors. + // In this case the subset load balancer should not be used. + if (cluster.has_lb_subset_config() && !cluster.lb_subset_config().subset_selectors().empty()) { + auto* lb_factory = Config::Utility::getFactoryByName( + "envoy.load_balancing_policies.subset"); + if (lb_factory != nullptr) { + return Result{lb_factory, lb_factory->loadConfig(cluster, visitor)}; + } + return absl::InvalidArgumentError("No subset load balancer factory found"); + } + + return getTypedLbConfigFromLegacyProtoWithoutSubset(cluster, visitor); +} + LBPolicyConfig::LBPolicyConfig(const envoy::config::cluster::v3::Cluster& config) { switch (config.lb_config_case()) { case envoy::config::cluster::v3::Cluster::kRoundRobinLbConfig: @@ -1066,8 +1071,6 @@ ClusterInfoImpl::ClusterInfoImpl( server_context)), network_filter_config_provider_manager_( createSingletonUpstreamNetworkFilterConfigProviderManager(server_context)), - factory_context_( - std::make_unique(*stats_scope_, runtime, factory_context)), upstream_context_(server_context, init_manager, *stats_scope_), per_connection_buffer_limit_bytes_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, per_connection_buffer_limit_bytes, 1024 * 1024)), @@ -1083,23 +1086,60 @@ ClusterInfoImpl::ClusterInfoImpl( common_lb_config_->ignore_new_hosts_until_first_hc()), set_local_interface_name_on_upstream_connections_( config.upstream_connection_options().set_local_interface_name_on_upstream_connections()), - added_via_api_(added_via_api), has_configured_http_filters_(false) { + added_via_api_(added_via_api), has_configured_http_filters_(false), + per_endpoint_stats_(config.has_track_cluster_stats() && + config.track_cluster_stats().per_endpoint_stats()) { #ifdef WIN32 if (set_local_interface_name_on_upstream_connections_) { - throw EnvoyException("set_local_interface_name_on_upstream_connections_ cannot be set to true " - "on Windows platforms"); + throwEnvoyExceptionOrPanic( + "set_local_interface_name_on_upstream_connections_ cannot be set to true " + "on Windows platforms"); } #endif + // Both LoadStatsReporter and per_endpoint_stats need to `latch()` the counters, so if both are + // configured they will interfere with each other and both get incorrect values. + if (perEndpointStatsEnabled() && + server_context.bootstrap().cluster_manager().has_load_stats_config()) { + throwEnvoyExceptionOrPanic("Only one of cluster per_endpoint_stats and cluster manager " + "load_stats_config can be specified"); + } + if (config.has_max_requests_per_connection() && http_protocol_options_->common_http_protocol_options_.has_max_requests_per_connection()) { - throw EnvoyException("Only one of max_requests_per_connection from Cluster or " - "HttpProtocolOptions can be specified"); + throwEnvoyExceptionOrPanic("Only one of max_requests_per_connection from Cluster or " + "HttpProtocolOptions can be specified"); } // If load_balancing_policy is set we will use it directly, ignoring lb_policy. - if (config.has_load_balancing_policy()) { + if (config.has_load_balancing_policy() || + config.lb_policy() == envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG) { configureLbPolicies(config, server_context); + } else if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.convert_legacy_lb_config")) { + auto lb_pair = LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProto( + config, server_context.messageValidationVisitor()); + + if (!lb_pair.ok()) { + throwEnvoyExceptionOrPanic(std::string(lb_pair.status().message())); + } + + load_balancer_config_ = std::move(lb_pair->config); + load_balancer_factory_ = lb_pair->factory; + lb_type_ = LoadBalancerType::LoadBalancingPolicyConfig; + + RELEASE_ASSERT( + load_balancer_factory_, + fmt::format( + "No load balancer factory found from legacy LB configuration (type: {}, subset: {}).", + envoy::config::cluster::v3::Cluster::LbPolicy_Name(config.lb_policy()), + config.has_lb_subset_config())); + + // Clear unnecessary legacy config because all legacy config is wrapped in load_balancer_config_ + // except the original_dst_lb_config. + lb_subset_ = nullptr; + if (config.lb_policy() != envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED) { + lb_policy_config_ = nullptr; + } } else { switch (config.lb_policy()) { PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; @@ -1120,7 +1160,7 @@ ClusterInfoImpl::ClusterInfoImpl( break; case envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED: if (config.has_lb_subset_config()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", envoy::config::cluster::v3::Cluster::LbPolicy_Name(config.lb_policy()))); } @@ -1128,7 +1168,9 @@ ClusterInfoImpl::ClusterInfoImpl( lb_type_ = LoadBalancerType::ClusterProvided; break; case envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG: { - configureLbPolicies(config, server_context); + // 'LOAD_BALANCING_POLICY_CONFIG' should be handled by the 'configureLbPolicies' + // function in previous branch and should not reach here. + PANIC("Should not reach here"); break; } } @@ -1136,9 +1178,9 @@ ClusterInfoImpl::ClusterInfoImpl( if (config.lb_subset_config().locality_weight_aware() && !config.common_lb_config().has_locality_weighted_lb_config()) { - throw EnvoyException(fmt::format("Locality weight aware subset LB requires that a " - "locality_weighted_lb_config be set in {}", - name_)); + throwEnvoyExceptionOrPanic(fmt::format("Locality weight aware subset LB requires that a " + "locality_weighted_lb_config be set in {}", + name_)); } // Use default (1h) or configured `idle_timeout`, unless it's set to 0, indicating that no @@ -1184,7 +1226,7 @@ ClusterInfoImpl::ClusterInfoImpl( if (config.has_eds_cluster_config()) { if (config.type() != envoy::config::cluster::v3::Cluster::EDS) { - throw EnvoyException("eds_cluster_config set in a non-EDS cluster"); + throwEnvoyExceptionOrPanic("eds_cluster_config set in a non-EDS cluster"); } } @@ -1193,18 +1235,18 @@ ClusterInfoImpl::ClusterInfoImpl( // early validation of sanity of fields that we should catch at config ingestion. DurationUtil::durationToMilliseconds(common_lb_config_->update_merge_window()); - // Create upstream filter factories + // Create upstream network filter factories const auto& filters = config.filters(); ASSERT(filter_factories_.empty()); filter_factories_.reserve(filters.size()); for (ssize_t i = 0; i < filters.size(); i++) { const auto& proto_config = filters[i]; const bool is_terminal = i == filters.size() - 1; - ENVOY_LOG(debug, " upstream filter #{}:", i); + ENVOY_LOG(debug, " upstream network filter #{}:", i); if (proto_config.has_config_discovery()) { if (proto_config.has_typed_config()) { - throw EnvoyException("Only one of typed_config or config_discovery can be used"); + throwEnvoyExceptionOrPanic("Only one of typed_config or config_discovery can be used"); } ENVOY_LOG(debug, " dynamic filter name: {}", proto_config.name()); @@ -1239,14 +1281,14 @@ ClusterInfoImpl::ClusterInfoImpl( envoy::extensions::filters::http::upstream_codec::v3::UpstreamCodec::default_instance()); } if (http_filters[http_filters.size() - 1].name() != "envoy.filters.http.upstream_codec") { - throw EnvoyException( - fmt::format("The codec filter is the only valid terminal upstream filter")); + throwEnvoyExceptionOrPanic( + fmt::format("The codec filter is the only valid terminal upstream HTTP filter")); } std::string prefix = stats_scope_->symbolTable().toString(stats_scope_->prefix()); Http::FilterChainHelper - helper(*http_filter_config_provider_manager_, upstream_context_.getServerFactoryContext(), + helper(*http_filter_config_provider_manager_, upstream_context_.serverFactoryContext(), factory_context.clusterManager(), upstream_context_, prefix); THROW_IF_NOT_OK(helper.processFilters(http_filters, "upstream http", "upstream http", http_filter_factories_)); @@ -1258,18 +1300,19 @@ void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Clus Server::Configuration::ServerFactoryContext& context) { // Check if load_balancing_policy is set first. if (!config.has_load_balancing_policy()) { - throw EnvoyException("cluster: field load_balancing_policy need to be set"); + throwEnvoyExceptionOrPanic("cluster: field load_balancing_policy need to be set"); } if (config.has_lb_subset_config()) { - throw EnvoyException("cluster: load_balancing_policy cannot be combined with lb_subset_config"); + throwEnvoyExceptionOrPanic( + "cluster: load_balancing_policy cannot be combined with lb_subset_config"); } if (config.has_common_lb_config()) { const auto& lb_config = config.common_lb_config(); if (lb_config.has_zone_aware_lb_config() || lb_config.has_locality_weighted_lb_config() || lb_config.has_consistent_hashing_lb_config()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "cluster: load_balancing_policy cannot be combined with partial fields " "(zone_aware_lb_config, " "locality_weighted_lb_config, consistent_hashing_lb_config) of common_lb_config"); @@ -1288,7 +1331,7 @@ void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Clus context.messageValidationVisitor(), *proto_message); load_balancer_config_ = - factory->loadConfig(std::move(proto_message), context.messageValidationVisitor()); + factory->loadConfig(*proto_message, context.messageValidationVisitor()); load_balancer_factory_ = factory; break; @@ -1297,9 +1340,10 @@ void ClusterInfoImpl::configureLbPolicies(const envoy::config::cluster::v3::Clus } if (load_balancer_factory_ == nullptr) { - throw EnvoyException(fmt::format("cluster: didn't find a registered load balancer factory " - "implementation for cluster: '{}' with names from [{}]", - name_, absl::StrJoin(missing_policies, ", "))); + throwEnvoyExceptionOrPanic( + fmt::format("cluster: didn't find a registered load balancer factory " + "implementation for cluster: '{}' with names from [{}]", + name_, absl::StrJoin(missing_policies, ", "))); } lb_type_ = LoadBalancerType::LoadBalancingPolicyConfig; @@ -1433,13 +1477,13 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus if ((info_->features() & ClusterInfoImpl::Features::USE_ALPN)) { if (!raw_factory_pointer->supportsAlpn()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("ALPN configured for cluster {} which has a non-ALPN transport socket: {}", cluster.name(), cluster.DebugString())); } if (!matcher_supports_alpn && !runtime_.snapshot().featureEnabled(ClusterImplBase::DoNotValidateAlpnRuntimeKey, 0)) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "ALPN configured for cluster {} which has a non-ALPN transport socket matcher: {}", cluster.name(), cluster.DebugString())); } @@ -1448,12 +1492,12 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus if (info_->features() & ClusterInfoImpl::Features::HTTP3) { #if defined(ENVOY_ENABLE_QUIC) if (!validateTransportSocketSupportsQuic(cluster.transport_socket())) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("HTTP3 requires a QuicUpstreamTransport transport socket: {} {}", cluster.name(), cluster.transport_socket().DebugString())); } #else - throw EnvoyException("HTTP3 configured but not enabled in the build."); + throwEnvoyExceptionOrPanic("HTTP3 configured but not enabled in the build."); #endif } @@ -1481,6 +1525,11 @@ ClusterImplBase::ClusterImplBase(const envoy::config::cluster::v3::Cluster& clus info_->endpointStats().membership_degraded_.set(degraded_hosts); info_->endpointStats().membership_excluded_.set(excluded_hosts); }); + // Drop overload configuration parsing. + absl::Status status = parseDropOverloadConfig(cluster.load_assignment()); + if (!status.ok()) { + throwEnvoyExceptionOrPanic(std::string(status.message())); + } } namespace { @@ -1600,6 +1649,45 @@ void ClusterImplBase::finishInitialization() { } } +absl::Status ClusterImplBase::parseDropOverloadConfig( + const envoy::config::endpoint::v3::ClusterLoadAssignment& cluster_load_assignment) { + // Default drop_overload_ to zero. + drop_overload_ = UnitFloat(0); + + if (!cluster_load_assignment.has_policy()) { + return absl::OkStatus(); + } + auto policy = cluster_load_assignment.policy(); + if (policy.drop_overloads().size() == 0) { + return absl::OkStatus(); + } + if (policy.drop_overloads().size() > kDropOverloadSize) { + return absl::InvalidArgumentError( + fmt::format("Cluster drop_overloads config has {} categories. Envoy only support one.", + policy.drop_overloads().size())); + } + + const auto drop_percentage = policy.drop_overloads(0).drop_percentage(); + float denominator = 100; + switch (drop_percentage.denominator()) { + case envoy::type::v3::FractionalPercent::HUNDRED: + denominator = 100; + break; + case envoy::type::v3::FractionalPercent::TEN_THOUSAND: + denominator = 10000; + break; + case envoy::type::v3::FractionalPercent::MILLION: + denominator = 1000000; + break; + default: + return absl::InvalidArgumentError(fmt::format( + "Cluster drop_overloads config denominator setting is invalid : {}. Valid range 0~2.", + drop_percentage.denominator())); + } + drop_overload_ = UnitFloat(float(drop_percentage.numerator()) / (denominator)); + return absl::OkStatus(); +} + void ClusterImplBase::setHealthChecker(const HealthCheckerSharedPtr& health_checker) { ASSERT(!health_checker_); health_checker_ = health_checker; @@ -1658,9 +1746,10 @@ ClusterImplBase::resolveProtoAddress(const envoy::config::core::v3::Address& add CATCH(EnvoyException & e, { if (info_->type() == envoy::config::cluster::v3::Cluster::STATIC || info_->type() == envoy::config::cluster::v3::Cluster::EDS) { - throw EnvoyException(fmt::format("{}. Consider setting resolver_name or setting cluster type " - "to 'STRICT_DNS' or 'LOGICAL_DNS'", - e.what())); + throwEnvoyExceptionOrPanic( + fmt::format("{}. Consider setting resolver_name or setting cluster type " + "to 'STRICT_DNS' or 'LOGICAL_DNS'", + e.what())); } throw e; }); @@ -1669,7 +1758,7 @@ ClusterImplBase::resolveProtoAddress(const envoy::config::core::v3::Address& add void ClusterImplBase::validateEndpointsForZoneAwareRouting( const envoy::config::endpoint::v3::LocalityLbEndpoints& endpoints) const { if (local_cluster_ && endpoints.priority() > 0) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Unexpected non-zero priority for local cluster '{}'.", info()->name())); } } @@ -1857,7 +1946,7 @@ ClusterInfoImpl::ResourceManagers::load(const envoy::config::cluster::v3::Cluste if (per_host_it->has_max_pending_requests() || per_host_it->has_max_requests() || per_host_it->has_max_retries() || per_host_it->has_max_connection_pools() || per_host_it->has_retry_budget()) { - throw EnvoyException("Unsupported field in per_host_thresholds"); + throwEnvoyExceptionOrPanic("Unsupported field in per_host_thresholds"); } if (per_host_it->has_max_connections()) { max_connections_per_host = per_host_it->max_connections().value(); diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 454b08cc3194..7a2a7cc30616 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -71,10 +71,28 @@ namespace Envoy { namespace Upstream { +using ClusterProto = envoy::config::cluster::v3::Cluster; + using UpstreamNetworkFilterConfigProviderManager = Filter::FilterConfigProviderManager; +class LegacyLbPolicyConfigHelper { +public: + struct Result { + TypedLoadBalancerFactory* factory; + LoadBalancerConfigPtr config; + }; + + static absl::StatusOr + getTypedLbConfigFromLegacyProtoWithoutSubset(const ClusterProto& cluster, + ProtobufMessage::ValidationVisitor& visitor); + + static absl::StatusOr + getTypedLbConfigFromLegacyProto(const ClusterProto& cluster, + ProtobufMessage::ValidationVisitor& visitor); +}; + /** * Class for LBPolicies * Uses a absl::variant to store pointers for the LBPolicy @@ -878,22 +896,42 @@ class ClusterInfoImpl : public ClusterInfo, } OptRef lbRoundRobinConfig() const override { + if (lb_policy_config_ == nullptr) { + return {}; + } + return lb_policy_config_->lbRoundRobinConfig(); } OptRef lbLeastRequestConfig() const override { + if (lb_policy_config_ == nullptr) { + return {}; + } + return lb_policy_config_->lbLeastRequestConfig(); } OptRef lbRingHashConfig() const override { + if (lb_policy_config_ == nullptr) { + return {}; + } + return lb_policy_config_->lbRingHashConfig(); } OptRef lbMaglevConfig() const override { + if (lb_policy_config_ == nullptr) { + return {}; + } + return lb_policy_config_->lbMaglevConfig(); } OptRef lbOriginalDstConfig() const override { + if (lb_policy_config_ == nullptr) { + return {}; + } + return lb_policy_config_->lbOriginalDstConfig(); } OptRef upstreamConfig() const override { @@ -942,6 +980,8 @@ class ClusterInfoImpl : public ClusterInfo, return std::ref(*(optional_cluster_stats_->timeout_budget_stats_)); } + bool perEndpointStatsEnabled() const override { return per_endpoint_stats_; } + UpstreamLocalAddressSelectorConstSharedPtr getUpstreamLocalAddressSelector() const override { return upstream_local_address_selector_; } @@ -1004,7 +1044,7 @@ class ClusterInfoImpl : public ClusterInfo, } bool createUpgradeFilterChain(absl::string_view, const UpgradeMap*, Http::FilterChainManager&) const override { - // Upgrade filter chains not yet supported for upstream filters. + // Upgrade filter chains not yet supported for upstream HTTP filters. return false; } @@ -1076,7 +1116,7 @@ class ClusterInfoImpl : public ClusterInfo, mutable ResourceManagers resource_managers_; const std::string maintenance_mode_runtime_key_; UpstreamLocalAddressSelectorConstSharedPtr upstream_local_address_selector_; - const std::unique_ptr lb_policy_config_; + std::unique_ptr lb_policy_config_; std::unique_ptr upstream_config_; std::unique_ptr lb_subset_; std::unique_ptr metadata_; @@ -1094,7 +1134,6 @@ class ClusterInfoImpl : public ClusterInfo, std::shared_ptr http_filter_config_provider_manager_; std::shared_ptr network_filter_config_provider_manager_; - const std::unique_ptr factory_context_; Filter::NetworkFilterFactoriesList filter_factories_; Http::FilterChainUtility::FilterFactoriesList http_filter_factories_; mutable Http::Http1::CodecStats::AtomicPtr http1_codec_stats_; @@ -1115,6 +1154,7 @@ class ClusterInfoImpl : public ClusterInfo, const bool added_via_api_ : 1; // true iff the cluster proto specified upstream http filters. bool has_configured_http_filters_ : 1; + const bool per_endpoint_stats_ : 1; }; /** @@ -1179,6 +1219,8 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable callback) override; + UnitFloat dropOverload() const override { return drop_overload_; } + void setDropOverload(UnitFloat drop_overload) override { drop_overload_ = drop_overload; } protected: ClusterImplBase(const envoy::config::cluster::v3::Cluster& cluster, @@ -1206,6 +1248,9 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable; diff --git a/source/common/version/BUILD b/source/common/version/BUILD index 7f88244cb351..8591bf4c047e 100644 --- a/source/common/version/BUILD +++ b/source/common/version/BUILD @@ -66,6 +66,7 @@ envoy_cc_library( ["-DENVOY_SSL_VERSION=\\\"BoringSSL\\\""], ), external_deps = ["ssl"], + tags = ["notidy"], deps = [ ":version_includes", "//source/common/common:macros", diff --git a/source/docs/fine_grain_log.md b/source/docs/fine_grain_log.md index 64d30052e5b5..6a370981d10f 100644 --- a/source/docs/fine_grain_log.md +++ b/source/docs/fine_grain_log.md @@ -40,7 +40,25 @@ Runtime update of Fine-Grain Logger is supported with administration interface, 3. `POST /logging?paths=file_path1:level1,file_path2:level2...`: Change log level of multiple file paths at once; 4. `POST /logging?level=`: Change levels of all loggers. -Users can view and change the log level in a granularity of file in runtime through admin page. Note that `file_path` is determined by `__FILE__` macro, which is the path seen by preprocessor. +Users can view and change the log level by file granularity at runtime through the admin page. + +Envoy admin `/logging` supports the file basename, glob ``*`` and ``?`` match for the fine-grain loggers update as well. For example, the following loggers are active in Envoy right now, and 0 means trace level. + +``` + source/server/admin/admin_filter.cc: 0 + source/common/event/dispatcher_impl.cc: 0 + source/common/network/tcp_listener_impl.cc: 0 + source/common/network/udp_listener_impl.cc: 0 +``` + + 1. ``POST /logging?paths=source/common/event/dispatcher_impl.cc:debug`` will make the level of ``source/common/event/dispatcher_impl.cc`` be debug. + 2. ``POST /logging?admin_filter=info`` will make the level of ``source/server/admin/admin_filter.cc`` be info, and other unmatched loggers will be the default trace. + 3. ``POST /logging?paths=source/common*:warning`` will make the level of ``source/common/event/dispatcher_impl.cc:``, ``source/common/network/tcp_listener_impl.cc`` be warning. + Other unmatched loggers will be the default trace, e.g., `admin_filter.cc`, even it was updated to info from the previous post update. + 4. ``POST /logging?paths=???_listener_impl:info`` will make the level of ``source/common/network/tcp_listener_impl.cc``, ``source/common/network/udp_listener_impl.cc`` be info. + 5. ``POST /logging?paths=???_listener_impl:info,tcp_listener_impl:warning``, the level of ``source/common/network/tcp_listener_impl.cc`` will be info, since the first match will take effect. + 6. ``POST /logging?level=info`` will change the default verbosity level to info. All the unmatched loggers in the following update will be this default level. + ### Implementation Details Fine-Grain Logger can be divided into two parts: @@ -59,4 +77,5 @@ Fine-Grain Logger provides control interfaces through command line and admin pag To pass the arguments such as log format and default log level to Fine-Grain Logger, `fine_grain_log_format` and `fine_grain_default_level` are added in `class Context` and they are updated when a new logging context is activated. `getFineGrainLogContext().setDefaultFineGrainLogLevelFormat(level, format)` is called in `Context::activate()` to set log format and update loggers' previous default level to the new level. -To support the runtime update in admin page, log handler in admin page uses `getFineGrainLogContext().listFineGrainLoggers()` to show all Fine-Grain Loggers, `getFineGrainLogContext().setFineGrainLogger(name, level)` to set a specific logger and `getFineGrainLogContext().setAllFineGrainLoggers(level)` to set all loggers. +To support the runtime update in admin page, log handler in admin page uses `getFineGrainLogContext().listFineGrainLoggers()` to show all Fine-Grain Loggers, `getFineGrainLogContext().setFineGrainLogger(name, level)` to set a specific logger, `getFineGrainLogContext().setAllFineGrainLoggers(level)` to set all loggers, and `getFineGrainLogContext().updateVerbositySetting(glob_levels)` to set glob matched loggers. + diff --git a/source/docs/upstream_filters.md b/source/docs/upstream_filters.md index f7ad713f70fb..34f7d9e69f8d 100644 --- a/source/docs/upstream_filters.md +++ b/source/docs/upstream_filters.md @@ -1,6 +1,6 @@ -# Converting a downstream filter to be a dual filter +# Converting a downstream HTTP filter to be a dual filter -Before coverting a filter to be an upstream filter you should do some basic +Before coverting a filter to be an upstream HTTP filter you should do some basic functionality analysis. Make sure * The filter does not use any downstream-only functionality, accessed via @@ -11,7 +11,7 @@ functionality analysis. Make sure * Either the filter does not access streamInfo in a non-const way, or you test and document how the filter interacts with hedging and retries. Note that for hedging, a single downstream StreamInfo is accessible in parallel to - both instances of the upstream filter instance, so it must be resiliant to + both instances of the upstream HTTP filter instance, so it must be resiliant to parallel access. * Any code accessing the downstream connection checks to make sure it is present. The downstream connection will not be available for mirrored/shadowed requests. @@ -33,6 +33,6 @@ Assuming your filter inherits from FactoryBase: * If your filter is listed in ``source/extensions/extensions_metadata.yaml`` add ``envoy.filters.http.upstream`` to the filter category. -Your filter should now be available as an upstream filter. +Your filter should now be available as an upstream HTTP filter. An example PR for conversion is [23071](https://github.com/envoyproxy/envoy/pull/23071) diff --git a/source/exe/BUILD b/source/exe/BUILD index 32fdd616dd26..a4651aae9b4f 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -10,7 +10,7 @@ load( "envoy_select_enable_http3", "envoy_select_signal_trace", ) -load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS") +load("//bazel:repositories.bzl", "NO_HTTP3_SKIP_TARGETS", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS") load("//source/extensions:all_extensions.bzl", "envoy_all_core_extensions", "envoy_all_extensions") licenses(["notice"]) # Apache 2 @@ -33,7 +33,7 @@ envoy_cc_binary( ) envoy_cc_library( - name = "envoy_common_lib", + name = "all_extensions_lib", deps = [ "//source/common/event:libevent_lib", "//source/common/network:utility_lib", @@ -41,11 +41,12 @@ envoy_cc_library( "//source/common/stats:thread_local_store_lib", "//source/server:drain_manager_lib", "//source/server:listener_hooks_lib", - "//source/server:options_lib", - "//source/server:server_lib", + "//source/server:options_base", + "//source/server:server_base_lib", ] + select({ "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), + "//bazel:disable_http3": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS), "//conditions:default": envoy_all_extensions(), }), ) @@ -60,6 +61,7 @@ envoy_cc_library( ":envoy_main_common_lib", ":platform_impl_lib", ":scm_impl_lib", + "//source/server:options_lib", ], ) @@ -68,16 +70,24 @@ envoy_cc_library( srcs = ["stripped_main_base.cc"], hdrs = ["stripped_main_base.h"], deps = [ - ":envoy_common_with_core_extensions_lib", ":platform_impl_lib", ":process_wide_lib", "//source/common/api:os_sys_calls_lib", "//source/common/common:compiler_requirements_lib", "//source/common/common:perf_annotation_lib", + "//source/common/event:libevent_lib", + "//source/common/event:real_time_system_lib", "//source/common/grpc:google_grpc_context_lib", + "//source/common/network:utility_lib", + "//source/common/stats:stats_lib", + "//source/common/stats:thread_local_store_lib", "//source/common/thread_local:thread_local_lib", + "//source/server:drain_manager_lib", "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", + "//source/server:listener_hooks_lib", + "//source/server:options_base", + "//source/server:server_base_lib", ] + envoy_select_signal_trace([ "//source/common/signal:sigaction_lib", ":terminate_handler_lib", @@ -88,39 +98,56 @@ envoy_cc_library( name = "main_common_lib", srcs = [ "main_common.cc", - "stripped_main_base.cc", ], hdrs = [ "main_common.h", - "stripped_main_base.h", ], deps = [ - ":envoy_common_lib", ":platform_impl_lib", ":process_wide_lib", + ":stripped_main_base_lib", "//source/common/api:os_sys_calls_lib", "//source/common/common:compiler_requirements_lib", "//source/common/common:perf_annotation_lib", "//source/common/grpc:google_grpc_context_lib", "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", + "//source/server:options_lib", "//source/server/config_validation:server_lib", - ] + envoy_select_signal_trace([ - "//source/common/signal:sigaction_lib", - ":terminate_handler_lib", - ]), + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "main_common_with_all_extensions_lib", + deps = [ + ":all_extensions_lib", + ":main_common_lib", + ":platform_impl_lib", + ":process_wide_lib", + ":stripped_main_base_lib", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:compiler_requirements_lib", + "//source/common/common:perf_annotation_lib", + "//source/common/grpc:google_grpc_context_lib", + "//source/server:hot_restart_lib", + "//source/server:hot_restart_nop_lib", + "//source/server:options_lib", + "//source/server/config_validation:server_lib", + ], ) # provides a library target for Envoy server builds with the versioning information set up correctly. envoy_cc_library( name = "envoy_main_common_lib", deps = [ - ":main_common_lib", + ":main_common_with_all_extensions_lib", # These are compiled as extensions so Envoy Mobile doesn't have to link them in. # Envoy requires them. - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//source/common/listener_manager:listener_manager_lib", "//source/extensions/listener_managers/validation_listener_manager:validation_listener_manager_lib", "//source/common/version:version_linkstamp", + "//source/server:options_lib", ] + envoy_select_enable_http3(["//source/common/quic:server_codec_lib"]), ) @@ -134,16 +161,23 @@ envoy_cc_library( ) envoy_cc_library( - name = "envoy_common_with_core_extensions_lib", + name = "envoy_main_common_with_core_extensions_lib", deps = [ - "//source/common/event:libevent_lib", - "//source/common/network:utility_lib", - "//source/common/stats:stats_lib", - "//source/common/stats:thread_local_store_lib", - "//source/server:drain_manager_lib", - "//source/server:listener_hooks_lib", + ":main_common_lib", + ":platform_impl_lib", + ":process_wide_lib", + ":stripped_main_base_lib", + "//envoy/server:platform_interface", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:compiler_requirements_lib", + "//source/common/common:perf_annotation_lib", + "//source/common/grpc:google_grpc_context_lib", + "//source/extensions/listener_managers/validation_listener_manager:validation_listener_manager_lib", + "//source/server:hot_restart_lib", + "//source/server:hot_restart_nop_lib", "//source/server:options_lib", "//source/server:server_lib", + "//source/server/config_validation:server_lib", ] + envoy_all_core_extensions() + # TODO(rojkov): drop io_uring dependency when it's fully integrated. select({ @@ -152,35 +186,6 @@ envoy_cc_library( }), ) -envoy_cc_library( - name = "envoy_main_common_with_core_extensions_lib", - srcs = [ - "main_common.cc", - "stripped_main_base.cc", - ], - hdrs = [ - "main_common.h", - "stripped_main_base.h", - ], - deps = [ - ":envoy_common_with_core_extensions_lib", - ":platform_impl_lib", - ":process_wide_lib", - "//envoy/server:platform_interface", - "//source/common/api:os_sys_calls_lib", - "//source/common/common:compiler_requirements_lib", - "//source/common/common:perf_annotation_lib", - "//source/common/grpc:google_grpc_context_lib", - "//source/extensions/listener_managers/validation_listener_manager:validation_listener_manager_lib", - "//source/server:hot_restart_lib", - "//source/server:hot_restart_nop_lib", - "//source/server/config_validation:server_lib", - ] + envoy_select_signal_trace([ - "//source/common/signal:sigaction_lib", - ":terminate_handler_lib", - ]), -) - envoy_cc_library( name = "process_wide_lib", srcs = ["process_wide.cc"], @@ -263,12 +268,13 @@ envoy_cc_library( }), deps = select({ "//bazel:windows_x86_64": [ - ":main_common_lib", + ":main_common_with_all_extensions_lib", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:thread_lib", "//source/common/common:win32_event_logger_impl_lib", "//source/common/event:signal_lib", + "//source/server:options_lib", ], "//conditions:default": [], }), diff --git a/source/exe/main_common.cc b/source/exe/main_common.cc index ba0edd482cac..d18a575f2dd0 100644 --- a/source/exe/main_common.cc +++ b/source/exe/main_common.cc @@ -16,9 +16,9 @@ #include "source/server/config_validation/server.h" #include "source/server/drain_manager_impl.h" #include "source/server/hot_restart_nop_impl.h" +#include "source/server/instance_impl.h" #include "source/server/listener_hooks.h" -#include "source/server/options_impl.h" -#include "source/server/server.h" +#include "source/server/options_impl_base.h" #include "absl/debugging/symbolize.h" #include "absl/strings/str_split.h" @@ -29,6 +29,35 @@ namespace Envoy { +StrippedMainBase::CreateInstanceFunction createFunction() { + return + [](Init::Manager& init_manager, const Server::Options& options, + Event::TimeSystem& time_system, ListenerHooks& hooks, Server::HotRestart& restarter, + Stats::StoreRoot& store, Thread::BasicLockable& access_log_lock, + Server::ComponentFactory& component_factory, Random::RandomGeneratorPtr&& random_generator, + ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, + Filesystem::Instance& file_system, std::unique_ptr process_context, + Buffer::WatermarkFactorySharedPtr watermark_factory) { + auto local_address = Network::Utility::getLocalAddress(options.localAddressIpVersion()); + auto server = std::make_unique( + init_manager, options, time_system, hooks, restarter, store, access_log_lock, + std::move(random_generator), tls, thread_factory, file_system, + std::move(process_context), watermark_factory); + server->initialize(local_address, component_factory); + return server; + }; +} + +MainCommonBase::MainCommonBase(const Server::Options& options, Event::TimeSystem& time_system, + ListenerHooks& listener_hooks, + Server::ComponentFactory& component_factory, + std::unique_ptr platform_impl, + std::unique_ptr&& random_generator, + std::unique_ptr process_context) + : StrippedMainBase(options, time_system, listener_hooks, component_factory, + std::move(platform_impl), std::move(random_generator), + std::move(process_context), createFunction()) {} + bool MainCommonBase::run() { switch (options_.mode()) { case Server::Mode::Serve: @@ -37,7 +66,8 @@ bool MainCommonBase::run() { case Server::Mode::Validate: return Server::validateConfig( options_, Network::Utility::getLocalAddress(options_.localAddressIpVersion()), - component_factory_, platform_impl_->threadFactory(), platform_impl_->fileSystem()); + component_factory_, platform_impl_->threadFactory(), platform_impl_->fileSystem(), + process_context_ ? ProcessContextOptRef(std::ref(*process_context_)) : absl::nullopt); case Server::Mode::InitOnly: PERF_DUMP(); return true; diff --git a/source/exe/main_common.h b/source/exe/main_common.h index f101e578986a..349decdec0cc 100644 --- a/source/exe/main_common.h +++ b/source/exe/main_common.h @@ -25,7 +25,11 @@ namespace Envoy { class MainCommonBase : public StrippedMainBase { public: - using StrippedMainBase::StrippedMainBase; + MainCommonBase(const Server::Options& options, Event::TimeSystem& time_system, + ListenerHooks& listener_hooks, Server::ComponentFactory& component_factory, + std::unique_ptr platform_impl, + std::unique_ptr&& random_generator, + std::unique_ptr process_context); bool run(); diff --git a/source/exe/stripped_main_base.cc b/source/exe/stripped_main_base.cc index 878e72a75735..502963e897d1 100644 --- a/source/exe/stripped_main_base.cc +++ b/source/exe/stripped_main_base.cc @@ -16,7 +16,7 @@ #include "source/server/drain_manager_impl.h" #include "source/server/hot_restart_nop_impl.h" #include "source/server/listener_hooks.h" -#include "source/server/options_impl.h" +#include "source/server/options_impl_base.h" #include "source/server/server.h" #include "absl/debugging/symbolize.h" @@ -46,12 +46,13 @@ StrippedMainBase::StrippedMainBase(const Server::Options& options, Event::TimeSy Server::ComponentFactory& component_factory, std::unique_ptr platform_impl, std::unique_ptr&& random_generator, - std::unique_ptr process_context) + std::unique_ptr process_context, + CreateInstanceFunction createInstance) : platform_impl_(std::move(platform_impl)), options_(options), component_factory_(component_factory), stats_allocator_(symbol_table_) { // Process the option to disable extensions as early as possible, // before we do any configuration loading. - OptionsImpl::disableExtensions(options.disabledExtensions()); + OptionsImplBase::disableExtensions(options.disabledExtensions()); // Enable core dumps as early as possible. if (options_.coreDumpEnabled()) { @@ -71,24 +72,22 @@ StrippedMainBase::StrippedMainBase(const Server::Options& options, Event::TimeSy tls_ = std::make_unique(); Thread::BasicLockable& log_lock = restarter_->logLock(); Thread::BasicLockable& access_log_lock = restarter_->accessLogLock(); - auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion()); logging_context_ = std::make_unique(options_.logLevel(), options_.logFormat(), log_lock, options_.logFormatEscaped(), options_.enableFineGrainLogging()); configureComponentLogLevels(); - // Provide consistent behavior for out-of-memory, regardless of whether it occurs in a try/catch - // block or not. + // Provide consistent behavior for out-of-memory, regardless of whether it occurs in a + // try/catch block or not. std::set_new_handler([]() { PANIC("out of memory"); }); stats_store_ = std::make_unique(stats_allocator_); - server_ = std::make_unique( - *init_manager_, options_, time_system, local_address, listener_hooks, *restarter_, - *stats_store_, access_log_lock, component_factory, std::move(random_generator), *tls_, - platform_impl_->threadFactory(), platform_impl_->fileSystem(), std::move(process_context)); - + server_ = createInstance(*init_manager_, options_, time_system, listener_hooks, *restarter_, + *stats_store_, access_log_lock, component_factory, + std::move(random_generator), *tls_, platform_impl_->threadFactory(), + platform_impl_->fileSystem(), std::move(process_context), nullptr); break; } case Server::Mode::Validate: @@ -96,6 +95,7 @@ StrippedMainBase::StrippedMainBase(const Server::Options& options, Event::TimeSy logging_context_ = std::make_unique(options_.logLevel(), options_.logFormat(), restarter_->logLock(), options_.logFormatEscaped()); + process_context_ = std::move(process_context); break; } } diff --git a/source/exe/stripped_main_base.h b/source/exe/stripped_main_base.h index 10a2bcf0f73b..0da36500ff44 100644 --- a/source/exe/stripped_main_base.h +++ b/source/exe/stripped_main_base.h @@ -12,7 +12,6 @@ #include "source/common/thread_local/thread_local_impl.h" #include "source/exe/process_wide.h" #include "source/server/listener_hooks.h" -#include "source/server/options_impl.h" #include "source/server/server.h" #ifdef ENVOY_HANDLE_SIGNALS @@ -36,6 +35,15 @@ class ProdComponentFactory : public Server::ComponentFactory { // separate for legacy reasons. class StrippedMainBase { public: + using CreateInstanceFunction = std::function( + Init::Manager& init_manager, const Server::Options& options, Event::TimeSystem& time_system, + ListenerHooks& hooks, Server::HotRestart& restarter, Stats::StoreRoot& store, + Thread::BasicLockable& access_log_lock, Server::ComponentFactory& component_factory, + Random::RandomGeneratorPtr&& random_generator, ThreadLocal::Instance& tls, + Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system, + std::unique_ptr process_context, + Buffer::WatermarkFactorySharedPtr watermark_factory)>; + static std::string hotRestartVersion(bool hot_restart_enabled); // Consumer must guarantee that all passed references are alive until this object is @@ -44,7 +52,8 @@ class StrippedMainBase { ListenerHooks& listener_hooks, Server::ComponentFactory& component_factory, std::unique_ptr platform_impl, std::unique_ptr&& random_generator, - std::unique_ptr process_context); + std::unique_ptr process_context, + CreateInstanceFunction createInstance); void runServer() { ASSERT(options_.mode() == Server::Mode::Serve); @@ -71,7 +80,10 @@ class StrippedMainBase { Stats::ThreadLocalStoreImplPtr stats_store_; std::unique_ptr logging_context_; std::unique_ptr init_manager_{std::make_unique("Server")}; - std::unique_ptr server_; + std::unique_ptr server_; + + // Only used for validation mode + std::unique_ptr process_context_; private: void configureComponentLogLevels(); diff --git a/source/extensions/BUILD b/source/extensions/BUILD index 152c70189418..dfb91fa3b185 100644 --- a/source/extensions/BUILD +++ b/source/extensions/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:utils.bzl", "json_data") +load("@envoy_toolshed//:macros.bzl", "json_data") load("//bazel:envoy_build_system.bzl", "envoy_extension_package") load(":all_extensions.bzl", "envoy_all_extensions") load(":extensions_build_config.bzl", "EXTENSIONS") diff --git a/source/extensions/access_loggers/common/access_log_base.cc b/source/extensions/access_loggers/common/access_log_base.cc index f90781a86125..01f114297e21 100644 --- a/source/extensions/access_loggers/common/access_log_base.cc +++ b/source/extensions/access_loggers/common/access_log_base.cc @@ -8,26 +8,14 @@ namespace Extensions { namespace AccessLoggers { namespace Common { -void ImplBase::log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { - if (!request_headers) { - request_headers = Http::StaticEmptyHeaders::get().request_headers.get(); - } - if (!response_headers) { - response_headers = Http::StaticEmptyHeaders::get().response_headers.get(); - } - if (!response_trailers) { - response_trailers = Http::StaticEmptyHeaders::get().response_trailers.get(); - } - if (filter_ && !filter_->evaluate(stream_info, *request_headers, *response_headers, - *response_trailers, access_log_type)) { +void ImplBase::log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + + if (filter_ && !filter_->evaluate(log_context, stream_info)) { return; } - return emitLog(*request_headers, *response_headers, *response_trailers, stream_info, - access_log_type); + + return emitLog(log_context, stream_info); } } // namespace Common diff --git a/source/extensions/access_loggers/common/access_log_base.h b/source/extensions/access_loggers/common/access_log_base.h index c15b6003c310..35ba33344f72 100644 --- a/source/extensions/access_loggers/common/access_log_base.h +++ b/source/extensions/access_loggers/common/access_log_base.h @@ -6,8 +6,8 @@ #include "envoy/access_log/access_log.h" #include "envoy/runtime/runtime.h" -#include "envoy/server/access_log_config.h" +#include "source/common/access_log/access_log_impl.h" #include "source/common/http/header_utility.h" #include "source/common/protobuf/protobuf.h" @@ -26,26 +26,18 @@ class ImplBase : public AccessLog::Instance { /** * Log a completed request if the underlying AccessLog `filter_` allows it. */ - void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override; + void log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) override; private: /** * Log a completed request. - * @param request_headers supplies the incoming request headers after filtering. - * @param response_headers supplies response headers. - * @param response_trailers supplies response trailers. + * @param context supplies the necessary context to log. * @param stream_info supplies additional information about the request not * contained in the request headers. */ - virtual void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) PURE; + virtual void emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& stream_info) PURE; AccessLog::FilterPtr filter_; }; diff --git a/source/extensions/access_loggers/common/file_access_log_impl.cc b/source/extensions/access_loggers/common/file_access_log_impl.cc index c06fff4fa8ee..e8dd581660ef 100644 --- a/source/extensions/access_loggers/common/file_access_log_impl.cc +++ b/source/extensions/access_loggers/common/file_access_log_impl.cc @@ -12,15 +12,9 @@ FileAccessLog::FileAccessLog(const Filesystem::FilePathAndType& access_log_file_ log_file_ = log_manager.createAccessLog(access_log_file_info); } -void FileAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { - log_file_->write( - formatter_->formatWithContext({&request_headers, &response_headers, &response_trailers, - absl::string_view(), access_log_type}, - stream_info)); +void FileAccessLog::emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& stream_info) { + log_file_->write(formatter_->formatWithContext(context, stream_info)); } } // namespace File diff --git a/source/extensions/access_loggers/common/file_access_log_impl.h b/source/extensions/access_loggers/common/file_access_log_impl.h index 4b866879fbbb..457cdc50a07b 100644 --- a/source/extensions/access_loggers/common/file_access_log_impl.h +++ b/source/extensions/access_loggers/common/file_access_log_impl.h @@ -19,11 +19,8 @@ class FileAccessLog : public Common::ImplBase { private: // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override; + void emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& stream_info) override; AccessLog::AccessLogFileSharedPtr log_file_; Formatter::FormatterPtr formatter_; diff --git a/source/extensions/access_loggers/common/stream_access_log_common_impl.h b/source/extensions/access_loggers/common/stream_access_log_common_impl.h index 635a3ce6604e..1713b9182520 100644 --- a/source/extensions/access_loggers/common/stream_access_log_common_impl.h +++ b/source/extensions/access_loggers/common/stream_access_log_common_impl.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" #include "source/common/formatter/substitution_format_string.h" #include "source/common/formatter/substitution_formatter.h" @@ -13,7 +13,7 @@ namespace AccessLoggers { template AccessLog::InstanceSharedPtr createStreamAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { const auto& fal_config = MessageUtil::downcastAndValidate(config, context.messageValidationVisitor()); Formatter::FormatterPtr formatter; @@ -26,7 +26,8 @@ createStreamAccessLogInstance(const Protobuf::Message& config, AccessLog::Filter } Filesystem::FilePathAndType file_info{destination_type, ""}; return std::make_shared( - file_info, std::move(filter), std::move(formatter), context.accessLogManager()); + file_info, std::move(filter), std::move(formatter), + context.serverFactoryContext().accessLogManager()); } } // namespace AccessLoggers diff --git a/source/extensions/access_loggers/file/config.cc b/source/extensions/access_loggers/file/config.cc index 008b715489ea..a21c69343693 100644 --- a/source/extensions/access_loggers/file/config.cc +++ b/source/extensions/access_loggers/file/config.cc @@ -19,17 +19,10 @@ namespace Extensions { namespace AccessLoggers { namespace File { -AccessLog::InstanceSharedPtr FileAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); -} - -AccessLog::InstanceSharedPtr FileAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { +AccessLog::InstanceSharedPtr +FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { const auto& fal_config = MessageUtil::downcastAndValidate< const envoy::extensions::access_loggers::file::v3::FileAccessLog&>( config, context.messageValidationVisitor()); @@ -68,7 +61,7 @@ AccessLog::InstanceSharedPtr FileAccessLogFactory::createAccessLogInstance( Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, fal_config.path()}; return std::make_shared(file_info, std::move(filter), std::move(formatter), - context.accessLogManager()); + context.serverFactoryContext().accessLogManager()); } ProtobufTypes::MessagePtr FileAccessLogFactory::createEmptyConfigProto() { @@ -81,7 +74,7 @@ std::string FileAccessLogFactory::name() const { return "envoy.access_loggers.fi /** * Static registration for the file access log. @see RegisterFactory. */ -LEGACY_REGISTER_FACTORY(FileAccessLogFactory, Server::Configuration::AccessLogInstanceFactory, +LEGACY_REGISTER_FACTORY(FileAccessLogFactory, AccessLog::AccessLogInstanceFactory, "envoy.file_access_log"); } // namespace File diff --git a/source/extensions/access_loggers/file/config.h b/source/extensions/access_loggers/file/config.h index 8a18d2903f31..5d6cab9ee464 100644 --- a/source/extensions/access_loggers/file/config.h +++ b/source/extensions/access_loggers/file/config.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" namespace Envoy { namespace Extensions { @@ -10,15 +10,11 @@ namespace File { /** * Config registration for the file access log. @see AccessLogInstanceFactory. */ -class FileAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class FileAccessLogFactory : public AccessLog::AccessLogInstanceFactory { public: AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - - AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/access_loggers/filters/cel/cel.cc b/source/extensions/access_loggers/filters/cel/cel.cc index 25684a54afdd..20f80b70e2a0 100644 --- a/source/extensions/access_loggers/filters/cel/cel.cc +++ b/source/extensions/access_loggers/filters/cel/cel.cc @@ -9,19 +9,18 @@ namespace CEL { namespace Expr = Envoy::Extensions::Filters::Common::Expr; CELAccessLogExtensionFilter::CELAccessLogExtensionFilter( - Expr::BuilderInstanceSharedPtr builder, const google::api::expr::v1alpha1::Expr& input_expr) - : builder_(builder), parsed_expr_(input_expr) { + const ::Envoy::LocalInfo::LocalInfo& local_info, Expr::BuilderInstanceSharedPtr builder, + const google::api::expr::v1alpha1::Expr& input_expr) + : local_info_(local_info), builder_(builder), parsed_expr_(input_expr) { compiled_expr_ = Expr::createExpression(builder_->builder(), parsed_expr_); } -bool CELAccessLogExtensionFilter::evaluate(const StreamInfo::StreamInfo& stream_info, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - AccessLog::AccessLogType) const { +bool CELAccessLogExtensionFilter::evaluate(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) const { Protobuf::Arena arena; - auto eval_status = Expr::evaluate(*compiled_expr_, arena, stream_info, &request_headers, - &response_headers, &response_trailers); + auto eval_status = Expr::evaluate(*compiled_expr_, arena, &local_info_, stream_info, + &log_context.requestHeaders(), &log_context.responseHeaders(), + &log_context.responseTrailers()); if (!eval_status.has_value() || eval_status.value().IsError()) { return false; } diff --git a/source/extensions/access_loggers/filters/cel/cel.h b/source/extensions/access_loggers/filters/cel/cel.h index 2cce2d68b44e..74247d5187d6 100644 --- a/source/extensions/access_loggers/filters/cel/cel.h +++ b/source/extensions/access_loggers/filters/cel/cel.h @@ -19,15 +19,15 @@ namespace CEL { class CELAccessLogExtensionFilter : public AccessLog::Filter { public: - CELAccessLogExtensionFilter(Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr, + CELAccessLogExtensionFilter(const ::Envoy::LocalInfo::LocalInfo& local_info, + Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr, const google::api::expr::v1alpha1::Expr&); - bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - AccessLog::AccessLogType access_log_type) const override; + bool evaluate(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) const override; private: + const ::Envoy::LocalInfo::LocalInfo& local_info_; Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr builder_; const google::api::expr::v1alpha1::Expr parsed_expr_; Extensions::Filters::Common::Expr::ExpressionPtr compiled_expr_; diff --git a/source/extensions/access_loggers/filters/cel/config.cc b/source/extensions/access_loggers/filters/cel/config.cc index 2431a2e9dbf2..9a1a1be32435 100644 --- a/source/extensions/access_loggers/filters/cel/config.cc +++ b/source/extensions/access_loggers/filters/cel/config.cc @@ -16,7 +16,7 @@ namespace CEL { Envoy::AccessLog::FilterPtr CELAccessLogExtensionFilterFactory::createFilter( const envoy::config::accesslog::v3::ExtensionFilter& config, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { auto factory_config = Config::Utility::translateToFactoryConfig(config, context.messageValidationVisitor(), *this); @@ -33,7 +33,9 @@ Envoy::AccessLog::FilterPtr CELAccessLogExtensionFilterFactory::createFilter( } return std::make_unique( - Extensions::Filters::Common::Expr::getBuilder(context), parse_status.value().expr()); + context.serverFactoryContext().localInfo(), + Extensions::Filters::Common::Expr::getBuilder(context.serverFactoryContext()), + parse_status.value().expr()); #else throw EnvoyException("CEL is not available for use in this environment."); #endif diff --git a/source/extensions/access_loggers/filters/cel/config.h b/source/extensions/access_loggers/filters/cel/config.h index 45dbe79dc3ca..62dfb342491c 100644 --- a/source/extensions/access_loggers/filters/cel/config.h +++ b/source/extensions/access_loggers/filters/cel/config.h @@ -22,7 +22,7 @@ class CELAccessLogExtensionFilterFactory : public Envoy::AccessLog::ExtensionFil public: Envoy::AccessLog::FilterPtr createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override { return "envoy.access_loggers.extension_filters.cel"; } }; diff --git a/source/extensions/access_loggers/grpc/BUILD b/source/extensions/access_loggers/grpc/BUILD index 45bec6334796..976ada1ba7c5 100644 --- a/source/extensions/access_loggers/grpc/BUILD +++ b/source/extensions/access_loggers/grpc/BUILD @@ -52,6 +52,7 @@ envoy_cc_library( "//source/common/stream_info:stream_info_lib", "//source/common/stream_info:utility_lib", "//source/common/tracing:custom_tag_lib", + "//source/common/tracing:http_tracer_lib", "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto", ], @@ -100,7 +101,7 @@ envoy_cc_extension( hdrs = ["http_config.h"], deps = [ ":config_utils", - "//envoy/server:access_log_config_interface", + "//envoy/access_log:access_log_config_interface", "//source/common/common:assert_lib", "//source/common/grpc:async_client_lib", "//source/common/protobuf", @@ -116,7 +117,7 @@ envoy_cc_extension( hdrs = ["tcp_config.h"], deps = [ ":config_utils", - "//envoy/server:access_log_config_interface", + "//envoy/access_log:access_log_config_interface", "//source/common/common:assert_lib", "//source/common/grpc:async_client_lib", "//source/common/protobuf", diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc index 7155a13ee5fe..c9f08685f94a 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -2,11 +2,13 @@ #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" +#include "envoy/stream_info/filter_state.h" #include "envoy/upstream/upstream.h" #include "source/common/network/utility.h" #include "source/common/stream_info/utility.h" #include "source/common/tracing/custom_tag_impl.h" +#include "source/common/tracing/http_tracer_impl.h" namespace Envoy { namespace Extensions { @@ -39,7 +41,7 @@ void Utility::responseFlagsToAccessLogResponseFlags( envoy::data::accesslog::v3::AccessLogCommon& common_access_log, const StreamInfo::StreamInfo& stream_info) { - static_assert(StreamInfo::ResponseFlag::LastFlag == 0x4000000, + static_assert(StreamInfo::ResponseFlag::LastFlag == 0x8000000, "A flag has been added. Fix this code."); if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::FailedLocalHealthCheck)) { @@ -300,21 +302,17 @@ void Utility::extractCommonAccessLogProperties( } for (const auto& key : config.filter_state_objects_to_log()) { - if (auto state = stream_info.filterState().getDataReadOnlyGeneric(key); state != nullptr) { - ProtobufTypes::MessagePtr serialized_proto = state->serializeAsProto(); - if (serialized_proto != nullptr) { - auto& filter_state_objects = *common_access_log.mutable_filter_state_objects(); - ProtobufWkt::Any& any = filter_state_objects[key]; - if (dynamic_cast(serialized_proto.get()) != nullptr) { - any.Swap(dynamic_cast(serialized_proto.get())); - } else { - any.PackFrom(*serialized_proto); - } + if (!(extractFilterStateData(stream_info.filterState(), key, common_access_log))) { + if (stream_info.upstreamInfo().has_value() && + stream_info.upstreamInfo()->upstreamFilterState() != nullptr) { + extractFilterStateData(*(stream_info.upstreamInfo()->upstreamFilterState()), key, + common_access_log); } } } - Tracing::CustomTagContext ctx{&request_header, stream_info}; + Tracing::ReadOnlyHttpTraceContext trace_context(request_header); + Tracing::CustomTagContext ctx{trace_context, stream_info}; for (const auto& custom_tag : config.custom_tags()) { const auto tag_applier = Tracing::CustomTagUtility::createCustomTag(custom_tag); tag_applier->applyLog(common_access_log, ctx); @@ -342,6 +340,24 @@ void Utility::extractCommonAccessLogProperties( common_access_log.set_access_log_type(access_log_type); } +bool extractFilterStateData(const StreamInfo::FilterState& filter_state, const std::string& key, + envoy::data::accesslog::v3::AccessLogCommon& common_access_log) { + if (auto state = filter_state.getDataReadOnlyGeneric(key); state != nullptr) { + ProtobufTypes::MessagePtr serialized_proto = state->serializeAsProto(); + if (serialized_proto != nullptr) { + auto& filter_state_objects = *common_access_log.mutable_filter_state_objects(); + ProtobufWkt::Any& any = filter_state_objects[key]; + if (dynamic_cast(serialized_proto.get()) != nullptr) { + any.Swap(dynamic_cast(serialized_proto.get())); + } else { + any.PackFrom(*serialized_proto); + } + } + return true; + } + return false; +} + } // namespace GrpcCommon } // namespace AccessLoggers } // namespace Extensions diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.h b/source/extensions/access_loggers/grpc/grpc_access_log_utils.h index 9f4f1e07fbd5..beec1e719a8f 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.h @@ -24,6 +24,9 @@ class Utility { const StreamInfo::StreamInfo& stream_info); }; +bool extractFilterStateData(const StreamInfo::FilterState& filter_state, const std::string& key, + envoy::data::accesslog::v3::AccessLogCommon& common_access_log); + } // namespace GrpcCommon } // namespace AccessLoggers } // namespace Extensions diff --git a/source/extensions/access_loggers/grpc/http_config.cc b/source/extensions/access_loggers/grpc/http_config.cc index 7b2e1fbedab9..5e9dc3f07ae3 100644 --- a/source/extensions/access_loggers/grpc/http_config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -18,17 +18,10 @@ namespace Extensions { namespace AccessLoggers { namespace HttpGrpc { -AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); -} - -AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { +AccessLog::InstanceSharedPtr +HttpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { GrpcCommon::validateProtoDescriptors(); const auto& proto_config = MessageUtil::downcastAndValidate< @@ -36,8 +29,8 @@ AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance( config, context.messageValidationVisitor()); return std::make_shared( - std::move(filter), proto_config, context.threadLocal(), - GrpcCommon::getGrpcAccessLoggerCacheSingleton(context)); + std::move(filter), proto_config, context.serverFactoryContext().threadLocal(), + GrpcCommon::getGrpcAccessLoggerCacheSingleton(context.serverFactoryContext())); } ProtobufTypes::MessagePtr HttpGrpcAccessLogFactory::createEmptyConfigProto() { @@ -49,7 +42,7 @@ std::string HttpGrpcAccessLogFactory::name() const { return "envoy.access_logger /** * Static registration for the HTTP gRPC access log. @see RegisterFactory. */ -LEGACY_REGISTER_FACTORY(HttpGrpcAccessLogFactory, Server::Configuration::AccessLogInstanceFactory, +LEGACY_REGISTER_FACTORY(HttpGrpcAccessLogFactory, AccessLog::AccessLogInstanceFactory, "envoy.http_grpc_access_log"); } // namespace HttpGrpc diff --git a/source/extensions/access_loggers/grpc/http_config.h b/source/extensions/access_loggers/grpc/http_config.h index 066f0d495bb2..8615b1cc7c98 100644 --- a/source/extensions/access_loggers/grpc/http_config.h +++ b/source/extensions/access_loggers/grpc/http_config.h @@ -2,7 +2,7 @@ #include -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" namespace Envoy { namespace Extensions { @@ -12,14 +12,11 @@ namespace HttpGrpc { /** * Config registration for the HTTP gRPC access log. @see AccessLogInstanceFactory. */ -class HttpGrpcAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class HttpGrpcAccessLogFactory : public AccessLog::AccessLogInstanceFactory { public: AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc index b89f2cab0134..afa78ac36a43 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -50,17 +50,17 @@ HttpGrpcAccessLog::HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, }); } -void HttpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { +void HttpGrpcAccessLog::emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& stream_info) { // Common log properties. // TODO(mattklein123): Populate sample_rate field. envoy::data::accesslog::v3::HTTPAccessLogEntry log_entry; - GrpcCommon::Utility::extractCommonAccessLogProperties(*log_entry.mutable_common_properties(), - request_headers, stream_info, - config_->common_config(), access_log_type); + + const auto& request_headers = context.requestHeaders(); + + GrpcCommon::Utility::extractCommonAccessLogProperties( + *log_entry.mutable_common_properties(), request_headers, stream_info, + config_->common_config(), context.accessLogType()); if (stream_info.protocol()) { switch (stream_info.protocol().value()) { @@ -135,6 +135,9 @@ void HttpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, } // HTTP response properties. + const auto& response_headers = context.responseHeaders(); + const auto& response_trailers = context.responseTrailers(); + auto* response_properties = log_entry.mutable_response(); if (stream_info.responseCode()) { response_properties->mutable_response_code()->set_value(stream_info.responseCode().value()); diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h index f11d22cbd61c..d99add367f7c 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h @@ -44,11 +44,8 @@ class HttpGrpcAccessLog : public Common::ImplBase { }; // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override; + void emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) override; const HttpGrpcAccessLogConfigConstSharedPtr config_; const ThreadLocal::SlotPtr tls_slot_; diff --git a/source/extensions/access_loggers/grpc/tcp_config.cc b/source/extensions/access_loggers/grpc/tcp_config.cc index 9f1fdb55a737..7aeec7fef813 100644 --- a/source/extensions/access_loggers/grpc/tcp_config.cc +++ b/source/extensions/access_loggers/grpc/tcp_config.cc @@ -18,25 +18,19 @@ namespace Extensions { namespace AccessLoggers { namespace TcpGrpc { -AccessLog::InstanceSharedPtr TcpGrpcAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); -} - -AccessLog::InstanceSharedPtr TcpGrpcAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { +AccessLog::InstanceSharedPtr +TcpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { GrpcCommon::validateProtoDescriptors(); const auto& proto_config = MessageUtil::downcastAndValidate< const envoy::extensions::access_loggers::grpc::v3::TcpGrpcAccessLogConfig&>( config, context.messageValidationVisitor()); - return std::make_shared(std::move(filter), proto_config, context.threadLocal(), - GrpcCommon::getGrpcAccessLoggerCacheSingleton(context)); + return std::make_shared( + std::move(filter), proto_config, context.serverFactoryContext().threadLocal(), + GrpcCommon::getGrpcAccessLoggerCacheSingleton(context.serverFactoryContext())); } ProtobufTypes::MessagePtr TcpGrpcAccessLogFactory::createEmptyConfigProto() { @@ -48,7 +42,7 @@ std::string TcpGrpcAccessLogFactory::name() const { return "envoy.access_loggers /** * Static registration for the TCP gRPC access log. @see RegisterFactory. */ -LEGACY_REGISTER_FACTORY(TcpGrpcAccessLogFactory, Server::Configuration::AccessLogInstanceFactory, +LEGACY_REGISTER_FACTORY(TcpGrpcAccessLogFactory, AccessLog::AccessLogInstanceFactory, "envoy.tcp_grpc_access_log"); } // namespace TcpGrpc diff --git a/source/extensions/access_loggers/grpc/tcp_config.h b/source/extensions/access_loggers/grpc/tcp_config.h index 1e3481e5fea2..7fccd3e84da8 100644 --- a/source/extensions/access_loggers/grpc/tcp_config.h +++ b/source/extensions/access_loggers/grpc/tcp_config.h @@ -2,7 +2,7 @@ #include -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" namespace Envoy { namespace Extensions { @@ -12,15 +12,11 @@ namespace TcpGrpc { /** * Config registration for the TCP gRPC access log. @see AccessLogInstanceFactory. */ -class TcpGrpcAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class TcpGrpcAccessLogFactory : public AccessLog::AccessLogInstanceFactory { public: AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - - AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc index 10f9caa14a5d..288c002a40c5 100644 --- a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc @@ -32,15 +32,13 @@ TcpGrpcAccessLog::TcpGrpcAccessLog(AccessLog::FilterPtr&& filter, }); } -void TcpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_header, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { +void TcpGrpcAccessLog::emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& stream_info) { // Common log properties. envoy::data::accesslog::v3::TCPAccessLogEntry log_entry; - GrpcCommon::Utility::extractCommonAccessLogProperties(*log_entry.mutable_common_properties(), - request_header, stream_info, - config_->common_config(), access_log_type); + GrpcCommon::Utility::extractCommonAccessLogProperties( + *log_entry.mutable_common_properties(), context.requestHeaders(), stream_info, + config_->common_config(), context.accessLogType()); envoy::data::accesslog::v3::ConnectionProperties& connection_properties = *log_entry.mutable_connection_properties(); diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h index 71b5dfd81f9b..0bedd395cf6b 100644 --- a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h @@ -43,11 +43,8 @@ class TcpGrpcAccessLog : public Common::ImplBase { }; // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override; + void emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) override; const TcpGrpcAccessLogConfigConstSharedPtr config_; const ThreadLocal::SlotPtr tls_slot_; diff --git a/source/extensions/access_loggers/open_telemetry/BUILD b/source/extensions/access_loggers/open_telemetry/BUILD index fef41675f552..d782d8252deb 100644 --- a/source/extensions/access_loggers/open_telemetry/BUILD +++ b/source/extensions/access_loggers/open_telemetry/BUILD @@ -62,7 +62,7 @@ envoy_cc_extension( srcs = ["config.cc"], hdrs = ["config.h"], deps = [ - "//envoy/server:access_log_config_interface", + "//envoy/access_log:access_log_config_interface", "//source/common/common:assert_lib", "//source/common/grpc:async_client_lib", "//source/common/protobuf", diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc index 77c72c84fbd5..c742935af18e 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc @@ -76,11 +76,8 @@ AccessLog::AccessLog( attributes_formatter_ = std::make_unique(config.attributes()); } -void AccessLog::emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - Envoy::AccessLog::AccessLogType access_log_type) { +void AccessLog::emitLog(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { opentelemetry::proto::logs::v1::LogRecord log_entry; log_entry.set_time_unix_nano(std::chrono::duration_cast( stream_info.startTime().time_since_epoch()) @@ -88,14 +85,10 @@ void AccessLog::emitLog(const Http::RequestHeaderMap& request_headers, // Unpacking the body "KeyValueList" to "AnyValue". if (body_formatter_) { - const auto formatted_body = - unpackBody(body_formatter_->format(request_headers, response_headers, response_trailers, - stream_info, absl::string_view(), access_log_type)); + const auto formatted_body = unpackBody(body_formatter_->format(log_context, stream_info)); *log_entry.mutable_body() = formatted_body; } - const auto formatted_attributes = - attributes_formatter_->format(request_headers, response_headers, response_trailers, - stream_info, absl::string_view(), access_log_type); + const auto formatted_attributes = attributes_formatter_->format(log_context, stream_info); *log_entry.mutable_attributes() = formatted_attributes.values(); tls_slot_->getTyped().logger_->log(std::move(log_entry)); diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.h b/source/extensions/access_loggers/open_telemetry/access_log_impl.h index be5456d08c8d..7d1013488eb4 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.h @@ -49,11 +49,8 @@ class AccessLog : public Common::ImplBase { }; // Common::ImplBase - void emitLog(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, - Envoy::AccessLog::AccessLogType access_log_type) override; + void emitLog(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) override; const ThreadLocal::SlotPtr tls_slot_; const GrpcAccessLoggerCacheSharedPtr access_logger_cache_; diff --git a/source/extensions/access_loggers/open_telemetry/config.cc b/source/extensions/access_loggers/open_telemetry/config.cc index fd3477689fc2..c371b5b724a4 100644 --- a/source/extensions/access_loggers/open_telemetry/config.cc +++ b/source/extensions/access_loggers/open_telemetry/config.cc @@ -3,7 +3,6 @@ #include "envoy/extensions/access_loggers/open_telemetry/v3/logs_service.pb.h" #include "envoy/extensions/access_loggers/open_telemetry/v3/logs_service.pb.validate.h" #include "envoy/registry/registry.h" -#include "envoy/server/access_log_config.h" #include "envoy/server/filter_config.h" #include "source/common/common/assert.h" @@ -31,26 +30,19 @@ getAccessLoggerCacheSingleton(Server::Configuration::CommonFactoryContext& conte }); } -::Envoy::AccessLog::InstanceSharedPtr AccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, ::Envoy::AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); -} - ::Envoy::AccessLog::InstanceSharedPtr AccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, ::Envoy::AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::FactoryContext& context) { validateProtoDescriptors(); const auto& proto_config = MessageUtil::downcastAndValidate< const envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig&>( config, context.messageValidationVisitor()); - return std::make_shared(std::move(filter), proto_config, context.threadLocal(), - getAccessLoggerCacheSingleton(context)); + return std::make_shared(std::move(filter), proto_config, + context.serverFactoryContext().threadLocal(), + getAccessLoggerCacheSingleton(context.serverFactoryContext())); } ProtobufTypes::MessagePtr AccessLogFactory::createEmptyConfigProto() { @@ -63,8 +55,8 @@ std::string AccessLogFactory::name() const { return "envoy.access_loggers.open_t /** * Static registration for the OpenTelemetry (gRPC) access log. @see RegisterFactory. */ -REGISTER_FACTORY(AccessLogFactory, Server::Configuration::AccessLogInstanceFactory){ - "envoy.open_telemetry_access_log"}; +REGISTER_FACTORY(AccessLogFactory, + Envoy::AccessLog::AccessLogInstanceFactory){"envoy.open_telemetry_access_log"}; } // namespace OpenTelemetry } // namespace AccessLoggers diff --git a/source/extensions/access_loggers/open_telemetry/config.h b/source/extensions/access_loggers/open_telemetry/config.h index f71c541316d2..b82e5b7b6900 100644 --- a/source/extensions/access_loggers/open_telemetry/config.h +++ b/source/extensions/access_loggers/open_telemetry/config.h @@ -2,7 +2,7 @@ #include -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" namespace Envoy { namespace Extensions { @@ -12,15 +12,11 @@ namespace OpenTelemetry { /** * Config registration for the OpenTelemetry (gRPC) access log. @see AccessLogInstanceFactory. */ -class AccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class AccessLogFactory : public Envoy::AccessLog::AccessLogInstanceFactory { public: ::Envoy::AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, ::Envoy::AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - - ::Envoy::AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, ::Envoy::AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc index 2d3c76544469..e48522bc54ef 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc @@ -82,21 +82,17 @@ OpenTelemetryFormatter::FormatBuilder::toFormatStringValue(const std::string& st ::opentelemetry::proto::common::v1::AnyValue OpenTelemetryFormatter::providersCallback( const std::vector& providers, - const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, AccessLog::AccessLogType access_log_type) const { + const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& info) const { ASSERT(!providers.empty()); ::opentelemetry::proto::common::v1::AnyValue output; std::vector bits(providers.size()); - const Formatter::HttpFormatterContext formatter_context{ - &request_headers, &response_headers, &response_trailers, local_reply_body, access_log_type}; + std::transform( + providers.begin(), providers.end(), bits.begin(), + [&](const Formatter::FormatterProviderPtr& provider) { + return provider->formatWithContext(context, info).value_or(DefaultUnspecifiedValueString); + }); - std::transform(providers.begin(), providers.end(), bits.begin(), - [&](const Formatter::FormatterProviderPtr& provider) { - return provider->formatWithContext(formatter_context, stream_info) - .value_or(DefaultUnspecifiedValueString); - }); output.set_string_value(absl::StrJoin(bits, "")); return output; } @@ -128,14 +124,12 @@ OpenTelemetryFormatter::openTelemetryFormatListCallback( return output; } -::opentelemetry::proto::common::v1::KeyValueList OpenTelemetryFormatter::format( - const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body, AccessLog::AccessLogType access_log_type) const { +::opentelemetry::proto::common::v1::KeyValueList +OpenTelemetryFormatter::format(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const { OpenTelemetryFormatMapVisitor visitor{ [&](const std::vector& providers) { - return providersCallback(providers, request_headers, response_headers, response_trailers, - stream_info, local_reply_body, access_log_type); + return providersCallback(providers, context, info); }, [&, this](const OpenTelemetryFormatter::OpenTelemetryFormatMapWrapper& format_map) { return openTelemetryFormatMapCallback(format_map, visitor); diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h index 3ae2188f8461..ea797fff13e2 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h @@ -29,11 +29,7 @@ class OpenTelemetryFormatter { OpenTelemetryFormatter(const ::opentelemetry::proto::common::v1::KeyValueList& format_mapping); ::opentelemetry::proto::common::v1::KeyValueList - format(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type) const; + format(const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& info) const; private: struct OpenTelemetryFormatMapWrapper; @@ -77,11 +73,8 @@ class OpenTelemetryFormatter { // Methods for doing the actual formatting. ::opentelemetry::proto::common::v1::AnyValue providersCallback(const std::vector& providers, - const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, - const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body, - AccessLog::AccessLogType access_log_type) const; + const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) const; ::opentelemetry::proto::common::v1::AnyValue openTelemetryFormatMapCallback( const OpenTelemetryFormatter::OpenTelemetryFormatMapWrapper& format_map, const OpenTelemetryFormatMapVisitor& visitor) const; diff --git a/source/extensions/access_loggers/stream/config.cc b/source/extensions/access_loggers/stream/config.cc index e9159e6f3556..37409202a97c 100644 --- a/source/extensions/access_loggers/stream/config.cc +++ b/source/extensions/access_loggers/stream/config.cc @@ -20,17 +20,10 @@ namespace Extensions { namespace AccessLoggers { namespace File { -AccessLog::InstanceSharedPtr StdoutAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); -} - -AccessLog::InstanceSharedPtr StdoutAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { +AccessLog::InstanceSharedPtr +StdoutAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { return AccessLoggers::createStreamAccessLogInstance< envoy::extensions::access_loggers::stream::v3::StdoutAccessLog, Filesystem::DestinationType::Stdout>(config, std::move(filter), context); @@ -46,20 +39,13 @@ std::string StdoutAccessLogFactory::name() const { return "envoy.access_loggers. /** * Static registration for the file access log. @see RegisterFactory. */ -LEGACY_REGISTER_FACTORY(StdoutAccessLogFactory, Server::Configuration::AccessLogInstanceFactory, +LEGACY_REGISTER_FACTORY(StdoutAccessLogFactory, AccessLog::AccessLogInstanceFactory, "envoy.stdout_access_log"); -AccessLog::InstanceSharedPtr StderrAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - config, std::move(filter), - static_cast(context)); -} - -AccessLog::InstanceSharedPtr StderrAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { +AccessLog::InstanceSharedPtr +StderrAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { return createStreamAccessLogInstance< envoy::extensions::access_loggers::stream::v3::StderrAccessLog, Filesystem::DestinationType::Stderr>(config, std::move(filter), context); @@ -75,7 +61,7 @@ std::string StderrAccessLogFactory::name() const { return "envoy.access_loggers. /** * Static registration for the `stderr` access log. @see RegisterFactory. */ -LEGACY_REGISTER_FACTORY(StderrAccessLogFactory, Server::Configuration::AccessLogInstanceFactory, +LEGACY_REGISTER_FACTORY(StderrAccessLogFactory, AccessLog::AccessLogInstanceFactory, "envoy.stderr_access_log"); } // namespace File diff --git a/source/extensions/access_loggers/stream/config.h b/source/extensions/access_loggers/stream/config.h index 6fb9522fd229..c50cb12ab7b3 100644 --- a/source/extensions/access_loggers/stream/config.h +++ b/source/extensions/access_loggers/stream/config.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" namespace Envoy { namespace Extensions { @@ -10,15 +10,11 @@ namespace File { /** * Config registration for the standard output access log. @see AccessLogInstanceFactory. */ -class StdoutAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class StdoutAccessLogFactory : public AccessLog::AccessLogInstanceFactory { public: AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - - AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; @@ -28,15 +24,11 @@ class StdoutAccessLogFactory : public Server::Configuration::AccessLogInstanceFa /** * Config registration for the standard error access log. @see AccessLogInstanceFactory. */ -class StderrAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class StderrAccessLogFactory : public AccessLog::AccessLogInstanceFactory { public: AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - - AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/access_loggers/wasm/BUILD b/source/extensions/access_loggers/wasm/BUILD index 5765746237c4..077e8f0239ec 100644 --- a/source/extensions/access_loggers/wasm/BUILD +++ b/source/extensions/access_loggers/wasm/BUILD @@ -28,8 +28,8 @@ envoy_cc_extension( hdrs = ["config.h"], deps = [ ":wasm_access_log_lib", + "//envoy/access_log:access_log_config_interface", "//envoy/registry", - "//envoy/server:access_log_config_interface", "//source/common/config:datasource_lib", "//source/common/protobuf", "//source/extensions/common/wasm:wasm_lib", diff --git a/source/extensions/access_loggers/wasm/config.cc b/source/extensions/access_loggers/wasm/config.cc index e9c2772f92d5..f9559951b0c8 100644 --- a/source/extensions/access_loggers/wasm/config.cc +++ b/source/extensions/access_loggers/wasm/config.cc @@ -16,31 +16,24 @@ namespace Wasm { using Common::Wasm::PluginHandleSharedPtrThreadLocal; -AccessLog::InstanceSharedPtr WasmAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& proto_config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) { - return createAccessLogInstance( - proto_config, std::move(filter), - static_cast(context)); -} - -AccessLog::InstanceSharedPtr WasmAccessLogFactory::createAccessLogInstance( - const Protobuf::Message& proto_config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) { +AccessLog::InstanceSharedPtr +WasmAccessLogFactory::createAccessLogInstance(const Protobuf::Message& proto_config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { const auto& config = MessageUtil::downcastAndValidate< const envoy::extensions::access_loggers::wasm::v3::WasmAccessLog&>( proto_config, context.messageValidationVisitor()); auto plugin = std::make_shared( - config.config(), envoy::config::core::v3::TrafficDirection::UNSPECIFIED, context.localInfo(), - nullptr /* listener_metadata */); + config.config(), envoy::config::core::v3::TrafficDirection::UNSPECIFIED, + context.serverFactoryContext().localInfo(), nullptr /* listener_metadata */); auto access_log = std::make_shared(plugin, nullptr, std::move(filter)); auto callback = [access_log, &context, plugin](Common::Wasm::WasmHandleSharedPtr base_wasm) { // NB: the Slot set() call doesn't complete inline, so all arguments must outlive this call. - auto tls_slot = - ThreadLocal::TypedSlot::makeUnique(context.threadLocal()); + auto tls_slot = ThreadLocal::TypedSlot::makeUnique( + context.serverFactoryContext().threadLocal()); tls_slot->set([base_wasm, plugin](Event::Dispatcher& dispatcher) { return std::make_shared( Common::Wasm::getOrCreateThreadLocalPlugin(base_wasm, plugin, dispatcher)); @@ -48,15 +41,16 @@ AccessLog::InstanceSharedPtr WasmAccessLogFactory::createAccessLogInstance( access_log->setTlsSlot(std::move(tls_slot)); }; - if (!Common::Wasm::createWasm(plugin, context.scope().createScope(""), context.clusterManager(), - context.initManager(), context.mainThreadDispatcher(), - context.api(), context.lifecycleNotifier(), remote_data_provider_, - std::move(callback))) { + if (!Common::Wasm::createWasm( + plugin, context.scope().createScope(""), context.serverFactoryContext().clusterManager(), + context.initManager(), context.serverFactoryContext().mainThreadDispatcher(), + context.serverFactoryContext().api(), context.serverFactoryContext().lifecycleNotifier(), + remote_data_provider_, std::move(callback))) { throw Common::Wasm::WasmException( fmt::format("Unable to create Wasm access log {}", plugin->name_)); } - context.api().customStatNamespaces().registerStatNamespace( + context.serverFactoryContext().api().customStatNamespaces().registerStatNamespace( Extensions::Common::Wasm::CustomStatNamespace); return access_log; } @@ -71,7 +65,7 @@ std::string WasmAccessLogFactory::name() const { return "envoy.access_loggers.wa /** * Static registration for the wasm access log. @see RegisterFactory. */ -LEGACY_REGISTER_FACTORY(WasmAccessLogFactory, Server::Configuration::AccessLogInstanceFactory, +LEGACY_REGISTER_FACTORY(WasmAccessLogFactory, Envoy::AccessLog::AccessLogInstanceFactory, "envoy.wasm_access_log"); } // namespace Wasm diff --git a/source/extensions/access_loggers/wasm/config.h b/source/extensions/access_loggers/wasm/config.h index e198f906f489..e6b69e951f1d 100644 --- a/source/extensions/access_loggers/wasm/config.h +++ b/source/extensions/access_loggers/wasm/config.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/server/access_log_config.h" +#include "envoy/access_log/access_log_config.h" #include "source/common/config/datasource.h" @@ -12,16 +12,12 @@ namespace Wasm { /** * Config registration for the file access log. @see AccessLogInstanceFactory. */ -class WasmAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory, +class WasmAccessLogFactory : public AccessLog::AccessLogInstanceFactory, Logger::Loggable { public: AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::ListenerAccessLogFactoryContext& context) override; - - AccessLog::InstanceSharedPtr - createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::FactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; diff --git a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h index 21dd30273eeb..fbfb17fa70dc 100644 --- a/source/extensions/access_loggers/wasm/wasm_access_log_impl.h +++ b/source/extensions/access_loggers/wasm/wasm_access_log_impl.h @@ -20,28 +20,20 @@ class WasmAccessLog : public AccessLog::Instance { AccessLog::FilterPtr filter) : plugin_(plugin), tls_slot_(std::move(tls_slot)), filter_(std::move(filter)) {} - void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override { - if (filter_ && request_headers && response_headers && response_trailers) { - if (!filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers, - access_log_type)) { + void log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) override { + if (filter_) { + if (!filter_->evaluate(log_context, stream_info)) { return; } } - if (!tls_slot_) { - return; - } - auto handle = tls_slot_->get()->handle(); - if (!handle) { - return; - } - if (handle->wasmHandle()) { - handle->wasmHandle()->wasm()->log(plugin_, request_headers, response_headers, - response_trailers, stream_info, access_log_type); + if (tls_slot_ != nullptr) { + if (auto handle = tls_slot_->get()->handle(); handle != nullptr) { + if (handle->wasmHandle()) { + handle->wasmHandle()->wasm()->log(plugin_, log_context, stream_info); + } + } } } diff --git a/source/extensions/all_extensions.bzl b/source/extensions/all_extensions.bzl index 1a18c2cf44e1..8cbdeb844d81 100644 --- a/source/extensions/all_extensions.bzl +++ b/source/extensions/all_extensions.bzl @@ -7,6 +7,8 @@ _required_extensions = { "envoy.http.original_ip_detection.xff": "//source/extensions/http/original_ip_detection/xff:config", "envoy.request_id.uuid": "//source/extensions/request_id/uuid:config", "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", + # To provide default round robin load balancer. + "envoy.load_balancing_policies.round_robin": "//source/extensions/load_balancing_policies/round_robin:config", } # Return the extension cc_library target after select @@ -33,6 +35,7 @@ _core_extensions = [ "envoy.transport_sockets.raw_buffer", "envoy.network.dns_resolver.cares", "envoy.network.dns_resolver.apple", + "envoy.load_balancing_policies.round_robin", ] # Return all core extensions to be compiled into Envoy. diff --git a/source/extensions/bootstrap/internal_listener/BUILD b/source/extensions/bootstrap/internal_listener/BUILD index b5d4d9d6535b..d15e5072d4fa 100644 --- a/source/extensions/bootstrap/internal_listener/BUILD +++ b/source/extensions/bootstrap/internal_listener/BUILD @@ -30,7 +30,7 @@ envoy_cc_extension( deps = [ ":thread_local_registry", "//envoy/server:bootstrap_extension_config_interface", - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//source/common/listener_manager:listener_manager_lib", "@envoy_api//envoy/extensions/bootstrap/internal_listener/v3:pkg_cc_proto", ], alwayslink = 1, @@ -93,12 +93,12 @@ envoy_cc_library( "//source/common/common:linked_object", "//source/common/common:non_copyable", "//source/common/event:deferred_task", + "//source/common/listener_manager:active_stream_listener_base", + "//source/common/listener_manager:active_tcp_listener", "//source/common/network:connection_lib", "//source/common/network:listener_filter_buffer_lib", "//source/common/stats:timespan_lib", "//source/common/stream_info:stream_info_lib", "//source/extensions/io_socket/user_space:config", - "//source/extensions/listener_managers/listener_manager:active_stream_listener_base", - "//source/extensions/listener_managers/listener_manager:active_tcp_listener", ], ) diff --git a/source/extensions/bootstrap/internal_listener/active_internal_listener.cc b/source/extensions/bootstrap/internal_listener/active_internal_listener.cc index d14a80dccf3c..f2fe0a069567 100644 --- a/source/extensions/bootstrap/internal_listener/active_internal_listener.cc +++ b/source/extensions/bootstrap/internal_listener/active_internal_listener.cc @@ -3,10 +3,10 @@ #include "envoy/network/filter.h" #include "envoy/stats/scope.h" +#include "source/common/listener_manager/active_stream_listener_base.h" #include "source/common/network/address_impl.h" #include "source/common/stats/timespan_impl.h" #include "source/extensions/io_socket/user_space/io_handle.h" -#include "source/extensions/listener_managers/listener_manager/active_stream_listener_base.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/bootstrap/internal_listener/active_internal_listener.h b/source/extensions/bootstrap/internal_listener/active_internal_listener.h index 03ef517b36cf..2d71d771e43d 100644 --- a/source/extensions/bootstrap/internal_listener/active_internal_listener.h +++ b/source/extensions/bootstrap/internal_listener/active_internal_listener.h @@ -17,8 +17,8 @@ #include "source/common/common/linked_object.h" #include "source/common/common/non_copyable.h" +#include "source/common/listener_manager/active_stream_listener_base.h" #include "source/common/stream_info/stream_info_impl.h" -#include "source/extensions/listener_managers/listener_manager/active_stream_listener_base.h" #include "spdlog/spdlog.h" diff --git a/source/extensions/bootstrap/internal_listener/client_connection_factory.cc b/source/extensions/bootstrap/internal_listener/client_connection_factory.cc index 6ba936911f60..8c218bdd5e21 100644 --- a/source/extensions/bootstrap/internal_listener/client_connection_factory.cc +++ b/source/extensions/bootstrap/internal_listener/client_connection_factory.cc @@ -61,8 +61,8 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec return client_conn; } - auto accepted_socket = std::make_unique(std::move(io_handle_server), - address, source_address); + auto accepted_socket = std::make_unique( + std::move(io_handle_server), address, source_address, absl::nullopt, false); internal_listener->onAccept(std::move(accepted_socket)); return client_conn; } diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index 939a77cf5726..1200a49052da 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -53,12 +53,12 @@ REGISTER_FACTORY(DynamicPortObjectFactory, StreamInfo::FilterState::ObjectFactor Cluster::Cluster( const envoy::config::cluster::v3::Cluster& cluster, + Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr&& cache, const envoy::extensions::clusters::dynamic_forward_proxy::v3::ClusterConfig& config, Upstream::ClusterFactoryContext& context, - Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory) + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr&& cache_manager) : Upstream::BaseDynamicClusterImpl(cluster, context), - dns_cache_manager_(cache_manager_factory.get()), - dns_cache_(dns_cache_manager_->getCache(config.dns_cache_config())), + dns_cache_manager_(std::move(cache_manager)), dns_cache_(std::move(cache)), update_callbacks_handle_(dns_cache_->addUpdateCallbacks(*this)), local_info_(context.serverFactoryContext().localInfo()), main_thread_dispatcher_(context.serverFactoryContext().mainThreadDispatcher()), @@ -478,18 +478,9 @@ ClusterFactory::createClusterWithConfig( const envoy::extensions::clusters::dynamic_forward_proxy::v3::ClusterConfig& proto_config, Upstream::ClusterFactoryContext& context) { - auto& server_context = context.serverFactoryContext(); - - // The message validation visitor of Upstream::ClusterFactoryContext should be used for - // validating the cluster config. - Server::FactoryContextBaseImpl factory_context_base( - server_context.options(), server_context.mainThreadDispatcher(), server_context.api(), - server_context.localInfo(), server_context.admin(), server_context.runtime(), - server_context.singletonManager(), context.messageValidationVisitor(), - server_context.serverScope().store(), server_context.threadLocal()); - Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( - factory_context_base); + context.serverFactoryContext(), context.messageValidationVisitor()); + envoy::config::cluster::v3::Cluster cluster_config = cluster; if (!cluster_config.has_upstream_http_protocol_options()) { // This sets defaults which will only apply if using old style http config. @@ -499,11 +490,17 @@ ClusterFactory::createClusterWithConfig( cluster_config.mutable_upstream_http_protocol_options()->set_auto_san_validation(true); } - auto new_cluster = std::shared_ptr( - new Cluster(cluster_config, proto_config, context, cache_manager_factory)); + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr cache_manager = + cache_manager_factory.get(); + auto dns_cache_or_error = cache_manager->getCache(proto_config.dns_cache_config()); + RETURN_IF_STATUS_NOT_OK(dns_cache_or_error); + + auto new_cluster = + std::shared_ptr(new Cluster(cluster_config, std::move(dns_cache_or_error.value()), + proto_config, context, std::move(cache_manager))); Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( - factory_context_base); + context.serverFactoryContext().singletonManager()); cluster_store_factory.get()->save(new_cluster->info()->name(), new_cluster); auto& options = new_cluster->info()->upstreamHttpProtocolOptions(); diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h index ebb61d344d50..f974474b3e90 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.h +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h @@ -57,9 +57,10 @@ class Cluster : public Upstream::BaseDynamicClusterImpl, friend class ClusterTest; Cluster(const envoy::config::cluster::v3::Cluster& cluster, + Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr&& cacahe, const envoy::extensions::clusters::dynamic_forward_proxy::v3::ClusterConfig& config, Upstream::ClusterFactoryContext& context, - Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory); + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr&& cache_manager); struct ClusterInfo { ClusterInfo(std::string cluster_name, Cluster& parent); void touch(); diff --git a/source/extensions/clusters/eds/eds.cc b/source/extensions/clusters/eds/eds.cc index c2d1c504e5df..962b0f249e49 100644 --- a/source/extensions/clusters/eds/eds.cc +++ b/source/extensions/clusters/eds/eds.cc @@ -163,22 +163,31 @@ void EdsClusterImpl::BatchUpdateHelper::updateLocalityEndpoints( absl::Status EdsClusterImpl::onConfigUpdate(const std::vector& resources, const std::string&) { - if (!validateUpdateSize(resources.size())) { + if (resources.empty()) { + ENVOY_LOG(debug, "Missing ClusterLoadAssignment for {} in onConfigUpdate()", edsServiceName()); + info_->configUpdateStats().update_empty_.inc(); + onPreInitComplete(); return absl::OkStatus(); } + if (resources.size() != 1) { + return absl::InvalidArgumentError( + fmt::format("Unexpected EDS resource length: {}", resources.size())); + } + envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment = dynamic_cast( resources[0].get().resource()); if (cluster_load_assignment.cluster_name() != edsServiceName()) { - throw EnvoyException(fmt::format("Unexpected EDS cluster (expecting {}): {}", edsServiceName(), - cluster_load_assignment.cluster_name())); + return absl::InvalidArgumentError(fmt::format("Unexpected EDS cluster (expecting {}): {}", + edsServiceName(), + cluster_load_assignment.cluster_name())); } // Validate that each locality doesn't have both LEDS and endpoints defined. // TODO(adisuissa): This is only needed for the API v3 support. In future major versions // the oneof definition will take care of it. for (const auto& locality : cluster_load_assignment.endpoints()) { if (locality.has_leds_cluster_locality_config() && locality.lb_endpoints_size() > 0) { - throw EnvoyException(fmt::format( + return absl::InvalidArgumentError(fmt::format( "A ClusterLoadAssignment for cluster {} cannot include both LEDS (resource: {}) and a " "list of endpoints.", edsServiceName(), locality.leds_cluster_locality_config().leds_collection_name())); @@ -205,6 +214,12 @@ EdsClusterImpl::onConfigUpdate(const std::vector& re } } + // Drop overload configuration parsing. + absl::Status status = parseDropOverloadConfig(cluster_load_assignment); + if (!status.ok()) { + return status; + } + // Pause LEDS messages until the EDS config is finished processing. Config::ScopedResume maybe_resume_leds; if (transport_factory_context_->clusterManager().adsMux()) { @@ -287,24 +302,7 @@ void EdsClusterImpl::update( absl::Status EdsClusterImpl::onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField&, const std::string&) { - if (!validateUpdateSize(added_resources.size())) { - return absl::OkStatus(); - } - return onConfigUpdate(added_resources, added_resources[0].get().version()); -} - -bool EdsClusterImpl::validateUpdateSize(int num_resources) { - if (num_resources == 0) { - ENVOY_LOG(debug, "Missing ClusterLoadAssignment for {} in onConfigUpdate()", edsServiceName()); - info_->configUpdateStats().update_empty_.inc(); - onPreInitComplete(); - return false; - } - if (num_resources != 1) { - throw EnvoyException(fmt::format("Unexpected EDS resource length: {}", num_resources)); - // (would be a return false here) - } - return true; + return onConfigUpdate(added_resources, ""); } void EdsClusterImpl::onAssignmentTimeout() { diff --git a/source/extensions/clusters/eds/eds.h b/source/extensions/clusters/eds/eds.h index b20ec27bf0bb..18cfdb39c913 100644 --- a/source/extensions/clusters/eds/eds.h +++ b/source/extensions/clusters/eds/eds.h @@ -35,7 +35,7 @@ class EdsClusterImpl public: EdsClusterImpl(const envoy::config::cluster::v3::Cluster& cluster, ClusterFactoryContext& cluster_context); - virtual ~EdsClusterImpl(); + ~EdsClusterImpl() override; // Upstream::Cluster InitializePhase initializePhase() const override { return initialize_phase_; } diff --git a/source/extensions/clusters/logical_dns/logical_dns_cluster.cc b/source/extensions/clusters/logical_dns/logical_dns_cluster.cc index de67ee077bb1..f1e84526b68f 100644 --- a/source/extensions/clusters/logical_dns/logical_dns_cluster.cc +++ b/source/extensions/clusters/logical_dns/logical_dns_cluster.cc @@ -90,7 +90,7 @@ LogicalDnsCluster::~LogicalDnsCluster() { } void LogicalDnsCluster::startResolve() { - ENVOY_LOG(debug, "starting async DNS resolution for {}", dns_address_); + ENVOY_LOG(trace, "starting async DNS resolution for {}", dns_address_); info_->configUpdateStats().update_attempt_.inc(); active_dns_query_ = dns_resolver_->resolve( @@ -98,7 +98,7 @@ void LogicalDnsCluster::startResolve() { [this](Network::DnsResolver::ResolutionStatus status, std::list&& response) -> void { active_dns_query_ = nullptr; - ENVOY_LOG(debug, "async DNS resolution complete for {}", dns_address_); + ENVOY_LOG(trace, "async DNS resolution complete for {}", dns_address_); std::chrono::milliseconds final_refresh_rate = dns_refresh_rate_ms_; diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.cc b/source/extensions/clusters/original_dst/original_dst_cluster.cc index 3cee7f5c49b0..2b3313862933 100644 --- a/source/extensions/clusters/original_dst/original_dst_cluster.cc +++ b/source/extensions/clusters/original_dst/original_dst_cluster.cc @@ -109,9 +109,8 @@ OriginalDstCluster::LoadBalancer::filterStateOverrideHost(LoadBalancerContext* c if (streamInfo == nullptr) { continue; } - const auto* dst_address = - streamInfo->filterState().getDataReadOnly( - Network::DestinationAddress::key()); + const auto* dst_address = streamInfo->filterState().getDataReadOnly( + OriginalDstClusterFilterStateKey); if (dst_address) { return dst_address->address(); } @@ -354,5 +353,12 @@ OriginalDstClusterFactory::createClusterImpl(const envoy::config::cluster::v3::C */ REGISTER_FACTORY(OriginalDstClusterFactory, ClusterFactory); +class OriginalDstClusterFilterStateFactory : public Network::BaseAddressObjectFactory { +public: + std::string name() const override { return std::string(OriginalDstClusterFilterStateKey); } +}; + +REGISTER_FACTORY(OriginalDstClusterFilterStateFactory, StreamInfo::FilterState::ObjectFactory); + } // namespace Upstream } // namespace Envoy diff --git a/source/extensions/clusters/original_dst/original_dst_cluster.h b/source/extensions/clusters/original_dst/original_dst_cluster.h index db8990353b15..68a44cf5b945 100644 --- a/source/extensions/clusters/original_dst/original_dst_cluster.h +++ b/source/extensions/clusters/original_dst/original_dst_cluster.h @@ -185,6 +185,9 @@ class OriginalDstCluster : public ClusterImplBase { friend class OriginalDstClusterHandle; }; +constexpr absl::string_view OriginalDstClusterFilterStateKey = + "envoy.network.transport_socket.original_dst_address"; + class OriginalDstClusterFactory : public ClusterFactoryImplBase { public: OriginalDstClusterFactory() : ClusterFactoryImplBase("envoy.cluster.original_dst") {} diff --git a/source/extensions/common/aws/BUILD b/source/extensions/common/aws/BUILD index 96382e2095c2..1be1caab105d 100644 --- a/source/extensions/common/aws/BUILD +++ b/source/extensions/common/aws/BUILD @@ -2,6 +2,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_extension_package", + "envoy_select_boringssl", ) licenses(["notice"]) # Apache 2 @@ -40,6 +41,18 @@ envoy_cc_library( external_deps = ["abseil_optional"], ) +envoy_cc_library( + name = "metadata_fetcher_lib", + srcs = ["metadata_fetcher.cc"], + hdrs = ["metadata_fetcher.h"], + deps = [ + ":utility_lib", + "//envoy/upstream:cluster_manager_interface", + "//source/common/http:utility_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "credentials_provider_impl_lib", srcs = ["credentials_provider_impl.cc"], @@ -47,12 +60,16 @@ envoy_cc_library( external_deps = ["abseil_time"], deps = [ ":credentials_provider_interface", + ":metadata_fetcher_lib", ":utility_lib", "//envoy/api:api_interface", "//source/common/common:logger_lib", "//source/common/common:thread_lib", "//source/common/http:utility_lib", + "//source/common/init:target_lib", "//source/common/json:json_loader_lib", + "//source/common/runtime:runtime_features_lib", + "//source/common/tracing:http_tracer_lib", ], ) @@ -60,13 +77,23 @@ envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], hdrs = ["utility.h"], + copts = envoy_select_boringssl( + [ + "-DENVOY_SSL_FIPS", + ], + ), external_deps = ["curl"], deps = [ "//envoy/http:message_interface", + "//envoy/upstream:cluster_manager_interface", "//source/common/common:empty_string", "//source/common/common:matchers_lib", "//source/common/common:utility_lib", "//source/common/http:headers_lib", + "//source/common/http:utility_lib", + "//source/common/runtime:runtime_features_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/upstreams/http/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/common/aws/credentials_provider.h b/source/extensions/common/aws/credentials_provider.h index 9de0fe8b7a4d..dc06c0c77988 100644 --- a/source/extensions/common/aws/credentials_provider.h +++ b/source/extensions/common/aws/credentials_provider.h @@ -68,6 +68,8 @@ class CredentialsProvider { virtual Credentials getCredentials() PURE; }; +using CredentialsConstSharedPtr = std::shared_ptr; +using CredentialsConstUniquePtr = std::unique_ptr; using CredentialsProviderSharedPtr = std::shared_ptr; } // namespace Aws diff --git a/source/extensions/common/aws/credentials_provider_impl.cc b/source/extensions/common/aws/credentials_provider_impl.cc index 9cba9e82c265..731b155f2e5e 100644 --- a/source/extensions/common/aws/credentials_provider_impl.cc +++ b/source/extensions/common/aws/credentials_provider_impl.cc @@ -9,6 +9,7 @@ #include "source/common/http/utility.h" #include "source/common/json/json_loader.h" #include "source/common/runtime/runtime_features.h" +#include "source/common/tracing/http_tracer_impl.h" #include "source/extensions/common/aws/utility.h" #include "absl/strings/str_format.h" @@ -24,13 +25,20 @@ namespace { constexpr char AWS_ACCESS_KEY_ID[] = "AWS_ACCESS_KEY_ID"; constexpr char AWS_SECRET_ACCESS_KEY[] = "AWS_SECRET_ACCESS_KEY"; constexpr char AWS_SESSION_TOKEN[] = "AWS_SESSION_TOKEN"; +constexpr char AWS_ROLE_ARN[] = "AWS_ROLE_ARN"; +constexpr char AWS_WEB_IDENTITY_TOKEN_FILE[] = "AWS_WEB_IDENTITY_TOKEN_FILE"; +constexpr char AWS_ROLE_SESSION_NAME[] = "AWS_ROLE_SESSION_NAME"; +constexpr char CREDENTIALS[] = "Credentials"; constexpr char ACCESS_KEY_ID[] = "AccessKeyId"; constexpr char SECRET_ACCESS_KEY[] = "SecretAccessKey"; constexpr char TOKEN[] = "Token"; constexpr char EXPIRATION[] = "Expiration"; constexpr char EXPIRATION_FORMAT[] = "%E4Y-%m-%dT%H:%M:%S%z"; constexpr char TRUE[] = "true"; +constexpr char SESSION_TOKEN[] = "SessionToken"; +constexpr char WEB_IDENTITY_RESPONSE_ELEMENT[] = "AssumeRoleWithWebIdentityResponse"; +constexpr char WEB_IDENTITY_RESULT_ELEMENT[] = "AssumeRoleWithWebIdentityResult"; constexpr char AWS_SHARED_CREDENTIALS_FILE[] = "AWS_SHARED_CREDENTIALS_FILE"; constexpr char AWS_PROFILE[] = "AWS_PROFILE"; @@ -52,6 +60,10 @@ constexpr char EC2_IMDS_TOKEN_TTL_HEADER[] = "X-aws-ec2-metadata-token-ttl-secon constexpr char EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE[] = "21600"; constexpr char SECURITY_CREDENTIALS_PATH[] = "/latest/meta-data/iam/security-credentials"; +constexpr char EC2_METADATA_CLUSTER[] = "ec2_instance_metadata_server_internal"; +constexpr char CONTAINER_METADATA_CLUSTER[] = "ecs_task_metadata_server_internal"; +constexpr char STS_TOKEN_CLUSTER[] = "sts_token_service_internal"; + } // namespace Credentials EnvironmentCredentialsProvider::getCredentials() { @@ -80,6 +92,96 @@ void CachedCredentialsProviderBase::refreshIfNeeded() { } } +// TODO(suniltheta): The field context is of type ServerFactoryContextOptRef so that an +// optional empty value can be set. Especially in aws iam plugin the cluster manager +// obtained from server factory context object is not fully initialized due to the +// reasons explained in https://github.com/envoyproxy/envoy/issues/27586 which cannot +// utilize http async client here to fetch AWS credentials. For time being if context +// is empty then will use libcurl to fetch the credentials. +MetadataCredentialsProviderBase::MetadataCredentialsProviderBase( + Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, + const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, absl::string_view uri) + : api_(api), context_(context), fetch_metadata_using_curl_(fetch_metadata_using_curl), + create_metadata_fetcher_cb_(create_metadata_fetcher_cb), + cluster_name_(std::string(cluster_name)), cluster_type_(cluster_type), uri_(std::string(uri)), + cache_duration_(getCacheDuration()), + debug_name_(absl::StrCat("Fetching aws credentials from cluster=", cluster_name)) { + if (context_) { + context_->mainThreadDispatcher().post([this]() { + if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, + cluster_type_, uri_)) { + ENVOY_LOG(critical, + "Failed to add [STATIC cluster = {} with address = {}] or cluster not found", + cluster_name_, uri_); + return; + } + }); + + tls_ = ThreadLocal::TypedSlot::makeUnique(context_->threadLocal()); + tls_->set( + [](Envoy::Event::Dispatcher&) { return std::make_shared(); }); + + cache_duration_timer_ = context_->mainThreadDispatcher().createTimer([this]() -> void { + if (useHttpAsyncClient()) { + const Thread::LockGuard lock(lock_); + refresh(); + } + }); + + if (useHttpAsyncClient()) { + // Register with init_manager, force the listener to wait for fetching (refresh). + init_target_ = + std::make_unique(debug_name_, [this]() -> void { refresh(); }); + context_->initManager().add(*init_target_); + } + } +} + +Credentials MetadataCredentialsProviderBase::getCredentials() { + refreshIfNeeded(); + if (useHttpAsyncClient() && context_ && tls_) { + // If server factor context was supplied then we would have thread local slot initialized. + return *(*tls_)->credentials_.get(); + } else { + return cached_credentials_; + } +} + +std::chrono::seconds MetadataCredentialsProviderBase::getCacheDuration() { + return std::chrono::seconds( + REFRESH_INTERVAL * 60 * 60 - + REFRESH_GRACE_PERIOD /*TODO: Add jitter from context.api().randomGenerator()*/); +} + +void MetadataCredentialsProviderBase::handleFetchDone() { + if (useHttpAsyncClient() && context_) { + if (init_target_) { + init_target_->ready(); + init_target_.reset(); + } + if (cache_duration_timer_ && !cache_duration_timer_->enabled()) { + cache_duration_timer_->enableTimer(cache_duration_); + } + } +} + +void MetadataCredentialsProviderBase::setCredentialsToAllThreads( + CredentialsConstUniquePtr&& creds) { + CredentialsConstSharedPtr shared_credentials = std::move(creds); + if (tls_) { + tls_->runOnAllThreads([shared_credentials](OptRef obj) { + obj->credentials_ = shared_credentials; + }); + } +} + +bool MetadataCredentialsProviderBase::useHttpAsyncClient() { + return Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.use_http_client_to_fetch_aws_credentials"); +} + bool CredentialsFileCredentialsProvider::needsRefresh() { return api_.timeSource().systemTime() - last_updated_ > REFRESH_INTERVAL; } @@ -161,6 +263,14 @@ void CredentialsFileCredentialsProvider::extractCredentials(const std::string& c last_updated_ = api_.timeSource().systemTime(); } +InstanceProfileCredentialsProvider::InstanceProfileCredentialsProvider( + Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name) + : MetadataCredentialsProviderBase( + api, context, fetch_metadata_using_curl, create_metadata_fetcher_cb, cluster_name, + envoy::config::cluster::v3::Cluster::STATIC /*cluster_type*/, EC2_METADATA_HOST) {} + bool InstanceProfileCredentialsProvider::needsRefresh() { return api_.timeSource().systemTime() - last_updated_ > REFRESH_INTERVAL; } @@ -171,24 +281,49 @@ void InstanceProfileCredentialsProvider::refresh() { // First request for a session TOKEN so that we can call EC2MetadataService securely. // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html Http::RequestMessageImpl token_req_message; + token_req_message.headers().setScheme(Http::Headers::get().SchemeValues.Http); token_req_message.headers().setMethod(Http::Headers::get().MethodValues.Put); token_req_message.headers().setHost(EC2_METADATA_HOST); token_req_message.headers().setPath(EC2_IMDS_TOKEN_RESOURCE); token_req_message.headers().setCopy(Http::LowerCaseString(EC2_IMDS_TOKEN_TTL_HEADER), EC2_IMDS_TOKEN_TTL_DEFAULT_VALUE); - const auto token_string = metadata_fetcher_(token_req_message); - if (token_string) { - ENVOY_LOG(debug, "Obtained token to make secure call to EC2MetadataService"); - fetchInstanceRole(token_string.value()); + + if (!useHttpAsyncClient() || !context_) { + // Using curl to fetch the AWS credentials where we first get the token. + const auto token_string = fetch_metadata_using_curl_(token_req_message); + if (token_string) { + ENVOY_LOG(debug, "Obtained token to make secure call to EC2MetadataService"); + fetchInstanceRole(std::move(token_string.value())); + } else { + ENVOY_LOG(warn, + "Failed to get token from EC2MetadataService, falling back to less secure way"); + fetchInstanceRole(std::move("")); + } } else { - ENVOY_LOG(warn, "Failed to get token from EC2MetadataService, falling back to less secure way"); - fetchInstanceRole(""); + // Stop any existing timer. + if (cache_duration_timer_ && cache_duration_timer_->enabled()) { + cache_duration_timer_->disableTimer(); + } + // Using Http async client to fetch the AWS credentials where we first get the token. + if (!metadata_fetcher_) { + metadata_fetcher_ = create_metadata_fetcher_cb_(context_->clusterManager(), clusterName()); + } else { + metadata_fetcher_->cancel(); // Cancel if there is any inflight request. + } + on_async_fetch_cb_ = [this](const std::string&& arg) { + return this->fetchInstanceRoleAsync(std::move(arg)); + }; + continue_on_async_fetch_failure_ = true; + continue_on_async_fetch_failure_reason_ = "Token fetch failed so fall back to less secure way"; + metadata_fetcher_->fetch(token_req_message, Tracing::NullSpan::instance(), *this); } } -void InstanceProfileCredentialsProvider::fetchInstanceRole(const std::string& token_string) { +void InstanceProfileCredentialsProvider::fetchInstanceRole(const std::string&& token_string, + bool async /*default = false*/) { // Discover the Role of this instance. Http::RequestMessageImpl message; + message.headers().setScheme(Http::Headers::get().SchemeValues.Http); message.headers().setMethod(Http::Headers::get().MethodValues.Get); message.headers().setHost(EC2_METADATA_HOST); message.headers().setPath(SECURITY_CREDENTIALS_PATH); @@ -196,22 +331,43 @@ void InstanceProfileCredentialsProvider::fetchInstanceRole(const std::string& to message.headers().setCopy(Http::LowerCaseString(EC2_IMDS_TOKEN_HEADER), StringUtil::trim(token_string)); } - const auto instance_role_string = metadata_fetcher_(message); - if (!instance_role_string) { - ENVOY_LOG(error, "Could not retrieve credentials listing from the EC2MetadataService"); - return; + + if (!async) { + // Using curl to fetch the Instance Role. + const auto instance_role_string = fetch_metadata_using_curl_(message); + if (!instance_role_string) { + ENVOY_LOG(error, "Could not retrieve credentials listing from the EC2MetadataService"); + return; + } + fetchCredentialFromInstanceRole(std::move(instance_role_string.value()), + std::move(token_string)); + } else { + // Using Http async client to fetch the Instance Role. + metadata_fetcher_->cancel(); // Cancel if there is any inflight request. + on_async_fetch_cb_ = [this, token_string = std::move(token_string)](const std::string&& arg) { + return this->fetchCredentialFromInstanceRoleAsync(std::move(arg), std::move(token_string)); + }; + metadata_fetcher_->fetch(message, Tracing::NullSpan::instance(), *this); } - fetchCredentialFromInstanceRole(instance_role_string.value(), token_string); } void InstanceProfileCredentialsProvider::fetchCredentialFromInstanceRole( - const std::string& instance_role, const std::string& token_string) { + const std::string&& instance_role, const std::string&& token_string, + bool async /*default = false*/) { + if (instance_role.empty()) { + ENVOY_LOG(error, "No roles found to fetch AWS credentials from the EC2MetadataService"); + if (async) { + handleFetchDone(); + } return; } const auto instance_role_list = StringUtil::splitToken(StringUtil::trim(instance_role), "\n"); if (instance_role_list.empty()) { - ENVOY_LOG(error, "No AWS credentials were found in the EC2MetadataService"); + ENVOY_LOG(error, "No roles found to fetch AWS credentials from the EC2MetadataService"); + if (async) { + handleFetchDone(); + } return; } ENVOY_LOG(debug, "AWS credentials list:\n{}", instance_role); @@ -223,8 +379,8 @@ void InstanceProfileCredentialsProvider::fetchCredentialFromInstanceRole( std::string(instance_role_list[0].data(), instance_role_list[0].size()); ENVOY_LOG(debug, "AWS credentials path: {}", credential_path); - // Then fetch and parse the credentials Http::RequestMessageImpl message; + message.headers().setScheme(Http::Headers::get().SchemeValues.Http); message.headers().setMethod(Http::Headers::get().MethodValues.Get); message.headers().setHost(EC2_METADATA_HOST); message.headers().setPath(credential_path); @@ -232,23 +388,40 @@ void InstanceProfileCredentialsProvider::fetchCredentialFromInstanceRole( message.headers().setCopy(Http::LowerCaseString(EC2_IMDS_TOKEN_HEADER), StringUtil::trim(token_string)); } - const auto credential_document = metadata_fetcher_(message); - if (!credential_document) { - ENVOY_LOG(error, "Could not load AWS credentials document from the EC2MetadataService"); - return; + + if (!async) { + // Fetch and parse the credentials. + const auto credential_document = fetch_metadata_using_curl_(message); + if (!credential_document) { + ENVOY_LOG(error, "Could not load AWS credentials document from the EC2MetadataService"); + return; + } + extractCredentials(std::move(credential_document.value())); + } else { + // Using Http async client to fetch and parse the AWS credentials. + metadata_fetcher_->cancel(); // Cancel if there is any inflight request. + on_async_fetch_cb_ = [this](const std::string&& arg) { + return this->extractCredentialsAsync(std::move(arg)); + }; + metadata_fetcher_->fetch(message, Tracing::NullSpan::instance(), *this); } - extractCredentials(credential_document.value()); } void InstanceProfileCredentialsProvider::extractCredentials( - const std::string& credential_document_value) { + const std::string&& credential_document_value, bool async /*default = false*/) { if (credential_document_value.empty()) { + if (async) { + handleFetchDone(); + } return; } Json::ObjectSharedPtr document_json; TRY_NEEDS_AUDIT { document_json = Json::Factory::loadFromString(credential_document_value); } END_TRY catch (EnvoyException& e) { ENVOY_LOG(error, "Could not parse AWS credentials document: {}", e.what()); + if (async) { + handleFetchDone(); + } return; } @@ -262,10 +435,47 @@ void InstanceProfileCredentialsProvider::extractCredentials( secret_access_key.empty() ? "" : "*****", AWS_SESSION_TOKEN, session_token.empty() ? "" : "*****"); - cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); last_updated_ = api_.timeSource().systemTime(); + if (useHttpAsyncClient() && context_) { + setCredentialsToAllThreads( + std::make_unique(access_key_id, secret_access_key, session_token)); + } else { + cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); + } + handleFetchDone(); +} + +void InstanceProfileCredentialsProvider::onMetadataSuccess(const std::string&& body) { + // TODO(suniltheta): increment fetch success stats + ENVOY_LOG(debug, "AWS Instance metadata fetch success, calling callback func"); + on_async_fetch_cb_(std::move(body)); +} + +void InstanceProfileCredentialsProvider::onMetadataError(Failure reason) { + // TODO(suniltheta): increment fetch failed stats + if (continue_on_async_fetch_failure_) { + ENVOY_LOG(warn, "{}. Reason: {}", continue_on_async_fetch_failure_reason_, + metadata_fetcher_->failureToString(reason)); + continue_on_async_fetch_failure_ = false; + continue_on_async_fetch_failure_reason_ = ""; + on_async_fetch_cb_(std::move("")); + } else { + ENVOY_LOG(error, "AWS Instance metadata fetch failure: {}", + metadata_fetcher_->failureToString(reason)); + handleFetchDone(); + } } +TaskRoleCredentialsProvider::TaskRoleCredentialsProvider( + Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view credential_uri, + absl::string_view authorization_token = {}, absl::string_view cluster_name = {}) + : MetadataCredentialsProviderBase( + api, context, fetch_metadata_using_curl, create_metadata_fetcher_cb, cluster_name, + envoy::config::cluster::v3::Cluster::STATIC /*cluster_type*/, credential_uri), + credential_uri_(credential_uri), authorization_token_(authorization_token) {} + bool TaskRoleCredentialsProvider::needsRefresh() { const auto now = api_.timeSource().systemTime(); return (now - last_updated_ > REFRESH_INTERVAL) || @@ -280,26 +490,48 @@ void TaskRoleCredentialsProvider::refresh() { Http::Utility::extractHostPathFromUri(credential_uri_, host, path); Http::RequestMessageImpl message; + message.headers().setScheme(Http::Headers::get().SchemeValues.Http); message.headers().setMethod(Http::Headers::get().MethodValues.Get); message.headers().setHost(host); message.headers().setPath(path); message.headers().setCopy(Http::CustomHeaders::get().Authorization, authorization_token_); - const auto credential_document = metadata_fetcher_(message); - if (!credential_document) { - ENVOY_LOG(error, "Could not load AWS credentials document from the task role"); - return; + if (!useHttpAsyncClient() || !context_) { + // Using curl to fetch the AWS credentials. + const auto credential_document = fetch_metadata_using_curl_(message); + if (!credential_document) { + ENVOY_LOG(error, "Could not load AWS credentials document from the task role"); + return; + } + extractCredentials(std::move(credential_document.value())); + } else { + // Stop any existing timer. + if (cache_duration_timer_ && cache_duration_timer_->enabled()) { + cache_duration_timer_->disableTimer(); + } + // Using Http async client to fetch the AWS credentials. + if (!metadata_fetcher_) { + metadata_fetcher_ = create_metadata_fetcher_cb_(context_->clusterManager(), clusterName()); + } else { + metadata_fetcher_->cancel(); // Cancel if there is any inflight request. + } + on_async_fetch_cb_ = [this](const std::string&& arg) { + return this->extractCredentials(std::move(arg)); + }; + metadata_fetcher_->fetch(message, Tracing::NullSpan::instance(), *this); } - extractCredentials(credential_document.value()); } -void TaskRoleCredentialsProvider::extractCredentials(const std::string& credential_document_value) { +void TaskRoleCredentialsProvider::extractCredentials( + const std::string&& credential_document_value) { if (credential_document_value.empty()) { + handleFetchDone(); return; } Json::ObjectSharedPtr document_json; TRY_NEEDS_AUDIT { document_json = Json::Factory::loadFromString(credential_document_value); } END_TRY catch (EnvoyException& e) { ENVOY_LOG(error, "Could not parse AWS credentials document from the task role: {}", e.what()); + handleFetchDone(); return; } @@ -322,7 +554,178 @@ void TaskRoleCredentialsProvider::extractCredentials(const std::string& credenti } last_updated_ = api_.timeSource().systemTime(); - cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); + if (useHttpAsyncClient() && context_) { + setCredentialsToAllThreads( + std::make_unique(access_key_id, secret_access_key, session_token)); + } else { + cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); + } + handleFetchDone(); +} + +void TaskRoleCredentialsProvider::onMetadataSuccess(const std::string&& body) { + // TODO(suniltheta): increment fetch success stats + ENVOY_LOG(debug, "AWS Task metadata fetch success, calling callback func"); + on_async_fetch_cb_(std::move(body)); +} + +void TaskRoleCredentialsProvider::onMetadataError(Failure reason) { + // TODO(suniltheta): increment fetch failed stats + ENVOY_LOG(error, "AWS metadata fetch failure: {}", metadata_fetcher_->failureToString(reason)); + handleFetchDone(); +} + +WebIdentityCredentialsProvider::WebIdentityCredentialsProvider( + Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view token_file_path, + absl::string_view sts_endpoint, absl::string_view role_arn, absl::string_view role_session_name, + absl::string_view cluster_name = {}) + : MetadataCredentialsProviderBase( + api, context, fetch_metadata_using_curl, create_metadata_fetcher_cb, cluster_name, + envoy::config::cluster::v3::Cluster::LOGICAL_DNS /*cluster_type*/, sts_endpoint), + token_file_path_(token_file_path), sts_endpoint_(sts_endpoint), role_arn_(role_arn), + role_session_name_(role_session_name) {} + +bool WebIdentityCredentialsProvider::needsRefresh() { + const auto now = api_.timeSource().systemTime(); + return (now - last_updated_ > REFRESH_INTERVAL) || + (expiration_time_ - now < REFRESH_GRACE_PERIOD); +} + +void WebIdentityCredentialsProvider::refresh() { + // If http async client is not enabled then just set empty credentials and return. + if (!useHttpAsyncClient()) { + cached_credentials_ = Credentials(); + return; + } + + ENVOY_LOG(debug, "Getting AWS web identity credentials from STS: {}", sts_endpoint_); + + const auto web_token_file_or_error = api_.fileSystem().fileReadToEnd(token_file_path_); + THROW_IF_STATUS_NOT_OK(web_token_file_or_error, throw); + Http::RequestMessageImpl message; + message.headers().setScheme(Http::Headers::get().SchemeValues.Https); + message.headers().setMethod(Http::Headers::get().MethodValues.Get); + message.headers().setHost(Http::Utility::parseAuthority(sts_endpoint_).host_); + message.headers().setPath( + fmt::format("/?Action=AssumeRoleWithWebIdentity" + "&Version=2011-06-15" + "&RoleSessionName={}" + "&RoleArn={}" + "&WebIdentityToken={}", + Envoy::Http::Utility::PercentEncoding::encode(role_session_name_), + Envoy::Http::Utility::PercentEncoding::encode(role_arn_), + Envoy::Http::Utility::PercentEncoding::encode(web_token_file_or_error.value()))); + // Use the Accept header to ensure that AssumeRoleWithWebIdentityResponse is returned as JSON. + message.headers().setReference(Http::CustomHeaders::get().Accept, + Http::Headers::get().ContentTypeValues.Json); + // Stop any existing timer. + if (cache_duration_timer_ && cache_duration_timer_->enabled()) { + cache_duration_timer_->disableTimer(); + } + // Using Http async client to fetch the AWS credentials. + if (!metadata_fetcher_) { + metadata_fetcher_ = create_metadata_fetcher_cb_(context_->clusterManager(), clusterName()); + } else { + metadata_fetcher_->cancel(); // Cancel if there is any inflight request. + } + on_async_fetch_cb_ = [this](const std::string&& arg) { + return this->extractCredentials(std::move(arg)); + }; + metadata_fetcher_->fetch(message, Tracing::NullSpan::instance(), *this); +} + +void WebIdentityCredentialsProvider::extractCredentials( + const std::string&& credential_document_value) { + if (credential_document_value.empty()) { + handleFetchDone(); + ENVOY_LOG(error, "Could not load AWS credentials document from STS"); + return; + } + + Json::ObjectSharedPtr document_json; + TRY_NEEDS_AUDIT { document_json = Json::Factory::loadFromString(credential_document_value); } + END_TRY catch (EnvoyException& e) { + ENVOY_LOG(error, "Could not parse AWS credentials document from STS: {}", e.what()); + handleFetchDone(); + return; + } + + absl::StatusOr root_node = + document_json->getObjectNoThrow(WEB_IDENTITY_RESPONSE_ELEMENT); + if (!root_node.ok()) { + ENVOY_LOG(error, "AWS STS credentials document is empty"); + handleFetchDone(); + return; + } + absl::StatusOr result_node = + root_node.value()->getObjectNoThrow(WEB_IDENTITY_RESULT_ELEMENT); + if (!result_node.ok()) { + ENVOY_LOG(error, "AWS STS returned an unexpected result"); + handleFetchDone(); + return; + } + absl::StatusOr credentials = + result_node.value()->getObjectNoThrow(CREDENTIALS); + if (!credentials.ok()) { + ENVOY_LOG(error, "AWS STS credentials document does not contain any credentials"); + handleFetchDone(); + return; + } + + TRY_NEEDS_AUDIT { + const auto access_key_id = credentials.value()->getString(ACCESS_KEY_ID, ""); + const auto secret_access_key = credentials.value()->getString(SECRET_ACCESS_KEY, ""); + const auto session_token = credentials.value()->getString(SESSION_TOKEN, ""); + + ENVOY_LOG(debug, "Received the following AWS credentials from STS: {}={}, {}={}, {}={}", + AWS_ACCESS_KEY_ID, access_key_id, AWS_SECRET_ACCESS_KEY, + secret_access_key.empty() ? "" : "*****", AWS_SESSION_TOKEN, + session_token.empty() ? "" : "*****"); + setCredentialsToAllThreads( + std::make_unique(access_key_id, secret_access_key, session_token)); + } + END_TRY catch (EnvoyException& e) { + ENVOY_LOG(error, "Bad format, could not parse AWS credentials document from STS: {}", e.what()); + handleFetchDone(); + return; + } + + TRY_NEEDS_AUDIT { + const auto expiration = credentials.value()->getInteger(EXPIRATION, 0); + if (expiration != 0) { + expiration_time_ = + std::chrono::time_point(std::chrono::seconds(expiration)); + ENVOY_LOG(debug, "AWS STS credentials expiration time (unix timestamp): {}", expiration); + } else { + expiration_time_ = api_.timeSource().systemTime() + REFRESH_INTERVAL; + ENVOY_LOG(warn, "Could not get Expiration value of AWS credentials document from STS, so " + "setting expiration to 1 hour in future"); + } + } + END_TRY catch (EnvoyException& e) { + expiration_time_ = api_.timeSource().systemTime() + REFRESH_INTERVAL; + ENVOY_LOG(warn, + "Could not parse Expiration value of AWS credentials document from STS: {}, so " + "setting expiration to 1 hour in future", + e.what()); + } + + last_updated_ = api_.timeSource().systemTime(); + handleFetchDone(); +} + +void WebIdentityCredentialsProvider::onMetadataSuccess(const std::string&& body) { + // TODO(suniltheta): increment fetch success stats + ENVOY_LOG(debug, "AWS metadata fetch from STS success, calling callback func"); + on_async_fetch_cb_(std::move(body)); +} + +void WebIdentityCredentialsProvider::onMetadataError(Failure reason) { + // TODO(suniltheta): increment fetch failed stats + ENVOY_LOG(error, "AWS metadata fetch failure: {}", metadata_fetcher_->failureToString(reason)); + handleFetchDone(); } Credentials CredentialsProviderChain::getCredentials() { @@ -338,7 +741,8 @@ Credentials CredentialsProviderChain::getCredentials() { } DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( - Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + Api::Api& api, ServerFactoryContextOptRef context, absl::string_view region, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, const CredentialsProviderChainFactories& factories) { ENVOY_LOG(debug, "Using environment credentials provider"); add(factories.createEnvironmentCredentialsProvider()); @@ -350,6 +754,37 @@ DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( ENVOY_LOG(debug, "Not using credential file credentials provider because it is not enabled"); } + // WebIdentityCredentialsProvider can be used only if `context` is supplied which is required to + // use http async http client to make http calls to fetch the credentials. + if (context) { + const auto web_token_path = absl::NullSafeStringView(std::getenv(AWS_WEB_IDENTITY_TOKEN_FILE)); + const auto role_arn = absl::NullSafeStringView(std::getenv(AWS_ROLE_ARN)); + if (!web_token_path.empty() && !role_arn.empty()) { + const auto role_session_name = absl::NullSafeStringView(std::getenv(AWS_ROLE_SESSION_NAME)); + std::string actual_session_name; + if (!role_session_name.empty()) { + actual_session_name = std::string(role_session_name); + } else { + // In practice, this value will be provided by the environment, so the placeholder value is + // not important. Some AWS SDKs use time in nanoseconds, so we'll just use that. + const auto now_nanos = std::chrono::duration_cast( + api.timeSource().systemTime().time_since_epoch()) + .count(); + actual_session_name = fmt::format("{}", now_nanos); + } + const auto sts_endpoint = Utility::getSTSEndpoint(region) + ":443"; + ENVOY_LOG( + debug, + "Using web identity credentials provider with STS endpoint: {} and session name: {}", + sts_endpoint, actual_session_name); + add(factories.createWebIdentityCredentialsProvider( + api, context, fetch_metadata_using_curl, MetadataFetcher::create, STS_TOKEN_CLUSTER, + web_token_path, sts_endpoint, role_arn, actual_session_name)); + } + } + + // Even if WebIdentity is supported keep the fallback option open so that + // Envoy can use other credentials provider if available. const auto relative_uri = absl::NullSafeStringView(std::getenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI)); const auto full_uri = absl::NullSafeStringView(std::getenv(AWS_CONTAINER_CREDENTIALS_FULL_URI)); @@ -358,7 +793,9 @@ DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( if (!relative_uri.empty()) { const auto uri = absl::StrCat(CONTAINER_METADATA_HOST, relative_uri); ENVOY_LOG(debug, "Using task role credentials provider with URI: {}", uri); - add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, uri)); + add(factories.createTaskRoleCredentialsProvider(api, context, fetch_metadata_using_curl, + MetadataFetcher::create, + CONTAINER_METADATA_CLUSTER, uri)); } else if (!full_uri.empty()) { const auto authorization_token = absl::NullSafeStringView(std::getenv(AWS_CONTAINER_AUTHORIZATION_TOKEN)); @@ -367,15 +804,19 @@ DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( "Using task role credentials provider with URI: " "{} and authorization token", full_uri); - add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, full_uri, - authorization_token)); + add(factories.createTaskRoleCredentialsProvider( + api, context, fetch_metadata_using_curl, MetadataFetcher::create, + CONTAINER_METADATA_CLUSTER, full_uri, authorization_token)); } else { ENVOY_LOG(debug, "Using task role credentials provider with URI: {}", full_uri); - add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, full_uri)); + add(factories.createTaskRoleCredentialsProvider(api, context, fetch_metadata_using_curl, + MetadataFetcher::create, + CONTAINER_METADATA_CLUSTER, full_uri)); } } else if (metadata_disabled != TRUE) { ENVOY_LOG(debug, "Using instance profile credentials provider"); - add(factories.createInstanceProfileCredentialsProvider(api, metadata_fetcher)); + add(factories.createInstanceProfileCredentialsProvider( + api, context, fetch_metadata_using_curl, MetadataFetcher::create, EC2_METADATA_CLUSTER)); } } diff --git a/source/extensions/common/aws/credentials_provider_impl.h b/source/extensions/common/aws/credentials_provider_impl.h index 0b207620ad88..26996edfe98f 100644 --- a/source/extensions/common/aws/credentials_provider_impl.h +++ b/source/extensions/common/aws/credentials_provider_impl.h @@ -1,14 +1,23 @@ #pragma once #include +#include +#include #include "envoy/api/api.h" +#include "envoy/common/optref.h" #include "envoy/event/timer.h" #include "envoy/http/message.h" +#include "envoy/server/factory_context.h" +#include "source/common/common/lock_guard.h" #include "source/common/common/logger.h" #include "source/common/common/thread.h" +#include "source/common/init/target_impl.h" +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/utility.h" #include "source/extensions/common/aws/credentials_provider.h" +#include "source/extensions/common/aws/metadata_fetcher.h" #include "absl/strings/string_view.h" @@ -17,6 +26,13 @@ namespace Extensions { namespace Common { namespace Aws { +/** + * CreateMetadataFetcherCb is a callback interface for creating a MetadataFetcher instance. + */ +using CreateMetadataFetcherCb = + std::function; +using ServerFactoryContextOptRef = OptRef; + /** * Retrieve AWS credentials from the environment variables. * @@ -68,14 +84,78 @@ class CredentialsFileCredentialsProvider : public CachedCredentialsProviderBase class MetadataCredentialsProviderBase : public CachedCredentialsProviderBase { public: - using MetadataFetcher = std::function(Http::RequestMessage&)>; + using CurlMetadataFetcher = std::function(Http::RequestMessage&)>; + using OnAsyncFetchCb = std::function; - MetadataCredentialsProviderBase(Api::Api& api, const MetadataFetcher& metadata_fetcher) - : api_(api), metadata_fetcher_(metadata_fetcher) {} + MetadataCredentialsProviderBase( + Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, + const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, absl::string_view uri); + + Credentials getCredentials() override; + + // Get the Metadata credentials cache duration. + static std::chrono::seconds getCacheDuration(); protected: + struct ThreadLocalCredentialsCache : public ThreadLocal::ThreadLocalObject { + ThreadLocalCredentialsCache() { + credentials_ = std::make_shared(); // Creating empty credentials as default. + } + // The credentials object. + CredentialsConstSharedPtr credentials_; + }; + + const std::string& clusterName() const { return cluster_name_; } + + // Handle fetch done. + void handleFetchDone(); + + // Set Credentials shared_ptr on all threads. + void setCredentialsToAllThreads(CredentialsConstUniquePtr&& creds); + + // Returns true if http async client can be used instead of libcurl to fetch the aws credentials, + // else false. + bool useHttpAsyncClient(); + Api::Api& api_; - MetadataFetcher metadata_fetcher_; + // The optional server factory context. + ServerFactoryContextOptRef context_; + // Store the method to fetch metadata from libcurl (deprecated) + CurlMetadataFetcher fetch_metadata_using_curl_; + // The callback used to create a MetadataFetcher instance. + CreateMetadataFetcherCb create_metadata_fetcher_cb_; + // The cluster name to use for internal static cluster pointing towards the credentials provider. + const std::string cluster_name_; + // The cluster type to use for internal static cluster pointing towards the credentials provider. + const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type_; + // The uri of internal static cluster credentials provider. + const std::string uri_; + // The cache duration of the fetched credentials. + const std::chrono::seconds cache_duration_; + // The thread local slot for cache. + ThreadLocal::TypedSlotPtr tls_; + // The timer to trigger fetch due to cache duration. + Envoy::Event::TimerPtr cache_duration_timer_; + // The Metadata fetcher object. + MetadataFetcherPtr metadata_fetcher_; + // Callback function to call on successful metadata fetch. + OnAsyncFetchCb on_async_fetch_cb_; + // To determine if credentials fetching can continue even after metadata fetch failure. + bool continue_on_async_fetch_failure_ = false; + // Reason to log on fetch failure while continue. + std::string continue_on_async_fetch_failure_reason_ = ""; + // Last update time to determine expiration. + SystemTime last_updated_; + // Cache credentials when using libcurl. + Credentials cached_credentials_; + // Lock guard. + Thread::MutexBasicLockable lock_; + // The init target. + std::unique_ptr init_target_; + // Used in logs. + const std::string debug_name_; }; /** @@ -83,17 +163,35 @@ class MetadataCredentialsProviderBase : public CachedCredentialsProviderBase { * * https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials */ -class InstanceProfileCredentialsProvider : public MetadataCredentialsProviderBase { +class InstanceProfileCredentialsProvider : public MetadataCredentialsProviderBase, + public MetadataFetcher::MetadataReceiver { public: - InstanceProfileCredentialsProvider(Api::Api& api, const MetadataFetcher& metadata_fetcher) - : MetadataCredentialsProviderBase(api, metadata_fetcher) {} + InstanceProfileCredentialsProvider(Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, + absl::string_view cluster_name); + + // Following functions are for MetadataFetcher::MetadataReceiver interface + void onMetadataSuccess(const std::string&& body) override; + void onMetadataError(Failure reason) override; private: bool needsRefresh() override; void refresh() override; - void fetchInstanceRole(const std::string& token); - void fetchCredentialFromInstanceRole(const std::string& instance_role, const std::string& token); - void extractCredentials(const std::string& credential_document_value); + void fetchInstanceRole(const std::string&& token, bool async = false); + void fetchInstanceRoleAsync(const std::string&& token) { + fetchInstanceRole(std::move(token), true); + } + void fetchCredentialFromInstanceRole(const std::string&& instance_role, const std::string&& token, + bool async = false); + void fetchCredentialFromInstanceRoleAsync(const std::string&& instance_role, + const std::string&& token) { + fetchCredentialFromInstanceRole(std::move(instance_role), std::move(token), true); + } + void extractCredentials(const std::string&& credential_document_value, bool async = false); + void extractCredentialsAsync(const std::string&& credential_document_value) { + extractCredentials(std::move(credential_document_value), true); + } }; /** @@ -101,22 +199,58 @@ class InstanceProfileCredentialsProvider : public MetadataCredentialsProviderBas * * https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html#enable_task_iam_roles */ -class TaskRoleCredentialsProvider : public MetadataCredentialsProviderBase { +class TaskRoleCredentialsProvider : public MetadataCredentialsProviderBase, + public MetadataFetcher::MetadataReceiver { public: - TaskRoleCredentialsProvider(Api::Api& api, const MetadataFetcher& metadata_fetcher, + TaskRoleCredentialsProvider(Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view credential_uri, - absl::string_view authorization_token = {}) - : MetadataCredentialsProviderBase(api, metadata_fetcher), credential_uri_(credential_uri), - authorization_token_(authorization_token) {} + absl::string_view authorization_token, + absl::string_view cluster_name); + + // Following functions are for MetadataFetcher::MetadataReceiver interface + void onMetadataSuccess(const std::string&& body) override; + void onMetadataError(Failure reason) override; private: SystemTime expiration_time_; - std::string credential_uri_; - std::string authorization_token_; + const std::string credential_uri_; + const std::string authorization_token_; bool needsRefresh() override; void refresh() override; - void extractCredentials(const std::string& credential_document_value); + void extractCredentials(const std::string&& credential_document_value); +}; + +/** + * Retrieve AWS credentials from Security Token Service using a web identity token (e.g. OAuth, + * OpenID) + */ +class WebIdentityCredentialsProvider : public MetadataCredentialsProviderBase, + public MetadataFetcher::MetadataReceiver { +public: + WebIdentityCredentialsProvider(Api::Api& api, ServerFactoryContextOptRef context, + const CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, + absl::string_view token_file_path, absl::string_view sts_endpoint, + absl::string_view role_arn, absl::string_view role_session_name, + absl::string_view cluster_name); + + // Following functions are for MetadataFetcher::MetadataReceiver interface + void onMetadataSuccess(const std::string&& body) override; + void onMetadataError(Failure reason) override; + +private: + SystemTime expiration_time_; + const std::string token_file_path_; + const std::string sts_endpoint_; + const std::string role_arn_; + const std::string role_session_name_; + + bool needsRefresh() override; + void refresh() override; + void extractCredentials(const std::string&& credential_document_value); }; /** @@ -146,13 +280,24 @@ class CredentialsProviderChainFactories { virtual CredentialsProviderSharedPtr createCredentialsFileCredentialsProvider(Api::Api& api) const PURE; + virtual CredentialsProviderSharedPtr createWebIdentityCredentialsProvider( + Api::Api& api, ServerFactoryContextOptRef context, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, + absl::string_view token_file_path, absl::string_view sts_endpoint, absl::string_view role_arn, + absl::string_view role_session_name) const PURE; + virtual CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( - Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + Api::Api& api, ServerFactoryContextOptRef context, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, absl::string_view credential_uri, absl::string_view authorization_token = {}) const PURE; virtual CredentialsProviderSharedPtr createInstanceProfileCredentialsProvider( - Api::Api& api, - const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher) const PURE; + Api::Api& api, ServerFactoryContextOptRef context, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, + absl::string_view cluster_name) const PURE; }; /** @@ -165,11 +310,13 @@ class DefaultCredentialsProviderChain : public CredentialsProviderChain, public CredentialsProviderChainFactories { public: DefaultCredentialsProviderChain( - Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher) - : DefaultCredentialsProviderChain(api, metadata_fetcher, *this) {} + Api::Api& api, ServerFactoryContextOptRef context, absl::string_view region, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl) + : DefaultCredentialsProviderChain(api, context, region, fetch_metadata_using_curl, *this) {} DefaultCredentialsProviderChain( - Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + Api::Api& api, ServerFactoryContextOptRef context, absl::string_view region, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, const CredentialsProviderChainFactories& factories); private: @@ -183,19 +330,40 @@ class DefaultCredentialsProviderChain : public CredentialsProviderChain, } CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( - Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + Api::Api& api, ServerFactoryContextOptRef context, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, absl::string_view credential_uri, absl::string_view authorization_token = {}) const override { - return std::make_shared(api, metadata_fetcher, credential_uri, - authorization_token); + return std::make_shared(api, context, fetch_metadata_using_curl, + create_metadata_fetcher_cb, credential_uri, + authorization_token, cluster_name); } CredentialsProviderSharedPtr createInstanceProfileCredentialsProvider( - Api::Api& api, - const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher) const override { - return std::make_shared(api, metadata_fetcher); + Api::Api& api, ServerFactoryContextOptRef context, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, + absl::string_view cluster_name) const override { + return std::make_shared( + api, context, fetch_metadata_using_curl, create_metadata_fetcher_cb, cluster_name); + } + + CredentialsProviderSharedPtr createWebIdentityCredentialsProvider( + Api::Api& api, ServerFactoryContextOptRef context, + const MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, + absl::string_view token_file_path, absl::string_view sts_endpoint, absl::string_view role_arn, + absl::string_view role_session_name) const override { + return std::make_shared( + api, context, fetch_metadata_using_curl, create_metadata_fetcher_cb, token_file_path, + sts_endpoint, role_arn, role_session_name, cluster_name); } }; +using InstanceProfileCredentialsProviderPtr = std::shared_ptr; +using TaskRoleCredentialsProviderPtr = std::shared_ptr; +using WebIdentityCredentialsProviderPtr = std::shared_ptr; + } // namespace Aws } // namespace Common } // namespace Extensions diff --git a/source/extensions/common/aws/metadata_fetcher.cc b/source/extensions/common/aws/metadata_fetcher.cc new file mode 100644 index 000000000000..d5b260a16112 --- /dev/null +++ b/source/extensions/common/aws/metadata_fetcher.cc @@ -0,0 +1,184 @@ +#include "source/extensions/common/aws/metadata_fetcher.h" + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/core/v3/http_uri.pb.h" + +#include "source/common/common/enum_to_int.h" +#include "source/common/http/headers.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/utility.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Aws { + +namespace { + +class MetadataFetcherImpl : public MetadataFetcher, + public Logger::Loggable, + public Http::AsyncClient::Callbacks { + +public: + MetadataFetcherImpl(Upstream::ClusterManager& cm, absl::string_view cluster_name) + : cm_(cm), cluster_name_(std::string(cluster_name)) {} + + ~MetadataFetcherImpl() override { cancel(); } + + void cancel() override { + if (request_ && !complete_) { + request_->cancel(); + ENVOY_LOG(debug, "fetch AWS Metadata [cluster = {}]: cancelled", cluster_name_); + } + reset(); + } + + absl::string_view failureToString(MetadataFetcher::MetadataReceiver::Failure reason) override { + switch (reason) { + case MetadataFetcher::MetadataReceiver::Failure::Network: + return "Network"; + case MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata: + return "InvalidMetadata"; + case MetadataFetcher::MetadataReceiver::Failure::MissingConfig: + return "MissingConfig"; + default: + return ""; + } + } + + void fetch(Http::RequestMessage& message, Tracing::Span& parent_span, + MetadataFetcher::MetadataReceiver& receiver) override { + ASSERT(!request_); + complete_ = false; + receiver_ = makeOptRef(receiver); + const auto thread_local_cluster = cm_.getThreadLocalCluster(cluster_name_); + if (thread_local_cluster == nullptr) { + ENVOY_LOG(error, "{} AWS Metadata failed: [cluster = {}] not found", __func__, cluster_name_); + complete_ = true; + receiver_->onMetadataError(MetadataFetcher::MetadataReceiver::Failure::MissingConfig); + reset(); + return; + } + + constexpr uint64_t MAX_RETRIES = 3; + constexpr uint64_t RETRY_DELAY = 1000; + constexpr uint64_t TIMEOUT = 5 * 1000; + + const auto host_attributes = Http::Utility::parseAuthority(message.headers().getHostValue()); + const auto host = host_attributes.host_; + const auto path = message.headers().getPathValue(); + const auto scheme = message.headers().getSchemeValue(); + const auto method = message.headers().getMethodValue(); + + const size_t query_offset = path.find('?'); + // Sanitize the path before logging. + // However, the route debug log will still display the entire path. + // So safely store the Envoy logs at debug level. + const absl::string_view sanitized_path = + query_offset != absl::string_view::npos ? path.substr(0, query_offset) : path; + ENVOY_LOG(debug, "fetch AWS Metadata from the cluster {} at [uri = {}]", cluster_name_, + fmt::format("{}://{}{}", scheme, host, sanitized_path)); + + Http::RequestHeaderMapPtr headersPtr = + Envoy::Http::createHeaderMap( + {{Envoy::Http::Headers::get().Method, std::string(method)}, + {Envoy::Http::Headers::get().Host, std::string(host)}, + {Envoy::Http::Headers::get().Scheme, std::string(scheme)}, + {Envoy::Http::Headers::get().Path, std::string(path)}}); + + // Copy the remaining headers. + message.headers().iterate( + [&headersPtr](const Http::HeaderEntry& entry) -> Http::HeaderMap::Iterate { + // Skip pseudo-headers + if (!entry.key().getStringView().empty() && entry.key().getStringView()[0] == ':') { + return Http::HeaderMap::Iterate::Continue; + } + headersPtr->addCopy(Http::LowerCaseString(entry.key().getStringView()), + entry.value().getStringView()); + return Http::HeaderMap::Iterate::Continue; + }); + + auto messagePtr = std::make_unique(std::move(headersPtr)); + + auto options = Http::AsyncClient::RequestOptions() + .setTimeout(std::chrono::milliseconds(TIMEOUT)) + .setParentSpan(parent_span) + .setSendXff(false) + .setChildSpanName("AWS Metadata Fetch"); + + envoy::config::route::v3::RetryPolicy route_retry_policy; + route_retry_policy.mutable_num_retries()->set_value(MAX_RETRIES); + route_retry_policy.mutable_per_try_timeout()->CopyFrom( + Protobuf::util::TimeUtil::MillisecondsToDuration(TIMEOUT)); + route_retry_policy.mutable_per_try_idle_timeout()->CopyFrom( + Protobuf::util::TimeUtil::MillisecondsToDuration(RETRY_DELAY)); + route_retry_policy.set_retry_on("5xx,gateway-error,connect-failure,reset,refused-stream"); + + options.setRetryPolicy(route_retry_policy); + options.setBufferBodyForRetry(true); + request_ = makeOptRefFromPtr( + thread_local_cluster->httpAsyncClient().send(std::move(messagePtr), *this, options)); + } + + // HTTP async receive method on success. + void onSuccess(const Http::AsyncClient::Request&, Http::ResponseMessagePtr&& response) override { + ASSERT(receiver_); + complete_ = true; + const uint64_t status_code = Http::Utility::getResponseStatus(response->headers()); + if (status_code == enumToInt(Http::Code::OK)) { + ENVOY_LOG(debug, "{}: fetch AWS Metadata [cluster = {}]: success", __func__, cluster_name_); + if (response->body().length() != 0) { + const auto body = response->bodyAsString(); + receiver_->onMetadataSuccess(std::move(body)); + } else { + ENVOY_LOG(debug, "{}: fetch AWS Metadata [cluster = {}]: body is empty", __func__, + cluster_name_); + receiver_->onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + if (response->body().length() != 0) { + ENVOY_LOG(debug, "{}: fetch AWS Metadata [cluster = {}]: response status code {}, body: {}", + __func__, cluster_name_, status_code, response->bodyAsString()); + } else { + ENVOY_LOG(debug, + "{}: fetch AWS Metadata [cluster = {}]: response status code {}, body is empty", + __func__, cluster_name_, status_code); + } + receiver_->onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + reset(); + } + + // HTTP async receive method on failure. + void onFailure(const Http::AsyncClient::Request&, + Http::AsyncClient::FailureReason reason) override { + ASSERT(receiver_); + ENVOY_LOG(debug, "{}: fetch AWS Metadata [cluster = {}]: network error {}", __func__, + cluster_name_, enumToInt(reason)); + complete_ = true; + receiver_->onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + reset(); + } + + // TODO(suniltheta): Add metadata fetch status into the span like it is done on ext_authz filter. + void onBeforeFinalizeUpstreamSpan(Tracing::Span&, const Http::ResponseHeaderMap*) override {} + +private: + bool complete_{}; + Upstream::ClusterManager& cm_; + const std::string cluster_name_; + OptRef receiver_; + OptRef request_; + + void reset() { request_.reset(); } +}; +} // namespace + +MetadataFetcherPtr MetadataFetcher::create(Upstream::ClusterManager& cm, + absl::string_view cluster_name) { + return std::make_unique(cm, cluster_name); +} +} // namespace Aws +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/aws/metadata_fetcher.h b/source/extensions/common/aws/metadata_fetcher.h new file mode 100644 index 000000000000..a39d1480447c --- /dev/null +++ b/source/extensions/common/aws/metadata_fetcher.h @@ -0,0 +1,97 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" +#include "envoy/http/message.h" +#include "envoy/upstream/cluster_manager.h" + +#include "source/common/http/message_impl.h" +#include "source/extensions/common/aws/utility.h" + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Aws { + +class MetadataFetcher; +using MetadataFetcherPtr = std::unique_ptr; + +/** + * MetadataFetcher interface can be used to retrieve AWS Metadata from various providers. + * An instance of this interface is designed to retrieve one AWS Metadata at a time. + * The implementation of AWS Metadata Fetcher is similar to JwksFetcher. + */ + +class MetadataFetcher { +public: + class MetadataReceiver { + public: + enum class Failure { + /* A network error occurred causing AWS Metadata retrieval failure. */ + Network, + /* A failure occurred when trying to parse the retrieved AWS Metadata data. */ + InvalidMetadata, + /* A missing config causing AWS Metadata retrieval failure. */ + MissingConfig, + }; + + virtual ~MetadataReceiver() = default; + + /** + * @brief Successful retrieval callback of returned AWS Metadata. + * @param body Fetched AWS Metadata. + */ + virtual void onMetadataSuccess(const std::string&& body) PURE; + + /** + * @brief Retrieval error callback. + * @param reason the failure reason. + */ + virtual void onMetadataError(Failure reason) PURE; + }; + + virtual ~MetadataFetcher() = default; + + /** + * @brief Cancel any in-flight request. + */ + virtual void cancel() PURE; + + /** + * @brief Retrieve a AWS Metadata from a remote HTTP host. + * At most one outstanding request may be in-flight. + * i.e. from the invocation of `fetch()` until either + * a callback or `cancel()` is invoked, no additional + * `fetch()` may be issued. The URI to fetch is to pre + * determined based on the credentials provider source. + * + * @param receiver the receiver of the fetched AWS Metadata or error + */ + virtual void fetch(Http::RequestMessage& message, Tracing::Span& parent_span, + MetadataReceiver& receiver) PURE; + + /** + * @brief Return MetadataReceiver Failure enum as a string. + * + * @return absl::string_view + */ + virtual absl::string_view failureToString(MetadataReceiver::Failure) PURE; + + /** + * @brief Factory method for creating a Metadata Fetcher. + * + * @param cm the cluster manager to use during AWS Metadata retrieval + * @param provider the AWS Metadata provider + * @return a MetadataFetcher instance + */ + static MetadataFetcherPtr create(Upstream::ClusterManager& cm, absl::string_view cluster_name); +}; +} // namespace Aws +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/aws/utility.cc b/source/extensions/common/aws/utility.cc index 9d5669b2113a..4c4cf1eb6089 100644 --- a/source/extensions/common/aws/utility.cc +++ b/source/extensions/common/aws/utility.cc @@ -1,13 +1,18 @@ #include "source/extensions/common/aws/utility.h" +#include "envoy/upstream/cluster_manager.h" + #include "source/common/common/empty_string.h" #include "source/common/common/fmt.h" #include "source/common/common/utility.h" +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/utility.h" #include "absl/strings/match.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "curl/curl.h" +#include "fmt/printf.h" namespace Envoy { namespace Extensions { @@ -59,7 +64,7 @@ Utility::canonicalizeHeaders(const Http::RequestHeaderMap& headers, }); // The AWS SDK has a quirk where it removes "default ports" (80, 443) from the host headers // Additionally, we canonicalize the :authority header as "host" - // TODO(lavignes): This may need to be tweaked to canonicalize :authority for HTTP/2 requests + // TODO(suniltheta): This may need to be tweaked to canonicalize :authority for HTTP/2 requests const absl::string_view authority_header = headers.getHostValue(); if (!authority_header.empty()) { const auto parts = StringUtil::splitToken(authority_header, ":"); @@ -217,6 +222,22 @@ Utility::joinCanonicalHeaderNames(const std::map& cano }); } +std::string Utility::getSTSEndpoint(absl::string_view region) { + if (region == "cn-northwest-1" || region == "cn-north-1") { + return fmt::format("sts.{}.amazonaws.com.cn", region); + } +#ifdef ENVOY_SSL_FIPS + // Use AWS STS FIPS endpoints in FIPS mode https://docs.aws.amazon.com/general/latest/gr/sts.html. + // Note: AWS GovCloud doesn't have separate fips endpoints. + // TODO(suniltheta): Include `ca-central-1` when sts supports a dedicated FIPS endpoint. + if (region == "us-east-1" || region == "us-east-2" || region == "us-west-1" || + region == "us-west-2") { + return fmt::format("sts-fips.{}.amazonaws.com", region); + } +#endif + return fmt::format("sts.{}.amazonaws.com", region); +} + static size_t curlCallback(char* ptr, size_t, size_t nmemb, void* data) { auto buf = static_cast(data); buf->append(ptr, nmemb); @@ -236,8 +257,9 @@ absl::optional Utility::fetchMetadata(Http::RequestMessage& message const auto host = message.headers().getHostValue(); const auto path = message.headers().getPathValue(); const auto method = message.headers().getMethodValue(); + const auto scheme = message.headers().getSchemeValue(); - const std::string url = fmt::format("http://{}{}", host, path); + const std::string url = fmt::format("{}://{}{}", scheme, host, path); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT.count()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); @@ -294,6 +316,71 @@ absl::optional Utility::fetchMetadata(Http::RequestMessage& message return buffer.empty() ? absl::nullopt : absl::optional(buffer); } +bool Utility::addInternalClusterStatic( + Upstream::ClusterManager& cm, absl::string_view cluster_name, + const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, absl::string_view uri) { + // Check if local cluster exists with that name. + if (cm.getThreadLocalCluster(cluster_name) == nullptr) { + // Make sure we run this on main thread. + TRY_ASSERT_MAIN_THREAD { + envoy::config::cluster::v3::Cluster cluster; + absl::string_view host_port; + absl::string_view path; + Http::Utility::extractHostPathFromUri(uri, host_port, path); + const auto host_attributes = Http::Utility::parseAuthority(host_port); + const auto host = host_attributes.host_; + const auto port = host_attributes.port_ ? host_attributes.port_.value() : 80; + + cluster.set_name(cluster_name); + cluster.set_type(cluster_type); + cluster.mutable_connect_timeout()->set_seconds(5); + cluster.mutable_load_assignment()->set_cluster_name(cluster_name); + auto* endpoint = cluster.mutable_load_assignment() + ->add_endpoints() + ->add_lb_endpoints() + ->mutable_endpoint(); + auto* addr = endpoint->mutable_address(); + addr->mutable_socket_address()->set_address(host); + addr->mutable_socket_address()->set_port_value(port); + cluster.set_lb_policy(envoy::config::cluster::v3::Cluster::ROUND_ROBIN); + envoy::extensions::upstreams::http::v3::HttpProtocolOptions protocol_options; + auto* http_protocol_options = + protocol_options.mutable_explicit_http_config()->mutable_http_protocol_options(); + http_protocol_options->set_accept_http_10(true); + (*cluster.mutable_typed_extension_protocol_options()) + ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] + .PackFrom(protocol_options); + + // Add tls transport socket if cluster supports https over port 443. + if (port == 443) { + auto* socket = cluster.mutable_transport_socket(); + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_socket; + socket->set_name("envoy.transport_sockets.tls"); + socket->mutable_typed_config()->PackFrom(tls_socket); + } + + // TODO(suniltheta): use random number generator here for cluster version. + // While adding multiple clusters make sure that change in random version number across + // multiple clusters won't make Envoy delete/replace previously registered internal cluster. + cm.addOrUpdateCluster(cluster, "12345"); + + const auto cluster_type_str = envoy::config::cluster::v3::Cluster::DiscoveryType_descriptor() + ->FindValueByNumber(cluster_type) + ->name(); + ENVOY_LOG_MISC(info, + "Added a {} internal cluster [name: {}, address:{}] to fetch aws " + "credentials", + cluster_type_str, cluster_name, host_port); + } + END_TRY + CATCH(const EnvoyException& e, { + ENVOY_LOG_MISC(error, "Failed to add internal cluster {}: {}", cluster_name, e.what()); + return false; + }); + } + return true; +} + } // namespace Aws } // namespace Common } // namespace Extensions diff --git a/source/extensions/common/aws/utility.h b/source/extensions/common/aws/utility.h index 2ec7cae045cd..902d0c6fa61b 100644 --- a/source/extensions/common/aws/utility.h +++ b/source/extensions/common/aws/utility.h @@ -1,9 +1,13 @@ #pragma once +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/upstreams/http/v3/http_protocol_options.pb.h" +#include "envoy/extensions/upstreams/http/v3/http_protocol_options.pb.validate.h" #include "envoy/http/message.h" #include "source/common/common/matchers.h" #include "source/common/http/headers.h" +#include "source/common/http/utility.h" namespace Envoy { namespace Extensions { @@ -80,6 +84,14 @@ class Utility { static std::string joinCanonicalHeaderNames(const std::map& canonical_headers); + /** + * Get the Security Token Service endpoint for a given region: sts..amazonaws.com + * See: https://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region + * @param region An AWS region. + * @return an sts endpoint url. + */ + static std::string getSTSEndpoint(absl::string_view region); + /** * Fetch AWS instance or task metadata. * @@ -92,6 +104,24 @@ class Utility { * gRPC auth plugins that are able to schedule blocking plugins on a different thread. */ static absl::optional fetchMetadata(Http::RequestMessage& message); + + /** + * @brief Adds a static cluster towards a credentials provider + * to fetch the credentials using http async client. + * + * @param cm cluster manager + * @param cluster_name a name for credentials provider cluster + * @param cluster_type STATIC or STRICT_DNS or LOGICAL_DNS etc + * @param uri provider's IP (STATIC cluster) or URL (STRICT_DNS). Will use port 80 if the port is + * not specified in the uri or no matching cluster is found. + * @return true if successfully added the cluster or if a cluster with the cluster_name already + * exists. + * @return false if failed to add the cluster + */ + static bool + addInternalClusterStatic(Upstream::ClusterManager& cm, absl::string_view cluster_name, + const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, + absl::string_view uri); }; } // namespace Aws diff --git a/source/extensions/common/dynamic_forward_proxy/BUILD b/source/extensions/common/dynamic_forward_proxy/BUILD index df95f5254c54..bb3535dab9ec 100644 --- a/source/extensions/common/dynamic_forward_proxy/BUILD +++ b/source/extensions/common/dynamic_forward_proxy/BUILD @@ -29,7 +29,6 @@ envoy_cc_library( deps = [ ":dns_cache_impl", "//source/common/protobuf", - "//source/server:factory_context_base_impl_lib", "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", ], ) @@ -50,6 +49,7 @@ envoy_cc_library( "//source/common/network:resolver_lib", "//source/common/network:utility_lib", "//source/common/network/dns_resolver:dns_factory_util_lib", + "//source/server:generic_factory_context_lib", "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", ], ) @@ -77,7 +77,6 @@ envoy_cc_library( deps = [ "//envoy/server:factory_context_interface", "//envoy/upstream:upstream_interface", - "//source/server:factory_context_base_impl_lib", "@com_google_absl//absl/container:flat_hash_set", ], ) diff --git a/source/extensions/common/dynamic_forward_proxy/cluster_store.cc b/source/extensions/common/dynamic_forward_proxy/cluster_store.cc index 96ba5c151ad4..4c5de2114a10 100644 --- a/source/extensions/common/dynamic_forward_proxy/cluster_store.cc +++ b/source/extensions/common/dynamic_forward_proxy/cluster_store.cc @@ -30,7 +30,7 @@ void DFPClusterStore::remove(const std::string cluster_name) { } DFPClusterStoreSharedPtr DFPClusterStoreFactory::get() { - return context_.singletonManager().getTyped( + return singleton_manager_.getTyped( SINGLETON_MANAGER_REGISTERED_NAME(dynamic_forward_proxy_cluster_store), []() { return std::make_shared(); }); } diff --git a/source/extensions/common/dynamic_forward_proxy/cluster_store.h b/source/extensions/common/dynamic_forward_proxy/cluster_store.h index 47798d8416d0..de407599bd02 100644 --- a/source/extensions/common/dynamic_forward_proxy/cluster_store.h +++ b/source/extensions/common/dynamic_forward_proxy/cluster_store.h @@ -3,8 +3,6 @@ #include "envoy/server/factory_context.h" #include "envoy/upstream/upstream.h" -#include "source/server/factory_context_base_impl.h" - #include "absl/container/flat_hash_map.h" namespace Envoy { @@ -66,11 +64,12 @@ using DFPClusterStoreSharedPtr = std::shared_ptr; class DFPClusterStoreFactory { public: - DFPClusterStoreFactory(Server::Configuration::FactoryContextBase& context) : context_(context) {} + DFPClusterStoreFactory(Singleton::Manager& singleton_manager) + : singleton_manager_(singleton_manager) {} DFPClusterStoreSharedPtr get(); private: - Server::FactoryContextBaseImpl context_; + Singleton::Manager& singleton_manager_; }; } // namespace DynamicForwardProxy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h index 89670576681a..25af3e34e649 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -45,6 +45,11 @@ class DnsHostInfo { */ virtual Network::Address::InstanceConstSharedPtr address() const PURE; + /** + * Returns whether the first DNS resolving attempt is completed or not. + */ + virtual bool firstResolveComplete() const PURE; + /** * Returns the host's currently resolved address. These addresses may change periodically due to * async re-resolution. @@ -264,7 +269,7 @@ class DnsCacheManager { * @param config supplies the cache parameters. If a cache exists with the same parameters it * will be returned, otherwise a new one will be created. */ - virtual DnsCacheSharedPtr + virtual absl::StatusOr getCache(const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config) PURE; /** diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index b86510e92e9c..12a2446ecf2b 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -9,39 +9,47 @@ #include "source/common/network/dns_resolver/dns_factory_util.h" #include "source/common/network/resolver_impl.h" #include "source/common/network/utility.h" +#include "source/common/runtime/runtime_features.h" namespace Envoy { namespace Extensions { namespace Common { namespace DynamicForwardProxy { +absl::StatusOr> DnsCacheImpl::createDnsCacheImpl( + Server::Configuration::GenericFactoryContext& context, + const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config) { + const uint32_t max_hosts = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_hosts, 1024); + if (static_cast(config.preresolve_hostnames().size()) > max_hosts) { + return absl::InvalidArgumentError(fmt::format( + "DNS Cache [{}] configured with preresolve_hostnames={} larger than max_hosts={}", + config.name(), config.preresolve_hostnames().size(), max_hosts)); + } + + return std::shared_ptr(new DnsCacheImpl(context, config)); +} + DnsCacheImpl::DnsCacheImpl( - Server::Configuration::FactoryContextBase& context, + Server::Configuration::GenericFactoryContext& context, const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config) - : main_thread_dispatcher_(context.mainThreadDispatcher()), config_(config), - random_generator_(context.api().randomGenerator()), + : main_thread_dispatcher_(context.serverFactoryContext().mainThreadDispatcher()), + config_(config), random_generator_(context.serverFactoryContext().api().randomGenerator()), dns_lookup_family_(DnsUtils::getDnsLookupFamilyFromEnum(config.dns_lookup_family())), - resolver_(selectDnsResolver(config, main_thread_dispatcher_, context)), - tls_slot_(context.threadLocal()), + resolver_(selectDnsResolver(config, main_thread_dispatcher_, context.serverFactoryContext())), + tls_slot_(context.serverFactoryContext().threadLocal()), scope_(context.scope().createScope(fmt::format("dns_cache.{}.", config.name()))), stats_(generateDnsCacheStats(*scope_)), - resource_manager_(*scope_, context.runtime(), config.name(), + resource_manager_(*scope_, context.serverFactoryContext().runtime(), config.name(), config.dns_cache_circuit_breaker()), refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_refresh_rate, 60000)), min_refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_min_refresh_rate, 5000)), timeout_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_query_timeout, 5000)), - file_system_(context.api().fileSystem()), + file_system_(context.serverFactoryContext().api().fileSystem()), validation_visitor_(context.messageValidationVisitor()), host_ttl_(PROTOBUF_GET_MS_OR_DEFAULT(config, host_ttl, 300000)), max_hosts_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_hosts, 1024)) { tls_slot_.set([&](Event::Dispatcher&) { return std::make_shared(*this); }); - if (static_cast(config.preresolve_hostnames().size()) > max_hosts_) { - throw EnvoyException(fmt::format( - "DNS Cache [{}] configured with preresolve_hostnames={} larger than max_hosts={}", - config.name(), config.preresolve_hostnames().size(), max_hosts_)); - } - loadCacheEntries(config); // Preresolved hostnames are resolved without a read lock on primary hosts because it is done @@ -51,7 +59,13 @@ DnsCacheImpl::DnsCacheImpl( // cache to load an entry. Further if this particular resolution fails all the is lost is the // potential optimization of having the entry be preresolved the first time a true consumer of // this DNS cache asks for it. - startCacheLoad(hostname.address(), hostname.port_value(), false); + const std::string host = + (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.normalize_host_for_preresolve_dfp_dns")) + ? DnsHostInfo::normalizeHostForDfp(hostname.address(), hostname.port_value()) + : hostname.address(); + ENVOY_LOG(debug, "DNS pre-resolve starting for host {}", host); + startCacheLoad(host, hostname.port_value(), false); } } @@ -70,7 +84,8 @@ DnsCacheImpl::~DnsCacheImpl() { Network::DnsResolverSharedPtr DnsCacheImpl::selectDnsResolver( const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config, - Event::Dispatcher& main_thread_dispatcher, Server::Configuration::FactoryContextBase& context) { + Event::Dispatcher& main_thread_dispatcher, + Server::Configuration::CommonFactoryContext& context) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; Network::DnsResolverFactory& dns_resolver_factory = Network::createDnsResolverFactoryFromProto(config, typed_dns_resolver_config); @@ -291,8 +306,8 @@ void DnsCacheImpl::forceRefreshHosts() { } void DnsCacheImpl::startResolve(const std::string& host, PrimaryHostInfo& host_info) { - ENVOY_LOG(debug, "starting main thread resolve for host='{}' dns='{}' port='{}'", host, - host_info.host_info_->resolvedHost(), host_info.port_); + ENVOY_LOG(debug, "starting main thread resolve for host='{}' dns='{}' port='{}' timeout='{}'", + host, host_info.host_info_->resolvedHost(), host_info.port_, timeout_interval_.count()); ASSERT(host_info.active_query_ == nullptr); stats_.dns_query_attempt_.inc(); @@ -394,6 +409,10 @@ void DnsCacheImpl::finishResolve(const std::string& host, primary_host_info->host_info_->setAddresses(new_address, std::move(address_list)); runAddUpdateCallbacks(host, primary_host_info->host_info_); + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.dns_cache_set_first_resolve_complete")) { + primary_host_info->host_info_->setFirstResolveComplete(); + } address_changed = true; stats_.host_address_changed_.inc(); } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 531209aed9a0..c7bd9656897b 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -11,6 +11,7 @@ #include "source/common/common/cleanup.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache_resource_manager.h" +#include "source/server/generic_factory_context.h" #include "absl/container/flat_hash_map.h" @@ -46,14 +47,17 @@ class DnsCacheImplTest; class DnsCacheImpl : public DnsCache, Logger::Loggable { public: - DnsCacheImpl(Server::Configuration::FactoryContextBase& context, - const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config); + // Create a DnsCacheImpl or return a failed status; + static absl::StatusOr> createDnsCacheImpl( + Server::Configuration::GenericFactoryContext& context, + const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config); + ~DnsCacheImpl() override; static DnsCacheStats generateDnsCacheStats(Stats::Scope& scope); static Network::DnsResolverSharedPtr selectDnsResolver( const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config, Event::Dispatcher& main_thread_dispatcher, - Server::Configuration::FactoryContextBase& context); + Server::Configuration::CommonFactoryContext& context); // DnsCache LoadDnsCacheEntryResult loadDnsCacheEntry(absl::string_view host, uint16_t default_port, @@ -66,6 +70,8 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable { @@ -132,14 +138,17 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable&& list) { absl::WriterMutexLock lock{&resolve_lock_}; - first_resolve_complete_ = true; + if (!(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.dns_cache_set_first_resolve_complete"))) { + first_resolve_complete_ = true; + } address_ = address; address_list_ = std::move(list); } std::chrono::steady_clock::duration lastUsedTime() const { return last_used_time_.load(); } - bool firstResolveComplete() const { + bool firstResolveComplete() const override { absl::ReaderMutexLock lock{&resolve_lock_}; return first_resolve_complete_; } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc index 3fea4fcec98a..e15cfe325af1 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc @@ -14,25 +14,27 @@ namespace DynamicForwardProxy { SINGLETON_MANAGER_REGISTRATION(dns_cache_manager); -DnsCacheSharedPtr DnsCacheManagerImpl::getCache( +absl::StatusOr DnsCacheManagerImpl::getCache( const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config) { const auto& existing_cache = caches_.find(config.name()); if (existing_cache != caches_.end()) { if (!Protobuf::util::MessageDifferencer::Equivalent(config, existing_cache->second.config_)) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("config specified DNS cache '{}' with different settings", config.name())); } return existing_cache->second.cache_; } - DnsCacheSharedPtr new_cache = std::make_shared(context_, config); + auto cache_or_status = DnsCacheImpl::createDnsCacheImpl(context_, config); + RETURN_IF_STATUS_NOT_OK(cache_or_status); + DnsCacheSharedPtr new_cache = std::move(cache_or_status.value()); caches_.emplace(config.name(), ActiveCache{config, new_cache}); return new_cache; } DnsCacheSharedPtr DnsCacheManagerImpl::lookUpCacheByName(absl::string_view cache_name) { - ASSERT(context_.mainThreadDispatcher().isThreadSafe()); + ASSERT(context_.serverFactoryContext().mainThreadDispatcher().isThreadSafe()); const auto& existing_cache = caches_.find(cache_name); if (existing_cache != caches_.end()) { return existing_cache->second.cache_; @@ -42,7 +44,7 @@ DnsCacheSharedPtr DnsCacheManagerImpl::lookUpCacheByName(absl::string_view cache } DnsCacheManagerSharedPtr DnsCacheManagerFactoryImpl::get() { - return context_.singletonManager().getTyped( + return context_.serverFactoryContext().singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(dns_cache_manager), [this] { return std::make_shared(context_); }); } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h index 9ec6c434e468..1bc1c2cb7307 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h @@ -4,7 +4,7 @@ #include "envoy/server/factory_context.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache.h" -#include "source/server/factory_context_base_impl.h" +#include "source/server/generic_factory_context.h" #include "absl/container/flat_hash_map.h" @@ -15,10 +15,10 @@ namespace DynamicForwardProxy { class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance { public: - DnsCacheManagerImpl(Server::Configuration::FactoryContextBase& context) : context_(context) {} + DnsCacheManagerImpl(Server::Configuration::GenericFactoryContext& context) : context_(context) {} // DnsCacheManager - DnsCacheSharedPtr getCache( + absl::StatusOr getCache( const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config) override; DnsCacheSharedPtr lookUpCacheByName(absl::string_view cache_name) override; @@ -32,19 +32,22 @@ class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance { DnsCacheSharedPtr cache_; }; - Server::FactoryContextBaseImpl context_; + Server::GenericFactoryContextImpl context_; absl::flat_hash_map caches_; }; class DnsCacheManagerFactoryImpl : public DnsCacheManagerFactory { public: - DnsCacheManagerFactoryImpl(Server::Configuration::FactoryContextBase& context) + DnsCacheManagerFactoryImpl(Server::Configuration::ServerFactoryContext& server_context, + ProtobufMessage::ValidationVisitor& validation_visitor) + : context_(server_context, validation_visitor) {} + DnsCacheManagerFactoryImpl(Server::Configuration::GenericFactoryContext& context) : context_(context) {} DnsCacheManagerSharedPtr get() override; private: - Server::FactoryContextBaseImpl context_; + Server::GenericFactoryContextImpl context_; }; } // namespace DynamicForwardProxy diff --git a/source/extensions/common/matcher/BUILD b/source/extensions/common/matcher/BUILD index a4b96c3f92bf..9303688ee5a3 100644 --- a/source/extensions/common/matcher/BUILD +++ b/source/extensions/common/matcher/BUILD @@ -25,6 +25,10 @@ envoy_cc_extension( name = "trie_matcher_lib", srcs = ["trie_matcher.cc"], hdrs = ["trie_matcher.h"], + extra_visibility = [ + "//source/common/listener_manager:__subpackages__", + "//test:__subpackages__", + ], deps = [ "//envoy/matcher:matcher_interface", "//envoy/network:filter_interface", @@ -33,6 +37,6 @@ envoy_cc_extension( "//source/common/matcher:matcher_lib", "//source/common/network:lc_trie_lib", "//source/common/network:utility_lib", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/common/tap/tap_config_base.cc b/source/extensions/common/tap/tap_config_base.cc index 7f75f4636f77..953d96f1c669 100644 --- a/source/extensions/common/tap/tap_config_base.cc +++ b/source/extensions/common/tap/tap_config_base.cc @@ -109,7 +109,8 @@ TapConfigBaseImpl::TapConfigBaseImpl(const envoy::config::tap::v3::TapConfig& pr absl::get(context).get(); config = Config::Utility::translateAnyToFactoryConfig( sinks[0].custom_sink().typed_config(), - http_context.messageValidationContext().staticValidationVisitor(), tap_sink_factory); + http_context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), + tap_sink_factory); } sink_ = tap_sink_factory.createSinkPtr(*config, context); diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index e43f296255e9..57c0c24146b9 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -1476,10 +1476,8 @@ void Context::initializeWriteFilterCallbacks(Network::WriteFilterCallbacks& call network_write_filter_callbacks_ = &callbacks; } -void Context::log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { +void Context::log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { // `log` may be called multiple times due to mid-request logging -- we only want to run on the // last call. if (!stream_info.requestComplete().has_value()) { @@ -1495,10 +1493,10 @@ void Context::log(const Http::RequestHeaderMap* request_headers, } access_log_phase_ = true; - access_log_request_headers_ = request_headers; + access_log_request_headers_ = &log_context.requestHeaders(); // ? request_trailers ? - access_log_response_headers_ = response_headers; - access_log_response_trailers_ = response_trailers; + access_log_response_headers_ = &log_context.responseHeaders(); + access_log_response_trailers_ = &log_context.responseTrailers(); access_log_stream_info_ = &stream_info; onLog(); diff --git a/source/extensions/common/wasm/context.h b/source/extensions/common/wasm/context.h index e755d26a0871..7bc4322e54ab 100644 --- a/source/extensions/common/wasm/context.h +++ b/source/extensions/common/wasm/context.h @@ -148,11 +148,8 @@ class Context : public proxy_wasm::ContextBase, const std::shared_ptr& plugin); // deprecated // AccessLog::Instance - void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override; + void log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& info) override; uint32_t getLogLevel() override; @@ -247,7 +244,7 @@ class Context : public proxy_wasm::ContextBase, // HTTP WasmResult httpCall(std::string_view cluster, const Pairs& request_headers, std::string_view request_body, const Pairs& request_trailers, - int timeout_millisconds, uint32_t* token_ptr) override; + int timeout_milliseconds, uint32_t* token_ptr) override; // Stats/Metrics WasmResult defineMetric(uint32_t type, std::string_view name, uint32_t* metric_id_ptr) override; diff --git a/source/extensions/common/wasm/wasm.cc b/source/extensions/common/wasm/wasm.cc index 17d40d003a9c..c8e969f0793b 100644 --- a/source/extensions/common/wasm/wasm.cc +++ b/source/extensions/common/wasm/wasm.cc @@ -226,13 +226,10 @@ ContextBase* Wasm::createRootContext(const std::shared_ptr& plugin) ContextBase* Wasm::createVmContext() { return new Context(this); } -void Wasm::log(const PluginSharedPtr& plugin, const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { +void Wasm::log(const PluginSharedPtr& plugin, const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& info) { auto context = getRootContext(plugin, true); - context->log(request_headers, response_headers, response_trailers, stream_info, access_log_type); + context->log(log_context, info); } void Wasm::onStatsUpdate(const PluginSharedPtr& plugin, Envoy::Stats::MetricSnapshot& snapshot) { diff --git a/source/extensions/common/wasm/wasm.h b/source/extensions/common/wasm/wasm.h index dc2b5d704d16..17cf43e028b0 100644 --- a/source/extensions/common/wasm/wasm.h +++ b/source/extensions/common/wasm/wasm.h @@ -66,10 +66,8 @@ class Wasm : public WasmBase, Logger::Loggable { void getFunctions() override; // AccessLog::Instance - void log(const PluginSharedPtr& plugin, const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType access_log_type); + void log(const PluginSharedPtr& plugin, const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& info); void onStatsUpdate(const PluginSharedPtr& plugin, Envoy::Stats::MetricSnapshot& snapshot); diff --git a/source/extensions/compression/brotli/common/base.cc b/source/extensions/compression/brotli/common/base.cc index edbd9be90d9f..2f35aeb72cab 100644 --- a/source/extensions/compression/brotli/common/base.cc +++ b/source/extensions/compression/brotli/common/base.cc @@ -8,8 +8,8 @@ namespace Common { BrotliContext::BrotliContext(uint32_t chunk_size, uint32_t max_output_size) : max_output_size_{max_output_size}, chunk_size_{chunk_size}, - chunk_ptr_{std::make_unique(chunk_size)}, next_in_{}, next_out_{chunk_ptr_.get()}, - avail_in_{0}, avail_out_{chunk_size} {} + chunk_ptr_{std::make_unique(chunk_size)}, next_out_{chunk_ptr_.get()}, + avail_out_{chunk_size} {} void BrotliContext::updateOutput(Buffer::Instance& output_buffer) { if (avail_out_ == 0) { diff --git a/source/extensions/compression/brotli/common/base.h b/source/extensions/compression/brotli/common/base.h index 929081c6ef7f..2ec86153b28a 100644 --- a/source/extensions/compression/brotli/common/base.h +++ b/source/extensions/compression/brotli/common/base.h @@ -20,9 +20,9 @@ struct BrotliContext { const uint32_t max_output_size_; const uint32_t chunk_size_; std::unique_ptr chunk_ptr_; - const uint8_t* next_in_; + const uint8_t* next_in_{}; uint8_t* next_out_; - size_t avail_in_; + size_t avail_in_{0}; size_t avail_out_; private: diff --git a/source/extensions/compression/zstd/common/dictionary_manager.h b/source/extensions/compression/zstd/common/dictionary_manager.h index 06358b594876..c70f2aba25b0 100644 --- a/source/extensions/compression/zstd/common/dictionary_manager.h +++ b/source/extensions/compression/zstd/common/dictionary_manager.h @@ -89,7 +89,9 @@ template class public ThreadLocal::ThreadLocalObject {}; void onDictionaryUpdate(unsigned origin_id, const std::string& filename) { - const auto data = api_.fileSystem().fileReadToEnd(filename); + auto file_or_error = api_.fileSystem().fileReadToEnd(filename); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + const auto data = file_or_error.value(); if (!data.empty()) { auto dictionary = DictionarySharedPtr(builder_(data.data(), data.length())); auto id = getDictId(dictionary.get()); diff --git a/source/extensions/compression/zstd/compressor/config.cc b/source/extensions/compression/zstd/compressor/config.cc index db5c72574e37..978d2b69200c 100644 --- a/source/extensions/compression/zstd/compressor/config.cc +++ b/source/extensions/compression/zstd/compressor/config.cc @@ -33,8 +33,10 @@ Envoy::Compression::Compressor::CompressorFactoryPtr ZstdCompressorLibraryFactory::createCompressorFactoryFromProtoTyped( const envoy::extensions::compression::zstd::compressor::v3::Zstd& proto_config, Server::Configuration::FactoryContext& context) { - return std::make_unique(proto_config, context.mainThreadDispatcher(), - context.api(), context.threadLocal()); + auto& server_context = context.serverFactoryContext(); + return std::make_unique( + proto_config, server_context.mainThreadDispatcher(), server_context.api(), + server_context.threadLocal()); } /** diff --git a/source/extensions/compression/zstd/decompressor/config.cc b/source/extensions/compression/zstd/decompressor/config.cc index 694bff443815..92e1b5c29fe7 100644 --- a/source/extensions/compression/zstd/decompressor/config.cc +++ b/source/extensions/compression/zstd/decompressor/config.cc @@ -29,9 +29,10 @@ Envoy::Compression::Decompressor::DecompressorFactoryPtr ZstdDecompressorLibraryFactory::createDecompressorFactoryFromProtoTyped( const envoy::extensions::compression::zstd::decompressor::v3::Zstd& proto_config, Server::Configuration::FactoryContext& context) { - return std::make_unique(proto_config, context.scope(), - context.mainThreadDispatcher(), context.api(), - context.threadLocal()); + auto& server_context = context.serverFactoryContext(); + return std::make_unique( + proto_config, context.scope(), server_context.mainThreadDispatcher(), server_context.api(), + server_context.threadLocal()); } /** diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD index a91ef4e40507..4b373e96b073 100644 --- a/source/extensions/config_subscription/grpc/BUILD +++ b/source/extensions/config_subscription/grpc/BUILD @@ -171,10 +171,19 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "grpc_stream_interface", + hdrs = ["grpc_stream_interface.h"], + deps = [ + "//source/common/grpc:async_client_lib", + ], +) + envoy_cc_library( name = "grpc_stream_lib", hdrs = ["grpc_stream.h"], deps = [ + ":grpc_stream_interface", "//envoy/config:grpc_mux_interface", "//envoy/config:subscription_interface", "//envoy/grpc:async_client_interface", diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc index 435c98f53c0c..4f21f98a400e 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc @@ -47,7 +47,9 @@ bool isXdsTpWildcard(const std::string& resource_name) { // Must only be called on XdsTp resource names. std::string convertToWildcard(const std::string& resource_name) { ASSERT(XdsResourceIdentifier::hasXdsTpScheme(resource_name)); - xds::core::v3::ResourceName xdstp_resource = XdsResourceIdentifier::decodeUrn(resource_name); + auto resource_or_error = XdsResourceIdentifier::decodeUrn(resource_name); + THROW_IF_STATUS_NOT_OK(resource_or_error, throw); + xds::core::v3::ResourceName xdstp_resource = resource_or_error.value(); const auto pos = xdstp_resource.id().find_last_of('/'); xdstp_resource.set_id( pos == std::string::npos ? "*" : absl::StrCat(xdstp_resource.id().substr(0, pos), "/*")); @@ -67,7 +69,7 @@ GrpcMuxImpl::GrpcMuxImpl(GrpcMuxContext& grpc_mux_context, bool skip_subsequent_ xds_config_tracker_(grpc_mux_context.xds_config_tracker_), xds_resources_delegate_(grpc_mux_context.xds_resources_delegate_), eds_resources_cache_(std::move(grpc_mux_context.eds_resources_cache_)), - target_xds_authority_(grpc_mux_context.target_xds_authority_), first_stream_request_(true), + target_xds_authority_(grpc_mux_context.target_xds_authority_), dispatcher_(grpc_mux_context.dispatcher_), dynamic_update_callback_handle_( grpc_mux_context.local_info_.contextProvider().addDynamicContextUpdateCallback( @@ -91,7 +93,14 @@ void GrpcMuxImpl::onDynamicContextUpdate(absl::string_view resource_type_url) { queueDiscoveryRequest(resource_type_url); } -void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } +void GrpcMuxImpl::start() { + ASSERT(!started_); + if (started_) { + return; + } + started_ = true; + grpc_stream_.establishNewStream(); +} void GrpcMuxImpl::sendDiscoveryRequest(absl::string_view type_url) { if (shutdown_) { @@ -130,6 +139,15 @@ void GrpcMuxImpl::sendDiscoveryRequest(absl::string_view type_url) { } } +void GrpcMuxImpl::clearNonce() { + // Iterate over all api_states (for each type_url), and clear its nonce. + for (auto& [type_url, api_state] : api_state_) { + if (api_state) { + api_state->request_.clear_response_nonce(); + } + } +} + void GrpcMuxImpl::loadConfigFromDelegate(const std::string& type_url, const absl::flat_hash_set& resource_names) { if (!xds_resources_delegate_.has_value()) { @@ -368,8 +386,9 @@ void GrpcMuxImpl::processDiscoveryResources(const std::vectorname())) { // Sort the context params of an xdstp resource, so we can compare them easily. - xds::core::v3::ResourceName xdstp_resource = - XdsResourceIdentifier::decodeUrn(resource->name()); + auto resource_or_error = XdsResourceIdentifier::decodeUrn(resource->name()); + THROW_IF_STATUS_NOT_OK(resource_or_error, throw); + xds::core::v3::ResourceName xdstp_resource = resource_or_error.value(); XdsResourceIdentifier::EncodeOptions options; options.sort_context_params_ = true; resource_ref_map.emplace(XdsResourceIdentifier::encodeUrn(xdstp_resource, options), @@ -451,6 +470,7 @@ void GrpcMuxImpl::onWriteable() { drainRequests(); } void GrpcMuxImpl::onStreamEstablished() { first_stream_request_ = true; grpc_stream_.maybeUpdateQueueSizeStat(0); + clearNonce(); request_queue_ = std::make_unique>(); for (const auto& type_url : subscriptions_) { queueDiscoveryRequest(type_url); diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h index 92a557ef9604..f2a5b1fb3f9f 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h @@ -94,6 +94,8 @@ class GrpcMuxImpl : public GrpcMux, void drainRequests(); void setRetryTimer(); void sendDiscoveryRequest(absl::string_view type_url); + // Clears the nonces of all subscribed types in this gRPC mux. + void clearNonce(); struct GrpcMuxWatchImpl : public GrpcMuxWatch { GrpcMuxWatchImpl(const absl::flat_hash_set& resources, @@ -151,7 +153,9 @@ class GrpcMuxImpl : public GrpcMux, resources.begin(), resources.end(), std::inserter(resources_, resources_.begin()), [this](const std::string& resource_name) -> std::string { if (XdsResourceIdentifier::hasXdsTpScheme(resource_name)) { - auto xdstp_resource = XdsResourceIdentifier::decodeUrn(resource_name); + auto xdstp_resource_or_error = XdsResourceIdentifier::decodeUrn(resource_name); + THROW_IF_STATUS_NOT_OK(xdstp_resource_or_error, throw); + auto xdstp_resource = xdstp_resource_or_error.value(); if (subscription_options_.add_xdstp_node_context_params_) { const auto context = XdsContextParams::encodeResource( local_info_.contextProvider().nodeContext(), xdstp_resource.context(), {}, {}); @@ -263,7 +267,7 @@ class GrpcMuxImpl : public GrpcMux, XdsResourcesDelegateOptRef xds_resources_delegate_; EdsResourcesCachePtr eds_resources_cache_; const std::string target_xds_authority_; - bool first_stream_request_; + bool first_stream_request_{true}; // Helper function for looking up and potentially allocating a new ApiState. ApiState& apiStateFor(absl::string_view type_url); @@ -282,6 +286,7 @@ class GrpcMuxImpl : public GrpcMux, Event::Dispatcher& dispatcher_; Common::CallbackHandlePtr dynamic_update_callback_handle_; + bool started_{false}; // True iff Envoy is shutting down; no messages should be sent on the `grpc_stream_` when this is // true because it may contain dangling pointers. std::atomic shutdown_{false}; diff --git a/source/extensions/config_subscription/grpc/grpc_stream.h b/source/extensions/config_subscription/grpc/grpc_stream.h index 91449aefe090..4ae05e554cd7 100644 --- a/source/extensions/config_subscription/grpc/grpc_stream.h +++ b/source/extensions/config_subscription/grpc/grpc_stream.h @@ -11,6 +11,7 @@ #include "source/common/common/token_bucket_impl.h" #include "source/common/config/utility.h" #include "source/common/grpc/typed_async_client.h" +#include "source/extensions/config_subscription/grpc/grpc_stream_interface.h" namespace Envoy { namespace Config { @@ -21,7 +22,7 @@ template using ResponseProtoPtr = std::unique_ptr -class GrpcStream : public Grpc::AsyncStreamCallbacks, +class GrpcStream : public GrpcStreamInterface, public Logger::Loggable { public: GrpcStream(GrpcStreamCallbacks* callbacks, Grpc::RawAsyncClientPtr async_client, @@ -46,7 +47,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, } } - void establishNewStream() { + void establishNewStream() override { ENVOY_LOG(debug, "Establishing new gRPC bidi stream to {} for {}", async_client_.destination(), service_method_.DebugString()); if (stream_ != nullptr) { @@ -66,9 +67,9 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, callbacks_->onStreamEstablished(); } - bool grpcStreamAvailable() const { return stream_ != nullptr; } + bool grpcStreamAvailable() const override { return stream_ != nullptr; } - void sendMessage(const RequestProto& request) { stream_->sendMessage(request, false); } + void sendMessage(const RequestProto& request) override { stream_->sendMessage(request, false); } // Grpc::AsyncStreamCallbacks void onCreateInitialMetadata(Http::RequestHeaderMap& metadata) override { @@ -104,7 +105,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, setRetryTimer(); } - void maybeUpdateQueueSizeStat(uint64_t size) { + void maybeUpdateQueueSizeStat(uint64_t size) override { // Although request_queue_.push() happens elsewhere, the only time the queue is non-transiently // non-empty is when it remains non-empty after a drain attempt. (The push() doesn't matter // because we always attempt this drain immediately after the push). Basically, a change in @@ -117,7 +118,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, } } - bool checkRateLimitAllowsDrain() { + bool checkRateLimitAllowsDrain() override { if (!rate_limiting_enabled_ || limit_request_->consume(1, false)) { return true; } @@ -130,7 +131,9 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, return false; } - absl::optional getCloseStatus() { return last_close_status_; } + absl::optional getCloseStatusForTest() const override { + return last_close_status_; + } private: void setRetryTimer() { diff --git a/source/extensions/config_subscription/grpc/grpc_stream_interface.h b/source/extensions/config_subscription/grpc/grpc_stream_interface.h new file mode 100644 index 000000000000..d339c2f46e8c --- /dev/null +++ b/source/extensions/config_subscription/grpc/grpc_stream_interface.h @@ -0,0 +1,39 @@ +#pragma once + +#include "source/common/grpc/typed_async_client.h" + +namespace Envoy { +namespace Config { + +// Oversees communication for gRPC xDS implementations (parent to both SotW xDS and delta +// xDS variants). Reestablishes the gRPC channel when necessary, and provides rate limiting of +// requests. +template +class GrpcStreamInterface : public Grpc::AsyncStreamCallbacks { +public: + // Attempt to establish a new gRPC stream to the xDS server. + virtual void establishNewStream() PURE; + + // Returns true if the gRPC stream is available and messages can be sent over it. + virtual bool grpcStreamAvailable() const PURE; + + // Sends a request to the xDS server over the stream. + virtual void sendMessage(const RequestProto& request) PURE; + + // Updates the control_plane_stats `pending_requests` value. Note that the + // update will not be taken into effect if the size is 0, and the + // `pending_request` value was not set previously to non-zero value. + // This is done to avoid updating the queue's length until the first + // meaningful value is given. + virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; + + // Returns true if a message can be sent from the rate-limiting perspective. + // The rate-limiting counters may be updated by this method. + virtual bool checkRateLimitAllowsDrain() PURE; + + // Returns the current close-status, if set. + virtual absl::optional getCloseStatusForTest() const PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc index 5a45f97abd8f..e0f73f807f0b 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc @@ -158,7 +158,14 @@ void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { } // TODO(fredlas) to be removed from the GrpcMux interface very soon. -void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } +void NewGrpcMuxImpl::start() { + ASSERT(!started_); + if (started_) { + return; + } + started_ = true; + grpc_stream_.establishNewStream(); +} GrpcMuxWatchPtr NewGrpcMuxImpl::addWatch(const std::string& type_url, const absl::flat_hash_set& resources, @@ -193,7 +200,9 @@ void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, absl::flat_hash_set effective_resources; for (const auto& resource : resources) { if (XdsResourceIdentifier::hasXdsTpScheme(resource)) { - auto xdstp_resource = XdsResourceIdentifier::decodeUrn(resource); + auto xdstp_resource_or_error = XdsResourceIdentifier::decodeUrn(resource); + THROW_IF_STATUS_NOT_OK(xdstp_resource_or_error, throw); + auto xdstp_resource = xdstp_resource_or_error.value(); if (options.add_xdstp_node_context_params_) { const auto context = XdsContextParams::encodeResource( local_info_.contextProvider().nodeContext(), xdstp_resource.context(), {}, {}); diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h index 421cbd9ae05b..788879136560 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h @@ -192,6 +192,7 @@ class NewGrpcMuxImpl XdsConfigTrackerOptRef xds_config_tracker_; EdsResourcesCachePtr eds_resources_cache_; + bool started_{false}; // True iff Envoy is shutting down; no messages should be sent on the `grpc_stream_` when this is // true because it may contain dangling pointers. std::atomic shutdown_{false}; diff --git a/source/extensions/config_subscription/grpc/watch_map.cc b/source/extensions/config_subscription/grpc/watch_map.cc index 50a9a2909719..88bbf01125de 100644 --- a/source/extensions/config_subscription/grpc/watch_map.cc +++ b/source/extensions/config_subscription/grpc/watch_map.cc @@ -95,7 +95,9 @@ absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& res // This is not very efficient; it is possible to canonicalize etc. much faster with raw string // operations, but this implementation provides a reference for later optimization while we // adopt xdstp://. - xdstp_resource = XdsResourceIdentifier::decodeUrn(resource_name); + auto resource_or_error = XdsResourceIdentifier::decodeUrn(resource_name); + THROW_IF_STATUS_NOT_OK(resource_or_error, throw); + xdstp_resource = resource_or_error.value(); } auto watches_interested = watch_interest_.find( is_xdstp ? XdsResourceIdentifier::encodeUrn(xdstp_resource, encode_options) : resource_name); diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc index e13695971fde..ebcb675b62a0 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc @@ -129,7 +129,9 @@ void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* absl::flat_hash_set effective_resources; for (const auto& resource : resources) { if (XdsResourceIdentifier::hasXdsTpScheme(resource)) { - auto xdstp_resource = XdsResourceIdentifier::decodeUrn(resource); + auto xdstp_resource_or_error = XdsResourceIdentifier::decodeUrn(resource); + THROW_IF_STATUS_NOT_OK(xdstp_resource_or_error, throw); + auto xdstp_resource = xdstp_resource_or_error.value(); if (options.add_xdstp_node_context_params_) { const auto context = XdsContextParams::encodeResource( local_info_.contextProvider().nodeContext(), xdstp_resource.context(), {}, {}); @@ -222,6 +224,11 @@ void GrpcMuxImpl::genericHandleResponse(const std::string& type_ur } template void GrpcMuxImpl::start() { + ASSERT(!started_); + if (started_) { + return; + } + started_ = true; ENVOY_LOG(debug, "GrpcMuxImpl now trying to establish a stream"); grpc_stream_.establishNewStream(); } diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h index f7063dc76f7c..8af1675ba5cb 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h @@ -211,6 +211,7 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, EdsResourcesCachePtr eds_resources_cache_; const std::string target_xds_authority_; + bool started_{false}; // True iff Envoy is shutting down; no messages should be sent on the `grpc_stream_` when this is // true because it may contain dangling pointers. std::atomic shutdown_{false}; diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index d6164dacef35..a67bf3be4744 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -124,6 +124,7 @@ EXTENSIONS = { "envoy.filters.http.aws_lambda": "//source/extensions/filters/http/aws_lambda:config", "envoy.filters.http.aws_request_signing": "//source/extensions/filters/http/aws_request_signing:config", "envoy.filters.http.bandwidth_limit": "//source/extensions/filters/http/bandwidth_limit:config", + "envoy.filters.http.basic_auth": "//source/extensions/filters/http/basic_auth:config", "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cache": "//source/extensions/filters/http/cache:config", "envoy.filters.http.cdn_loop": "//source/extensions/filters/http/cdn_loop:config", @@ -140,7 +141,7 @@ EXTENSIONS = { "envoy.filters.http.fault": "//source/extensions/filters/http/fault:config", "envoy.filters.http.file_system_buffer": "//source/extensions/filters/http/file_system_buffer:config", "envoy.filters.http.gcp_authn": "//source/extensions/filters/http/gcp_authn:config", - "envoy.filters.http.geoip": "//source/extensions/filters/http/geoip:config", + "envoy.filters.http.geoip": "//source/extensions/filters/http/geoip:config", "envoy.filters.http.grpc_field_extraction": "//source/extensions/filters/http/grpc_field_extraction:config", "envoy.filters.http.grpc_http1_bridge": "//source/extensions/filters/http/grpc_http1_bridge:config", "envoy.filters.http.grpc_http1_reverse_bridge": "//source/extensions/filters/http/grpc_http1_reverse_bridge:config", @@ -153,7 +154,9 @@ EXTENSIONS = { "envoy.filters.http.json_to_metadata": "//source/extensions/filters/http/json_to_metadata:config", "envoy.filters.http.jwt_authn": "//source/extensions/filters/http/jwt_authn:config", "envoy.filters.http.rate_limit_quota": "//source/extensions/filters/http/rate_limit_quota:config", - # Disabled by default + # Disabled by default. kill_request is not built into most prebuilt images. + # For instructions for building with disabled-by-default filters enabled, see + # https://github.com/envoyproxy/envoy/blob/main/bazel/README.md#enabling-and-disabling-extensions "envoy.filters.http.kill_request": "//source/extensions/filters/http/kill_request:kill_request_config", "envoy.filters.http.local_ratelimit": "//source/extensions/filters/http/local_ratelimit:config", "envoy.filters.http.lua": "//source/extensions/filters/http/lua:config", @@ -163,6 +166,7 @@ EXTENSIONS = { "envoy.filters.http.ratelimit": "//source/extensions/filters/http/ratelimit:config", "envoy.filters.http.rbac": "//source/extensions/filters/http/rbac:config", "envoy.filters.http.router": "//source/extensions/filters/http/router:config", + "envoy.filters.http.set_filter_state": "//source/extensions/filters/http/set_filter_state:config", "envoy.filters.http.set_metadata": "//source/extensions/filters/http/set_metadata:config", "envoy.filters.http.tap": "//source/extensions/filters/http/tap:config", "envoy.filters.http.wasm": "//source/extensions/filters/http/wasm:config", @@ -201,6 +205,7 @@ EXTENSIONS = { "envoy.filters.network.redis_proxy": "//source/extensions/filters/network/redis_proxy:config", "envoy.filters.network.tcp_proxy": "//source/extensions/filters/network/tcp_proxy:config", "envoy.filters.network.thrift_proxy": "//source/extensions/filters/network/thrift_proxy:config", + "envoy.filters.network.set_filter_state": "//source/extensions/filters/network/set_filter_state:config", "envoy.filters.network.sni_cluster": "//source/extensions/filters/network/sni_cluster:config", "envoy.filters.network.sni_dynamic_forward_proxy": "//source/extensions/filters/network/sni_dynamic_forward_proxy:config", "envoy.filters.network.wasm": "//source/extensions/filters/network/wasm:config", @@ -213,6 +218,13 @@ EXTENSIONS = { "envoy.filters.udp.dns_filter": "//source/extensions/filters/udp/dns_filter:config", "envoy.filters.udp_listener.udp_proxy": "//source/extensions/filters/udp/udp_proxy:config", + # + # UDP Session filters + # + + "envoy.filters.udp.session.http_capsule": "//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:config", + "envoy.filters.udp.session.dynamic_forward_proxy": "//source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy:config", + # # Resource monitors # @@ -254,6 +266,19 @@ EXTENSIONS = { "envoy.tracers.skywalking": "//source/extensions/tracers/skywalking:config", "envoy.tracers.opentelemetry": "//source/extensions/tracers/opentelemetry:config", + # + # OpenTelemetry Resource Detectors + # + + "envoy.tracers.opentelemetry.resource_detectors.environment": "//source/extensions/tracers/opentelemetry/resource_detectors/environment:config", + "envoy.tracers.opentelemetry.resource_detectors.dynatrace": "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:config", + + # + # OpenTelemetry tracer samplers + # + + "envoy.tracers.opentelemetry.samplers.always_on": "//source/extensions/tracers/opentelemetry/samplers/always_on:config", + # # Transport sockets # @@ -464,6 +489,17 @@ EXTENSIONS = { "envoy.config_subscription.ads_collection": "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", "envoy.config_mux.delta_grpc_mux_factory": "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", "envoy.config_mux.sotw_grpc_mux_factory": "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", + + # + # Geolocation Provider + # + "envoy.geoip_providers.maxmind": "//source/extensions/geoip_providers/maxmind:config", + + # + # cluster specifier plugin + # + + "envoy.router.cluster_specifier_plugin.lua": "//source/extensions/router/cluster_specifiers/lua:config", } # These can be changed to ["//visibility:public"], for downstream builds which diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index f63117e8aff8..587fb00c0363 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -217,6 +217,13 @@ envoy.filters.http.bandwidth_limit: status: stable type_urls: - envoy.extensions.filters.http.bandwidth_limit.v3.BandwidthLimit +envoy.filters.http.basic_auth: + categories: + - envoy.filters.http + security_posture: robust_to_untrusted_downstream + status: alpha + type_urls: + - envoy.extensions.filters.http.basic_auth.v3.BasicAuth envoy.filters.http.buffer: categories: - envoy.filters.http @@ -319,8 +326,8 @@ envoy.filters.http.ext_authz: envoy.filters.http.ext_proc: categories: - envoy.filters.http - security_posture: unknown - status: alpha + security_posture: robust_to_untrusted_downstream_and_upstream + status: stable type_urls: - envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute - envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor @@ -577,7 +584,7 @@ envoy.filters.listener.proxy_protocol: envoy.filters.listener.tls_inspector: categories: - envoy.filters.listener - security_posture: robust_to_untrusted_downstream + security_posture: robust_to_untrusted_downstream_and_upstream status: stable type_urls: - envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector @@ -747,6 +754,20 @@ envoy.filters.udp_listener.udp_proxy: status: stable type_urls: - envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig +envoy.filters.udp.session.http_capsule: + categories: + - envoy.filters.udp.session + security_posture: robust_to_untrusted_downstream + status: alpha + type_urls: + - envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3.FilterConfig +envoy.filters.udp.session.dynamic_forward_proxy: + categories: + - envoy.filters.udp.session + security_posture: robust_to_untrusted_downstream + status: alpha + type_urls: + - envoy.extensions.filters.udp.udp_proxy.session.dynamic_forward_proxy.v3.FilterConfig envoy.formatter.cel: categories: - envoy.formatter @@ -768,6 +789,13 @@ envoy.formatter.req_without_query: status: alpha type_urls: - envoy.extensions.formatter.req_without_query.v3.ReqWithoutQuery +envoy.geoip_providers.maxmind: + categories: + - envoy.geoip_providers + security_posture: unknown + status: wip + type_urls: + - envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig envoy.grpc_credentials.aws_iam: categories: - envoy.grpc_credentials @@ -993,7 +1021,7 @@ envoy.resource_monitors.downstream_connections: categories: - envoy.resource_monitors security_posture: data_plane_agnostic - status: wip + status: alpha type_urls: - envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig envoy.resource_monitors.fixed_heap: @@ -1045,6 +1073,13 @@ envoy.route.early_data_policy.default: status: stable type_urls: - envoy.extensions.early_data.v3.DefaultEarlyDataPolicy +envoy.router.cluster_specifier_plugin.lua: + categories: + - envoy.router.cluster_specifier_plugin + security_posture: robust_to_untrusted_downstream_and_upstream + status: alpha + type_urls: + - envoy.extensions.router.cluster_specifiers.lua.v3.LuaConfig envoy.stat_sinks.dog_statsd: categories: - envoy.stats_sinks @@ -1127,6 +1162,13 @@ envoy.tracers.opentelemetry: status: wip type_urls: - envoy.config.trace.v3.OpenTelemetryConfig +envoy.tracers.opentelemetry.samplers.always_on: + categories: + - envoy.tracers.opentelemetry.samplers + security_posture: unknown + status: wip + type_urls: + - envoy.extensions.tracers.opentelemetry.samplers.v3.AlwaysOnSamplerConfig envoy.tracers.skywalking: categories: - envoy.tracers @@ -1355,42 +1397,42 @@ envoy.matching.inputs.request_headers: categories: - envoy.matching.http.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.type.matcher.v3.HttpRequestHeaderMatchInput envoy.matching.inputs.request_trailers: categories: - envoy.matching.http.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.type.matcher.v3.HttpRequestTrailerMatchInput envoy.matching.inputs.response_headers: categories: - envoy.matching.http.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.type.matcher.v3.HttpResponseHeaderMatchInput envoy.matching.inputs.response_trailers: categories: - envoy.matching.http.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.type.matcher.v3.HttpResponseTrailerMatchInput envoy.matching.inputs.query_params: categories: - envoy.matching.http.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.type.matcher.v3.HttpRequestQueryParamMatchInput envoy.matching.inputs.cel_data_input: categories: - envoy.matching.http.input security_posture: unknown - status: alpha + status: stable type_urls: - xds.type.matcher.v3.HttpAttributesCelMatchInput envoy.matching.inputs.destination_ip: @@ -1398,7 +1440,7 @@ envoy.matching.inputs.destination_ip: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.DestinationIPInput envoy.matching.inputs.destination_port: @@ -1406,7 +1448,7 @@ envoy.matching.inputs.destination_port: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.DestinationPortInput envoy.matching.inputs.source_ip: @@ -1414,7 +1456,7 @@ envoy.matching.inputs.source_ip: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.SourceIPInput envoy.matching.inputs.source_port: @@ -1422,7 +1464,7 @@ envoy.matching.inputs.source_port: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.SourcePortInput envoy.matching.inputs.direct_source_ip: @@ -1430,7 +1472,7 @@ envoy.matching.inputs.direct_source_ip: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.DirectSourceIPInput envoy.matching.inputs.source_type: @@ -1438,7 +1480,7 @@ envoy.matching.inputs.source_type: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.SourceTypeInput envoy.matching.inputs.server_name: @@ -1446,28 +1488,28 @@ envoy.matching.inputs.server_name: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.ServerNameInput envoy.matching.inputs.transport_protocol: categories: - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.TransportProtocolInput envoy.matching.inputs.application_protocol: categories: - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.ApplicationProtocolInput envoy.matching.inputs.filter_state: categories: - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.network.v3.FilterStateInput envoy.matching.inputs.uri_san: @@ -1475,7 +1517,7 @@ envoy.matching.inputs.uri_san: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.ssl.v3.UriSanInput envoy.matching.inputs.dns_san: @@ -1483,7 +1525,7 @@ envoy.matching.inputs.dns_san: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.ssl.v3.DnsSanInput envoy.matching.inputs.subject: @@ -1491,7 +1533,7 @@ envoy.matching.inputs.subject: - envoy.matching.http.input - envoy.matching.network.input security_posture: unknown - status: alpha + status: stable type_urls: - envoy.extensions.matching.common_inputs.ssl.v3.SubjectInput envoy.matching.custom_matchers.trie_matcher: @@ -1499,7 +1541,7 @@ envoy.matching.custom_matchers.trie_matcher: - envoy.matching.http.custom_matchers - envoy.matching.network.custom_matchers security_posture: unknown - status: alpha + status: stable type_urls: - xds.type.matcher.v3.IPMatcher envoy.load_balancing_policies.least_request: @@ -1569,7 +1611,7 @@ envoy.matching.actions.format_string: categories: - envoy.matching.action security_posture: unknown - status: alpha + status: stable type_urls: - envoy.config.core.v3.SubstitutionFormatString envoy.http.custom_response.redirect_policy: @@ -1641,3 +1683,31 @@ envoy.config_mux.sotw_grpc_mux_factory: - envoy.config_mux security_posture: unknown status: stable +envoy.filters.http.set_filter_state: + categories: + - envoy.filters.http + security_posture: unknown + status: alpha + type_urls: + - envoy.extensions.filters.http.set_filter_state.v3.Config +envoy.filters.network.set_filter_state: + categories: + - envoy.filters.network + security_posture: unknown + status: alpha + type_urls: + - envoy.extensions.filters.network.set_filter_state.v3.Config +envoy.tracers.opentelemetry.resource_detectors.environment: + categories: + - envoy.tracers.opentelemetry.resource_detectors + security_posture: unknown + status: wip + type_urls: + - envoy.extensions.tracers.opentelemetry.resource_detectors.v3.EnvironmentResourceDetectorConfig +envoy.tracers.opentelemetry.resource_detectors.dynatrace: + categories: + - envoy.tracers.opentelemetry.resource_detectors + security_posture: unknown + status: wip + type_urls: + - envoy.extensions.tracers.opentelemetry.resource_detectors.v3.DynatraceResourceDetectorConfig diff --git a/source/extensions/filters/common/expr/context.cc b/source/extensions/filters/common/expr/context.cc index 3a7277c200dd..a609e8be884a 100644 --- a/source/extensions/filters/common/expr/context.cc +++ b/source/extensions/filters/common/expr/context.cc @@ -210,6 +210,11 @@ absl::optional ConnectionWrapper::operator[](CelValue key) const { return CelValue::CreateString(&info_.connectionTerminationDetails().value()); } return {}; + } else if (value == DownstreamTransportFailureReason) { + if (!info_.downstreamTransportFailureReason().empty()) { + return CelValue::CreateStringView(info_.downstreamTransportFailureReason()); + } + return {}; } auto ssl_info = info_.downstreamAddressProvider().sslConnection(); @@ -380,6 +385,20 @@ absl::optional XDSWrapper::operator[](CelValue key) const { const absl::string_view filter_chain_name = filter_chain_info.has_value() ? filter_chain_info->name() : absl::string_view{}; return CelValue::CreateStringView(filter_chain_name); + } else if (value == ListenerMetadata) { + const auto listener_info = info_.downstreamAddressProvider().listenerInfo(); + if (listener_info) { + return CelProtoWrapper::CreateMessage(&listener_info->metadata(), &arena_); + } + } else if (value == ListenerDirection) { + const auto listener_info = info_.downstreamAddressProvider().listenerInfo(); + if (listener_info) { + return CelValue::CreateInt64(listener_info->direction()); + } + } else if (value == Node) { + if (local_info_) { + return CelProtoWrapper::CreateMessage(&local_info_->node(), &arena_); + } } return {}; } diff --git a/source/extensions/filters/common/expr/context.h b/source/extensions/filters/common/expr/context.h index 7257655ffb77..ff1f127373cf 100644 --- a/source/extensions/filters/common/expr/context.h +++ b/source/extensions/filters/common/expr/context.h @@ -66,6 +66,7 @@ constexpr absl::string_view URISanPeerCertificate = "uri_san_peer_certificate"; constexpr absl::string_view DNSSanLocalCertificate = "dns_san_local_certificate"; constexpr absl::string_view DNSSanPeerCertificate = "dns_san_peer_certificate"; constexpr absl::string_view SHA256PeerCertificateDigest = "sha256_peer_certificate_digest"; +constexpr absl::string_view DownstreamTransportFailureReason = "transport_failure_reason"; // Source properties constexpr absl::string_view Source = "source"; @@ -88,6 +89,9 @@ constexpr absl::string_view RouteName = "route_name"; constexpr absl::string_view RouteMetadata = "route_metadata"; constexpr absl::string_view UpstreamHostMetadata = "upstream_host_metadata"; constexpr absl::string_view FilterChainName = "filter_chain_name"; +constexpr absl::string_view ListenerMetadata = "listener_metadata"; +constexpr absl::string_view ListenerDirection = "listener_direction"; +constexpr absl::string_view Node = "node"; class WrapperFieldValues { public: @@ -231,12 +235,14 @@ class FilterStateWrapper : public BaseWrapper { class XDSWrapper : public BaseWrapper { public: - XDSWrapper(Protobuf::Arena& arena, const StreamInfo::StreamInfo& info) - : BaseWrapper(arena), info_(info) {} + XDSWrapper(Protobuf::Arena& arena, const StreamInfo::StreamInfo& info, + const LocalInfo::LocalInfo* local_info) + : BaseWrapper(arena), info_(info), local_info_(local_info) {} absl::optional operator[](CelValue key) const override; private: const StreamInfo::StreamInfo& info_; + const LocalInfo::LocalInfo* local_info_; }; } // namespace Expr diff --git a/source/extensions/filters/common/expr/evaluator.cc b/source/extensions/filters/common/expr/evaluator.cc index f7f965e9b3b9..a6eec90a949a 100644 --- a/source/extensions/filters/common/expr/evaluator.cc +++ b/source/extensions/filters/common/expr/evaluator.cc @@ -64,23 +64,26 @@ absl::optional StreamActivation::FindValue(absl::string_view name, return CelValue::CreateMap( Protobuf::Arena::Create(arena, *arena, info.filterState())); case ActivationToken::XDS: - return CelValue::CreateMap(Protobuf::Arena::Create(arena, *arena, info)); - }; + return CelValue::CreateMap( + Protobuf::Arena::Create(arena, *arena, info, local_info_)); + } return {}; } void StreamActivation::resetActivation() const { + local_info_ = nullptr; activation_info_ = nullptr; activation_request_headers_ = nullptr; activation_response_headers_ = nullptr; activation_response_trailers_ = nullptr; } -ActivationPtr createActivation(const StreamInfo::StreamInfo& info, +ActivationPtr createActivation(const LocalInfo::LocalInfo* local_info, + const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers) { - return std::make_unique(info, request_headers, response_headers, + return std::make_unique(local_info, info, request_headers, response_headers, response_trailers); } @@ -131,11 +134,13 @@ ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alph } absl::optional evaluate(const Expression& expr, Protobuf::Arena& arena, + const LocalInfo::LocalInfo* local_info, const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const Http::ResponseTrailerMap* response_trailers) { - auto activation = createActivation(info, request_headers, response_headers, response_trailers); + auto activation = + createActivation(local_info, info, request_headers, response_headers, response_trailers); auto eval_status = expr.Evaluate(*activation, &arena); if (!eval_status.ok()) { return {}; @@ -147,7 +152,7 @@ absl::optional evaluate(const Expression& expr, Protobuf::Arena& arena bool matches(const Expression& expr, const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& headers) { Protobuf::Arena arena; - auto eval_status = Expr::evaluate(expr, arena, info, &headers, nullptr, nullptr); + auto eval_status = Expr::evaluate(expr, arena, nullptr, info, &headers, nullptr, nullptr); if (!eval_status.has_value()) { return false; } diff --git a/source/extensions/filters/common/expr/evaluator.h b/source/extensions/filters/common/expr/evaluator.h index 656a9c7e676f..e250469298fe 100644 --- a/source/extensions/filters/common/expr/evaluator.h +++ b/source/extensions/filters/common/expr/evaluator.h @@ -26,11 +26,13 @@ using ExpressionPtr = std::unique_ptr; // Base class for the context used by the CEL evaluator to look up attributes. class StreamActivation : public google::api::expr::runtime::BaseActivation { public: - StreamActivation(const StreamInfo::StreamInfo& info, + StreamActivation(const ::Envoy::LocalInfo::LocalInfo* local_info, + const StreamInfo::StreamInfo& info, const ::Envoy::Http::RequestHeaderMap* request_headers, const ::Envoy::Http::ResponseHeaderMap* response_headers, const ::Envoy::Http::ResponseTrailerMap* response_trailers) - : activation_info_(&info), activation_request_headers_(request_headers), + : local_info_(local_info), activation_info_(&info), + activation_request_headers_(request_headers), activation_response_headers_(response_headers), activation_response_trailers_(response_trailers) {} @@ -44,6 +46,7 @@ class StreamActivation : public google::api::expr::runtime::BaseActivation { protected: void resetActivation() const; + mutable const ::Envoy::LocalInfo::LocalInfo* local_info_{nullptr}; mutable const StreamInfo::StreamInfo* activation_info_{nullptr}; mutable const ::Envoy::Http::RequestHeaderMap* activation_request_headers_{nullptr}; mutable const ::Envoy::Http::ResponseHeaderMap* activation_response_headers_{nullptr}; @@ -52,7 +55,8 @@ class StreamActivation : public google::api::expr::runtime::BaseActivation { // Creates an activation providing the common context attributes. // The activation lazily creates wrappers during an evaluation using the evaluation arena. -ActivationPtr createActivation(const StreamInfo::StreamInfo& info, +ActivationPtr createActivation(const ::Envoy::LocalInfo::LocalInfo* local_info, + const StreamInfo::StreamInfo& info, const ::Envoy::Http::RequestHeaderMap* request_headers, const ::Envoy::Http::ResponseHeaderMap* response_headers, const ::Envoy::Http::ResponseTrailerMap* response_trailers); @@ -84,6 +88,7 @@ ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alph // Evaluates an expression for a request. The arena is used to hold intermediate computational // results and potentially the final value. absl::optional evaluate(const Expression& expr, Protobuf::Arena& arena, + const ::Envoy::LocalInfo::LocalInfo* local_info, const StreamInfo::StreamInfo& info, const ::Envoy::Http::RequestHeaderMap* request_headers, const ::Envoy::Http::ResponseHeaderMap* response_headers, diff --git a/source/extensions/filters/common/ext_authz/check_request_utils.cc b/source/extensions/filters/common/ext_authz/check_request_utils.cc index 6d65dc7d4a4d..8a62aa1cdc98 100644 --- a/source/extensions/filters/common/ext_authz/check_request_utils.cc +++ b/source/extensions/filters/common/ext_authz/check_request_utils.cc @@ -192,6 +192,7 @@ void CheckRequestUtils::createHttpCheck( const Envoy::Http::RequestHeaderMap& headers, Protobuf::Map&& context_extensions, envoy::config::core::v3::Metadata&& metadata_context, + envoy::config::core::v3::Metadata&& route_metadata_context, envoy::service::auth::v3::CheckRequest& request, uint64_t max_request_bytes, bool pack_as_bytes, bool include_peer_certificate, bool include_tls_session, const Protobuf::Map& destination_labels, @@ -224,6 +225,7 @@ void CheckRequestUtils::createHttpCheck( // Fill in the context extensions and metadata context. (*attrs->mutable_context_extensions()) = std::move(context_extensions); (*attrs->mutable_metadata_context()) = std::move(metadata_context); + (*attrs->mutable_route_metadata_context()) = std::move(route_metadata_context); } void CheckRequestUtils::createTcpCheck( diff --git a/source/extensions/filters/common/ext_authz/check_request_utils.h b/source/extensions/filters/common/ext_authz/check_request_utils.h index 96d10334bcbb..1390485c0ae0 100644 --- a/source/extensions/filters/common/ext_authz/check_request_utils.h +++ b/source/extensions/filters/common/ext_authz/check_request_utils.h @@ -93,6 +93,7 @@ class CheckRequestUtils { const Envoy::Http::RequestHeaderMap& headers, Protobuf::Map&& context_extensions, envoy::config::core::v3::Metadata&& metadata_context, + envoy::config::core::v3::Metadata&& route_metadata_context, envoy::service::auth::v3::CheckRequest& request, uint64_t max_request_bytes, bool pack_as_bytes, bool include_peer_certificate, bool include_tls_session, diff --git a/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.cc b/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.cc index 71d095aacaba..603a61eca0d5 100644 --- a/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.cc +++ b/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.cc @@ -17,11 +17,13 @@ LocalRateLimiterImpl::LocalRateLimiterImpl( const std::chrono::milliseconds fill_interval, const uint32_t max_tokens, const uint32_t tokens_per_fill, Event::Dispatcher& dispatcher, const Protobuf::RepeatedPtrField< - envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptors) + envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptors, + bool always_consume_default_token_bucket) : fill_timer_(fill_interval > std::chrono::milliseconds(0) ? dispatcher.createTimer([this] { onFillTimer(); }) : nullptr), - time_source_(dispatcher.timeSource()) { + time_source_(dispatcher.timeSource()), + always_consume_default_token_bucket_(always_consume_default_token_bucket) { if (fill_timer_ && fill_interval < std::chrono::milliseconds(50)) { throw EnvoyException("local rate limit token bucket fill timer must be >= 50ms"); } @@ -175,10 +177,12 @@ bool LocalRateLimiterImpl::requestAllowed( // Matched descriptors will be sorted by tokens per second and tokens consumed in order. // In most cases, if one of them is limited the remaining descriptors will not consume // their tokens. + bool matched_descriptor = false; if (!descriptors_.empty() && !request_descriptors.empty()) { for (const auto& descriptor : sorted_descriptors_) { for (const auto& request_descriptor : request_descriptors) { if (descriptor == request_descriptor) { + matched_descriptor = true; // Descriptor token is not enough. if (!requestAllowedHelper(*descriptor.token_state_)) { return false; @@ -188,8 +192,12 @@ bool LocalRateLimiterImpl::requestAllowed( } } } - // Since global tokens are not sorted, it should be larger than other descriptors. - return requestAllowedHelper(tokens_); + + if (!matched_descriptor || always_consume_default_token_bucket_) { + // Since global tokens are not sorted, it should be larger than other descriptors. + return requestAllowedHelper(tokens_); + } + return true; } int LocalRateLimiterImpl::tokensFillPerSecond(LocalDescriptorImpl& descriptor) { diff --git a/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.h b/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.h index 9ab00b727ac7..c0cc182a492a 100644 --- a/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.h +++ b/source/extensions/filters/common/local_ratelimit/local_ratelimit_impl.h @@ -22,7 +22,8 @@ class LocalRateLimiterImpl { const std::chrono::milliseconds fill_interval, const uint32_t max_tokens, const uint32_t tokens_per_fill, Event::Dispatcher& dispatcher, const Protobuf::RepeatedPtrField< - envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptors); + envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptors, + bool always_consume_default_token_bucket = true); ~LocalRateLimiterImpl(); bool requestAllowed(absl::Span request_descriptors) const; @@ -84,6 +85,7 @@ class LocalRateLimiterImpl { absl::flat_hash_set descriptors_; std::vector sorted_descriptors_; mutable Thread::ThreadSynchronizer synchronizer_; // Used for testing only. + const bool always_consume_default_token_bucket_{}; friend class LocalRateLimiterImplTest; }; diff --git a/source/extensions/filters/common/ratelimit/ratelimit_impl.cc b/source/extensions/filters/common/ratelimit/ratelimit_impl.cc index b0904a5db86c..3097f1b62e5e 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit_impl.cc +++ b/source/extensions/filters/common/ratelimit/ratelimit_impl.cc @@ -123,13 +123,15 @@ void GrpcClientImpl::onFailure(Grpc::Status::GrpcStatus status, const std::strin } ClientPtr rateLimitClient(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::GrpcService& grpc_service, + const Grpc::GrpcServiceConfigWithHashKey& config_with_hash_key, const std::chrono::milliseconds timeout) { // TODO(ramaraochavali): register client to singleton when GrpcClientImpl supports concurrent // requests. return std::make_unique( - context.clusterManager().grpcAsyncClientManager().getOrCreateRawAsyncClient( - grpc_service, context.scope(), true), + context.serverFactoryContext() + .clusterManager() + .grpcAsyncClientManager() + .getOrCreateRawAsyncClientWithHashKey(config_with_hash_key, context.scope(), true), timeout); } diff --git a/source/extensions/filters/common/ratelimit/ratelimit_impl.h b/source/extensions/filters/common/ratelimit/ratelimit_impl.h index 2e8fc1e2134e..79502ec2ef78 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit_impl.h +++ b/source/extensions/filters/common/ratelimit/ratelimit_impl.h @@ -81,7 +81,7 @@ class GrpcClientImpl : public Client, * Builds the rate limit client. */ ClientPtr rateLimitClient(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::GrpcService& grpc_service, + const Grpc::GrpcServiceConfigWithHashKey& config_with_hash_key, const std::chrono::milliseconds timeout); } // namespace RateLimit diff --git a/source/extensions/filters/common/ratelimit/stat_names.h b/source/extensions/filters/common/ratelimit/stat_names.h index 65a8b0614d5d..4af10617f0ac 100644 --- a/source/extensions/filters/common/ratelimit/stat_names.h +++ b/source/extensions/filters/common/ratelimit/stat_names.h @@ -1,5 +1,6 @@ #pragma once +#include "source/common/common/empty_string.h" #include "source/common/stats/symbol_table.h" namespace Envoy { @@ -12,10 +13,19 @@ namespace RateLimit { // filters. These should generally be initialized once per process, and // not per-request, to avoid lock contention. struct StatNames { - explicit StatNames(Stats::SymbolTable& symbol_table) - : pool_(symbol_table), ok_(pool_.add("ratelimit.ok")), error_(pool_.add("ratelimit.error")), - failure_mode_allowed_(pool_.add("ratelimit.failure_mode_allowed")), - over_limit_(pool_.add("ratelimit.over_limit")) {} + explicit StatNames(Stats::SymbolTable& symbol_table, + const std::string& stat_prefix = EMPTY_STRING) + : pool_(symbol_table), ok_(pool_.add(createPoolStatName(stat_prefix, "ok"))), + error_(pool_.add(createPoolStatName(stat_prefix, "error"))), + failure_mode_allowed_(pool_.add(createPoolStatName(stat_prefix, "failure_mode_allowed"))), + over_limit_(pool_.add(createPoolStatName(stat_prefix, "over_limit"))) {} + + // This generates ratelimit..name + const std::string createPoolStatName(const std::string& stat_prefix, const std::string& name) { + return absl::StrCat("ratelimit", + stat_prefix.empty() ? EMPTY_STRING : absl::StrCat(".", stat_prefix), ".", + name); + } Stats::StatNamePool pool_; Stats::StatName ok_; Stats::StatName error_; diff --git a/source/extensions/filters/common/set_filter_state/BUILD b/source/extensions/filters/common/set_filter_state/BUILD new file mode 100644 index 000000000000..7c7a20a61c51 --- /dev/null +++ b/source/extensions/filters/common/set_filter_state/BUILD @@ -0,0 +1,22 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "filter_config_lib", + srcs = ["filter_config.cc"], + hdrs = ["filter_config.h"], + deps = [ + "//envoy/formatter:substitution_formatter_interface", + "//envoy/stream_info:filter_state_interface", + "//source/common/formatter:substitution_format_string_lib", + "//source/common/protobuf", + "@envoy_api//envoy/extensions/filters/common/set_filter_state/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/common/set_filter_state/filter_config.cc b/source/extensions/filters/common/set_filter_state/filter_config.cc new file mode 100644 index 000000000000..9bd0497aaed9 --- /dev/null +++ b/source/extensions/filters/common/set_filter_state/filter_config.cc @@ -0,0 +1,67 @@ +#include "source/extensions/filters/common/set_filter_state/filter_config.h" + +#include "source/common/formatter/substitution_format_string.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace SetFilterState { + +std::vector +Config::parse(const Protobuf::RepeatedPtrField& proto_values, + Server::Configuration::GenericFactoryContext& context) const { + std::vector values; + values.reserve(proto_values.size()); + for (const auto& proto_value : proto_values) { + Value value; + value.key_ = proto_value.object_key(); + value.factory_ = + Registry::FactoryRegistry::getFactory(value.key_); + if (value.factory_ == nullptr) { + throw EnvoyException(fmt::format("'{}' does not have an object factory", value.key_)); + } + value.state_type_ = proto_value.read_only() ? StateType::ReadOnly : StateType::Mutable; + switch (proto_value.shared_with_upstream()) { + case FilterStateValueProto::ONCE: + value.stream_sharing_ = StreamSharing::SharedWithUpstreamConnectionOnce; + break; + case FilterStateValueProto::TRANSITIVE: + value.stream_sharing_ = StreamSharing::SharedWithUpstreamConnection; + break; + default: + value.stream_sharing_ = StreamSharing::None; + break; + } + value.skip_if_empty_ = proto_value.skip_if_empty(); + value.value_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + proto_value.format_string(), context); + values.push_back(std::move(value)); + } + return values; +} + +void Config::updateFilterState(const Formatter::HttpFormatterContext& context, + StreamInfo::StreamInfo& info) const { + for (const auto& value : values_) { + const std::string bytes_value = value.value_->formatWithContext(context, info); + if (bytes_value.empty() && value.skip_if_empty_) { + ENVOY_LOG(debug, "Skip empty value for an object '{}'", value.key_); + continue; + } + auto object = value.factory_->createFromBytes(bytes_value); + if (object == nullptr) { + ENVOY_LOG(debug, "Failed to create an object '{}' from value '{}'", value.key_, bytes_value); + continue; + } + ENVOY_LOG(debug, "Created the filter state '{}' from value '{}'", value.key_, bytes_value); + info.filterState()->setData(value.key_, std::move(object), value.state_type_, life_span_, + value.stream_sharing_); + } +} + +} // namespace SetFilterState +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/set_filter_state/filter_config.h b/source/extensions/filters/common/set_filter_state/filter_config.h new file mode 100644 index 000000000000..3626aea701cd --- /dev/null +++ b/source/extensions/filters/common/set_filter_state/filter_config.h @@ -0,0 +1,52 @@ +#pragma once + +#include "envoy/extensions/filters/common/set_filter_state/v3/value.pb.h" +#include "envoy/formatter/substitution_formatter.h" +#include "envoy/stream_info/filter_state.h" + +#include "source/common/common/logger.h" +#include "source/common/protobuf/protobuf.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace SetFilterState { + +using LifeSpan = StreamInfo::FilterState::LifeSpan; +using StateType = StreamInfo::FilterState::StateType; +using StreamSharing = StreamInfo::StreamSharingMayImpactPooling; +using FilterStateValueProto = + envoy::extensions::filters::common::set_filter_state::v3::FilterStateValue; + +struct Value { + std::string key_; + StreamInfo::FilterState::ObjectFactory* factory_; + StateType state_type_{StateType::ReadOnly}; + StreamSharing stream_sharing_{StreamSharing::None}; + bool skip_if_empty_; + Formatter::FormatterConstSharedPtr value_; +}; + +class Config : public Logger::Loggable { +public: + Config(const Protobuf::RepeatedPtrField& proto_values, LifeSpan life_span, + Server::Configuration::GenericFactoryContext& context) + : life_span_(life_span), values_(parse(proto_values, context)) {} + void updateFilterState(const Formatter::HttpFormatterContext& context, + StreamInfo::StreamInfo& info) const; + +private: + std::vector parse(const Protobuf::RepeatedPtrField& proto_values, + Server::Configuration::GenericFactoryContext& context) const; + const LifeSpan life_span_; + const std::vector values_; +}; + +using ConfigSharedPtr = std::shared_ptr; + +} // namespace SetFilterState +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/adaptive_concurrency/config.cc b/source/extensions/filters/http/adaptive_concurrency/config.cc index ed2182bafcde..8bd16a945cea 100644 --- a/source/extensions/filters/http/adaptive_concurrency/config.cc +++ b/source/extensions/filters/http/adaptive_concurrency/config.cc @@ -15,6 +15,7 @@ namespace AdaptiveConcurrency { Http::FilterFactoryCb AdaptiveConcurrencyFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::adaptive_concurrency::v3::AdaptiveConcurrency& config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); auto acc_stats_prefix = stats_prefix + "adaptive_concurrency."; @@ -22,16 +23,16 @@ Http::FilterFactoryCb AdaptiveConcurrencyFilterFactory::createFilterFactoryFromP using Proto = envoy::extensions::filters::http::adaptive_concurrency::v3::AdaptiveConcurrency; ASSERT(config.concurrency_controller_config_case() == Proto::ConcurrencyControllerConfigCase::kGradientControllerConfig); - auto gradient_controller_config = - Controller::GradientControllerConfig(config.gradient_controller_config(), context.runtime()); + auto gradient_controller_config = Controller::GradientControllerConfig( + config.gradient_controller_config(), server_context.runtime()); controller = std::make_shared( - std::move(gradient_controller_config), context.mainThreadDispatcher(), context.runtime(), - acc_stats_prefix + "gradient_controller.", context.scope(), context.api().randomGenerator(), - context.timeSource()); + std::move(gradient_controller_config), server_context.mainThreadDispatcher(), + server_context.runtime(), acc_stats_prefix + "gradient_controller.", context.scope(), + server_context.api().randomGenerator(), server_context.timeSource()); - AdaptiveConcurrencyFilterConfigSharedPtr filter_config( - new AdaptiveConcurrencyFilterConfig(config, context.runtime(), std::move(acc_stats_prefix), - context.scope(), context.timeSource())); + AdaptiveConcurrencyFilterConfigSharedPtr filter_config(new AdaptiveConcurrencyFilterConfig( + config, server_context.runtime(), std::move(acc_stats_prefix), context.scope(), + server_context.timeSource())); return [filter_config, controller](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter( diff --git a/source/extensions/filters/http/admission_control/admission_control.cc b/source/extensions/filters/http/admission_control/admission_control.cc index 1038e9361013..51fc97ef8303 100644 --- a/source/extensions/filters/http/admission_control/admission_control.cc +++ b/source/extensions/filters/http/admission_control/admission_control.cc @@ -75,8 +75,7 @@ double AdmissionControlFilterConfig::maxRejectionProbability() const { AdmissionControlFilter::AdmissionControlFilter(AdmissionControlFilterConfigSharedPtr config, const std::string& stats_prefix) - : config_(std::move(config)), stats_(generateStats(config_->scope(), stats_prefix)), - record_request_(true) {} + : config_(std::move(config)), stats_(generateStats(config_->scope(), stats_prefix)) {} Http::FilterHeadersStatus AdmissionControlFilter::decodeHeaders(Http::RequestHeaderMap&, bool) { if (!config_->filterEnabled() || decoder_callbacks_->streamInfo().healthCheck()) { diff --git a/source/extensions/filters/http/admission_control/admission_control.h b/source/extensions/filters/http/admission_control/admission_control.h index 22f976768316..56eb55ea40a9 100644 --- a/source/extensions/filters/http/admission_control/admission_control.h +++ b/source/extensions/filters/http/admission_control/admission_control.h @@ -122,7 +122,7 @@ class AdmissionControlFilter : public Http::PassThroughFilter, bool expect_grpc_status_in_trailer_{false}; // If false, the filter will forego recording a request success or failure during encoding. - bool record_request_; + bool record_request_{true}; }; } // namespace AdmissionControl diff --git a/source/extensions/filters/http/admission_control/config.cc b/source/extensions/filters/http/admission_control/config.cc index c94adb7a7883..a20bf1be0812 100644 --- a/source/extensions/filters/http/admission_control/config.cc +++ b/source/extensions/filters/http/admission_control/config.cc @@ -17,7 +17,8 @@ namespace AdmissionControl { static constexpr std::chrono::seconds defaultSamplingWindow{30}; -Http::FilterFactoryCb AdmissionControlFilterFactory::createFilterFactoryFromProtoTyped( +absl::StatusOr +AdmissionControlFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::admission_control::v3::AdmissionControl& config, const std::string& stats_prefix, DualInfo dual_info, Server::Configuration::ServerFactoryContext& context) { diff --git a/source/extensions/filters/http/admission_control/config.h b/source/extensions/filters/http/admission_control/config.h index 446b8860a79f..9ce460f9d268 100644 --- a/source/extensions/filters/http/admission_control/config.h +++ b/source/extensions/filters/http/admission_control/config.h @@ -19,7 +19,7 @@ class AdmissionControlFilterFactory public: AdmissionControlFilterFactory() : DualFactoryBase("envoy.filters.http.admission_control") {} - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::admission_control::v3::AdmissionControl& proto_config, const std::string& stats_prefix, DualInfo dual_info, Server::Configuration::ServerFactoryContext& context) override; diff --git a/source/extensions/filters/http/alternate_protocols_cache/config.cc b/source/extensions/filters/http/alternate_protocols_cache/config.cc index ea641d14e680..ae290b2e959b 100644 --- a/source/extensions/filters/http/alternate_protocols_cache/config.cc +++ b/source/extensions/filters/http/alternate_protocols_cache/config.cc @@ -15,11 +15,15 @@ Http::FilterFactoryCb AlternateProtocolsCacheFilterFactory::createFilterFactoryF const envoy::extensions::filters::http::alternate_protocols_cache::v3::FilterConfig& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { + + auto& server_context = context.serverFactoryContext(); + Http::HttpServerPropertiesCacheManagerFactoryImpl alternate_protocol_cache_manager_factory( - context.singletonManager(), context.threadLocal(), {context}); + server_context.singletonManager(), server_context.threadLocal(), + {context.serverFactoryContext(), context.messageValidationVisitor()}); FilterConfigSharedPtr filter_config( std::make_shared(proto_config, alternate_protocol_cache_manager_factory, - context.mainThreadDispatcher().timeSource())); + server_context.mainThreadDispatcher().timeSource())); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamEncoderFilter( diff --git a/source/extensions/filters/http/alternate_protocols_cache/filter.cc b/source/extensions/filters/http/alternate_protocols_cache/filter.cc index fb8339cb44b1..671c0f772831 100644 --- a/source/extensions/filters/http/alternate_protocols_cache/filter.cc +++ b/source/extensions/filters/http/alternate_protocols_cache/filter.cc @@ -86,7 +86,8 @@ Http::FilterHeadersStatus Filter::encodeHeaders(Http::ResponseHeaderMap& headers // available. hostname = encoder_callbacks_->streamInfo().upstreamInfo()->upstreamSslConnection()->sni(); } - const uint32_t port = host->address()->ip()->port(); + auto host_addr = host->address(); + const uint32_t port = (host_addr ? host_addr->ip()->port() : 443); Http::HttpServerPropertiesCache::Origin origin(Http::Headers::get().SchemeValues.Https, hostname, port); cache->setAlternatives(origin, protocols); diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc index f47f5b8915d2..d4132605b439 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc @@ -286,8 +286,9 @@ void Filter::jsonizeRequest(Http::RequestHeaderMap const& headers, const Buffer: // Wrap the Query String if (headers.Path()) { - for (auto&& kv_pair : Http::Utility::parseQueryString(headers.getPathValue())) { - json_req.mutable_query_string_parameters()->insert({kv_pair.first, kv_pair.second}); + auto queryParams = Http::Utility::QueryParamsMulti::parseQueryString(headers.getPathValue()); + for (const auto& kv_pair : queryParams.data()) { + json_req.mutable_query_string_parameters()->insert({kv_pair.first, kv_pair.second[0]}); } } diff --git a/source/extensions/filters/http/aws_lambda/config.cc b/source/extensions/filters/http/aws_lambda/config.cc index de5a1cfc0a39..9110bf8e0394 100644 --- a/source/extensions/filters/http/aws_lambda/config.cc +++ b/source/extensions/filters/http/aws_lambda/config.cc @@ -1,5 +1,6 @@ #include "source/extensions/filters/http/aws_lambda/config.h" +#include "envoy/common/optref.h" #include "envoy/extensions/filters/http/aws_lambda/v3/aws_lambda.pb.validate.h" #include "envoy/registry/registry.h" #include "envoy/stats/scope.h" @@ -36,6 +37,7 @@ getInvocationMode(const envoy::extensions::filters::http::aws_lambda::v3::Config Http::FilterFactoryCb AwsLambdaFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::aws_lambda::v3::Config& proto_config, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); const auto arn = parseArn(proto_config.arn()); if (!arn) { @@ -45,11 +47,12 @@ Http::FilterFactoryCb AwsLambdaFilterFactory::createFilterFactoryFromProtoTyped( auto credentials_provider = std::make_shared( - context.api(), Extensions::Common::Aws::Utility::fetchMetadata); + server_context.api(), makeOptRef(server_context), region, + Extensions::Common::Aws::Utility::fetchMetadata); auto signer = std::make_shared( service_name, region, std::move(credentials_provider), - context.mainThreadDispatcher().timeSource(), + server_context.mainThreadDispatcher().timeSource(), // TODO: extend API to allow specifying header exclusion. ref: // https://github.com/envoyproxy/envoy/pull/18998 Extensions::Common::Aws::AwsSigV4HeaderExclusionVector{}); diff --git a/source/extensions/filters/http/aws_request_signing/config.cc b/source/extensions/filters/http/aws_request_signing/config.cc index c277f8e600c6..0efdba970916 100644 --- a/source/extensions/filters/http/aws_request_signing/config.cc +++ b/source/extensions/filters/http/aws_request_signing/config.cc @@ -1,5 +1,6 @@ #include "source/extensions/filters/http/aws_request_signing/config.h" +#include "envoy/common/optref.h" #include "envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.pb.h" #include "envoy/extensions/filters/http/aws_request_signing/v3/aws_request_signing.pb.validate.h" #include "envoy/registry/registry.h" @@ -18,14 +19,17 @@ Http::FilterFactoryCb AwsRequestSigningFilterFactory::createFilterFactoryFromPro const AwsRequestSigningProtoConfig& config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + auto credentials_provider = std::make_shared( - context.api(), Extensions::Common::Aws::Utility::fetchMetadata); + server_context.api(), makeOptRef(server_context), config.region(), + Extensions::Common::Aws::Utility::fetchMetadata); const auto matcher_config = Extensions::Common::Aws::AwsSigV4HeaderExclusionVector( config.match_excluded_headers().begin(), config.match_excluded_headers().end()); auto signer = std::make_unique( config.service_name(), config.region(), credentials_provider, - context.mainThreadDispatcher().timeSource(), matcher_config); + server_context.mainThreadDispatcher().timeSource(), matcher_config); auto filter_config = std::make_shared(std::move(signer), stats_prefix, context.scope(), config.host_rewrite(), config.use_unsigned_payload()); @@ -41,7 +45,8 @@ AwsRequestSigningFilterFactory::createRouteSpecificFilterConfigTyped( Server::Configuration::ServerFactoryContext& context, ProtobufMessage::ValidationVisitor&) { auto credentials_provider = std::make_shared( - context.api(), Extensions::Common::Aws::Utility::fetchMetadata); + context.api(), makeOptRef(context), per_route_config.aws_request_signing().region(), + Extensions::Common::Aws::Utility::fetchMetadata); const auto matcher_config = Extensions::Common::Aws::AwsSigV4HeaderExclusionVector( per_route_config.aws_request_signing().match_excluded_headers().begin(), per_route_config.aws_request_signing().match_excluded_headers().end()); diff --git a/source/extensions/filters/http/bandwidth_limit/config.cc b/source/extensions/filters/http/bandwidth_limit/config.cc index d4068dea5af5..ed4aff4e5a8d 100644 --- a/source/extensions/filters/http/bandwidth_limit/config.cc +++ b/source/extensions/filters/http/bandwidth_limit/config.cc @@ -15,8 +15,10 @@ namespace BandwidthLimitFilter { Http::FilterFactoryCb BandwidthLimitFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::bandwidth_limit::v3::BandwidthLimit& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + FilterConfigSharedPtr filter_config = std::make_shared( - proto_config, context.scope(), context.runtime(), context.timeSource()); + proto_config, context.scope(), server_context.runtime(), server_context.timeSource()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/http/basic_auth/BUILD b/source/extensions/filters/http/basic_auth/BUILD new file mode 100644 index 000000000000..f610d4fee905 --- /dev/null +++ b/source/extensions/filters/http/basic_auth/BUILD @@ -0,0 +1,39 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "basic_auth_lib", + srcs = ["basic_auth_filter.cc"], + hdrs = ["basic_auth_filter.h"], + external_deps = ["ssl"], + deps = [ + "//envoy/server:filter_config_interface", + "//source/common/common:base64_lib", + "//source/common/config:utility_lib", + "//source/common/http:header_map_lib", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":basic_auth_lib", + "//envoy/registry", + "//source/common/config:datasource_lib", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/http/common:factory_base_lib", + "@envoy_api//envoy/extensions/filters/http/basic_auth/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/http/basic_auth/basic_auth_filter.cc b/source/extensions/filters/http/basic_auth/basic_auth_filter.cc new file mode 100644 index 000000000000..d9946ae45851 --- /dev/null +++ b/source/extensions/filters/http/basic_auth/basic_auth_filter.cc @@ -0,0 +1,93 @@ +#include "source/extensions/filters/http/basic_auth/basic_auth_filter.h" + +#include + +#include "source/common/common/base64.h" +#include "source/common/http/header_utility.h" +#include "source/common/http/headers.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { + +namespace { + +// Function to compute SHA1 hash +std::string computeSHA1(absl::string_view password) { + unsigned char hash[SHA_DIGEST_LENGTH]; + + // Calculate the SHA-1 hash + SHA1(reinterpret_cast(password.data()), password.length(), hash); + + // Encode the binary hash in Base64 + return Base64::encode(reinterpret_cast(hash), SHA_DIGEST_LENGTH); +} + +} // namespace + +FilterConfig::FilterConfig(UserMap&& users, const std::string& stats_prefix, Stats::Scope& scope) + : users_(std::move(users)), stats_(generateStats(stats_prefix + "basic_auth.", scope)) {} + +bool FilterConfig::validateUser(absl::string_view username, absl::string_view password) const { + auto user = users_.find(username); + if (user == users_.end()) { + return false; + } + + return computeSHA1(password) == user->second.hash; +} + +BasicAuthFilter::BasicAuthFilter(FilterConfigConstSharedPtr config) : config_(std::move(config)) {} + +Http::FilterHeadersStatus BasicAuthFilter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { + auto auth_header = headers.get(Http::CustomHeaders::get().Authorization); + + if (auth_header.empty()) { + return onDenied("User authentication failed. Missing username and password.", + "no_credential_for_basic_auth"); + } + + absl::string_view auth_value = auth_header[0]->value().getStringView(); + + if (!absl::StartsWith(auth_value, "Basic ")) { + return onDenied("User authentication failed. Expected 'Basic' authentication scheme.", + "invalid_scheme_for_basic_auth"); + } + + // Extract and decode the Base64 part of the header. + absl::string_view base64_token = auth_value.substr(6); + const std::string decoded = Base64::decodeWithoutPadding(base64_token); + + // The decoded string is in the format "username:password". + const size_t colon_pos = decoded.find(':'); + if (colon_pos == std::string::npos) { + return onDenied("User authentication failed. Invalid basic credential format.", + "invalid_format_for_basic_auth"); + } + + absl::string_view decoded_view = decoded; + absl::string_view username = decoded_view.substr(0, colon_pos); + absl::string_view password = decoded_view.substr(colon_pos + 1); + + if (!config_->validateUser(username, password)) { + return onDenied("User authentication failed. Invalid username/password combination.", + "invalid_credential_for_basic_auth"); + } + + config_->stats().allowed_.inc(); + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterHeadersStatus BasicAuthFilter::onDenied(absl::string_view body, + absl::string_view response_code_details) { + config_->stats().denied_.inc(); + decoder_callbacks_->sendLocalReply(Http::Code::Unauthorized, body, nullptr, absl::nullopt, + response_code_details); + return Http::FilterHeadersStatus::StopIteration; +} + +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/basic_auth/basic_auth_filter.h b/source/extensions/filters/http/basic_auth/basic_auth_filter.h new file mode 100644 index 000000000000..38553f655eb1 --- /dev/null +++ b/source/extensions/filters/http/basic_auth/basic_auth_filter.h @@ -0,0 +1,80 @@ +#pragma once + +#include "envoy/stats/stats_macros.h" + +#include "source/common/common/logger.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { + +/** + * All Basic Auth filter stats. @see stats_macros.h + */ +#define ALL_BASIC_AUTH_STATS(COUNTER) \ + COUNTER(allowed) \ + COUNTER(denied) + +/** + * Struct definition for Basic Auth stats. @see stats_macros.h + */ +struct BasicAuthStats { + ALL_BASIC_AUTH_STATS(GENERATE_COUNTER_STRUCT) +}; + +/** + * Struct definition for username password pairs. + */ +struct User { + // the user name + std::string name; + // the hashed password, see https://httpd.apache.org/docs/2.4/misc/password_encryptions.html + std::string hash; +}; + +using UserMap = absl::flat_hash_map; + +/** + * Configuration for the Basic Auth filter. + */ +class FilterConfig { +public: + FilterConfig(UserMap&& users, const std::string& stats_prefix, Stats::Scope& scope); + const BasicAuthStats& stats() const { return stats_; } + bool validateUser(absl::string_view username, absl::string_view password) const; + +private: + static BasicAuthStats generateStats(const std::string& prefix, Stats::Scope& scope) { + return BasicAuthStats{ALL_BASIC_AUTH_STATS(POOL_COUNTER_PREFIX(scope, prefix))}; + } + + const UserMap users_; + BasicAuthStats stats_; +}; +using FilterConfigConstSharedPtr = std::shared_ptr; + +// The Envoy filter to process HTTP basic auth. +class BasicAuthFilter : public Http::PassThroughDecoderFilter, + public Logger::Loggable { +public: + BasicAuthFilter(FilterConfigConstSharedPtr config); + + // Http::StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, bool) override; + +private: + Http::FilterHeadersStatus onDenied(absl::string_view body, + absl::string_view response_code_details); + + // The callback function. + FilterConfigConstSharedPtr config_; +}; + +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/basic_auth/config.cc b/source/extensions/filters/http/basic_auth/config.cc new file mode 100644 index 000000000000..d7064730bccb --- /dev/null +++ b/source/extensions/filters/http/basic_auth/config.cc @@ -0,0 +1,80 @@ +#include "source/extensions/filters/http/basic_auth/config.h" + +#include "source/common/config/datasource.h" +#include "source/extensions/filters/http/basic_auth/basic_auth_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { + +using envoy::extensions::filters::http::basic_auth::v3::BasicAuth; + +namespace { + +UserMap readHtpasswd(const std::string& htpasswd) { + UserMap users; + + std::istringstream htpsswd_ss(htpasswd); + std::string line; + + while (std::getline(htpsswd_ss, line)) { + // TODO(wbpcode): should we trim the spaces or empty chars? + + // Skip empty lines and comments. + if (line.empty() || line[0] == '#') { + continue; + } + + const size_t colon_pos = line.find(':'); + if (colon_pos == std::string::npos) { + throw EnvoyException("basic auth: invalid htpasswd format, username:password is expected"); + } + + std::string name = line.substr(0, colon_pos); + std::string hash = line.substr(colon_pos + 1); + + if (name.empty() || hash.empty()) { + throw EnvoyException("basic auth: empty user name or password"); + } + + if (users.contains(name)) { + throw EnvoyException("basic auth: duplicate users"); + } + + if (!absl::StartsWith(hash, "{SHA}")) { + throw EnvoyException("basic auth: unsupported htpasswd format: please use {SHA}"); + } + + hash = hash.substr(5); + // The base64 encoded SHA1 hash is 28 bytes long + if (hash.length() != 28) { + throw EnvoyException("basic auth: invalid htpasswd format, invalid SHA hash length"); + } + + users.insert({name, {name, hash}}); + } + + return users; +} + +} // namespace + +Http::FilterFactoryCb BasicAuthFilterFactory::createFilterFactoryFromProtoTyped( + const BasicAuth& proto_config, const std::string& stats_prefix, + Server::Configuration::FactoryContext& context) { + UserMap users = readHtpasswd( + Config::DataSource::read(proto_config.users(), false, context.serverFactoryContext().api())); + FilterConfigConstSharedPtr config = + std::make_unique(std::move(users), stats_prefix, context.scope()); + return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter(std::make_shared(config)); + }; +} + +REGISTER_FACTORY(BasicAuthFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/basic_auth/config.h b/source/extensions/filters/http/basic_auth/config.h new file mode 100644 index 000000000000..7abebaaa789c --- /dev/null +++ b/source/extensions/filters/http/basic_auth/config.h @@ -0,0 +1,27 @@ +#pragma once + +#include "envoy/extensions/filters/http/basic_auth/v3/basic_auth.pb.h" +#include "envoy/extensions/filters/http/basic_auth/v3/basic_auth.pb.validate.h" + +#include "source/extensions/filters/http/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { + +class BasicAuthFilterFactory + : public Common::FactoryBase { +public: + BasicAuthFilterFactory() : FactoryBase("envoy.filters.http.basic_auth") {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::basic_auth::v3::BasicAuth& config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/buffer/config.cc b/source/extensions/filters/http/buffer/config.cc index e1c8bc4b48cb..dcc93ff5a95b 100644 --- a/source/extensions/filters/http/buffer/config.cc +++ b/source/extensions/filters/http/buffer/config.cc @@ -15,7 +15,7 @@ namespace Extensions { namespace HttpFilters { namespace BufferFilter { -Http::FilterFactoryCb BufferFilterFactory::createFilterFactoryFromProtoTyped( +absl::StatusOr BufferFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::buffer::v3::Buffer& proto_config, const std::string&, DualInfo, Server::Configuration::ServerFactoryContext&) { ASSERT(proto_config.has_max_request_bytes()); diff --git a/source/extensions/filters/http/buffer/config.h b/source/extensions/filters/http/buffer/config.h index 844ad4a9de34..1401d78ae1f4 100644 --- a/source/extensions/filters/http/buffer/config.h +++ b/source/extensions/filters/http/buffer/config.h @@ -20,7 +20,7 @@ class BufferFilterFactory BufferFilterFactory() : DualFactoryBase("envoy.filters.http.buffer") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::buffer::v3::Buffer& proto_config, const std::string& stats_prefix, DualInfo, Server::Configuration::ServerFactoryContext& context) override; diff --git a/source/extensions/filters/http/cache/BUILD b/source/extensions/filters/http/cache/BUILD index fbe778851c59..7eaf3a4114f4 100644 --- a/source/extensions/filters/http/cache/BUILD +++ b/source/extensions/filters/http/cache/BUILD @@ -102,7 +102,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", - "//source/common/protobuf:utility_lib", + "//source/common/protobuf:deterministic_hash_lib", "@envoy_api//envoy/extensions/filters/http/cache/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/cache/cache_filter.cc b/source/extensions/filters/http/cache/cache_filter.cc index 1f7167e9d10d..8fe7fa56dbf7 100644 --- a/source/extensions/filters/http/cache/cache_filter.cc +++ b/source/extensions/filters/http/cache/cache_filter.cc @@ -540,9 +540,12 @@ void CacheFilter::processSuccessfulValidation(Http::ResponseHeaderMap& response_ filter_state_ = FilterState::EncodeServingFromCache; - // Update the 304 response status code and content-length + // Replace the 304 response status code with the cached status code. response_headers.setStatus(lookup_result_->headers_->getStatusValue()); - response_headers.setContentLength(lookup_result_->headers_->getContentLengthValue()); + + // Remove content length header if the 304 had one; if the cache entry had a + // content length header it will be added by the header adding block below. + response_headers.removeContentLength(); // A response that has been validated should not contain an Age header as it is equivalent to a // freshly served response from the origin, unless the 304 response has an Age header, which @@ -553,7 +556,7 @@ void CacheFilter::processSuccessfulValidation(Http::ResponseHeaderMap& response_ // Add any missing headers from the cached response to the 304 response. lookup_result_->headers_->iterate([&response_headers](const Http::HeaderEntry& cached_header) { // TODO(yosrym93): Try to avoid copying the header key twice. - Http::LowerCaseString key(std::string(cached_header.key().getStringView())); + Http::LowerCaseString key(cached_header.key().getStringView()); absl::string_view value = cached_header.value().getStringView(); if (response_headers.get(key).empty()) { response_headers.setCopy(key, value); diff --git a/source/extensions/filters/http/cache/cache_headers_utils.cc b/source/extensions/filters/http/cache/cache_headers_utils.cc index a02632d7c5ac..582746215dc2 100644 --- a/source/extensions/filters/http/cache/cache_headers_utils.cc +++ b/source/extensions/filters/http/cache/cache_headers_utils.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include "envoy/http/header_map.h" @@ -125,6 +126,63 @@ bool operator==(const ResponseCacheControl& lhs, const ResponseCacheControl& rhs (lhs.is_public_ == rhs.is_public_) && (lhs.max_age_ == rhs.max_age_); } +std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control) { + std::vector fields; + + if (request_cache_control.must_validate_) { + fields.push_back("must_validate"); + } + if (request_cache_control.no_store_) { + fields.push_back("no_store"); + } + if (request_cache_control.no_transform_) { + fields.push_back("no_transform"); + } + if (request_cache_control.only_if_cached_) { + fields.push_back("only_if_cached"); + } + if (request_cache_control.max_age_.has_value()) { + fields.push_back( + absl::StrCat("max-age=", std::to_string(request_cache_control.max_age_->count()))); + } + if (request_cache_control.min_fresh_.has_value()) { + fields.push_back( + absl::StrCat("min-fresh=", std::to_string(request_cache_control.min_fresh_->count()))); + } + if (request_cache_control.max_stale_.has_value()) { + fields.push_back( + absl::StrCat("max-stale=", std::to_string(request_cache_control.max_stale_->count()))); + } + + return os << "{" << absl::StrJoin(fields, ", ") << "}"; +} + +std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control) { + std::vector fields; + + if (response_cache_control.must_validate_) { + fields.push_back("must_validate"); + } + if (response_cache_control.no_store_) { + fields.push_back("no_store"); + } + if (response_cache_control.no_transform_) { + fields.push_back("no_transform"); + } + if (response_cache_control.no_stale_) { + fields.push_back("no_stale"); + } + if (response_cache_control.is_public_) { + fields.push_back("public"); + } + if (response_cache_control.max_age_.has_value()) { + fields.push_back( + absl::StrCat("max-age=", std::to_string(response_cache_control.max_age_->count()))); + } + + return os << "{" << absl::StrJoin(fields, ", ") << "}"; +} + SystemTime CacheHeadersUtils::httpTime(const Http::HeaderEntry* header_entry) { if (!header_entry) { return {}; diff --git a/source/extensions/filters/http/cache/cache_headers_utils.h b/source/extensions/filters/http/cache/cache_headers_utils.h index c5a219f0fedd..440b58843d1a 100644 --- a/source/extensions/filters/http/cache/cache_headers_utils.h +++ b/source/extensions/filters/http/cache/cache_headers_utils.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/common/time.h" #include "envoy/extensions/filters/http/cache/v3/cache.pb.h" #include "envoy/http/header_map.h" @@ -93,6 +95,8 @@ struct ResponseCacheControl { bool operator==(const RequestCacheControl& lhs, const RequestCacheControl& rhs); bool operator==(const ResponseCacheControl& lhs, const ResponseCacheControl& rhs); +std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control); +std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control); namespace CacheHeadersUtils { // Parses header_entry as an HTTP time. Returns SystemTime() if diff --git a/source/extensions/filters/http/cache/cache_policy.h b/source/extensions/filters/http/cache/cache_policy.h index a283f20fc10b..43a13b8dfae7 100644 --- a/source/extensions/filters/http/cache/cache_policy.h +++ b/source/extensions/filters/http/cache/cache_policy.h @@ -1,9 +1,10 @@ #pragma once +#include + #include "envoy/http/header_map.h" #include "envoy/stream_info/filter_state.h" -#include "source/extensions/filters/http/cache/cache_headers_utils.h" #include "source/extensions/filters/http/cache/http_cache.h" namespace Envoy { @@ -23,82 +24,138 @@ struct CacheEntryUsability { * Value to be put in the Age header for cache responses. */ Seconds age = Seconds::max(); + /** + * Remaining freshness lifetime--how long from now until the response is stale. (If the response + * is already stale, `ttl` should be negative.) + */ + Seconds ttl = Seconds::max(); + + friend bool operator==(const CacheEntryUsability& a, const CacheEntryUsability& b) { + return std::tie(a.status, a.age, a.ttl) == std::tie(b.status, b.age, b.ttl); + } + + friend bool operator!=(const CacheEntryUsability& a, const CacheEntryUsability& b) { + return !(a == b); + } }; -class CachePolicyCallbacks { -public: - virtual ~CachePolicyCallbacks() = default; +enum class RequestCacheability { + // This request is eligible for serving from cache, and for having its response stored. + Cacheable, + // Don't respond to this request from cache, or store its response into cache. + Bypass, + // This request is eligible for serving from cache, but its response + // must not be stored. Consider the following sequence: + // - Request 1: "curl http://example.com/ -H 'cache-control: no-store'" + // - `requestCacheability` returns `NoStore`. + // - CacheFilter finds nothing in cache, so request 1 is proxied upstream. + // - Origin responds with a cacheable response 1. + // - CacheFilter does not store response 1 into cache. + // - Request 2: "curl http://example.com/" + // - `requestCacheability` returns `Cacheable`. + // - CacheFilter finds nothing in cache, so request 2 is proxied upstream. + // - Origin responds with a cacheable response 2. + // - CacheFilter stores response 2 into cache. + // - Request 3: "curl http://example.com/ -H 'cache-control: no-store'" + // - `requestCacheability` returns `NoStore`. + // - CacheFilter looks in cache and finds response 2, which matches. + // - CacheFilter serves response 2 from cache. + // To summarize, all 3 requests were eligible for serving from cache (though only request 3 found + // a match to serve), but only request 2 was allowed to have its response stored into cache. + NoStore, +}; + +inline std::ostream& operator<<(std::ostream& os, RequestCacheability cacheability) { + switch (cacheability) { + using enum RequestCacheability; + case Cacheable: + return os << "Cacheable"; + case Bypass: + return os << "Bypass"; + case NoStore: + return os << "NoStore"; + } +} - virtual const StreamInfo::FilterStateSharedPtr& filterState() PURE; +enum class ResponseCacheability { + // Don't store this response in cache. + DoNotStore, + // Store the full response in cache. + StoreFullResponse, + // Store a cache entry indicating that the response was uncacheable, and that future responses are + // likely to be uncacheable. (CacheFilter and/or HttpCache implementations will treat such entries + // as cache misses, but may enable optimizations based on expecting uncacheable responses. If a + // future response is cacheable, it will overwrite this "uncacheable" entry.) + MarkUncacheable, }; -/** - * An extension point for deployment specific caching behavior. - */ -class CachePolicy { +inline std::ostream& operator<<(std::ostream& os, ResponseCacheability cacheability) { + switch (cacheability) { + using enum ResponseCacheability; + case DoNotStore: + return os << "DoNotStore"; + case StoreFullResponse: + return os << "StoreFullResponse"; + case MarkUncacheable: + return os << "MarkUncacheable"; + } +} + +// Create cache key, calculate cache content freshness and +// response cacheability. This can be a straight RFC compliant implementation +// but can also be used to implement deployment specific cache policies. +// +// NOT YET IMPLEMENTED: To make CacheFilter use a custom cache policy, store a mutable CachePolicy +// in FilterState before CacheFilter::decodeHeaders is called. +class CachePolicy : public StreamInfo::FilterState::Object { public: + // For use in FilterState. + static constexpr absl::string_view Name = + "io.envoyproxy.extensions.filters.http.cache.cache_policy"; + virtual ~CachePolicy() = default; /** * Calculates the lookup key for storing the entry in the cache. * @param request_headers - headers from the request the CacheFilter is currently processing. */ - virtual Key createCacheKey(const Http::RequestHeaderMap& request_headers) PURE; + virtual Key cacheKey(const Http::RequestHeaderMap& request_headers) PURE; /** - * Determines the cacheability of the response during decoding. + * Determines whether the request is eligible for serving from cache and/or having its response + * stored in cache. * @param request_headers - headers from the request the CacheFilter is currently processing. - * @param request_cache_control - the result of parsing the request's Cache-Control header, parsed - * by the caller. - * @return true if the response may be cached, based on the contents of the request. + * @return an enum indicating whether the request is eligible for serving from cache and/or having + * its response stored in cache. */ - virtual bool requestCacheable(const Http::RequestHeaderMap& request_headers, - const RequestCacheControl& request_cache_control) PURE; + virtual RequestCacheability + requestCacheability(const Http::RequestHeaderMap& request_headers) PURE; /** * Determines the cacheability of the response during encoding. * @param request_headers - headers from the request the CacheFilter is currently processing. * @param response_headers - headers from the upstream response the CacheFilter is currently * processing. - * @param response_cache_control - the result of parsing the response's Cache-Control header, - * parsed by the caller. - * @param vary_allow_list - list of headers that the cache will respect when creating the Key for - * Vary-differentiated responses. - * @return true if the response may be cached. + * @return an enum indicating how the response should be handled. */ - virtual bool responseCacheable(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const ResponseCacheControl& response_cache_control, - const VaryHeader& vary_allow_list) PURE; + virtual ResponseCacheability + responseCacheability(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers) PURE; /** * Determines whether the cached entry may be used directly or must be validated with upstream. * @param request_headers - request headers associated with the response_headers. * @param cached_response_headers - headers from the cached response. - * @param request_cache_control - the parsed result of the request's Cache-Control header, parsed - * by the caller. - * @param cached_response_cache_control - the parsed result of the response's Cache-Control - * header, parsed by the caller. * @param content_length - the byte length of the cached content. * @param cached_metadata - the metadata that has been stored along side the cached entry. * @param now - the timestamp for this request. * @return details about whether or not the cached entry can be used. */ virtual CacheEntryUsability - computeCacheEntryUsability(const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& cached_response_headers, - const RequestCacheControl& request_cache_control, - const ResponseCacheControl& cached_response_cache_control, - const uint64_t content_length, const ResponseMetadata& cached_metadata, - SystemTime now) PURE; - - /** - * Performs actions when StreamInfo and FilterState become available, for - * example for logging and observability, or to adapt CacheFilter behavior based on - * route-specific CacheFilter config. - * @param callbacks - Gives access to StreamInfo and FilterState - */ - virtual void setCallbacks(CachePolicyCallbacks& callbacks) PURE; + cacheEntryUsability(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& cached_response_headers, + const uint64_t content_length, const ResponseMetadata& cached_metadata, + SystemTime now) PURE; }; } // namespace Cache diff --git a/source/extensions/filters/http/cache/config.cc b/source/extensions/filters/http/cache/config.cc index 4d93f126cd5b..bfb64ad4bfb7 100644 --- a/source/extensions/filters/http/cache/config.cc +++ b/source/extensions/filters/http/cache/config.cc @@ -28,8 +28,8 @@ Http::FilterFactoryCb CacheFilterFactory::createFilterFactoryFromProtoTyped( return [config, stats_prefix, &context, cache](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter(std::make_shared(config, stats_prefix, context.scope(), - context.timeSource(), cache)); + callbacks.addStreamFilter(std::make_shared( + config, stats_prefix, context.scope(), context.serverFactoryContext().timeSource(), cache)); }; } diff --git a/source/extensions/filters/http/cache/http_cache.cc b/source/extensions/filters/http/cache/http_cache.cc index 3f5659e88495..73a2365a5f99 100644 --- a/source/extensions/filters/http/cache/http_cache.cc +++ b/source/extensions/filters/http/cache/http_cache.cc @@ -10,7 +10,7 @@ #include "source/common/http/header_utility.h" #include "source/common/http/headers.h" #include "source/common/http/utility.h" -#include "source/common/protobuf/utility.h" +#include "source/common/protobuf/deterministic_hash.h" #include "source/extensions/filters/http/cache/cache_custom_headers.h" #include "source/extensions/filters/http/cache/cache_headers_utils.h" @@ -54,7 +54,13 @@ LookupRequest::LookupRequest(const Http::RequestHeaderMap& request_headers, Syst // Unless this API is still alpha, calls to stableHashKey() must always return // the same result, or a way must be provided to deal with a complete cache // flush. -size_t stableHashKey(const Key& key) { return MessageUtil::hash(key); } +size_t stableHashKey(const Key& key) { + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.use_fast_protobuf_hash")) { + return DeterministicProtoHash::hash(key); + } else { + return MessageUtil::hash(key); + } +} void LookupRequest::initializeRequestCacheControl(const Http::RequestHeaderMap& request_headers) { const absl::string_view cache_control = diff --git a/source/extensions/filters/http/cache/http_cache.h b/source/extensions/filters/http/cache/http_cache.h index f5b670c25199..177d12863e33 100644 --- a/source/extensions/filters/http/cache/http_cache.h +++ b/source/extensions/filters/http/cache/http_cache.h @@ -69,8 +69,6 @@ using LookupResultPtr = std::unique_ptr; // // When providing a cached response, Caches must ensure that the keys (and not // just their hashes) match. -// -// TODO(toddmgreer): Ensure that stability guarantees above are accurate. size_t stableHashKey(const Key& key); // LookupRequest holds everything about a request that's needed to look for a diff --git a/source/extensions/filters/http/cache/range_utils.cc b/source/extensions/filters/http/cache/range_utils.cc index 44a7eff02ed3..6fab49640460 100644 --- a/source/extensions/filters/http/cache/range_utils.cc +++ b/source/extensions/filters/http/cache/range_utils.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -23,6 +24,10 @@ namespace Extensions { namespace HttpFilters { namespace Cache { +std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range) { + return os << "[" << range.begin() << "," << range.end() << ")"; +} + absl::optional RangeUtils::createRangeDetails(const Envoy::Http::RequestHeaderMap& request_headers, uint64_t content_length) { diff --git a/source/extensions/filters/http/cache/range_utils.h b/source/extensions/filters/http/cache/range_utils.h index 0f9d1ae90a83..4d7aa4728d2a 100644 --- a/source/extensions/filters/http/cache/range_utils.h +++ b/source/extensions/filters/http/cache/range_utils.h @@ -88,6 +88,8 @@ inline bool operator==(const AdjustedByteRange& lhs, const AdjustedByteRange& rh return lhs.begin() == rhs.begin() && lhs.end() == rhs.end(); } +std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range); + // Contains details about whether the ranges requested can be satisfied and, if // so, what those ranges are after being adjusted to fit the content. struct RangeDetails { diff --git a/source/extensions/filters/http/common/factory_base.h b/source/extensions/filters/http/common/factory_base.h index c334e6135482..ae0288c8e1fc 100644 --- a/source/extensions/filters/http/common/factory_base.h +++ b/source/extensions/filters/http/common/factory_base.h @@ -58,13 +58,14 @@ class CommonFactoryBase : public virtual Server::Configuration::HttpFilterConfig const std::string name_; }; + template class FactoryBase : public CommonFactoryBase, public Server::Configuration::NamedHttpFilterConfigFactory { public: FactoryBase(const std::string& name) : CommonFactoryBase(name) {} - Envoy::Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override { @@ -95,6 +96,27 @@ class FactoryBase : public CommonFactoryBase, } }; +template +class ExceptionFreeFactoryBase : public CommonFactoryBase, + public Server::Configuration::NamedHttpFilterConfigFactory { +public: + ExceptionFreeFactoryBase(const std::string& name) + : CommonFactoryBase(name) {} + + absl::StatusOr + createFilterFactoryFromProto(const Protobuf::Message& proto_config, + const std::string& stats_prefix, + Server::Configuration::FactoryContext& context) override { + return createFilterFactoryFromProtoTyped(MessageUtil::downcastAndValidate( + proto_config, context.messageValidationVisitor()), + stats_prefix, context); + } + virtual absl::StatusOr + createFilterFactoryFromProtoTyped(const ConfigProto& proto_config, + const std::string& stats_prefix, + Server::Configuration::FactoryContext& context) PURE; +}; + template class DualFactoryBase : public CommonFactoryBase, public Server::Configuration::NamedHttpFilterConfigFactory, @@ -112,27 +134,27 @@ class DualFactoryBase : public CommonFactoryBase, Stats::Scope& scope; }; - Envoy::Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override { return createFilterFactoryFromProtoTyped(MessageUtil::downcastAndValidate( proto_config, context.messageValidationVisitor()), stats_prefix, DualInfo(context), - context.getServerFactoryContext()); + context.serverFactoryContext()); } - Envoy::Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::UpstreamFactoryContext& context) override { return createFilterFactoryFromProtoTyped( MessageUtil::downcastAndValidate( - proto_config, context.getServerFactoryContext().messageValidationVisitor()), - stats_prefix, DualInfo(context), context.getServerFactoryContext()); + proto_config, context.serverFactoryContext().messageValidationVisitor()), + stats_prefix, DualInfo(context), context.serverFactoryContext()); } - virtual Envoy::Http::FilterFactoryCb + virtual absl::StatusOr createFilterFactoryFromProtoTyped(const ConfigProto& proto_config, const std::string& stats_prefix, DualInfo info, Server::Configuration::ServerFactoryContext& context) PURE; diff --git a/source/extensions/filters/http/composite/BUILD b/source/extensions/filters/http/composite/BUILD index 512c9b054913..bf636545cf4b 100644 --- a/source/extensions/filters/http/composite/BUILD +++ b/source/extensions/filters/http/composite/BUILD @@ -16,6 +16,7 @@ envoy_cc_library( srcs = ["action.cc"], hdrs = ["action.h"], deps = [ + "//source/common/http:filter_chain_helper_lib", "//source/common/http/matching:data_impl_lib", "//source/common/matcher:matcher_lib", "@envoy_api//envoy/extensions/filters/http/composite/v3:pkg_cc_proto", diff --git a/source/extensions/filters/http/composite/action.cc b/source/extensions/filters/http/composite/action.cc index 40fdbe1bf45f..d7cb694e7452 100644 --- a/source/extensions/filters/http/composite/action.cc +++ b/source/extensions/filters/http/composite/action.cc @@ -4,6 +4,10 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Composite { + +using HttpExtensionConfigProviderSharedPtr = std::shared_ptr< + Config::DynamicExtensionConfigProvider>; + void ExecuteFilterAction::createFilters(Http::FilterChainFactoryCallbacks& callbacks) const { cb_(callbacks); } @@ -15,6 +19,39 @@ Matcher::ActionFactoryCb ExecuteFilterActionFactory::createActionFactoryCb( const envoy::extensions::filters::http::composite::v3::ExecuteFilterAction&>( config, validation_visitor); + if (composite_action.has_dynamic_config() && composite_action.has_typed_config()) { + throw EnvoyException( + fmt::format("Error: Only one of `dynamic_config` or `typed_config` can be set.")); + } + + if (composite_action.has_dynamic_config()) { + if (!context.factory_context_.has_value() || !context.server_factory_context_.has_value()) { + throw EnvoyException(fmt::format("Failed to get factory context or server factory context.")); + } + // Create a dynamic filter config provider and register it with the server factory context. + auto config_discovery = composite_action.dynamic_config().config_discovery(); + Server::Configuration::FactoryContext& factory_context = context.factory_context_.value(); + Server::Configuration::ServerFactoryContext& server_factory_context = + context.server_factory_context_.value(); + auto provider_manager = + Envoy::Http::FilterChainUtility::createSingletonDownstreamFilterConfigProviderManager( + server_factory_context); + HttpExtensionConfigProviderSharedPtr provider = + provider_manager->createDynamicFilterConfigProvider( + config_discovery, composite_action.dynamic_config().name(), server_factory_context, + factory_context, server_factory_context.clusterManager(), false, "http", nullptr); + return [provider = std::move(provider)]() -> Matcher::ActionPtr { + auto config_value = provider->config(); + if (config_value.has_value()) { + auto factory_cb = config_value.value().get().factory_cb; + return std::make_unique(factory_cb); + } + // There is no dynamic config available. Apply missing config filter. + auto factory_cb = Envoy::Http::MissingConfigFilterFactory; + return std::make_unique(factory_cb); + }; + } + auto& factory = Config::Utility::getAndCheckFactory( composite_action.typed_config()); @@ -25,8 +62,10 @@ Matcher::ActionFactoryCb ExecuteFilterActionFactory::createActionFactoryCb( // First, try to create the filter factory creation function from factory context (if exists). if (context.factory_context_.has_value()) { - callback = factory.createFilterFactoryFromProto(*message, context.stat_prefix_, - context.factory_context_.value()); + auto callback_or_status = factory.createFilterFactoryFromProto( + *message, context.stat_prefix_, context.factory_context_.value()); + THROW_IF_STATUS_NOT_OK(callback_or_status, throw); + callback = callback_or_status.value(); } // If above failed, try to create the filter factory creation function from server factory diff --git a/source/extensions/filters/http/composite/action.h b/source/extensions/filters/http/composite/action.h index 725eadada76d..b78553d8ab77 100644 --- a/source/extensions/filters/http/composite/action.h +++ b/source/extensions/filters/http/composite/action.h @@ -2,6 +2,7 @@ #include "envoy/extensions/filters/http/composite/v3/composite.pb.validate.h" +#include "source/common/http/filter_chain_helper.h" #include "source/common/http/matching/data_impl.h" #include "source/common/matcher/matcher.h" diff --git a/source/extensions/filters/http/composite/filter.h b/source/extensions/filters/http/composite/filter.h index bd0e56c8571a..9f73a566b034 100644 --- a/source/extensions/filters/http/composite/filter.h +++ b/source/extensions/filters/http/composite/filter.h @@ -75,13 +75,10 @@ class Filter : public Http::StreamFilter, void onMatchCallback(const Matcher::Action& action) override; // AccessLog::Instance - void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override { + void log(const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& info) override { for (const auto& log : access_loggers_) { - log->log(request_headers, response_headers, response_trailers, stream_info, access_log_type); + log->log(log_context, info); } } diff --git a/source/extensions/filters/http/compressor/compressor_filter.cc b/source/extensions/filters/http/compressor/compressor_filter.cc index 5c068872b2c8..09a1d0c50493 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.cc +++ b/source/extensions/filters/http/compressor/compressor_filter.cc @@ -179,6 +179,10 @@ CompressorPerRouteFilterConfig::CompressorPerRouteFilterConfig( // Consequently, if `response_direction_config.common_direction_config.enabled` ever gets // added, its absence must enable compression. response_compression_enabled_ = true; + if (config.overrides().response_direction_config().has_remove_accept_encoding_header()) { + remove_accept_encoding_header_ = + config.overrides().response_direction_config().remove_accept_encoding_header().value(); + } } break; case CompressorPerRoute::OVERRIDE_NOT_SET: @@ -198,7 +202,11 @@ Http::FilterHeadersStatus CompressorFilter::decodeHeaders(Http::RequestHeaderMap } const auto& response_config = config_->responseDirectionConfig(); - if (compressionEnabled(response_config) && response_config.removeAcceptEncodingHeader()) { + const auto* per_route_config = + Http::Utility::resolveMostSpecificPerFilterConfig( + decoder_callbacks_); + if (compressionEnabled(response_config, per_route_config) && + removeAcceptEncodingHeader(response_config, per_route_config)) { headers.removeInline(accept_encoding_handle.handle()); } @@ -266,10 +274,13 @@ void CompressorFilter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallba Http::FilterHeadersStatus CompressorFilter::encodeHeaders(Http::ResponseHeaderMap& headers, bool end_stream) { const auto& config = config_->responseDirectionConfig(); + const auto* per_route_config = + Http::Utility::resolveMostSpecificPerFilterConfig( + decoder_callbacks_); // This is used to decide whether stats for accept-encoding header should be touched. const bool isEnabledAndContentLengthBigEnough = - compressionEnabled(config) && config.isMinimumContentLength(headers); + compressionEnabled(config, per_route_config) && config.isMinimumContentLength(headers); const bool isCompressible = isEnabledAndContentLengthBigEnough && !Http::Utility::isUpgrade(headers) && @@ -630,15 +641,21 @@ void CompressorFilter::sanitizeEtagHeader(Http::ResponseHeaderMap& headers) { // True if response compression is enabled. bool CompressorFilter::compressionEnabled( - const CompressorFilterConfig::ResponseDirectionConfig& config) const { - const CompressorPerRouteFilterConfig* per_route_config = - Http::Utility::resolveMostSpecificPerFilterConfig( - decoder_callbacks_); + const CompressorFilterConfig::ResponseDirectionConfig& config, + const CompressorPerRouteFilterConfig* per_route_config) const { return per_route_config && per_route_config->responseCompressionEnabled().has_value() ? *per_route_config->responseCompressionEnabled() : config.compressionEnabled(); } +bool CompressorFilter::removeAcceptEncodingHeader( + const CompressorFilterConfig::ResponseDirectionConfig& config, + const CompressorPerRouteFilterConfig* per_route_config) const { + return per_route_config && per_route_config->removeAcceptEncodingHeader().has_value() + ? *per_route_config->removeAcceptEncodingHeader() + : config.removeAcceptEncodingHeader(); +} + } // namespace Compressor } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/compressor/compressor_filter.h b/source/extensions/filters/http/compressor/compressor_filter.h index ce5634b9d13d..f63903195606 100644 --- a/source/extensions/filters/http/compressor/compressor_filter.h +++ b/source/extensions/filters/http/compressor/compressor_filter.h @@ -170,9 +170,11 @@ class CompressorPerRouteFilterConfig : public Router::RouteSpecificFilterConfig // If a value is present, that value overrides // ResponseDirectionConfig::compressionEnabled. absl::optional responseCompressionEnabled() const { return response_compression_enabled_; } + absl::optional removeAcceptEncodingHeader() const { return remove_accept_encoding_header_; } private: absl::optional response_compression_enabled_; + absl::optional remove_accept_encoding_header_; }; /** @@ -196,7 +198,10 @@ class CompressorFilter : public Http::PassThroughFilter { Http::FilterTrailersStatus encodeTrailers(Http::ResponseTrailerMap&) override; private: - bool compressionEnabled(const CompressorFilterConfig::ResponseDirectionConfig& config) const; + bool compressionEnabled(const CompressorFilterConfig::ResponseDirectionConfig& config, + const CompressorPerRouteFilterConfig* per_route_config) const; + bool removeAcceptEncodingHeader(const CompressorFilterConfig::ResponseDirectionConfig& config, + const CompressorPerRouteFilterConfig* per_route_config) const; bool hasCacheControlNoTransform(Http::ResponseHeaderMap& headers) const; bool isAcceptEncodingAllowed(bool maybe_compress, const Http::ResponseHeaderMap& headers) const; bool isEtagAllowed(Http::ResponseHeaderMap& headers) const; diff --git a/source/extensions/filters/http/compressor/config.cc b/source/extensions/filters/http/compressor/config.cc index 3b16724a5d66..18e54b637d62 100644 --- a/source/extensions/filters/http/compressor/config.cc +++ b/source/extensions/filters/http/compressor/config.cc @@ -10,7 +10,7 @@ namespace Extensions { namespace HttpFilters { namespace Compressor { -Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped( +absl::StatusOr CompressorFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::v3::Compressor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { const std::string type{TypeUtil::typeUrlToDescriptorFullName( @@ -19,7 +19,7 @@ Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped Registry::FactoryRegistry< Compression::Compressor::NamedCompressorLibraryConfigFactory>::getFactoryByType(type); if (config_factory == nullptr) { - throw EnvoyException( + return absl::InvalidArgumentError( fmt::format("Didn't find a registered implementation for type: '{}'", type)); } ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( @@ -27,9 +27,9 @@ Http::FilterFactoryCb CompressorFilterFactory::createFilterFactoryFromProtoTyped *config_factory); Compression::Compressor::CompressorFactoryPtr compressor_factory = config_factory->createCompressorFactoryFromProto(*message, context); - CompressorFilterConfigSharedPtr config = - std::make_shared(proto_config, stats_prefix, context.scope(), - context.runtime(), std::move(compressor_factory)); + CompressorFilterConfigSharedPtr config = std::make_shared( + proto_config, stats_prefix, context.scope(), context.serverFactoryContext().runtime(), + std::move(compressor_factory)); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(config)); }; diff --git a/source/extensions/filters/http/compressor/config.h b/source/extensions/filters/http/compressor/config.h index ef25ddd51408..95e2054defd6 100644 --- a/source/extensions/filters/http/compressor/config.h +++ b/source/extensions/filters/http/compressor/config.h @@ -14,14 +14,14 @@ namespace Compressor { * Config registration for the compressor filter. @see NamedHttpFilterConfigFactory. */ class CompressorFilterFactory - : public Common::FactoryBase< + : public Common::ExceptionFreeFactoryBase< envoy::extensions::filters::http::compressor::v3::Compressor, envoy::extensions::filters::http::compressor::v3::CompressorPerRoute> { public: - CompressorFilterFactory() : FactoryBase("envoy.filters.http.compressor") {} + CompressorFilterFactory() : ExceptionFreeFactoryBase("envoy.filters.http.compressor") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::compressor::v3::Compressor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; diff --git a/source/extensions/filters/http/connect_grpc_bridge/BUILD b/source/extensions/filters/http/connect_grpc_bridge/BUILD index ebb849195cdc..245ab3140043 100644 --- a/source/extensions/filters/http/connect_grpc_bridge/BUILD +++ b/source/extensions/filters/http/connect_grpc_bridge/BUILD @@ -7,7 +7,7 @@ load( licenses(["notice"]) # Apache 2 -# L7 HTTP filter that translates Buf Connect to gRPC. +# L7 HTTP filter that translates Connect RPC to gRPC. envoy_extension_package() diff --git a/source/extensions/filters/http/connect_grpc_bridge/config.cc b/source/extensions/filters/http/connect_grpc_bridge/config.cc index 8a0b45957759..b20865677c82 100644 --- a/source/extensions/filters/http/connect_grpc_bridge/config.cc +++ b/source/extensions/filters/http/connect_grpc_bridge/config.cc @@ -20,7 +20,7 @@ Http::FilterFactoryCb ConnectGrpcFilterConfigFactory::createFilterFactoryFromPro } /** - * Static registration for the Buf Connect stats filter. @see RegisterFactory. + * Static registration for the Connect RPC stats filter. @see RegisterFactory. */ REGISTER_FACTORY(ConnectGrpcFilterConfigFactory, Server::Configuration::NamedHttpFilterConfigFactory); diff --git a/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.cc b/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.cc index c5fd392edc3d..a228db503b7b 100644 --- a/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.cc +++ b/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.cc @@ -13,7 +13,7 @@ namespace { /** * Returns the appropriate error status code for a given gRPC status. * - * https://connect.build/docs/protocol#error-codes + * https://connectrpc.com/docs/protocol#error-codes */ std::string statusCodeToString(const Grpc::Status::GrpcStatus status) { using WellKnownGrpcStatus = Grpc::Status::WellKnownGrpcStatus; @@ -122,7 +122,7 @@ bool serializeJson(const EndStreamResponse& response, std::string& out) { /** * Gets the appropriate HTTP status code for a given gRPC status. * - * https://connect.build/docs/protocol#error-codes + * https://connectrpc.com/docs/protocol#error-codes */ uint64_t statusCodeToConnectUnaryStatus(const Grpc::Status::GrpcStatus status) { using WellKnownGrpcStatus = Grpc::Status::WellKnownGrpcStatus; diff --git a/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.h b/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.h index 8840c905cbdf..7a0600fc7bee 100644 --- a/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.h +++ b/source/extensions/filters/http/connect_grpc_bridge/end_stream_response.h @@ -28,9 +28,9 @@ struct EndStreamResponse { }; /** - * Serializes a Buf Connect Error object to JSON. + * Serializes a Connect Error object to JSON. * - * https://connect.build/docs/protocol#error-end-stream + * https://connectrpc.com/docs/protocol#error-end-stream * * @param error Error object to serialize. * @param out String to store result in. @@ -39,9 +39,9 @@ struct EndStreamResponse { bool serializeJson(const Error& error, std::string& out); /** - * Serializes a Buf Connect EndStreamResponse object to JSON. + * Serializes a Connect EndStreamResponse object to JSON. * - * https://connect.build/docs/protocol#error-end-stream + * https://connectrpc.com/docs/protocol#error-end-stream * * @param response EndStreamResponse object to serialize. * @param out String to store result in. diff --git a/source/extensions/filters/http/connect_grpc_bridge/filter.cc b/source/extensions/filters/http/connect_grpc_bridge/filter.cc index 5382f9dcea27..2bac2b2c8f1e 100644 --- a/source/extensions/filters/http/connect_grpc_bridge/filter.cc +++ b/source/extensions/filters/http/connect_grpc_bridge/filter.cc @@ -18,10 +18,10 @@ namespace ConnectGrpcBridge { namespace { struct RcDetailsValues { - // The Buf Connect gRPC bridge tried to buffer a unary request that was too large for the + // The Connect RPC to gRPC bridge tried to buffer a unary request that was too large for the // configured decoder buffer limit. const std::string ConnectBridgeUnaryRequestTooLarge = "connect_bridge_unary_request_too_large"; - // The Buf Connect gRPC bridge tried to buffer a unary response that was too large for the + // The Connect RPC to gRPC bridge tried to buffer a unary response that was too large for the // configured encoder buffer limit. const std::string ConnectBridgeUnaryResponseTooLarge = "connect_bridge_unary_response_too_large"; }; @@ -289,20 +289,31 @@ Http::FilterHeadersStatus ConnectGrpcBridgeFilter::decodeHeaders(Http::RequestHe compression[0]->value() != Http::CustomHeaders::get().AcceptEncodingValues.Identity) { unary_payload_frame_flags_ |= Envoy::Grpc::GRPC_FH_COMPRESSED; } + + headers.removeContentLength(); + + if (end_stream) { + Grpc::Encoder().prependFrameHeader(unary_payload_frame_flags_, request_buffer_); + decoder_callbacks_->addDecodedData(request_buffer_, true); + } } else { - Http::Utility::QueryParams query_parameters = - Http::Utility::parseAndDecodeQueryString(headers.getPathValue()); - if (query_parameters[ConnectGetParams::get().APIKey] == ConnectGetParams::get().APIValue) { + Http::Utility::QueryParamsMulti query_parameters = + Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(headers.getPathValue()); + if (query_parameters.getFirstValue(ConnectGetParams::get().APIKey).value_or("") == + ConnectGetParams::get().APIValue) { // Unary Connect Get protocol is_connect_unary_ = true; headers.setMethod(Http::Headers::get().MethodValues.Post); headers.setPath(removeQueryParameters(headers.getPathValue())); - auto message = query_parameters[ConnectGetParams::get().MessageKey]; - auto base64 = query_parameters[ConnectGetParams::get().Base64Key]; - auto encoding = query_parameters[ConnectGetParams::get().EncodingKey]; - auto compression = query_parameters[ConnectGetParams::get().CompressionKey]; + auto message = + query_parameters.getFirstValue(ConnectGetParams::get().MessageKey).value_or(""); + auto base64 = query_parameters.getFirstValue(ConnectGetParams::get().Base64Key).value_or(""); + auto encoding = + query_parameters.getFirstValue(ConnectGetParams::get().EncodingKey).value_or(""); + auto compression = + query_parameters.getFirstValue(ConnectGetParams::get().CompressionKey).value_or(""); if (base64 == "1") { message = Base64Url::decode(message); diff --git a/source/extensions/filters/http/csrf/config.cc b/source/extensions/filters/http/csrf/config.cc index bd5bd5b135cc..a36139e17bb7 100644 --- a/source/extensions/filters/http/csrf/config.cc +++ b/source/extensions/filters/http/csrf/config.cc @@ -14,8 +14,8 @@ namespace Csrf { Http::FilterFactoryCb CsrfFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::csrf::v3::CsrfPolicy& policy, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { - CsrfFilterConfigSharedPtr config = - std::make_shared(policy, stats_prefix, context.scope(), context.runtime()); + CsrfFilterConfigSharedPtr config = std::make_shared( + policy, stats_prefix, context.scope(), context.serverFactoryContext().runtime()); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(config)); }; diff --git a/source/extensions/filters/http/custom_response/factory.cc b/source/extensions/filters/http/custom_response/factory.cc index d8a55efceee1..4a25e8f52e40 100644 --- a/source/extensions/filters/http/custom_response/factory.cc +++ b/source/extensions/filters/http/custom_response/factory.cc @@ -12,7 +12,7 @@ ::Envoy::Http::FilterFactoryCb CustomResponseFilterFactory::createFilterFactoryF const std::string& stats_prefix, Envoy::Server::Configuration::FactoryContext& context) { Stats::StatNameManagedStorage prefix(stats_prefix, context.scope().symbolTable()); auto config_ptr = - std::make_shared(config, context.getServerFactoryContext(), prefix.statName()); + std::make_shared(config, context.serverFactoryContext(), prefix.statName()); return [config_ptr](::Envoy::Http::FilterChainFactoryCallbacks& callbacks) mutable -> void { callbacks.addStreamFilter(std::make_shared(config_ptr)); }; diff --git a/source/extensions/filters/http/decompressor/config.cc b/source/extensions/filters/http/decompressor/config.cc index e15257c7d2e3..74e9687afac0 100644 --- a/source/extensions/filters/http/decompressor/config.cc +++ b/source/extensions/filters/http/decompressor/config.cc @@ -10,7 +10,7 @@ namespace Extensions { namespace HttpFilters { namespace Decompressor { -Http::FilterFactoryCb DecompressorFilterFactory::createFilterFactoryFromProtoTyped( +absl::StatusOr DecompressorFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::decompressor::v3::Decompressor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { const std::string decompressor_library_type{TypeUtil::typeUrlToDescriptorFullName( @@ -20,8 +20,8 @@ Http::FilterFactoryCb DecompressorFilterFactory::createFilterFactoryFromProtoTyp Compression::Decompressor::NamedDecompressorLibraryConfigFactory>:: getFactoryByType(decompressor_library_type); if (decompressor_library_factory == nullptr) { - throw EnvoyException(fmt::format("Didn't find a registered implementation for type: '{}'", - decompressor_library_type)); + return absl::InvalidArgumentError(fmt::format( + "Didn't find a registered implementation for type: '{}'", decompressor_library_type)); } ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( proto_config.decompressor_library().typed_config(), context.messageValidationVisitor(), @@ -29,7 +29,7 @@ Http::FilterFactoryCb DecompressorFilterFactory::createFilterFactoryFromProtoTyp Compression::Decompressor::DecompressorFactoryPtr decompressor_factory = decompressor_library_factory->createDecompressorFactoryFromProto(*message, context); DecompressorFilterConfigSharedPtr filter_config = std::make_shared( - proto_config, stats_prefix, context.scope(), context.runtime(), + proto_config, stats_prefix, context.scope(), context.serverFactoryContext().runtime(), std::move(decompressor_factory)); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); diff --git a/source/extensions/filters/http/decompressor/config.h b/source/extensions/filters/http/decompressor/config.h index 6c3bdd5c2278..43a716160f53 100644 --- a/source/extensions/filters/http/decompressor/config.h +++ b/source/extensions/filters/http/decompressor/config.h @@ -14,12 +14,13 @@ namespace Decompressor { * Config registration for the decompressor filter. @see NamedHttpFilterConfigFactory. */ class DecompressorFilterFactory - : public Common::FactoryBase { + : public Common::ExceptionFreeFactoryBase< + envoy::extensions::filters::http::decompressor::v3::Decompressor> { public: - DecompressorFilterFactory() : FactoryBase("envoy.filters.http.decompressor") {} + DecompressorFilterFactory() : ExceptionFreeFactoryBase("envoy.filters.http.decompressor") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::decompressor::v3::Decompressor& config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; }; diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.cc b/source/extensions/filters/http/dynamic_forward_proxy/config.cc index 3717a38fdda5..eded3686d206 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/config.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.cc @@ -11,14 +11,24 @@ namespace Extensions { namespace HttpFilters { namespace DynamicForwardProxy { -Http::FilterFactoryCb DynamicForwardProxyFilterFactory::createFilterFactoryFromProtoTyped( +absl::StatusOr +DynamicForwardProxyFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( context); - Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory(context); + Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( + context.serverFactoryContext().singletonManager()); + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr cache_manager = + cache_manager_factory.get(); + auto cache_or_error = cache_manager->getCache(proto_config.dns_cache_config()); + if (!cache_or_error.status().ok()) { + return cache_or_error.status(); + } ProxyFilterConfigSharedPtr filter_config(std::make_shared( - proto_config, cache_manager_factory, cluster_store_factory, context)); + proto_config, std::move(cache_or_error.value()), std::move(cache_manager), + cluster_store_factory, context)); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.h b/source/extensions/filters/http/dynamic_forward_proxy/config.h index fe723ad0f537..a86ac3b194d6 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/config.h +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.h @@ -14,14 +14,15 @@ namespace DynamicForwardProxy { * Config registration for the dynamic forward proxy filter. */ class DynamicForwardProxyFilterFactory - : public Common::FactoryBase< + : public Common::ExceptionFreeFactoryBase< envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig, envoy::extensions::filters::http::dynamic_forward_proxy::v3::PerRouteConfig> { public: - DynamicForwardProxyFilterFactory() : FactoryBase("envoy.filters.http.dynamic_forward_proxy") {} + DynamicForwardProxyFilterFactory() + : ExceptionFreeFactoryBase("envoy.filters.http.dynamic_forward_proxy") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; Router::RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfigTyped( diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc index c1314d18c90d..f9cf7c253f47 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc @@ -52,13 +52,15 @@ using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCa ProxyFilterConfig::ProxyFilterConfig( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config, - Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr&& cache, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr&& cache_manager, Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory& cluster_store_factory, Server::Configuration::FactoryContext& context) - : cluster_store_(cluster_store_factory.get()), dns_cache_manager_(cache_manager_factory.get()), - dns_cache_(dns_cache_manager_->getCache(proto_config.dns_cache_config())), - cluster_manager_(context.clusterManager()), - main_thread_dispatcher_(context.mainThreadDispatcher()), tls_slot_(context.threadLocal()), + : cluster_store_(cluster_store_factory.get()), dns_cache_manager_(std::move(cache_manager)), + dns_cache_(std::move(cache)), + cluster_manager_(context.serverFactoryContext().clusterManager()), + main_thread_dispatcher_(context.serverFactoryContext().mainThreadDispatcher()), + tls_slot_(context.serverFactoryContext().threadLocal()), cluster_init_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(proto_config.sub_cluster_config(), cluster_init_timeout, 5000)), save_upstream_address_(proto_config.save_upstream_address()) { diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h index e63b96107db8..a84548995e2a 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h @@ -39,7 +39,8 @@ class ProxyFilterConfig : public Upstream::ClusterUpdateCallbacks, public: ProxyFilterConfig( const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config, - Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr&& cache, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr&& cache_manager, Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory& cluster_store_factory, Server::Configuration::FactoryContext& context); diff --git a/source/extensions/filters/http/ext_authz/config.cc b/source/extensions/filters/http/ext_authz/config.cc index 2b6cca8265a6..bbb99f63cc82 100644 --- a/source/extensions/filters/http/ext_authz/config.cc +++ b/source/extensions/filters/http/ext_authz/config.cc @@ -23,9 +23,11 @@ namespace ExtAuthz { Http::FilterFactoryCb ExtAuthzFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::ext_authz::v3::ExtAuthz& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + const auto filter_config = std::make_shared( - proto_config, context.scope(), context.runtime(), context.httpContext(), stats_prefix, - context.getServerFactoryContext().bootstrap()); + proto_config, context.scope(), server_context.runtime(), server_context.httpContext(), + stats_prefix, server_context.bootstrap()); // The callback is created in main thread and executed in worker thread, variables except factory // context must be captured by value into the callback. Http::FilterFactoryCb callback; @@ -38,9 +40,9 @@ Http::FilterFactoryCb ExtAuthzFilterConfig::createFilterFactoryFromProtoTyped( std::make_shared( proto_config, timeout_ms, proto_config.http_service().path_prefix()); callback = [filter_config, client_config, - &context](Http::FilterChainFactoryCallbacks& callbacks) { + &server_context](Http::FilterChainFactoryCallbacks& callbacks) { auto client = std::make_unique( - context.clusterManager(), client_config); + server_context.clusterManager(), client_config); callbacks.addStreamFilter(std::make_shared(filter_config, std::move(client))); }; } else { @@ -54,8 +56,10 @@ Http::FilterFactoryCb ExtAuthzFilterConfig::createFilterFactoryFromProtoTyped( callback = [&context, filter_config, timeout_ms, config_with_hash_key](Http::FilterChainFactoryCallbacks& callbacks) { auto client = std::make_unique( - context.clusterManager().grpcAsyncClientManager().getOrCreateRawAsyncClientWithHashKey( - config_with_hash_key, context.scope(), true), + context.serverFactoryContext() + .clusterManager() + .grpcAsyncClientManager() + .getOrCreateRawAsyncClientWithHashKey(config_with_hash_key, context.scope(), true), std::chrono::milliseconds(timeout_ms)); callbacks.addStreamFilter(std::make_shared(filter_config, std::move(client))); }; diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 68f327e9774d..fe713f7e98e4 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -1,6 +1,9 @@ #include "source/extensions/filters/http/ext_authz/ext_authz.h" #include +#include +#include +#include #include "envoy/config/core/v3/base.pb.h" @@ -15,6 +18,46 @@ namespace Extensions { namespace HttpFilters { namespace ExtAuthz { +namespace { + +using MetadataProto = ::envoy::config::core::v3::Metadata; + +void fillMetadataContext(const std::vector& source_metadata, + const std::vector& metadata_context_namespaces, + const std::vector& typed_metadata_context_namespaces, + MetadataProto& metadata_context) { + for (const auto& context_key : metadata_context_namespaces) { + for (const MetadataProto* metadata : source_metadata) { + if (metadata == nullptr) { + continue; + } + const auto& filter_metadata = metadata->filter_metadata(); + if (const auto metadata_it = filter_metadata.find(context_key); + metadata_it != filter_metadata.end()) { + (*metadata_context.mutable_filter_metadata())[metadata_it->first] = metadata_it->second; + break; + } + } + } + + for (const auto& context_key : typed_metadata_context_namespaces) { + for (const MetadataProto* metadata : source_metadata) { + if (metadata == nullptr) { + continue; + } + const auto& typed_filter_metadata = metadata->typed_filter_metadata(); + if (const auto metadata_it = typed_filter_metadata.find(context_key); + metadata_it != typed_filter_metadata.end()) { + (*metadata_context.mutable_typed_filter_metadata())[metadata_it->first] = + metadata_it->second; + break; + } + } + } +} + +} // namespace + void FilterConfigPerRoute::merge(const FilterConfigPerRoute& other) { // We only merge context extensions here, and leave boolean flags untouched since those flags are // not used from the merged config. @@ -41,45 +84,27 @@ void Filter::initiateCall(const Http::RequestHeaderMap& headers) { context_extensions = maybe_merged_per_route_config.value().takeContextExtensions(); } + // If metadata_context_namespaces or typed_metadata_context_namespaces is specified, + // pass matching filter metadata to the ext_authz service. + // If metadata key is set in both the connection and request metadata, + // then the value will be the request metadata value. envoy::config::core::v3::Metadata metadata_context; - - // If metadata_context_namespaces is specified, pass matching filter metadata to the ext_authz - // service. If metadata key is set in both the connection and request metadata then the value - // will be the request metadata value. - const auto& connection_metadata = - decoder_callbacks_->connection()->streamInfo().dynamicMetadata().filter_metadata(); - const auto& request_metadata = - decoder_callbacks_->streamInfo().dynamicMetadata().filter_metadata(); - for (const auto& context_key : config_->metadataContextNamespaces()) { - if (const auto metadata_it = request_metadata.find(context_key); - metadata_it != request_metadata.end()) { - (*metadata_context.mutable_filter_metadata())[metadata_it->first] = metadata_it->second; - } else if (const auto metadata_it = connection_metadata.find(context_key); - metadata_it != connection_metadata.end()) { - (*metadata_context.mutable_filter_metadata())[metadata_it->first] = metadata_it->second; - } - } - - // If typed_metadata_context_namespaces is specified, pass matching typed filter metadata to the - // ext_authz service. If metadata key is set in both the connection and request metadata then - // the value will be the request metadata value. - const auto& connection_typed_metadata = - decoder_callbacks_->connection()->streamInfo().dynamicMetadata().typed_filter_metadata(); - const auto& request_typed_metadata = - decoder_callbacks_->streamInfo().dynamicMetadata().typed_filter_metadata(); - for (const auto& context_key : config_->typedMetadataContextNamespaces()) { - if (const auto metadata_it = request_typed_metadata.find(context_key); - metadata_it != request_typed_metadata.end()) { - (*metadata_context.mutable_typed_filter_metadata())[metadata_it->first] = metadata_it->second; - } else if (const auto metadata_it = connection_typed_metadata.find(context_key); - metadata_it != connection_typed_metadata.end()) { - (*metadata_context.mutable_typed_filter_metadata())[metadata_it->first] = metadata_it->second; - } + fillMetadataContext({&decoder_callbacks_->streamInfo().dynamicMetadata(), + &decoder_callbacks_->connection()->streamInfo().dynamicMetadata()}, + config_->metadataContextNamespaces(), + config_->typedMetadataContextNamespaces(), metadata_context); + + // Fill route_metadata_context from the selected route's metadata. + envoy::config::core::v3::Metadata route_metadata_context; + if (decoder_callbacks_->route() != nullptr) { + fillMetadataContext({&decoder_callbacks_->route()->metadata()}, + config_->routeMetadataContextNamespaces(), + config_->routeTypedMetadataContextNamespaces(), route_metadata_context); } Filters::Common::ExtAuthz::CheckRequestUtils::createHttpCheck( decoder_callbacks_, headers, std::move(context_extensions), std::move(metadata_context), - check_request_, config_->maxRequestBytes(), config_->packAsBytes(), + std::move(route_metadata_context), check_request_, max_request_bytes_, config_->packAsBytes(), config_->includePeerCertificate(), config_->includeTLSSession(), config_->destinationLabels(), config_->requestHeaderMatchers()); @@ -121,14 +146,24 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, } request_headers_ = &headers; - buffer_data_ = config_->withRequestBody() && !per_route_flags.skip_request_body_buffering_ && + const auto check_settings = per_route_flags.check_settings_; + buffer_data_ = (config_->withRequestBody() || check_settings.has_with_request_body()) && + !check_settings.disable_request_body_buffering() && !(end_stream || Http::Utility::isWebSocketUpgradeRequest(headers) || Http::Utility::isH2UpgradeRequest(headers)); if (buffer_data_) { ENVOY_STREAM_LOG(debug, "ext_authz filter is buffering the request", *decoder_callbacks_); - if (!config_->allowPartialMessage()) { - decoder_callbacks_->setDecoderBufferLimit(config_->maxRequestBytes()); + + allow_partial_message_ = check_settings.has_with_request_body() + ? check_settings.with_request_body().allow_partial_message() + : config_->allowPartialMessage(); + max_request_bytes_ = check_settings.has_with_request_body() + ? check_settings.with_request_body().max_request_bytes() + : config_->maxRequestBytes(); + + if (!allow_partial_message_) { + decoder_callbacks_->setDecoderBufferLimit(max_request_bytes_); } return Http::FilterHeadersStatus::StopIteration; } @@ -369,19 +404,20 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { if (cluster_) { config_->incCounter(cluster_->statsScope(), config_->ext_authz_denied_); - - Http::CodeStats::ResponseStatInfo info{config_->scope(), - cluster_->statsScope(), - empty_stat_name, - enumToInt(response->status_code), - true, - empty_stat_name, - empty_stat_name, - empty_stat_name, - empty_stat_name, - empty_stat_name, - false}; - config_->httpContext().codeStats().chargeResponseStat(info, false); + if (config_->chargeClusterResponseStats()) { + Http::CodeStats::ResponseStatInfo info{config_->scope(), + cluster_->statsScope(), + empty_stat_name, + enumToInt(response->status_code), + true, + empty_stat_name, + empty_stat_name, + empty_stat_name, + empty_stat_name, + empty_stat_name, + false}; + config_->httpContext().codeStats().chargeResponseStat(info, false); + } } // setResponseFlag must be called before sendLocalReply @@ -421,8 +457,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { if (cluster_) { config_->incCounter(cluster_->statsScope(), config_->ext_authz_failure_mode_allowed_); } - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.http_ext_auth_failure_mode_allow_header_add")) { + if (config_->failureModeAllowHeaderAdd()) { request_headers_->addReferenceKey( Filters::Common::ExtAuthz::Headers::get().EnvoyAuthFailureModeAllowed, "true"); } @@ -443,7 +478,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { } bool Filter::isBufferFull(uint64_t num_bytes_processing) const { - if (!config_->allowPartialMessage()) { + if (!allow_partial_message_) { return false; } @@ -453,7 +488,7 @@ bool Filter::isBufferFull(uint64_t num_bytes_processing) const { num_bytes_buffered += buffer->length(); } - return num_bytes_buffered >= config_->maxRequestBytes(); + return num_bytes_buffered >= max_request_bytes_; } void Filter::continueDecoding() { @@ -468,17 +503,21 @@ void Filter::continueDecoding() { Filter::PerRouteFlags Filter::getPerRouteFlags(const Router::RouteConstSharedPtr& route) const { if (route == nullptr) { - return PerRouteFlags{true /*skip_check_*/, false /*skip_request_body_buffering_*/}; + return PerRouteFlags{ + true /*skip_check_*/, + envoy::extensions::filters::http::ext_authz::v3::CheckSettings() /*check_settings_*/}; } - const auto* specific_per_route_config = + const auto* specific_check_settings = Http::Utility::resolveMostSpecificPerFilterConfig(decoder_callbacks_); - if (specific_per_route_config != nullptr) { - return PerRouteFlags{specific_per_route_config->disabled(), - specific_per_route_config->disableRequestBodyBuffering()}; + if (specific_check_settings != nullptr) { + return PerRouteFlags{specific_check_settings->disabled(), + specific_check_settings->checkSettings()}; } - return PerRouteFlags{false /*skip_check_*/, false /*skip_request_body_buffering_*/}; + return PerRouteFlags{ + false /*skip_check_*/, + envoy::extensions::filters::http::ext_authz::v3::CheckSettings() /*check_settings_*/}; } } // namespace ExtAuthz diff --git a/source/extensions/filters/http/ext_authz/ext_authz.h b/source/extensions/filters/http/ext_authz/ext_authz.h index 91482a93eca7..d18aa27ccccc 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.h +++ b/source/extensions/filters/http/ext_authz/ext_authz.h @@ -60,10 +60,11 @@ class FilterConfig { const std::string& stats_prefix, envoy::config::bootstrap::v3::Bootstrap& bootstrap) : allow_partial_message_(config.with_request_body().allow_partial_message()), failure_mode_allow_(config.failure_mode_allow()), + failure_mode_allow_header_add_(config.failure_mode_allow_header_add()), clear_route_cache_(config.clear_route_cache()), max_request_bytes_(config.with_request_body().max_request_bytes()), - // `pack_as_bytes_` should be true when configured with an http service because there is no + // `pack_as_bytes_` should be true when configured with the HTTP service because there is no // difference to where the body is written in http requests, and a value of false here will // cause non UTF-8 body content to be changed when it doesn't need to. pack_as_bytes_(config.has_http_service() || config.with_request_body().pack_as_bytes()), @@ -87,8 +88,15 @@ class FilterConfig { config.metadata_context_namespaces().end()), typed_metadata_context_namespaces_(config.typed_metadata_context_namespaces().begin(), config.typed_metadata_context_namespaces().end()), + route_metadata_context_namespaces_(config.route_metadata_context_namespaces().begin(), + config.route_metadata_context_namespaces().end()), + route_typed_metadata_context_namespaces_( + config.route_typed_metadata_context_namespaces().begin(), + config.route_typed_metadata_context_namespaces().end()), include_peer_certificate_(config.include_peer_certificate()), include_tls_session_(config.include_tls_session()), + charge_cluster_response_stats_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, charge_cluster_response_stats, true)), stats_(generateStats(stats_prefix, config.stat_prefix(), scope)), ext_authz_ok_(pool_.add(createPoolStatName(config.stat_prefix(), "ok"))), ext_authz_denied_(pool_.add(createPoolStatName(config.stat_prefix(), "denied"))), @@ -133,6 +141,8 @@ class FilterConfig { bool failureModeAllow() const { return failure_mode_allow_; } + bool failureModeAllowHeaderAdd() const { return failure_mode_allow_header_add_; } + bool clearRouteCache() const { return clear_route_cache_; } uint32_t maxRequestBytes() const { return max_request_bytes_; } @@ -164,6 +174,14 @@ class FilterConfig { return typed_metadata_context_namespaces_; } + const std::vector& routeMetadataContextNamespaces() { + return route_metadata_context_namespaces_; + } + + const std::vector& routeTypedMetadataContextNamespaces() { + return route_typed_metadata_context_namespaces_; + } + const ExtAuthzFilterStats& stats() const { return stats_; } void incCounter(Stats::Scope& scope, Stats::StatName name) { @@ -174,6 +192,8 @@ class FilterConfig { bool includeTLSSession() const { return include_tls_session_; } const LabelsMap& destinationLabels() const { return destination_labels_; } + bool chargeClusterResponseStats() const { return charge_cluster_response_stats_; } + const Filters::Common::ExtAuthz::MatcherSharedPtr& requestHeaderMatchers() const { return request_header_matchers_; } @@ -205,6 +225,7 @@ class FilterConfig { const bool allow_partial_message_; const bool failure_mode_allow_; + const bool failure_mode_allow_header_add_; const bool clear_route_cache_; const uint32_t max_request_bytes_; const bool pack_as_bytes_; @@ -223,9 +244,12 @@ class FilterConfig { const std::vector metadata_context_namespaces_; const std::vector typed_metadata_context_namespaces_; + const std::vector route_metadata_context_namespaces_; + const std::vector route_typed_metadata_context_namespaces_; const bool include_peer_certificate_; const bool include_tls_session_; + const bool charge_cluster_response_stats_; // The stats for the filter. ExtAuthzFilterStats stats_; @@ -256,9 +280,17 @@ class FilterConfigPerRoute : public Router::RouteSpecificFilterConfig { : context_extensions_(config.has_check_settings() ? config.check_settings().context_extensions() : ContextExtensionsMap()), - disable_request_body_buffering_(config.has_check_settings() && - config.check_settings().disable_request_body_buffering()), - disabled_(config.disabled()) {} + check_settings_(config.has_check_settings() + ? config.check_settings() + : envoy::extensions::filters::http::ext_authz::v3::CheckSettings()), + disabled_(config.disabled()) { + if (config.has_check_settings() && config.check_settings().disable_request_body_buffering() && + config.check_settings().has_with_request_body()) { + ExceptionUtil::throwEnvoyException( + "Invalid configuration for check_settings. Only one of disable_request_body_buffering or " + "with_request_body can be set."); + } + } void merge(const FilterConfigPerRoute& other); @@ -271,13 +303,15 @@ class FilterConfigPerRoute : public Router::RouteSpecificFilterConfig { bool disabled() const { return disabled_; } - bool disableRequestBodyBuffering() const { return disable_request_body_buffering_; } + envoy::extensions::filters::http::ext_authz::v3::CheckSettings checkSettings() const { + return check_settings_; + } private: - // We save the context extensions as a protobuf map instead of an std::map as this allows us to + // We save the context extensions as a protobuf map instead of a std::map as this allows us to // move it to the CheckRequest, thus avoiding a copy that would incur by converting it. ContextExtensionsMap context_extensions_; - bool disable_request_body_buffering_; + envoy::extensions::filters::http::ext_authz::v3::CheckSettings check_settings_; bool disabled_; }; @@ -324,7 +358,7 @@ class Filter : public Logger::Loggable, // This holds a set of flags defined in per-route configuration. struct PerRouteFlags { const bool skip_check_; - const bool skip_request_body_buffering_; + const envoy::extensions::filters::http::ext_authz::v3::CheckSettings check_settings_; }; PerRouteFlags getPerRouteFlags(const Router::RouteConstSharedPtr& route) const; @@ -335,7 +369,7 @@ class Filter : public Logger::Loggable, // FilterReturn is used to capture what the return code should be to the filter chain. // if this filter is either in the middle of calling the service or the result is denied then - // the filter chain should stop. Otherwise the filter chain can continue to the next filter. + // the filter chain should stop. Otherwise, the filter chain can continue to the next filter. enum class FilterReturn { ContinueDecoding, StopDecoding }; Http::HeaderMapPtr getHeaderMap(const Filters::Common::ExtAuthz::ResponsePtr& response); @@ -352,6 +386,10 @@ class Filter : public Logger::Loggable, // The stats for the filter. ExtAuthzFilterStats stats_; + // This is used to hold the final configs after we merge them with per-route configs. + bool allow_partial_message_{}; + uint32_t max_request_bytes_; + // Used to identify if the callback to onComplete() is synchronous (on the stack) or asynchronous. bool initiating_call_{}; bool buffer_data_{}; diff --git a/source/extensions/filters/http/ext_proc/config.cc b/source/extensions/filters/http/ext_proc/config.cc index 7a0240e84e03..d11953036843 100644 --- a/source/extensions/filters/http/ext_proc/config.cc +++ b/source/extensions/filters/http/ext_proc/config.cc @@ -22,7 +22,7 @@ Http::FilterFactoryCb ExternalProcessingFilterConfig::createFilterFactoryFromPro return [filter_config, grpc_service = proto_config.grpc_service(), &context](Http::FilterChainFactoryCallbacks& callbacks) { auto client = std::make_unique( - context.clusterManager().grpcAsyncClientManager(), context.scope()); + context.serverFactoryContext().clusterManager().grpcAsyncClientManager(), context.scope()); callbacks.addStreamFilter(Http::StreamFilterSharedPtr{ std::make_shared(filter_config, std::move(client), grpc_service)}); diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index b0e16cba79bb..21e57f4be82b 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -235,13 +235,17 @@ FilterHeadersStatus Filter::decodeHeaders(RequestHeaderMap& headers, bool end_st decoding_state_.setCompleteBodyAvailable(true); } - if (!decoding_state_.sendHeaders()) { - ENVOY_LOG(trace, "decodeHeaders: Skipped"); - return FilterHeadersStatus::Continue; + FilterHeadersStatus status = FilterHeadersStatus::Continue; + if (decoding_state_.sendHeaders()) { + status = onHeaders(decoding_state_, headers, end_stream); + ENVOY_LOG(trace, "onHeaders returning {}", static_cast(status)); + } else { + ENVOY_LOG(trace, "decodeHeaders: Skipped header processing"); } - const auto status = onHeaders(decoding_state_, headers, end_stream); - ENVOY_LOG(trace, "decodeHeaders returning {}", static_cast(status)); + if (!processing_complete_ && decoding_state_.shouldRemoveContentLength()) { + headers.removeContentLength(); + } return status; } @@ -271,22 +275,6 @@ FilterDataStatus Filter::onData(ProcessorState& state, Buffer::Instance& data, b state.requestWatermark(); return FilterDataStatus::StopIterationAndWatermark; } - if (state.callbackState() == ProcessorState::CallbackState::StreamedBodyCallbackFinishing) { - // We were previously streaming the body, but there are more chunks waiting - // to be processed, so we can't send the body yet. - // Move the data for our chunk into a queue so that we can re-inject it later - // when the processor returns. See the comments below for more details on how - // this works in general. - ENVOY_LOG(trace, "Enqueuing data while we wait for processing to finish"); - state.enqueueStreamingChunk(data, end_stream, false); - if (end_stream) { - // But we need to buffer the last chunk because it's our last chance to do stuff - state.setPaused(true); - return FilterDataStatus::StopIterationNoBuffer; - } else { - return FilterDataStatus::Continue; - } - } FilterDataStatus result; switch (state.bodyMode()) { @@ -305,8 +293,8 @@ FilterDataStatus Filter::onData(ProcessorState& state, Buffer::Instance& data, b // The body has been buffered and we need to send the buffer ENVOY_LOG(debug, "Sending request body message"); state.addBufferedData(data); - sendBodyChunk(state, *state.bufferedData(), - ProcessorState::CallbackState::BufferedBodyCallback, true); + auto req = setupBodyChunk(state, *state.bufferedData(), end_stream); + sendBodyChunk(state, ProcessorState::CallbackState::BufferedBodyCallback, req); // Since we just just moved the data into the buffer, return NoBuffer // so that we do not buffer this chunk twice. state.setPaused(true); @@ -343,10 +331,11 @@ FilterDataStatus Filter::onData(ProcessorState& state, Buffer::Instance& data, b // Fall through break; } - // Send the chunk on the gRPC stream - sendBodyChunk(state, data, ProcessorState::CallbackState::StreamedBodyCallback, end_stream); - // Move the data to the queue and optionally raise the watermark. - state.enqueueStreamingChunk(data, end_stream, true); + + // Need to first enqueue the data into the chunk queue before sending. + auto req = setupBodyChunk(state, data, end_stream); + state.enqueueStreamingChunk(data, end_stream); + sendBodyChunk(state, ProcessorState::CallbackState::StreamedBodyCallback, req); // At this point we will continue, but with no data, because that will come later if (end_stream) { @@ -383,12 +372,12 @@ FilterDataStatus Filter::onData(ProcessorState& state, Buffer::Instance& data, b ProcessorState::CallbackState::BufferedPartialBodyCallback) { // More data came in while we were waiting for a callback result. We need // to queue it and deliver it later in case the callback changes the data. - state.enqueueStreamingChunk(data, end_stream, false); + state.enqueueStreamingChunk(data, end_stream); ENVOY_LOG(trace, "Call in progress for partial mode"); state.setPaused(true); result = FilterDataStatus::StopIterationNoBuffer; } else { - state.enqueueStreamingChunk(data, end_stream, false); + state.enqueueStreamingChunk(data, end_stream); if (end_stream || state.queueOverHighLimit()) { // At either end of stream or when the buffer is full, it's time to send what we have // to the processor. @@ -425,8 +414,8 @@ std::pair Filter::sendStreamChunk(ProcessorState& const auto& all_data = state.consolidateStreamedChunks(); ENVOY_LOG(debug, "Sending {} bytes of data in buffered partial mode. end_stream = {}", state.chunkQueue().receivedData().length(), all_data.end_stream); - sendBodyChunk(state, state.chunkQueue().receivedData(), - ProcessorState::CallbackState::BufferedPartialBodyCallback, all_data.end_stream); + auto req = setupBodyChunk(state, state.chunkQueue().receivedData(), all_data.end_stream); + sendBodyChunk(state, ProcessorState::CallbackState::BufferedPartialBodyCallback, req); state.setPaused(true); return {false, FilterDataStatus::StopIterationNoBuffer}; } @@ -469,17 +458,20 @@ FilterTrailersStatus Filter::onTrailers(ProcessorState& state, Http::HeaderMap& break; } + // We would like to process the body in a buffered way, but until now the complete + // body has not arrived. With the arrival of trailers, we now know that the body + // has arrived. if (state.bodyMode() == ProcessingMode::BUFFERED) { // Sending data left over in the buffer. - sendBodyChunk(state, *state.bufferedData(), - ProcessorState::CallbackState::BufferedBodyCallback, false); + auto req = setupBodyChunk(state, *state.bufferedData(), false); + sendBodyChunk(state, ProcessorState::CallbackState::BufferedBodyCallback, req); } else { // Sending data left over in the queue. const auto& all_data = state.consolidateStreamedChunks(); + auto req = setupBodyChunk(state, state.chunkQueue().receivedData(), false); ENVOY_LOG(debug, "Sending {} bytes of data in buffered partial mode. end_stream = {}", state.chunkQueue().receivedData().length(), all_data.end_stream); - sendBodyChunk(state, state.chunkQueue().receivedData(), - ProcessorState::CallbackState::BufferedPartialBodyCallback, false); + sendBodyChunk(state, ProcessorState::CallbackState::BufferedPartialBodyCallback, req); } state.setPaused(true); return FilterTrailersStatus::StopIteration; @@ -521,13 +513,22 @@ FilterHeadersStatus Filter::encodeHeaders(ResponseHeaderMap& headers, bool end_s encoding_state_.setCompleteBodyAvailable(true); } - if (processing_complete_ || !encoding_state_.sendHeaders()) { - ENVOY_LOG(trace, "encodeHeaders: Continue"); - return FilterHeadersStatus::Continue; + FilterHeadersStatus status = FilterHeadersStatus::Continue; + if (!processing_complete_ && encoding_state_.sendHeaders()) { + status = onHeaders(encoding_state_, headers, end_stream); + ENVOY_LOG(trace, "onHeaders returns {}", static_cast(status)); + } else { + ENVOY_LOG(trace, "encodeHeaders: Skipped header processing"); } - const auto status = onHeaders(encoding_state_, headers, end_stream); - ENVOY_LOG(trace, "encodeHeaders returns {}", static_cast(status)); + // The content-length header will be kept when either one of the following conditions is met: + // (1) `shouldRemoveContentLength` returns false. + // (2) side stream processing has been completed. For example, it could be caused by stream error + // that triggers the local reply or due to spurious message that skips the side stream + // mutation. + if (!processing_complete_ && encoding_state_.shouldRemoveContentLength()) { + headers.removeContentLength(); + } return status; } @@ -545,15 +546,20 @@ FilterTrailersStatus Filter::encodeTrailers(ResponseTrailerMap& trailers) { return status; } -void Filter::sendBodyChunk(ProcessorState& state, const Buffer::Instance& data, - ProcessorState::CallbackState new_state, bool end_stream) { - ENVOY_LOG(debug, "Sending a body chunk of {} bytes, end_stram {}", data.length(), end_stream); - state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), config_->messageTimeout(), - new_state); +ProcessingRequest Filter::setupBodyChunk(ProcessorState& state, const Buffer::Instance& data, + bool end_stream) { + ENVOY_LOG(debug, "Sending a body chunk of {} bytes, end_stream {}", data.length(), end_stream); ProcessingRequest req; auto* body_req = state.mutableBody(req); body_req->set_end_of_stream(end_stream); body_req->set_body(data.toString()); + return req; +} + +void Filter::sendBodyChunk(ProcessorState& state, ProcessorState::CallbackState new_state, + ProcessingRequest& req) { + state.onStartProcessorCall(std::bind(&Filter::onMessageTimeout, this), config_->messageTimeout(), + new_state); stream_->send(std::move(req), false); stats_.stream_msgs_sent_.inc(); } @@ -631,8 +637,9 @@ void Filter::onReceiveMessage(std::unique_ptr&& r) { // Update processing mode now because filter callbacks check it // and the various "handle" methods below may result in callbacks // being invoked in line. This only happens when filter has allow_mode_override - // set to true. Otherwise, the response mode_override proto field is ignored. - if (config_->allowModeOverride() && response->has_mode_override()) { + // set to true and filter is waiting for header processing response. + // Otherwise, the response mode_override proto field is ignored. + if (config_->allowModeOverride() && inHeaderProcessState() && response->has_mode_override()) { ENVOY_LOG(debug, "Processing mode overridden by server for this request"); decoding_state_.setProcessingMode(response->mode_override()); encoding_state_.setProcessingMode(response->mode_override()); @@ -669,14 +676,9 @@ void Filter::onReceiveMessage(std::unique_ptr&& r) { // We won't be sending anything more to the stream after we // receive this message. ENVOY_LOG(debug, "Sending immediate response"); - // TODO(tyxia) For immediate response case here and below, logging is needed because - // `onFinishProcessorCalls` is called after `closeStream` below. - // Investigate to see if we can switch the order of those two so that the logging here can be - // avoided. - logGrpcStreamInfo(); processing_complete_ = true; - closeStream(); onFinishProcessorCalls(Grpc::Status::Ok); + closeStream(); sendImmediateResponse(response->immediate_response()); processing_status = absl::OkStatus(); } @@ -709,9 +711,8 @@ void Filter::onReceiveMessage(std::unique_ptr&& r) { ENVOY_LOG(debug, "Sending immediate response: {}", processing_status.message()); stats_.stream_msgs_received_.inc(); processing_complete_ = true; - logGrpcStreamInfo(); - closeStream(); onFinishProcessorCalls(processing_status.raw_code()); + closeStream(); ImmediateResponse invalid_mutation_response; invalid_mutation_response.mutable_status()->set_code(StatusCode::InternalServerError); invalid_mutation_response.set_details(std::string(processing_status.message())); @@ -734,10 +735,10 @@ void Filter::onGrpcError(Grpc::Status::GrpcStatus status) { } else { processing_complete_ = true; - closeStream(); // Since the stream failed, there is no need to handle timeouts, so // make sure that they do not fire now. onFinishProcessorCalls(status); + closeStream(); ImmediateResponse errorResponse; errorResponse.mutable_status()->set_code(StatusCode::InternalServerError); errorResponse.set_details(absl::StrFormat("%s_gRPC_error_%i", ErrorPrefix, status)); diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index 9b74867aba47..6338cc4e2aae 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -279,10 +279,16 @@ class Filter : public Logger::Loggable, void onMessageTimeout(); void onNewTimeout(const ProtobufWkt::Duration& override_message_timeout); - void sendBodyChunk(ProcessorState& state, const Buffer::Instance& data, - ProcessorState::CallbackState new_state, bool end_stream); + envoy::service::ext_proc::v3::ProcessingRequest + setupBodyChunk(ProcessorState& state, const Buffer::Instance& data, bool end_stream); + void sendBodyChunk(ProcessorState& state, ProcessorState::CallbackState new_state, + envoy::service::ext_proc::v3::ProcessingRequest& req); void sendTrailers(ProcessorState& state, const Http::HeaderMap& trailers); + bool inHeaderProcessState() { + return (decoding_state_.callbackState() == ProcessorState::CallbackState::HeadersCallback || + encoding_state_.callbackState() == ProcessorState::CallbackState::HeadersCallback); + } private: void mergePerRouteConfig(); diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.cc b/source/extensions/filters/http/ext_proc/mutation_utils.cc index d56cd20c2fb7..ebd794aa41a9 100644 --- a/source/extensions/filters/http/ext_proc/mutation_utils.cc +++ b/source/extensions/filters/http/ext_proc/mutation_utils.cc @@ -114,7 +114,8 @@ absl::Status MutationUtils::headerMutationResultCheck(const Http::HeaderMap& hea absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation, Http::HeaderMap& headers, bool replacing_message, const Checker& checker, - Counter& rejected_mutations) { + Counter& rejected_mutations, + bool remove_content_length) { // Check whether the remove_headers or set_headers size exceed the HTTP connection manager limit. // Reject the mutation and return error status if either one does. const auto result = responseHeaderSizeCheck(headers, mutation, rejected_mutations); @@ -151,6 +152,12 @@ absl::Status MutationUtils::applyHeaderMutations(const HeaderMutation& mutation, continue; } + // Always skip setting content length header when `remove_content_length` is true. + if (remove_content_length && + absl::EqualsIgnoreCase(sh.header().key(), Http::Headers::get().ContentLength)) { + continue; + } + // Only one of value or raw_value in the HeaderValue message should be set. if (!sh.header().value().empty() && !sh.header().raw_value().empty()) { ENVOY_LOG(debug, "Only one of value or raw_value in the HeaderValue message should be set, " diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.h b/source/extensions/filters/http/ext_proc/mutation_utils.h index bc5c1e7c00d7..7e177befea74 100644 --- a/source/extensions/filters/http/ext_proc/mutation_utils.h +++ b/source/extensions/filters/http/ext_proc/mutation_utils.h @@ -28,11 +28,12 @@ class MutationUtils : public Logger::Loggable { // configured to reject failed mutations. The "rejected_mutations" counter // will be incremented with the number of invalid mutations, regardless of // whether an error is returned. + // TODO(tyxia) Normalizing the headers to lower-case in ext_proc's header mutation. static absl::Status applyHeaderMutations(const envoy::service::ext_proc::v3::HeaderMutation& mutation, Http::HeaderMap& headers, bool replacing_message, const Filters::Common::MutationRules::Checker& rule_checker, - Stats::Counter& rejected_mutations); + Stats::Counter& rejected_mutations, bool remove_content_length = false); // Modify a buffer based on a set of mutations from a protobuf static void applyBodyMutations(const envoy::service::ext_proc::v3::BodyMutation& mutation, diff --git a/source/extensions/filters/http/ext_proc/processor_state.cc b/source/extensions/filters/http/ext_proc/processor_state.cc index f57fcb407342..124d59a6f107 100644 --- a/source/extensions/filters/http/ext_proc/processor_state.cc +++ b/source/extensions/filters/http/ext_proc/processor_state.cc @@ -91,7 +91,8 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon const auto mut_status = MutationUtils::applyHeaderMutations( common_response.header_mutation(), *headers_, common_response.status() == CommonResponse::CONTINUE_AND_REPLACE, - filter_.config().mutationChecker(), filter_.stats().rejected_header_mutations_); + filter_.config().mutationChecker(), filter_.stats().rejected_header_mutations_, + shouldRemoveContentLength()); if (!mut_status.ok()) { return mut_status; } @@ -104,8 +105,9 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon ENVOY_LOG(debug, "Replacing complete message"); // Completely replace the body that may already exist. if (common_response.has_body_mutation()) { - // Always remove the content-length header if changing the body. - // The proxy can restore it later if it needs to. + // Remove the content length here because in this case external processor probably won't + // properly set the content-length header to match the length of the new body that replaces + // the original one. headers_->removeContentLength(); body_replaced_ = true; if (bufferedData() == nullptr) { @@ -136,9 +138,8 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon // flag of decodeData() can be determined by whether the trailers are received. // Also, bufferedData() is not nullptr means decodeData() is called, even though // the data can be an empty chunk. - filter_.sendBodyChunk(*this, *bufferedData(), - ProcessorState::CallbackState::BufferedBodyCallback, - !trailers_available_); + auto req = filter_.setupBodyChunk(*this, *bufferedData(), !trailers_available_); + filter_.sendBodyChunk(*this, ProcessorState::CallbackState::BufferedBodyCallback, req); clearWatermark(); return absl::OkStatus(); } @@ -157,9 +158,10 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon modifyBufferedData( [&buffered_chunk](Buffer::Instance& data) { buffered_chunk.move(data); }); ENVOY_LOG(debug, "Sending first chunk using buffered data ({})", buffered_chunk.length()); - filter_.sendBodyChunk(*this, buffered_chunk, - ProcessorState::CallbackState::StreamedBodyCallback, false); - enqueueStreamingChunk(buffered_chunk, false, true); + // Need to first enqueue the data into the chunk queue before sending. + auto req = filter_.setupBodyChunk(*this, buffered_chunk, false); + enqueueStreamingChunk(buffered_chunk, false); + filter_.sendBodyChunk(*this, ProcessorState::CallbackState::StreamedBodyCallback, req); } if (queueBelowLowLimit()) { clearWatermark(); @@ -174,7 +176,7 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon Buffer::OwnedImpl buffered_chunk; modifyBufferedData( [&buffered_chunk](Buffer::Instance& data) { buffered_chunk.move(data); }); - enqueueStreamingChunk(buffered_chunk, false, true); + enqueueStreamingChunk(buffered_chunk, false); } if (queueOverHighLimit()) { // We reached the limit so send what we have. This is different from the buffered @@ -185,8 +187,9 @@ absl::Status ProcessorState::handleHeadersResponse(const HeadersResponse& respon debug, "Sending {} bytes of data end_stream {} in buffered partial mode before end stream", chunkQueue().receivedData().length(), all_data.end_stream); - filter_.sendBodyChunk(*this, chunkQueue().receivedData(), - ProcessorState::CallbackState::BufferedPartialBodyCallback, false); + auto req = filter_.setupBodyChunk(*this, chunkQueue().receivedData(), false); + filter_.sendBodyChunk(*this, ProcessorState::CallbackState::BufferedPartialBodyCallback, + req); } else { // Let data continue to flow, but don't resume yet -- we would like to hold // the headers while we buffer the body up to the limit. @@ -219,7 +222,6 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { const auto& common_response = response.response(); if (callback_state_ == CallbackState::BufferedBodyCallback || callback_state_ == CallbackState::StreamedBodyCallback || - callback_state_ == CallbackState::StreamedBodyCallbackFinishing || callback_state_ == CallbackState::BufferedPartialBodyCallback) { ENVOY_LOG(debug, "Processing body response"); if (callback_state_ == CallbackState::BufferedBodyCallback) { @@ -229,7 +231,8 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { const auto mut_status = MutationUtils::applyHeaderMutations( common_response.header_mutation(), *headers_, common_response.status() == CommonResponse::CONTINUE_AND_REPLACE, - filter_.config().mutationChecker(), filter_.stats().rejected_header_mutations_); + filter_.config().mutationChecker(), filter_.stats().rejected_header_mutations_, + shouldRemoveContentLength()); if (!mut_status.ok()) { return mut_status; } @@ -237,13 +240,22 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { ENVOY_LOG(debug, "Response had header mutations but headers aren't available"); } } + if (common_response.has_body_mutation()) { + if (headers_ != nullptr && headers_->ContentLength() != nullptr) { + size_t content_length = 0; + // When body mutation by external processor is enabled, content-length header is only + // allowed in BUFFERED mode. If its value doesn't match the length of mutated body, the + // corresponding body mutation will be rejected and local reply will be sent with an error + // message. + if (absl::SimpleAtoi(headers_->getContentLengthValue(), &content_length) && + content_length != common_response.body_mutation().body().size()) { + return absl::InternalError( + "mismatch between content length and the length of the mutated body"); + } + } ENVOY_LOG(debug, "Applying body response to buffered data. State = {}", static_cast(callback_state_)); - if (headers_ != nullptr) { - // Always reset the content length here to prevent later problems. - headers_->removeContentLength(); - } modifyBufferedData([&common_response](Buffer::Instance& data) { MutationUtils::applyBodyMutations(common_response.body_mutation(), data); }); @@ -251,30 +263,20 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { clearWatermark(); onFinishProcessorCall(Grpc::Status::Ok); should_continue = true; - } else if (callback_state_ == CallbackState::StreamedBodyCallback || - callback_state_ == CallbackState::StreamedBodyCallbackFinishing) { - bool delivered_one = false; + } else if (callback_state_ == CallbackState::StreamedBodyCallback) { Buffer::OwnedImpl chunk_data; - while (auto queued_chunk = dequeueStreamingChunk(delivered_one, chunk_data)) { - // Loop through queue in case some of it is chunks that were never - // delivered because the processing mode changed. - auto chunk = std::move(*queued_chunk); - if (chunk->delivered) { - if (common_response.has_body_mutation()) { - ENVOY_LOG(debug, "Applying body response to chunk of data. Size = {}", chunk->length); - MutationUtils::applyBodyMutations(common_response.body_mutation(), chunk_data); - } - delivered_one = true; - // After we have delivered one chunk, don't process anything - // more from the queue unless it was never sent to the server. - } - should_continue = chunk->end_stream; - if (chunk_data.length() > 0) { - ENVOY_LOG(trace, "Injecting {} bytes of data to filter stream", chunk_data.length()); - injectDataToFilterChain(chunk_data, chunk->end_stream); - } - chunk_data.drain(chunk_data.length()); + auto chunk = dequeueStreamingChunk(chunk_data); + ENVOY_BUG(chunk, "Bad streamed body callback state"); + if (common_response.has_body_mutation()) { + ENVOY_LOG(debug, "Applying body response to chunk of data. Size = {}", chunk->length); + MutationUtils::applyBodyMutations(common_response.body_mutation(), chunk_data); } + should_continue = chunk->end_stream; + if (chunk_data.length() > 0) { + ENVOY_LOG(trace, "Injecting {} bytes of data to filter stream", chunk_data.length()); + injectDataToFilterChain(chunk_data, chunk->end_stream); + } + if (queueBelowLowLimit()) { clearWatermark(); } @@ -286,16 +288,16 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { } else if (callback_state_ == CallbackState::BufferedPartialBodyCallback) { // Apply changes to the buffer that we sent to the server Buffer::OwnedImpl chunk_data; - auto queued_chunk = dequeueStreamingChunk(false, chunk_data); - ENVOY_BUG(queued_chunk, "Bad partial body callback state"); - auto chunk = std::move(*queued_chunk); + auto chunk = dequeueStreamingChunk(chunk_data); + ENVOY_BUG(chunk, "Bad partial body callback state"); if (common_response.has_header_mutation()) { if (headers_ != nullptr) { ENVOY_LOG(debug, "Applying header mutations to buffered body message"); const auto mut_status = MutationUtils::applyHeaderMutations( common_response.header_mutation(), *headers_, common_response.status() == CommonResponse::CONTINUE_AND_REPLACE, - filter_.config().mutationChecker(), filter_.stats().rejected_header_mutations_); + filter_.config().mutationChecker(), filter_.stats().rejected_header_mutations_, + shouldRemoveContentLength()); if (!mut_status.ok()) { return mut_status; } @@ -304,10 +306,6 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { } } if (common_response.has_body_mutation()) { - if (headers_ != nullptr) { - // Always reset the content length here to prevent later problems. - headers_->removeContentLength(); - } MutationUtils::applyBodyMutations(common_response.body_mutation(), chunk_data); } if (chunk_data.length() > 0) { @@ -321,15 +319,11 @@ absl::Status ProcessorState::handleBodyResponse(const BodyResponse& response) { partial_body_processed_ = true; // If anything else is left on the queue, inject it too - Buffer::OwnedImpl leftover_data; - while (auto leftover_chunk = dequeueStreamingChunk(false, leftover_data)) { - auto chunk = std::move(*leftover_chunk); - if (leftover_data.length() > 0) { - ENVOY_LOG(trace, "Injecting {} bytes of leftover data to filter stream", - leftover_data.length()); - injectDataToFilterChain(leftover_data, chunk->end_stream); - } - leftover_data.drain(leftover_data.length()); + if (chunkQueue().receivedData().length() > 0) { + const auto& all_data = consolidateStreamedChunks(); + ENVOY_LOG(trace, "Injecting {} bytes of leftover data to filter stream", + chunkQueue().receivedData().length()); + injectDataToFilterChain(chunkQueue().receivedData(), all_data.end_stream); } } else { // Fake a grpc error when processor state and received message type doesn't match, beware this @@ -374,9 +368,8 @@ absl::Status ProcessorState::handleTrailersResponse(const TrailersResponse& resp return absl::FailedPreconditionError("spurious message"); } -void ProcessorState::enqueueStreamingChunk(Buffer::Instance& data, bool end_stream, - bool delivered) { - chunk_queue_.push(data, end_stream, delivered); +void ProcessorState::enqueueStreamingChunk(Buffer::Instance& data, bool end_stream) { + chunk_queue_.push(data, end_stream); if (queueOverHighLimit()) { requestWatermark(); } @@ -384,26 +377,16 @@ void ProcessorState::enqueueStreamingChunk(Buffer::Instance& data, bool end_stre void ProcessorState::clearAsyncState() { onFinishProcessorCall(Grpc::Status::Aborted); - Buffer::OwnedImpl chunk_data; - while (auto queued_chunk = dequeueStreamingChunk(false, chunk_data)) { - auto chunk = std::move(*queued_chunk); - ENVOY_LOG(trace, "Injecting leftover buffer of {} bytes", chunk_data.length()); - injectDataToFilterChain(chunk_data, chunk->end_stream); - chunk_data.drain(chunk_data.length()); + if (chunkQueue().receivedData().length() > 0) { + const auto& all_data = consolidateStreamedChunks(); + ENVOY_LOG(trace, "Injecting leftover buffer of {} bytes", chunkQueue().receivedData().length()); + injectDataToFilterChain(chunkQueue().receivedData(), all_data.end_stream); } clearWatermark(); continueIfNecessary(); } -void ProcessorState::setBodyMode(ProcessingMode_BodySendMode body_mode) { - body_mode_ = body_mode; - if (callback_state_ == CallbackState::StreamedBodyCallback && - body_mode != ProcessingMode::STREAMED) { - // Special handling for when the processing mode is changed while - // streaming. - callback_state_ = CallbackState::StreamedBodyCallbackFinishing; - } -} +void ProcessorState::setBodyMode(ProcessingMode_BodySendMode body_mode) { body_mode_ = body_mode; } void ProcessorState::continueIfNecessary() { if (paused_) { @@ -480,12 +463,11 @@ void EncodingProcessorState::clearWatermark() { } } -void ChunkQueue::push(Buffer::Instance& data, bool end_stream, bool delivered) { +void ChunkQueue::push(Buffer::Instance& data, bool end_stream) { // Adding the chunk into the queue. auto next_chunk = std::make_unique(); next_chunk->length = data.length(); next_chunk->end_stream = end_stream; - next_chunk->delivered = delivered; queue_.push_back(std::move(next_chunk)); bytes_enqueued_ += data.length(); @@ -493,13 +475,11 @@ void ChunkQueue::push(Buffer::Instance& data, bool end_stream, bool delivered) { received_data_.move(data); } -absl::optional ChunkQueue::pop(bool undelivered_only, Buffer::OwnedImpl& out_data) { +QueuedChunkPtr ChunkQueue::pop(Buffer::OwnedImpl& out_data) { if (queue_.empty()) { - return absl::nullopt; - } - if (undelivered_only && queue_.front()->delivered) { - return absl::nullopt; + return nullptr; } + QueuedChunkPtr chunk = std::move(queue_.front()); queue_.pop_front(); bytes_enqueued_ -= chunk->length; @@ -518,7 +498,6 @@ const QueuedChunk& ChunkQueue::consolidate() { queue_.push_front(std::move(new_chunk)); } auto& chunk = *(queue_.front()); - chunk.delivered = true; return chunk; } diff --git a/source/extensions/filters/http/ext_proc/processor_state.h b/source/extensions/filters/http/ext_proc/processor_state.h index b87031734c2b..de4628864941 100644 --- a/source/extensions/filters/http/ext_proc/processor_state.h +++ b/source/extensions/filters/http/ext_proc/processor_state.h @@ -27,8 +27,6 @@ class QueuedChunk { public: // True if this represents the last chunk in the stream bool end_stream = false; - // True if the chunk was actually sent to the gRPC stream - bool delivered = false; uint32_t length = 0; }; using QueuedChunkPtr = std::unique_ptr; @@ -40,16 +38,12 @@ class ChunkQueue { ChunkQueue& operator=(const ChunkQueue&) = delete; uint32_t bytesEnqueued() const { return bytes_enqueued_; } bool empty() const { return queue_.empty(); } - void push(Buffer::Instance& data, bool end_stream, bool delivered); - absl::optional pop(bool undelivered_only, Buffer::OwnedImpl& out_data); + void push(Buffer::Instance& data, bool end_stream); + QueuedChunkPtr pop(Buffer::OwnedImpl& out_data); const QueuedChunk& consolidate(); Buffer::OwnedImpl& receivedData() { return received_data_; } private: - // If we are in either streaming mode, store chunks that we received here, - // and use the "delivered" flag to keep track of which ones were pushed - // to the external processor. When matching responses come back for these - // chunks, then they will be removed. std::deque queue_; // The total size of chunks in the queue. uint32_t bytes_enqueued_{}; @@ -72,10 +66,6 @@ class ProcessorState : public Logger::Loggable { BufferedBodyCallback, // Waiting for a "body" response in streaming mode. StreamedBodyCallback, - // Waiting for a "body" response in streaming mode in the special case - // in which the processing mode was changed while there were outstanding - // messages sent to the processor. - StreamedBodyCallbackFinishing, // Waiting for a body callback in "buffered partial" mode. BufferedPartialBodyCallback, // Waiting for a "trailers" response. @@ -144,17 +134,30 @@ class ProcessorState : public Logger::Loggable { ChunkQueue& chunkQueue() { return chunk_queue_; } // Move the contents of "data" into a QueuedChunk object on the streaming queue. - void enqueueStreamingChunk(Buffer::Instance& data, bool end_stream, bool delivered); + void enqueueStreamingChunk(Buffer::Instance& data, bool end_stream); // If the queue has chunks, return the head of the queue. - absl::optional dequeueStreamingChunk(bool undelivered_only, - Buffer::OwnedImpl& out_data) { - // we should return both chunk, and the corresponding data as well here. - return chunk_queue_.pop(undelivered_only, out_data); + QueuedChunkPtr dequeueStreamingChunk(Buffer::OwnedImpl& out_data) { + return chunk_queue_.pop(out_data); } // Consolidate all the chunks on the queue into a single one and return a reference. const QueuedChunk& consolidateStreamedChunks() { return chunk_queue_.consolidate(); } bool queueOverHighLimit() const { return chunk_queue_.bytesEnqueued() > bufferLimit(); } bool queueBelowLowLimit() const { return chunk_queue_.bytesEnqueued() < bufferLimit() / 2; } + bool shouldRemoveContentLength() const { + // Always remove the content length in 3 cases below: + // 1) STREAMED BodySendMode + // 2) BUFFERED_PARTIAL BodySendMode + // 3) BUFFERED BodySendMode + SKIP HeaderSendMode + // In these modes, ext_proc filter can not guarantee to set the content length correctly if + // body is mutated by external processor later. + // In http1 codec, removing content length will enable chunked encoding whenever feasible. + return ( + body_mode_ == envoy::extensions::filters::http::ext_proc::v3::ProcessingMode::STREAMED || + body_mode_ == + envoy::extensions::filters::http::ext_proc::v3::ProcessingMode::BUFFERED_PARTIAL || + (body_mode_ == envoy::extensions::filters::http::ext_proc::v3::ProcessingMode::BUFFERED && + !send_headers_)); + } virtual Http::HeaderMap* addTrailers() PURE; diff --git a/source/extensions/filters/http/fault/config.cc b/source/extensions/filters/http/fault/config.cc index 33c83145e521..f57d234ee3e1 100644 --- a/source/extensions/filters/http/fault/config.cc +++ b/source/extensions/filters/http/fault/config.cc @@ -14,8 +14,11 @@ namespace Fault { Http::FilterFactoryCb FaultFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::fault::v3::HTTPFault& config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { - FaultFilterConfigSharedPtr filter_config(std::make_shared( - config, context.runtime(), stats_prefix, context.scope(), context.timeSource())); + auto& server_context = context.serverFactoryContext(); + + FaultFilterConfigSharedPtr filter_config( + std::make_shared(config, server_context.runtime(), stats_prefix, + context.scope(), server_context.timeSource())); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/http/file_system_buffer/config.cc b/source/extensions/filters/http/file_system_buffer/config.cc index d2f0f31ba778..555e1b31265a 100644 --- a/source/extensions/filters/http/file_system_buffer/config.cc +++ b/source/extensions/filters/http/file_system_buffer/config.cc @@ -22,7 +22,8 @@ Http::FilterFactoryCb FileSystemBufferFilterFactory::createFilterFactoryFromProt const ProtoFileSystemBufferFilterConfig& config, const std::string& stats_prefix ABSL_ATTRIBUTE_UNUSED, Server::Configuration::FactoryContext& context) { - auto factory = AsyncFileManagerFactory::singleton(&context.singletonManager()); + auto factory = + AsyncFileManagerFactory::singleton(&context.serverFactoryContext().singletonManager()); auto manager = config.has_manager_config() ? factory->getAsyncFileManager(config.manager_config()) : std::shared_ptr(); auto filter_config = std::make_shared(std::move(factory), diff --git a/source/extensions/filters/http/gcp_authn/gcp_authn_filter.cc b/source/extensions/filters/http/gcp_authn/gcp_authn_filter.cc index 4aadb57c7453..3ba81afc2f6e 100644 --- a/source/extensions/filters/http/gcp_authn/gcp_authn_filter.cc +++ b/source/extensions/filters/http/gcp_authn/gcp_authn_filter.cc @@ -42,7 +42,8 @@ Http::FilterHeadersStatus GcpAuthnFilter::decodeHeaders(Http::RequestHeaderMap& initiating_call_ = true; Envoy::Upstream::ThreadLocalCluster* cluster = - context_.clusterManager().getThreadLocalCluster(route->routeEntry()->clusterName()); + context_.serverFactoryContext().clusterManager().getThreadLocalCluster( + route->routeEntry()->clusterName()); if (cluster != nullptr) { // The `audience` is passed to filter through cluster metadata. diff --git a/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc b/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc index 95ce7104f366..d4eb3e52a79d 100644 --- a/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc +++ b/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc @@ -36,7 +36,8 @@ void GcpAuthnClient::fetchToken(RequestCallbacks& callbacks, Http::RequestMessag const std::string cluster = config_.http_uri().cluster(); const std::string uri = config_.http_uri().uri(); - const auto thread_local_cluster = context_.clusterManager().getThreadLocalCluster(cluster); + const auto thread_local_cluster = + context_.serverFactoryContext().clusterManager().getThreadLocalCluster(cluster); // Failed to fetch the token if the cluster is not configured. if (thread_local_cluster == nullptr) { diff --git a/source/extensions/filters/http/gcp_authn/token_cache.h b/source/extensions/filters/http/gcp_authn/token_cache.h index c104c8c3e99e..c0628e8f1588 100644 --- a/source/extensions/filters/http/gcp_authn/token_cache.h +++ b/source/extensions/filters/http/gcp_authn/token_cache.h @@ -59,7 +59,7 @@ class ThreadLocalCache : public Envoy::ThreadLocal::ThreadLocalObject { struct TokenCache { TokenCache(const envoy::extensions::filters::http::gcp_authn::v3::TokenCacheConfig& cache_config, Envoy::Server::Configuration::FactoryContext& context) - : tls(context.threadLocal()) { + : tls(context.serverFactoryContext().threadLocal()) { tls.set([cache_config](Envoy::Event::Dispatcher& dispatcher) { return std::make_shared(cache_config, dispatcher.timeSource()); }); diff --git a/source/extensions/filters/http/geoip/BUILD b/source/extensions/filters/http/geoip/BUILD index f12f88882178..7d47565f71d1 100644 --- a/source/extensions/filters/http/geoip/BUILD +++ b/source/extensions/filters/http/geoip/BUILD @@ -16,8 +16,9 @@ envoy_cc_library( name = "geoip_filter_lib", srcs = ["geoip_filter.cc"], hdrs = ["geoip_filter.h"], + tags = ["skip_on_windows"], deps = [ - ":provider_config", + "//envoy/geoip:geoip_provider_driver_interface", "//envoy/http:filter_interface", "//envoy/runtime:runtime_interface", "//source/common/common:assert_lib", @@ -29,28 +30,13 @@ envoy_cc_library( ], ) -#todo(nezdolik) may need to split into interface and impl -envoy_cc_extension( - name = "provider_config", - hdrs = [ - "geoip_provider_config.h", - "geoip_provider_config_impl.h", - ], - deps = [ - "//envoy/config:typed_config_interface", - "//envoy/network:address_interface", - "//envoy/protobuf:message_validator_interface", - "//source/common/common:hash_lib", - "@envoy_api//envoy/extensions/filters/http/geoip/v3:pkg_cc_proto", - ], -) - envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + tags = ["skip_on_windows"], deps = [ - ":provider_config", + "//envoy/geoip:geoip_provider_driver_interface", "//source/common/config:utility_lib", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", diff --git a/source/extensions/filters/http/geoip/config.cc b/source/extensions/filters/http/geoip/config.cc index b88138abbec2..86c83472dc91 100644 --- a/source/extensions/filters/http/geoip/config.cc +++ b/source/extensions/filters/http/geoip/config.cc @@ -5,7 +5,6 @@ #include "source/common/config/utility.h" #include "source/common/protobuf/utility.h" #include "source/extensions/filters/http/geoip/geoip_filter.h" -#include "source/extensions/filters/http/geoip/geoip_provider_config_impl.h" namespace Envoy { namespace Extensions { @@ -15,19 +14,16 @@ namespace Geoip { Http::FilterFactoryCb GeoipFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::geoip::v3::Geoip& proto_config, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { - if (!provider_context_) { - provider_context_ = - std::make_unique(context.messageValidationVisitor()); - } GeoipFilterConfigSharedPtr filter_config( std::make_shared(proto_config, stat_prefix, context.scope())); const auto& provider_config = proto_config.provider(); auto& geo_provider_factory = - Envoy::Config::Utility::getAndCheckFactory(provider_config); + Envoy::Config::Utility::getAndCheckFactory( + provider_config); ProtobufTypes::MessagePtr message = Envoy::Config::Utility::translateToFactoryConfig( - provider_config, provider_context_->messageValidationVisitor(), geo_provider_factory); - auto driver = geo_provider_factory.createGeoipProviderDriver(*message, provider_context_); + provider_config, context.messageValidationVisitor(), geo_provider_factory); + auto driver = geo_provider_factory.createGeoipProviderDriver(*message, stat_prefix, context); return [filter_config, driver](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(filter_config, driver)); }; diff --git a/source/extensions/filters/http/geoip/config.h b/source/extensions/filters/http/geoip/config.h index 8b6d782b8582..677cff6be55b 100644 --- a/source/extensions/filters/http/geoip/config.h +++ b/source/extensions/filters/http/geoip/config.h @@ -2,9 +2,9 @@ #include "envoy/extensions/filters/http/geoip/v3/geoip.pb.h" #include "envoy/extensions/filters/http/geoip/v3/geoip.pb.validate.h" +#include "envoy/geoip/geoip_provider_driver.h" #include "source/extensions/filters/http/common/factory_base.h" -#include "source/extensions/filters/http/geoip/geoip_provider_config.h" namespace Envoy { namespace Extensions { @@ -22,9 +22,6 @@ class GeoipFilterFactory Http::FilterFactoryCb createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::geoip::v3::Geoip& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; - -private: - GeoipProviderFactoryContextPtr provider_context_; }; } // namespace Geoip diff --git a/source/extensions/filters/http/geoip/geoip_filter.cc b/source/extensions/filters/http/geoip/geoip_filter.cc index cd3b2a423e35..2d197bb158fc 100644 --- a/source/extensions/filters/http/geoip/geoip_filter.cc +++ b/source/extensions/filters/http/geoip/geoip_filter.cc @@ -15,33 +15,10 @@ GeoipFilterConfig::GeoipFilterConfig( const envoy::extensions::filters::http::geoip::v3::Geoip& config, const std::string& stat_prefix, Stats::Scope& scope) : scope_(scope), stat_name_set_(scope.symbolTable().makeSet("Geoip")), - stats_prefix_(stat_name_set_->add(stat_prefix + "geoip")), - total_(stat_name_set_->add("total")), use_xff_(config.has_xff_config()), + stats_prefix_(stat_name_set_->add(stat_prefix + "geoip")), use_xff_(config.has_xff_config()), xff_num_trusted_hops_(config.has_xff_config() ? config.xff_config().xff_num_trusted_hops() : 0) { - const auto& geo_headers_to_add = config.geo_headers_to_add(); - geo_headers_ = processGeoHeaders({geo_headers_to_add.country(), geo_headers_to_add.city(), - geo_headers_to_add.region(), geo_headers_to_add.asn()}); - geo_anon_headers_ = - processGeoHeaders({geo_headers_to_add.is_anon(), geo_headers_to_add.anon_vpn(), - geo_headers_to_add.anon_hosting(), geo_headers_to_add.anon_tor(), - geo_headers_to_add.anon_proxy()}); - if (geo_headers_.empty() && geo_anon_headers_.empty()) { - throw EnvoyException("No geolocation headers configured"); - } -} - -absl::flat_hash_set -GeoipFilterConfig::processGeoHeaders(const absl::flat_hash_set& headers) const { - absl::flat_hash_set geo_headers; - for (auto header : headers) { - if (!header.empty()) { - stat_name_set_->rememberBuiltin(absl::StrCat(header, ".hit")); - stat_name_set_->rememberBuiltin(absl::StrCat(header, ".total")); - geo_headers.insert(std::string(header)); - } - } - return geo_headers; + stat_name_set_->rememberBuiltin("total"); } void GeoipFilterConfig::incCounter(Stats::StatName name) { @@ -49,7 +26,7 @@ void GeoipFilterConfig::incCounter(Stats::StatName name) { scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); } -GeoipFilter::GeoipFilter(GeoipFilterConfigSharedPtr config, DriverSharedPtr driver) +GeoipFilter::GeoipFilter(GeoipFilterConfigSharedPtr config, Geolocation::DriverSharedPtr driver) : config_(config), driver_(std::move(driver)) {} GeoipFilter::~GeoipFilter() = default; @@ -76,13 +53,9 @@ Http::FilterHeadersStatus GeoipFilter::decodeHeaders(Http::RequestHeaderMap& hea // This is a safe measure to protect against the case when filter gets deleted before the callback // is run. GeoipFilterWeakPtr self = weak_from_this(); - // Copy header values to pass to the driver lookup function (in case filter gets destroyed before - // lookup completes). - absl::flat_hash_set geo_headers = config_->geoHeaders(); - absl::flat_hash_set geo_anon_headers = config_->geoAnonHeaders(); driver_->lookup( - LookupRequest{std::move(remote_address), std::move(geo_headers), std::move(geo_anon_headers)}, - [self, &dispatcher = decoder_callbacks_->dispatcher()](LookupResult&& result) { + Geolocation::LookupRequest{std::move(remote_address)}, + [self, &dispatcher = decoder_callbacks_->dispatcher()](Geolocation::LookupResult&& result) { dispatcher.post([self, result]() { if (GeoipFilterSharedPtr filter = self.lock()) { filter->onLookupComplete(std::move(result)); @@ -106,18 +79,16 @@ void GeoipFilter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& decoder_callbacks_ = &callbacks; } -void GeoipFilter::onLookupComplete(LookupResult&& result) { +void GeoipFilter::onLookupComplete(Geolocation::LookupResult&& result) { ASSERT(request_headers_); for (auto it = result.cbegin(); it != result.cend();) { const auto& geo_header = it->first; const auto& lookup_result = it++->second; - if (lookup_result) { - request_headers_->setCopy(Http::LowerCaseString(geo_header), lookup_result.value()); - config_->incHit(geo_header); + if (!lookup_result.empty()) { + request_headers_->setCopy(Http::LowerCaseString(geo_header), lookup_result); } - config_->incTotal(geo_header); } - + config_->incTotal(); ENVOY_LOG(debug, "Geoip filter: finished decoding geolocation headers"); decoder_callbacks_->continueDecoding(); } diff --git a/source/extensions/filters/http/geoip/geoip_filter.h b/source/extensions/filters/http/geoip/geoip_filter.h index 254ab7c9a3d1..bbc2345b311e 100644 --- a/source/extensions/filters/http/geoip/geoip_filter.h +++ b/source/extensions/filters/http/geoip/geoip_filter.h @@ -3,19 +3,15 @@ #include "envoy/common/exception.h" #include "envoy/common/optref.h" #include "envoy/extensions/filters/http/geoip/v3/geoip.pb.h" +#include "envoy/geoip/geoip_provider_driver.h" #include "envoy/http/filter.h" #include "envoy/stats/scope.h" -#include "source/extensions/filters/http/geoip/geoip_provider_config.h" - namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Geoip { -using GeolocationHeadersToAdd = - envoy::extensions::filters::http::geoip::v3::Geoip_GeolocationHeadersToAdd; - /** * Configuration for the Geoip filter. */ @@ -24,32 +20,20 @@ class GeoipFilterConfig { GeoipFilterConfig(const envoy::extensions::filters::http::geoip::v3::Geoip& config, const std::string& stat_prefix, Stats::Scope& scope); - void incHit(absl::string_view geo_header) { - incCounter(stat_name_set_->getBuiltin(absl::StrCat(geo_header, ".hit"), unknown_hit_)); - } - void incTotal(absl::string_view geo_header) { - incCounter(stat_name_set_->getBuiltin(absl::StrCat(geo_header, ".total"), unknown_hit_)); - } + void incTotal() { incCounter(stat_name_set_->getBuiltin("total", unknown_hit_)); } bool useXff() const { return use_xff_; } uint32_t xffNumTrustedHops() const { return xff_num_trusted_hops_; } - const absl::flat_hash_set& geoHeaders() const { return geo_headers_; } - const absl::flat_hash_set& geoAnonHeaders() const { return geo_anon_headers_; } private: void incCounter(Stats::StatName name); - absl::flat_hash_set - processGeoHeaders(const absl::flat_hash_set& headers) const; Stats::Scope& scope_; Stats::StatNameSetPtr stat_name_set_; const Stats::StatName stats_prefix_; - const Stats::StatName total_; const Stats::StatName unknown_hit_; bool use_xff_; const uint32_t xff_num_trusted_hops_; - absl::flat_hash_set geo_headers_; - absl::flat_hash_set geo_anon_headers_; }; using GeoipFilterConfigSharedPtr = std::shared_ptr; @@ -58,7 +42,7 @@ class GeoipFilter : public Http::StreamDecoderFilter, public Logger::Loggable, public std::enable_shared_from_this { public: - GeoipFilter(GeoipFilterConfigSharedPtr config, DriverSharedPtr driver); + GeoipFilter(GeoipFilterConfigSharedPtr config, Geolocation::DriverSharedPtr driver); ~GeoipFilter() override; // Http::StreamFilterBase @@ -71,14 +55,14 @@ class GeoipFilter : public Http::StreamDecoderFilter, Http::FilterTrailersStatus decodeTrailers(Http::RequestTrailerMap& trailers) override; void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override; // Callbacks for geolocation filter when lookup is complete. - void onLookupComplete(LookupResult&& result); + void onLookupComplete(Geolocation::LookupResult&& result); private: // Allow the unit test to have access to private members. friend class GeoipFilterPeer; GeoipFilterConfigSharedPtr config_; Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; - DriverSharedPtr driver_; + Geolocation::DriverSharedPtr driver_; OptRef request_headers_; }; diff --git a/source/extensions/filters/http/geoip/geoip_provider_config_impl.h b/source/extensions/filters/http/geoip/geoip_provider_config_impl.h deleted file mode 100644 index 4b67084b0a13..000000000000 --- a/source/extensions/filters/http/geoip/geoip_provider_config_impl.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include "source/common/protobuf/utility.h" -#include "source/extensions/filters/http/geoip/geoip_provider_config.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Geoip { - -class GeoipProviderFactoryContextImpl : public GeoipProviderFactoryContext { -public: - GeoipProviderFactoryContextImpl(ProtobufMessage::ValidationVisitor& validation_visitor) - : validation_visitor_(validation_visitor) {} - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return validation_visitor_; - } - -private: - ProtobufMessage::ValidationVisitor& validation_visitor_; -}; - -/** - * Common base class for geolocation provider registrations. - */ -template class GeoipProviderFactoryBase : public GeoipProviderFactory { -public: - // Server::Configuration::GeoipProviderFactory - DriverSharedPtr createGeoipProviderDriver(const Protobuf::Message& config, - GeoipProviderFactoryContextPtr& context) override { - return createGeoipProviderDriverTyped(MessageUtil::downcastAndValidate( - config, context->messageValidationVisitor()), - context); - } - - ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); - } - - std::string name() const override { return name_; } - -protected: - GeoipProviderFactoryBase(const std::string& name) : name_(name) {} - -private: - virtual DriverSharedPtr - createGeoipProviderDriverTyped(const ConfigProto& proto_config, - GeoipProviderFactoryContextPtr& context) PURE; - - const std::string name_; -}; - -} // namespace Geoip -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/filters/http/grpc_field_extraction/config.cc b/source/extensions/filters/http/grpc_field_extraction/config.cc index c58a0eea211c..e387d451908f 100644 --- a/source/extensions/filters/http/grpc_field_extraction/config.cc +++ b/source/extensions/filters/http/grpc_field_extraction/config.cc @@ -25,7 +25,7 @@ Envoy::Http::FilterFactoryCb FilterFactoryCreator::createFilterFactoryFromProtoT const std::string&, Envoy::Server::Configuration::FactoryContext& context) { auto filter_config = std::make_shared( - proto_config, std::make_unique(), context.api()); + proto_config, std::make_unique(), context.serverFactoryContext().api()); return [filter_config](Envoy::Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(*filter_config)); }; diff --git a/source/extensions/filters/http/grpc_field_extraction/filter_config.cc b/source/extensions/filters/http/grpc_field_extraction/filter_config.cc index 96d3212511ec..996a5fff771c 100644 --- a/source/extensions/filters/http/grpc_field_extraction/filter_config.cc +++ b/source/extensions/filters/http/grpc_field_extraction/filter_config.cc @@ -56,8 +56,8 @@ void FilterConfig::initDescriptorPool(Api::Api& api) { switch (descriptor_config.specifier_case()) { case envoy::config::core::v3::DataSource::SpecifierCase::kFilename: { - if (!descriptor_set.ParseFromString( - api.fileSystem().fileReadToEnd(descriptor_config.filename()))) { + auto file_or_error = api.fileSystem().fileReadToEnd(descriptor_config.filename()); + if (!file_or_error.status().ok() || !descriptor_set.ParseFromString(file_or_error.value())) { throw Envoy::EnvoyException(fmt::format("unable to parse proto descriptor from file `{}`", descriptor_config.filename())); } diff --git a/source/extensions/filters/http/grpc_http1_bridge/config.cc b/source/extensions/filters/http/grpc_http1_bridge/config.cc index 411bdf6a85e1..e5138dd9456c 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/config.cc +++ b/source/extensions/filters/http/grpc_http1_bridge/config.cc @@ -13,8 +13,8 @@ Http::FilterFactoryCb GrpcHttp1BridgeFilterConfig::createFilterFactoryFromProtoT const envoy::extensions::filters::http::grpc_http1_bridge::v3::Config& proto_config, const std::string&, Server::Configuration::FactoryContext& factory_context) { return [&factory_context, proto_config](Http::FilterChainFactoryCallbacks& callbacks) { - callbacks.addStreamFilter( - std::make_shared(factory_context.grpcContext(), proto_config)); + callbacks.addStreamFilter(std::make_shared( + factory_context.serverFactoryContext().grpcContext(), proto_config)); }; } diff --git a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc index 5dd64feacb68..56d1eebe5a62 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc +++ b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc @@ -19,6 +19,18 @@ namespace Extensions { namespace HttpFilters { namespace GrpcHttp1Bridge { +// Some client requests' URLs may contain query params. gRPC upstream servers can not +// handle these requests, and may return error such as "unknown method". So we remove +// query params here. +void Http1BridgeFilter::ignoreQueryParams(Http::RequestHeaderMap& headers) { + absl::string_view path = headers.getPathValue(); + size_t pos = path.find("?"); + if (pos != absl::string_view::npos) { + absl::string_view new_path = path.substr(0, pos); + headers.setPath(new_path); + } +} + Http::FilterHeadersStatus Http1BridgeFilter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { const bool protobuf_request = Grpc::Common::isProtobufRequestHeaders(headers); if (upgrade_protobuf_ && protobuf_request) { @@ -30,13 +42,15 @@ Http::FilterHeadersStatus Http1BridgeFilter::decodeHeaders(Http::RequestHeaderMa } const bool grpc_request = Grpc::Common::isGrpcRequestHeaders(headers); - const absl::optional& protocol = decoder_callbacks_->streamInfo().protocol(); ASSERT(protocol); if (protocol.value() < Http::Protocol::Http2 && grpc_request) { do_bridging_ = true; } + if (do_bridging_ && ignore_query_parameters_) { + ignoreQueryParams(headers); + } return Http::FilterHeadersStatus::Continue; } diff --git a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h index da353a0b928d..70b0cd6fbf25 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h +++ b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h @@ -22,7 +22,8 @@ class Http1BridgeFilter : public Http::StreamFilter, Logger::LoggableaddEncodedData(data, false); + + // This call exists to ensure that the content-length is set correctly, + // regardless of whether we are withholding grpc frames above. + headers.setContentLength(Grpc::GRPC_FRAME_HEADER_SIZE); + + // Insert grpc-status trailers to communicate the error code. + auto& trailers = encoder_callbacks_->addEncodedTrailers(); + trailers.setGrpcStatus(grpc_status_); + } + } } return Http::FilterHeadersStatus::Continue; diff --git a/source/extensions/filters/http/grpc_json_transcoder/config.cc b/source/extensions/filters/http/grpc_json_transcoder/config.cc index 404a9643e211..85902b39ef05 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/config.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/config.cc @@ -16,7 +16,7 @@ Http::FilterFactoryCb GrpcJsonTranscoderFilterConfig::createFilterFactoryFromPro proto_config, const std::string&, Server::Configuration::FactoryContext& context) { JsonTranscoderConfigSharedPtr filter_config = - std::make_shared(proto_config, context.api()); + std::make_shared(proto_config, context.serverFactoryContext().api()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(*filter_config)); diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index 61b01703dc99..15aaeb4a0f31 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -120,12 +120,14 @@ JsonTranscoderConfig::JsonTranscoderConfig( switch (proto_config.descriptor_set_case()) { case envoy::extensions::filters::http::grpc_json_transcoder::v3::GrpcJsonTranscoder:: - DescriptorSetCase::kProtoDescriptor: - if (!descriptor_set.ParseFromString( - api.fileSystem().fileReadToEnd(proto_config.proto_descriptor()))) { + DescriptorSetCase::kProtoDescriptor: { + auto file_or_error = api.fileSystem().fileReadToEnd(proto_config.proto_descriptor()); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + if (!descriptor_set.ParseFromString(file_or_error.value())) { throw EnvoyException("transcoding_filter: Unable to parse proto descriptor"); } break; + } case envoy::extensions::filters::http::grpc_json_transcoder::v3::GrpcJsonTranscoder:: DescriptorSetCase::kProtoDescriptorBin: if (!descriptor_set.ParseFromString(proto_config.proto_descriptor_bin())) { diff --git a/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc b/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc index 186501a9e856..57dfd4e46762 100644 --- a/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc +++ b/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc @@ -93,7 +93,8 @@ class GrpcServiceMethodToRequestNamesMap { struct Config { Config(const envoy::extensions::filters::http::grpc_stats::v3::FilterConfig& proto_config, Server::Configuration::FactoryContext& context) - : context_(context.grpcContext()), emit_filter_state_(proto_config.emit_filter_state()), + : context_(context.serverFactoryContext().grpcContext()), + emit_filter_state_(proto_config.emit_filter_state()), enable_upstream_stats_(proto_config.enable_upstream_stats()), replace_dots_in_grpc_service_name_(proto_config.replace_dots_in_grpc_service_name()), stats_for_all_methods_( diff --git a/source/extensions/filters/http/grpc_stats/response_frame_counter.h b/source/extensions/filters/http/grpc_stats/response_frame_counter.h index 1be83db601d0..9689866b1bd5 100644 --- a/source/extensions/filters/http/grpc_stats/response_frame_counter.h +++ b/source/extensions/filters/http/grpc_stats/response_frame_counter.h @@ -11,7 +11,7 @@ namespace Extensions { namespace HttpFilters { namespace GrpcStats { -// An adaptation of the gRPC frame inspector that handles the Buf Connect end-of-stream frame. +// An adaptation of the gRPC frame inspector that handles the Connect end-of-stream frame. class ResponseFrameCounter : protected Grpc::FrameInspector { public: uint64_t inspect(const Buffer::Instance& input); diff --git a/source/extensions/filters/http/grpc_web/config.cc b/source/extensions/filters/http/grpc_web/config.cc index eaf8e432c96c..bd584a4e9c2a 100644 --- a/source/extensions/filters/http/grpc_web/config.cc +++ b/source/extensions/filters/http/grpc_web/config.cc @@ -13,7 +13,8 @@ Http::FilterFactoryCb GrpcWebFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::grpc_web::v3::GrpcWeb&, const std::string&, Server::Configuration::FactoryContext& factory_context) { return [&factory_context](Http::FilterChainFactoryCallbacks& callbacks) { - callbacks.addStreamFilter(std::make_shared(factory_context.grpcContext())); + callbacks.addStreamFilter( + std::make_shared(factory_context.serverFactoryContext().grpcContext())); }; } diff --git a/source/extensions/filters/http/header_mutation/config.cc b/source/extensions/filters/http/header_mutation/config.cc index 9705e149c747..c54ff79b642d 100644 --- a/source/extensions/filters/http/header_mutation/config.cc +++ b/source/extensions/filters/http/header_mutation/config.cc @@ -9,7 +9,8 @@ namespace Extensions { namespace HttpFilters { namespace HeaderMutation { -Http::FilterFactoryCb HeaderMutationFactoryConfig::createFilterFactoryFromProtoTyped( +absl::StatusOr +HeaderMutationFactoryConfig::createFilterFactoryFromProtoTyped( const ProtoConfig& config, const std::string&, DualInfo, Server::Configuration::ServerFactoryContext&) { auto filter_config = std::make_shared(config); diff --git a/source/extensions/filters/http/header_mutation/config.h b/source/extensions/filters/http/header_mutation/config.h index d7348241c703..1611b25ce590 100644 --- a/source/extensions/filters/http/header_mutation/config.h +++ b/source/extensions/filters/http/header_mutation/config.h @@ -20,7 +20,7 @@ class HeaderMutationFactoryConfig HeaderMutationFactoryConfig() : DualFactoryBase("envoy.filters.http.header_mutation") {} private: - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProtoTyped(const ProtoConfig& proto_config, const std::string& stats_prefix, DualInfo info, Server::Configuration::ServerFactoryContext& context) override; diff --git a/source/extensions/filters/http/header_mutation/header_mutation.cc b/source/extensions/filters/http/header_mutation/header_mutation.cc index b364f6099aaa..34ba63cc1d23 100644 --- a/source/extensions/filters/http/header_mutation/header_mutation.cc +++ b/source/extensions/filters/http/header_mutation/header_mutation.cc @@ -12,18 +12,16 @@ namespace Extensions { namespace HttpFilters { namespace HeaderMutation { -void Mutations::mutateRequestHeaders(Http::RequestHeaderMap& request_headers, +void Mutations::mutateRequestHeaders(Http::HeaderMap& headers, + const Formatter::HttpFormatterContext& ctx, const StreamInfo::StreamInfo& stream_info) const { - request_mutations_.evaluateHeaders(request_headers, request_headers, - *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + request_mutations_.evaluateHeaders(headers, ctx, stream_info); } -void Mutations::mutateResponseHeaders(const Http::RequestHeaderMap& request_headers, - Http::ResponseHeaderMap& response_headers, +void Mutations::mutateResponseHeaders(Http::HeaderMap& headers, + const Formatter::HttpFormatterContext& ctx, const StreamInfo::StreamInfo& stream_info) const { - response_mutations_.evaluateHeaders(response_headers, request_headers, response_headers, - stream_info); + response_mutations_.evaluateHeaders(headers, ctx, stream_info); } PerRouteHeaderMutation::PerRouteHeaderMutation(const PerRouteProtoConfig& config) @@ -33,7 +31,8 @@ HeaderMutationConfig::HeaderMutationConfig(const ProtoConfig& config) : mutations_(config.mutations()) {} Http::FilterHeadersStatus HeaderMutation::decodeHeaders(Http::RequestHeaderMap& headers, bool) { - config_->mutations().mutateRequestHeaders(headers, decoder_callbacks_->streamInfo()); + Formatter::HttpFormatterContext ctx{&headers}; + config_->mutations().mutateRequestHeaders(headers, ctx, decoder_callbacks_->streamInfo()); // Only the most specific route config is used. // TODO(wbpcode): It's possible to traverse all the route configs to merge the header mutations @@ -42,26 +41,15 @@ Http::FilterHeadersStatus HeaderMutation::decodeHeaders(Http::RequestHeaderMap& Http::Utility::resolveMostSpecificPerFilterConfig(decoder_callbacks_); if (route_config_ != nullptr) { - route_config_->mutations().mutateRequestHeaders(headers, decoder_callbacks_->streamInfo()); + route_config_->mutations().mutateRequestHeaders(headers, ctx, decoder_callbacks_->streamInfo()); } return Http::FilterHeadersStatus::Continue; } Http::FilterHeadersStatus HeaderMutation::encodeHeaders(Http::ResponseHeaderMap& headers, bool) { - // There is an corner case that the downstream request headers will nullptr when the request is - // reset (for example, reset by the stream idle timer) before the request headers are completely - // received. The filter chain will be created and the encodeHeaders() will be called but the - // downstream request headers will be nullptr. - const Http::RequestHeaderMap* downstream_request_headers = - encoder_callbacks_->streamInfo().getRequestHeaders(); - const Http::RequestHeaderMap& request_headers = - downstream_request_headers != nullptr - ? *downstream_request_headers - : *Http::StaticEmptyHeaders::get().request_headers.get(); - - config_->mutations().mutateResponseHeaders(request_headers, headers, - encoder_callbacks_->streamInfo()); + Formatter::HttpFormatterContext ctx{encoder_callbacks_->requestHeaders().ptr(), &headers}; + config_->mutations().mutateResponseHeaders(headers, ctx, encoder_callbacks_->streamInfo()); if (route_config_ == nullptr) { // If we haven't already resolved the route config, do so now. @@ -70,7 +58,7 @@ Http::FilterHeadersStatus HeaderMutation::encodeHeaders(Http::ResponseHeaderMap& } if (route_config_ != nullptr) { - route_config_->mutations().mutateResponseHeaders(request_headers, headers, + route_config_->mutations().mutateResponseHeaders(headers, ctx, encoder_callbacks_->streamInfo()); } diff --git a/source/extensions/filters/http/header_mutation/header_mutation.h b/source/extensions/filters/http/header_mutation/header_mutation.h index 4bb2c23eca41..b37766776ef8 100644 --- a/source/extensions/filters/http/header_mutation/header_mutation.h +++ b/source/extensions/filters/http/header_mutation/header_mutation.h @@ -29,15 +29,14 @@ class Mutations { : request_mutations_(config.request_mutations()), response_mutations_(config.response_mutations()) {} - void mutateRequestHeaders(Http::RequestHeaderMap& request_headers, + void mutateRequestHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext& ctx, const StreamInfo::StreamInfo& stream_info) const; - void mutateResponseHeaders(const Http::RequestHeaderMap& request_headers, - Http::ResponseHeaderMap& response_headers, + void mutateResponseHeaders(Http::HeaderMap& headers, const Formatter::HttpFormatterContext& ctx, const StreamInfo::StreamInfo& stream_info) const; private: - Http::HeaderMutations request_mutations_; - Http::HeaderMutations response_mutations_; + const Http::HeaderMutations request_mutations_; + const Http::HeaderMutations response_mutations_; }; class PerRouteHeaderMutation : public Router::RouteSpecificFilterConfig { diff --git a/source/extensions/filters/http/health_check/config.cc b/source/extensions/filters/http/health_check/config.cc index d7e9dc4293f6..714ca00220e8 100644 --- a/source/extensions/filters/http/health_check/config.cc +++ b/source/extensions/filters/http/health_check/config.cc @@ -33,7 +33,8 @@ Http::FilterFactoryCb HealthCheckFilterConfig::createFilterFactoryFromProtoTyped HealthCheckCacheManagerSharedPtr cache_manager; if (cache_time_ms > 0) { cache_manager = std::make_shared( - context.mainThreadDispatcher(), std::chrono::milliseconds(cache_time_ms)); + context.serverFactoryContext().mainThreadDispatcher(), + std::chrono::milliseconds(cache_time_ms)); } ClusterMinHealthyPercentagesConstSharedPtr cluster_min_healthy_percentages; @@ -47,9 +48,9 @@ Http::FilterFactoryCb HealthCheckFilterConfig::createFilterFactoryFromProtoTyped return [&context, pass_through_mode, cache_manager, header_match_data, cluster_min_healthy_percentages](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter(std::make_shared(context, pass_through_mode, - cache_manager, header_match_data, - cluster_min_healthy_percentages)); + callbacks.addStreamFilter(std::make_shared( + context.serverFactoryContext(), pass_through_mode, cache_manager, header_match_data, + cluster_min_healthy_percentages)); }; } diff --git a/source/extensions/filters/http/health_check/health_check.h b/source/extensions/filters/http/health_check/health_check.h index 4aebd69ebdbc..2779a4d16a52 100644 --- a/source/extensions/filters/http/health_check/health_check.h +++ b/source/extensions/filters/http/health_check/health_check.h @@ -60,7 +60,7 @@ using HeaderDataVectorSharedPtr = std::shared_ptr void { callbacks.addStreamDecoderFilter(std::make_shared(config)); diff --git a/source/extensions/filters/http/json_to_metadata/filter.cc b/source/extensions/filters/http/json_to_metadata/filter.cc index 9e0048fb9aff..18c31b1e5736 100644 --- a/source/extensions/filters/http/json_to_metadata/filter.cc +++ b/source/extensions/filters/http/json_to_metadata/filter.cc @@ -85,31 +85,40 @@ Rule::Rule(const ProtoRule& rule) : rule_(rule) { FilterConfig::FilterConfig( const envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata& proto_config, Stats::Scope& scope) - : stats_{ALL_JSON_TO_METADATA_FILTER_STATS(POOL_COUNTER_PREFIX(scope, "json_to_metadata."))}, - request_rules_(generateRequestRules(proto_config)), - request_allow_content_types_(generateRequestAllowContentTypes(proto_config)), - request_allow_empty_content_type_(proto_config.request_rules().allow_empty_content_type()) {} - -Rules FilterConfig::generateRequestRules( - const envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata& proto_config) - const { + : rqstats_{ALL_JSON_TO_METADATA_FILTER_STATS( + POOL_COUNTER_PREFIX(scope, "json_to_metadata.rq"))}, + respstats_{ + ALL_JSON_TO_METADATA_FILTER_STATS(POOL_COUNTER_PREFIX(scope, "json_to_metadata.resp"))}, + request_rules_(generateRules(proto_config.request_rules().rules())), + response_rules_(generateRules(proto_config.response_rules().rules())), + request_allow_content_types_( + generateAllowContentTypes(proto_config.request_rules().allow_content_types())), + response_allow_content_types_( + generateAllowContentTypes(proto_config.response_rules().allow_content_types())), + request_allow_empty_content_type_(proto_config.request_rules().allow_empty_content_type()), + response_allow_empty_content_type_(proto_config.response_rules().allow_empty_content_type()) { + if (request_rules_.empty() && response_rules_.empty()) { + throw EnvoyException("json_to_metadata_filter: Per filter configs must at least specify " + "either request or response rules"); + } +} + +Rules FilterConfig::generateRules(const ProtobufRepeatedRule& proto_rules) const { Rules rules; - for (const auto& rule : proto_config.request_rules().rules()) { + for (const auto& rule : proto_rules) { rules.emplace_back(rule); } return rules; } -absl::flat_hash_set FilterConfig::generateRequestAllowContentTypes( - const envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata& proto_config) - const { - if (proto_config.request_rules().allow_content_types().empty()) { +absl::flat_hash_set FilterConfig::generateAllowContentTypes( + const Protobuf::RepeatedPtrField& proto_allow_content_types) const { + if (proto_allow_content_types.empty()) { return {Http::Headers::get().ContentTypeValues.Json}; } absl::flat_hash_set allow_content_types; - for (const auto& request_allowed_content_type : - proto_config.request_rules().allow_content_types()) { + for (const auto& request_allowed_content_type : proto_allow_content_types) { allow_content_types.insert(request_allowed_content_type); } return allow_content_types; @@ -123,26 +132,35 @@ bool FilterConfig::requestContentTypeAllowed(absl::string_view content_type) con return request_allow_content_types_.contains(content_type); } +bool FilterConfig::responseContentTypeAllowed(absl::string_view content_type) const { + if (content_type.empty()) { + return response_allow_empty_content_type_; + } + + return response_allow_content_types_.contains(content_type); +} + void Filter::applyKeyValue(const std::string& value, const KeyValuePair& keyval, - StructMap& struct_map) { + StructMap& struct_map, Http::StreamFilterCallbacks& filter_callback) { ASSERT(!value.empty()); ProtobufWkt::Value val; val.set_string_value(value); - applyKeyValue(std::move(val), keyval, struct_map); + applyKeyValue(std::move(val), keyval, struct_map, filter_callback); } -void Filter::applyKeyValue(double value, const KeyValuePair& keyval, StructMap& struct_map) { +void Filter::applyKeyValue(double value, const KeyValuePair& keyval, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback) { ProtobufWkt::Value val; val.set_number_value(value); - applyKeyValue(std::move(val), keyval, struct_map); + applyKeyValue(std::move(val), keyval, struct_map, filter_callback); } void Filter::applyKeyValue(ProtobufWkt::Value value, const KeyValuePair& keyval, - StructMap& struct_map) { + StructMap& struct_map, Http::StreamFilterCallbacks& filter_callback) { const auto& nspace = decideNamespace(keyval.metadata_namespace()); addMetadata(nspace, keyval.key(), std::move(value), keyval.preserve_existing_metadata_value(), - struct_map); + struct_map, filter_callback); } const std::string& Filter::decideNamespace(const std::string& nspace) const { @@ -152,11 +170,10 @@ const std::string& Filter::decideNamespace(const std::string& nspace) const { bool Filter::addMetadata(const std::string& meta_namespace, const std::string& key, ProtobufWkt::Value val, const bool preserve_existing_metadata_value, - StructMap& struct_map) { + StructMap& struct_map, Http::StreamFilterCallbacks& filter_callback) { if (preserve_existing_metadata_value) { - // TODO(kuochunghsu): support encoding - auto& filter_metadata = decoder_callbacks_->streamInfo().dynamicMetadata().filter_metadata(); + auto& filter_metadata = filter_callback.streamInfo().dynamicMetadata().filter_metadata(); const auto entry_it = filter_metadata.find(meta_namespace); if (entry_it != filter_metadata.end()) { @@ -177,7 +194,8 @@ bool Filter::addMetadata(const std::string& meta_namespace, const std::string& k } void Filter::finalizeDynamicMetadata(Http::StreamFilterCallbacks& filter_callback, - const StructMap& struct_map, bool& processing_finished_flag) { + bool should_clear_route_cache, const StructMap& struct_map, + bool& processing_finished_flag) { ASSERT(!processing_finished_flag); processing_finished_flag = true; if (!struct_map.empty()) { @@ -185,46 +203,59 @@ void Filter::finalizeDynamicMetadata(Http::StreamFilterCallbacks& filter_callbac filter_callback.streamInfo().setDynamicMetadata(entry.first, entry.second); } - decoder_callbacks_->downstreamCallbacks()->clearRouteCache(); + if (should_clear_route_cache) { + decoder_callbacks_->downstreamCallbacks()->clearRouteCache(); + } } } -void Filter::handleAllOnMissing(const Rules& rules, bool& processing_finished_flag) { +void Filter::handleAllOnMissing(const Rules& rules, bool should_clear_route_cache, + Http::StreamFilterCallbacks& filter_callback, + bool& processing_finished_flag) { StructMap struct_map; for (const auto& rule : rules) { if (rule.rule_.has_on_missing()) { - applyKeyValue(rule.rule_.on_missing().value(), rule.rule_.on_missing(), struct_map); + applyKeyValue(rule.rule_.on_missing().value(), rule.rule_.on_missing(), struct_map, + filter_callback); } } - finalizeDynamicMetadata(*decoder_callbacks_, struct_map, processing_finished_flag); + finalizeDynamicMetadata(filter_callback, should_clear_route_cache, struct_map, + processing_finished_flag); } -void Filter::handleOnMissing(const Rule& rule, StructMap& struct_map) { +void Filter::handleOnMissing(const Rule& rule, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback) { if (rule.rule_.has_on_missing()) { - applyKeyValue(rule.rule_.on_missing().value(), rule.rule_.on_missing(), struct_map); + applyKeyValue(rule.rule_.on_missing().value(), rule.rule_.on_missing(), struct_map, + filter_callback); } } -void Filter::handleAllOnError(const Rules& rules, bool& processing_finished_flag) { +void Filter::handleAllOnError(const Rules& rules, bool should_clear_route_cache, + Http::StreamFilterCallbacks& filter_callback, + bool& processing_finished_flag) { StructMap struct_map; for (const auto& rule : rules) { if (rule.rule_.has_on_error()) { - applyKeyValue(rule.rule_.on_error().value(), rule.rule_.on_error(), struct_map); + applyKeyValue(rule.rule_.on_error().value(), rule.rule_.on_error(), struct_map, + filter_callback); } } - finalizeDynamicMetadata(*decoder_callbacks_, struct_map, processing_finished_flag); + finalizeDynamicMetadata(filter_callback, should_clear_route_cache, struct_map, + processing_finished_flag); } absl::Status Filter::handleOnPresent(Json::ObjectSharedPtr parent_node, const std::string& key, - const Rule& rule, StructMap& struct_map) { + const Rule& rule, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback) { if (!rule.rule_.has_on_present()) { return absl::OkStatus(); } auto& on_present_keyval = rule.rule_.on_present(); if (on_present_keyval.has_value()) { - applyKeyValue(on_present_keyval.value(), on_present_keyval, struct_map); + applyKeyValue(on_present_keyval.value(), on_present_keyval, struct_map, filter_callback); return absl::OkStatus(); } @@ -239,7 +270,7 @@ absl::Status Filter::handleOnPresent(Json::ObjectSharedPtr parent_node, const st if (auto value_result = absl::visit(JsonValueToProtobufValueConverter(), std::move(result.value())); value_result.ok()) { - applyKeyValue(value_result.value(), on_present_keyval, struct_map); + applyKeyValue(value_result.value(), on_present_keyval, struct_map, filter_callback); } else { return value_result.status(); } @@ -247,7 +278,7 @@ absl::Status Filter::handleOnPresent(Json::ObjectSharedPtr parent_node, const st case envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata::NUMBER: if (auto double_result = absl::visit(JsonValueToDoubleConverter(), std::move(result.value())); double_result.ok()) { - applyKeyValue(double_result.value(), on_present_keyval, struct_map); + applyKeyValue(double_result.value(), on_present_keyval, struct_map, filter_callback); } else { return double_result.status(); } @@ -265,19 +296,20 @@ absl::Status Filter::handleOnPresent(Json::ObjectSharedPtr parent_node, const st return absl::OkStatus(); } - applyKeyValue(std::move(str), on_present_keyval, struct_map); + applyKeyValue(std::move(str), on_present_keyval, struct_map, filter_callback); break; } return absl::OkStatus(); } void Filter::processBody(const Buffer::Instance* body, const Rules& rules, - bool& processing_finished_flag, Stats::Counter& success, - Stats::Counter& no_body, Stats::Counter& non_json) { + bool should_clear_route_cache, JsonToMetadataStats& stats, + Http::StreamFilterCallbacks& filter_callback, + bool& processing_finished_flag) { // In case we have trailers but no body. if (!body || body->length() == 0) { - handleAllOnMissing(rules, request_processing_finished_); - no_body.inc(); + handleAllOnMissing(rules, should_clear_route_cache, filter_callback, processing_finished_flag); + stats.no_body_.inc(); return; } @@ -285,8 +317,8 @@ void Filter::processBody(const Buffer::Instance* body, const Rules& rules, Json::Factory::loadFromStringNoThrow(body->toString()); if (!result.ok()) { ENVOY_LOG(debug, result.status().message()); - non_json.inc(); - handleAllOnError(rules, processing_finished_flag); + stats.invalid_json_body_.inc(); + handleAllOnError(rules, should_clear_route_cache, filter_callback, processing_finished_flag); return; } @@ -298,9 +330,9 @@ void Filter::processBody(const Buffer::Instance* body, const Rules& rules, ENVOY_LOG( debug, "Apply on_missing for all rules on a valid application/json body but not a json object."); - handleAllOnMissing(rules, request_processing_finished_); + handleAllOnMissing(rules, should_clear_route_cache, filter_callback, processing_finished_flag); // This JSON body is valid and successfully parsed. - success.inc(); + stats.success_.inc(); return; } @@ -313,7 +345,7 @@ void Filter::processBody(const Buffer::Instance* body, const Rules& rules, absl::StatusOr next_node_result = node->getObjectNoThrow(keys[i]); if (!next_node_result.ok()) { ENVOY_LOG(warn, result.status().message()); - handleOnMissing(rule, struct_map); + handleOnMissing(rule, struct_map, filter_callback); on_missing = true; break; } @@ -322,41 +354,71 @@ void Filter::processBody(const Buffer::Instance* body, const Rules& rules, if (on_missing) { continue; } - absl::Status result = handleOnPresent(std::move(node), keys.back(), rule, struct_map); + absl::Status result = + handleOnPresent(std::move(node), keys.back(), rule, struct_map, filter_callback); if (!result.ok()) { ENVOY_LOG(warn, fmt::format("{} key: {}", result.message(), keys.back())); - handleOnMissing(rule, struct_map); + handleOnMissing(rule, struct_map, filter_callback); } } - success.inc(); + stats.success_.inc(); - finalizeDynamicMetadata(*decoder_callbacks_, struct_map, processing_finished_flag); + finalizeDynamicMetadata(filter_callback, should_clear_route_cache, struct_map, + processing_finished_flag); } void Filter::processRequestBody() { - processBody(decoder_callbacks_->decodingBuffer(), config_->requestRules(), - request_processing_finished_, config_->stats().rq_success_, - config_->stats().rq_no_body_, config_->stats().rq_invalid_json_body_); + processBody(decoder_callbacks_->decodingBuffer(), config_->requestRules(), true, + config_->rqstats(), *decoder_callbacks_, request_processing_finished_); +} + +void Filter::processResponseBody() { + processBody(encoder_callbacks_->encodingBuffer(), config_->responseRules(), false, + config_->respstats(), *encoder_callbacks_, response_processing_finished_); } Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, bool end_stream) { - ASSERT(config_->doRequest()); + if (!config_->doRequest()) { + return Http::FilterHeadersStatus::Continue; + } if (!config_->requestContentTypeAllowed(headers.getContentTypeValue())) { request_processing_finished_ = true; - config_->stats().rq_mismatched_content_type_.inc(); + config_->rqstats().mismatched_content_type_.inc(); + return Http::FilterHeadersStatus::Continue; + } + + if (end_stream) { + handleAllOnMissing(config_->requestRules(), true, *decoder_callbacks_, + request_processing_finished_); + config_->rqstats().no_body_.inc(); + return Http::FilterHeadersStatus::Continue; + } + return Http::FilterHeadersStatus::StopIteration; +} + +Http::FilterHeadersStatus Filter::encodeHeaders(Http::ResponseHeaderMap& headers, bool end_stream) { + if (!config_->doResponse()) { + return Http::FilterHeadersStatus::Continue; + } + if (!config_->responseContentTypeAllowed(headers.getContentTypeValue())) { + response_processing_finished_ = true; + config_->respstats().mismatched_content_type_.inc(); return Http::FilterHeadersStatus::Continue; } if (end_stream) { - handleAllOnMissing(config_->requestRules(), request_processing_finished_); - config_->stats().rq_no_body_.inc(); + handleAllOnMissing(config_->responseRules(), false, *encoder_callbacks_, + response_processing_finished_); + config_->respstats().no_body_.inc(); return Http::FilterHeadersStatus::Continue; } return Http::FilterHeadersStatus::StopIteration; } Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_stream) { - ASSERT(config_->doRequest()); + if (!config_->doRequest()) { + return Http::FilterDataStatus::Continue; + } if (request_processing_finished_) { return Http::FilterDataStatus::Continue; } @@ -366,8 +428,9 @@ Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_strea if (!decoder_callbacks_->decodingBuffer() || decoder_callbacks_->decodingBuffer()->length() == 0) { - handleAllOnMissing(config_->requestRules(), request_processing_finished_); - config_->stats().rq_no_body_.inc(); + handleAllOnMissing(config_->requestRules(), true, *decoder_callbacks_, + request_processing_finished_); + config_->rqstats().no_body_.inc(); return Http::FilterDataStatus::Continue; } processRequestBody(); @@ -377,14 +440,51 @@ Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_strea return Http::FilterDataStatus::StopIterationAndBuffer; } +Http::FilterDataStatus Filter::encodeData(Buffer::Instance& data, bool end_stream) { + if (!config_->doResponse()) { + return Http::FilterDataStatus::Continue; + } + if (response_processing_finished_) { + return Http::FilterDataStatus::Continue; + } + + if (end_stream) { + encoder_callbacks_->addEncodedData(data, true); + + if (!encoder_callbacks_->encodingBuffer() || + encoder_callbacks_->encodingBuffer()->length() == 0) { + handleAllOnMissing(config_->responseRules(), false, *encoder_callbacks_, + response_processing_finished_); + config_->respstats().no_body_.inc(); + return Http::FilterDataStatus::Continue; + } + processResponseBody(); + return Http::FilterDataStatus::Continue; + } + + return Http::FilterDataStatus::StopIterationAndBuffer; +} + Http::FilterTrailersStatus Filter::decodeTrailers(Http::RequestTrailerMap&) { - ASSERT(config_->doRequest()); + if (!config_->doRequest()) { + return Http::FilterTrailersStatus::Continue; + } if (!request_processing_finished_) { processRequestBody(); } return Http::FilterTrailersStatus::Continue; } +Http::FilterTrailersStatus Filter::encodeTrailers(Http::ResponseTrailerMap&) { + if (!config_->doResponse()) { + return Http::FilterTrailersStatus::Continue; + } + if (!response_processing_finished_) { + processResponseBody(); + } + return Http::FilterTrailersStatus::Continue; +} + } // namespace JsonToMetadata } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/json_to_metadata/filter.h b/source/extensions/filters/http/json_to_metadata/filter.h index 562c6f0ab17a..99f4feea7047 100644 --- a/source/extensions/filters/http/json_to_metadata/filter.h +++ b/source/extensions/filters/http/json_to_metadata/filter.h @@ -24,10 +24,10 @@ namespace JsonToMetadata { * All stats for the Json to Metadata filter. @see stats_macros.h */ #define ALL_JSON_TO_METADATA_FILTER_STATS(COUNTER) \ - COUNTER(rq_success) \ - COUNTER(rq_mismatched_content_type) \ - COUNTER(rq_no_body) \ - COUNTER(rq_invalid_json_body) + COUNTER(success) \ + COUNTER(mismatched_content_type) \ + COUNTER(no_body) \ + COUNTER(invalid_json_body) /** * Wrapper struct for Json to Metadata filter stats. @see stats_macros.h @@ -61,24 +61,29 @@ class FilterConfig { const envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata& proto_config, Stats::Scope& scope); - JsonToMetadataStats& stats() { return stats_; } + JsonToMetadataStats& rqstats() { return rqstats_; } + JsonToMetadataStats& respstats() { return respstats_; } // True if we have rules for requests bool doRequest() const { return !request_rules_.empty(); } + bool doResponse() const { return !response_rules_.empty(); } const Rules& requestRules() const { return request_rules_; } + const Rules& responseRules() const { return response_rules_; } bool requestContentTypeAllowed(absl::string_view) const; + bool responseContentTypeAllowed(absl::string_view) const; private: using ProtobufRepeatedRule = Protobuf::RepeatedPtrField; - Rules generateRequestRules( - const envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata& proto_config) - const; - absl::flat_hash_set generateRequestAllowContentTypes( - const envoy::extensions::filters::http::json_to_metadata::v3::JsonToMetadata& proto_config) - const; - JsonToMetadataStats stats_; + Rules generateRules(const ProtobufRepeatedRule& proto_rule) const; + absl::flat_hash_set generateAllowContentTypes( + const Protobuf::RepeatedPtrField& proto_allow_content_types) const; + JsonToMetadataStats rqstats_; + JsonToMetadataStats respstats_; const Rules request_rules_; + const Rules response_rules_; const absl::flat_hash_set request_allow_content_types_; + const absl::flat_hash_set response_allow_content_types_; const bool request_allow_empty_content_type_; + const bool response_allow_empty_content_type_; }; const uint32_t MAX_PAYLOAD_VALUE_LEN = 8 * 1024; @@ -94,37 +99,54 @@ class Filter : public Http::PassThroughFilter, Logger::Loggable; // Handle on_missing case of the `rule` and store in `struct_map`. - void handleOnMissing(const Rule& rule, StructMap& struct_map); + void handleOnMissing(const Rule& rule, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback); // Handle on_present case of the `rule` and store in `struct_map`, which depends on // the value of `parent_node->key`. absl::Status handleOnPresent(Json::ObjectSharedPtr parent_node, const std::string& key, - const Rule& rule, StructMap& struct_map); + const Rule& rule, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback); // Process the case without body, i.e., on_missing is applied for all rules. - void handleAllOnMissing(const Rules& rules, bool& processing_finished_flag); + void handleAllOnMissing(const Rules& rules, bool should_clear_route_cache, + Http::StreamFilterCallbacks& filter_callback, + bool& processing_finished_flag); // Process the case with error, i.e., on_error is applied for all rules. - void handleAllOnError(const Rules& rules, bool& processing_finished_flag); + void handleAllOnError(const Rules& rules, bool should_clear_route_cache, + Http::StreamFilterCallbacks& filter_callback, + bool& processing_finished_flag); // Parse the body while we have the whole json. - void processBody(const Buffer::Instance* body, const Rules& rules, bool& processing_finished_flag, - Stats::Counter& success, Stats::Counter& no_body, Stats::Counter& non_json); + void processBody(const Buffer::Instance* body, const Rules& rules, bool should_clear_route_cache, + JsonToMetadataStats& stats, Http::StreamFilterCallbacks& filter_callback, + bool& processing_finished_flag); void processRequestBody(); + void processResponseBody(); const std::string& decideNamespace(const std::string& nspace) const; bool addMetadata(const std::string& meta_namespace, const std::string& key, ProtobufWkt::Value val, const bool preserve_existing_metadata_value, - StructMap& struct_map); - void applyKeyValue(const std::string& value, const KeyValuePair& keyval, StructMap& struct_map); - void applyKeyValue(double value, const KeyValuePair& keyval, StructMap& struct_map); - void applyKeyValue(ProtobufWkt::Value value, const KeyValuePair& keyval, StructMap& struct_map); + StructMap& struct_map, Http::StreamFilterCallbacks& filter_callback); + void applyKeyValue(const std::string& value, const KeyValuePair& keyval, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback); + void applyKeyValue(double value, const KeyValuePair& keyval, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback); + void applyKeyValue(ProtobufWkt::Value value, const KeyValuePair& keyval, StructMap& struct_map, + Http::StreamFilterCallbacks& filter_callback); void finalizeDynamicMetadata(Http::StreamFilterCallbacks& filter_callback, - const StructMap& struct_map, bool& processing_finished_flag); + bool should_clear_route_cache, const StructMap& struct_map, + bool& processing_finished_flag); std::shared_ptr config_; bool request_processing_finished_{false}; + bool response_processing_finished_{false}; }; } // namespace JsonToMetadata diff --git a/source/extensions/filters/http/jwt_authn/authenticator.cc b/source/extensions/filters/http/jwt_authn/authenticator.cc index d302e066f382..4239e0ebc6d3 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.cc +++ b/source/extensions/filters/http/jwt_authn/authenticator.cc @@ -12,6 +12,7 @@ #include "source/common/protobuf/protobuf.h" #include "source/common/tracing/http_tracer_impl.h" +#include "absl/strings/str_split.h" #include "jwt_verify_lib/jwt.h" #include "jwt_verify_lib/struct_utils.h" #include "jwt_verify_lib/verify.h" @@ -60,8 +61,8 @@ class AuthenticatorImpl : public Logger::Loggable, // Following functions are for Authenticator interface. void verify(Http::HeaderMap& headers, Tracing::Span& parent_span, std::vector&& tokens, - SetExtractedJwtDataCallback set_extracted_jwt_data_cb, - AuthenticatorCallback callback) override; + SetExtractedJwtDataCallback set_extracted_jwt_data_cb, AuthenticatorCallback callback, + ClearRouteCacheCallback clear_route_cb) override; void onDestroy() override; TimeSource& timeSource() { return time_source_; } @@ -76,6 +77,9 @@ class AuthenticatorImpl : public Logger::Loggable, // Handle Good Jwt either Cache JWT or verified public key. void handleGoodJwt(bool cache_hit); + // Normalize and set the payload metadata. + void setPayloadMetadata(const ProtobufWkt::Struct& jwt_payload); + // Calls the callback with status. void doneWithStatus(const Status& status); @@ -83,8 +87,8 @@ class AuthenticatorImpl : public Logger::Loggable, // finds one to verify with key. void startVerify(); - // Copy the JWT Claim to HTTP Header - void addJWTClaimToHeader(const std::string& claim_name, const std::string& header_name); + // Copy the JWT Claim to HTTP Header. Returns true iff header is added. + bool addJWTClaimToHeader(const std::string& claim_name, const std::string& header_name); // The jwks cache object. JwksCache& jwks_cache_; @@ -113,6 +117,10 @@ class AuthenticatorImpl : public Logger::Loggable, SetExtractedJwtDataCallback set_extracted_jwt_data_cb_; // The on_done function. AuthenticatorCallback callback_; + // Clear route cache callback function. + ClearRouteCacheCallback clear_route_cb_; + // Set to true to clear the route cache. + bool clear_route_cache_{false}; // check audience object. const CheckAudience* check_audience_; // specific provider or not when it is allow missing or failed. @@ -139,13 +147,16 @@ std::string AuthenticatorImpl::name() const { void AuthenticatorImpl::verify(Http::HeaderMap& headers, Tracing::Span& parent_span, std::vector&& tokens, SetExtractedJwtDataCallback set_extracted_jwt_data_cb, - AuthenticatorCallback callback) { + AuthenticatorCallback callback, + ClearRouteCacheCallback clear_route_cb) { ASSERT(!callback_); headers_ = &headers; parent_span_ = &parent_span; tokens_ = std::move(tokens); set_extracted_jwt_data_cb_ = std::move(set_extracted_jwt_data_cb); callback_ = std::move(callback); + clear_route_cb_ = std::move(clear_route_cb); + clear_route_cache_ = false; ENVOY_LOG(debug, "{}: JWT authentication starts (allow_failed={}), tokens size={}", name(), is_allow_failed_, tokens_.size()); @@ -299,31 +310,49 @@ void AuthenticatorImpl::verifyKey() { handleGoodJwt(/*cache_hit=*/false); } -void AuthenticatorImpl::addJWTClaimToHeader(const std::string& claim_name, +bool AuthenticatorImpl::addJWTClaimToHeader(const std::string& claim_name, const std::string& header_name) { StructUtils payload_getter(jwt_->payload_pb_); const ProtobufWkt::Value* claim_value; const auto status = payload_getter.GetValue(claim_name, claim_value); std::string str_claim_value; if (status == StructUtils::OK) { - if (claim_value->kind_case() == Envoy::ProtobufWkt::Value::kStringValue) { + switch (claim_value->kind_case()) { + case Envoy::ProtobufWkt::Value::kStringValue: str_claim_value = claim_value->string_value(); - } else if (claim_value->kind_case() == Envoy::ProtobufWkt::Value::kNumberValue) { + break; + case Envoy::ProtobufWkt::Value::kNumberValue: str_claim_value = convertClaimDoubleToString(claim_value->number_value()); - } else if (claim_value->kind_case() == Envoy::ProtobufWkt::Value::kBoolValue) { + break; + case Envoy::ProtobufWkt::Value::kBoolValue: str_claim_value = claim_value->bool_value() ? "true" : "false"; - } else { - ENVOY_LOG( - debug, - "--------claim : {} is not a primitive type of int, double, string, or bool -----------", - claim_name); + break; + case Envoy::ProtobufWkt::Value::kStructValue: + ABSL_FALLTHROUGH_INTENDED; + case Envoy::ProtobufWkt::Value::kListValue: { + std::string output; + auto status = claim_value->has_struct_value() + ? ProtobufUtil::MessageToJsonString(claim_value->struct_value(), &output) + : ProtobufUtil::MessageToJsonString(claim_value->list_value(), &output); + if (status.ok()) { + str_claim_value = Envoy::Base64::encode(output.data(), output.size()); + } + break; } + default: + ENVOY_LOG(debug, "[jwt_auth] claim : {} is of an unknown type '{}'", claim_name, + claim_value->kind_case()); + break; + } + if (!str_claim_value.empty()) { headers_->addCopy(Http::LowerCaseString(header_name), str_claim_value); - ENVOY_LOG(debug, "--------claim : {} with value : {} is added to the header : {} -----------", + ENVOY_LOG(debug, "[jwt_auth] claim : {} with value : {} is added to the header : {}", claim_name, str_claim_value, header_name); + return true; } } + return false; } void AuthenticatorImpl::handleGoodJwt(bool cache_hit) { @@ -343,8 +372,13 @@ void AuthenticatorImpl::handleGoodJwt(bool cache_hit) { } // Copy JWT claim to header + bool header_added = false; for (const auto& header_and_claim : provider.claim_to_headers()) { - addJWTClaimToHeader(header_and_claim.claim_name(), header_and_claim.header_name()); + header_added |= + addJWTClaimToHeader(header_and_claim.claim_name(), header_and_claim.header_name()); + } + if (provider.clear_route_cache() && (header_added || !provider.payload_in_metadata().empty())) { + clear_route_cache_ = true; } if (!provider.forward()) { @@ -357,9 +391,8 @@ void AuthenticatorImpl::handleGoodJwt(bool cache_hit) { if (!provider.header_in_metadata().empty()) { set_extracted_jwt_data_cb_(provider.header_in_metadata(), jwt_->header_pb_); } - if (!provider.payload_in_metadata().empty()) { - set_extracted_jwt_data_cb_(provider.payload_in_metadata(), jwt_->payload_pb_); + setPayloadMetadata(jwt_->payload_pb_); } } if (provider_ && !cache_hit) { @@ -369,6 +402,27 @@ void AuthenticatorImpl::handleGoodJwt(bool cache_hit) { doneWithStatus(Status::Ok); } +void AuthenticatorImpl::setPayloadMetadata(const ProtobufWkt::Struct& jwt_payload) { + const auto& provider = jwks_data_->getJwtProvider(); + const auto& normalize = provider.normalize_payload_in_metadata(); + if (normalize.space_delimited_claims().size() == 0) { + set_extracted_jwt_data_cb_(provider.payload_in_metadata(), jwt_payload); + } + // Make a temporary copy to normalize the JWT struct. + ProtobufWkt::Struct out_payload = jwt_payload; + for (const auto& claim : normalize.space_delimited_claims()) { + const auto& it = jwt_payload.fields().find(claim); + if (it != jwt_payload.fields().end() && it->second.has_string_value()) { + const auto list = absl::StrSplit(it->second.string_value(), ' ', absl::SkipEmpty()); + for (const auto& elt : list) { + (*out_payload.mutable_fields())[claim].mutable_list_value()->add_values()->set_string_value( + elt); + } + } + } + set_extracted_jwt_data_cb_(provider.payload_in_metadata(), out_payload); +} + void AuthenticatorImpl::doneWithStatus(const Status& status) { ENVOY_LOG(debug, "{}: JWT token verification completed with: {}", name(), ::google::jwt_verify::getStatusString(status)); @@ -413,8 +467,13 @@ void AuthenticatorImpl::doneWithStatus(const Status& status) { } else { callback_(status); } - callback_ = nullptr; + + if (clear_route_cache_ && clear_route_cb_) { + clear_route_cb_(); + } + clear_route_cb_ = nullptr; + return; } diff --git a/source/extensions/filters/http/jwt_authn/authenticator.h b/source/extensions/filters/http/jwt_authn/authenticator.h index 8e16c5044ce4..be5809aa0ba1 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.h +++ b/source/extensions/filters/http/jwt_authn/authenticator.h @@ -22,6 +22,8 @@ using AuthenticatorCallback = std::function; +using ClearRouteCacheCallback = std::function; + /** * Authenticator object to handle all JWT authentication flow. */ @@ -34,7 +36,7 @@ class Authenticator { virtual void verify(Http::HeaderMap& headers, Tracing::Span& parent_span, std::vector&& tokens, SetExtractedJwtDataCallback set_extracted_jwt_data_cb, - AuthenticatorCallback callback) PURE; + AuthenticatorCallback callback, ClearRouteCacheCallback clear_route_cb) PURE; // Called when the object is about to be destroyed. virtual void onDestroy() PURE; diff --git a/source/extensions/filters/http/jwt_authn/extractor.cc b/source/extensions/filters/http/jwt_authn/extractor.cc index c1d7c10e6744..805af897a612 100644 --- a/source/extensions/filters/http/jwt_authn/extractor.cc +++ b/source/extensions/filters/http/jwt_authn/extractor.cc @@ -273,14 +273,15 @@ ExtractorImpl::extract(const Http::RequestHeaderMap& headers) const { // Check query parameter locations only if query parameter locations specified and Path() is not // null if (!param_locations_.empty() && headers.Path() != nullptr) { - const auto& params = Http::Utility::parseAndDecodeQueryString(headers.getPathValue()); + const auto& params = + Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(headers.getPathValue()); for (const auto& location_it : param_locations_) { const auto& param_key = location_it.first; const auto& location_spec = location_it.second; - const auto& it = params.find(param_key); - if (it != params.end()) { + const auto& it = params.getFirstValue(param_key); + if (it.has_value()) { tokens.push_back(std::make_unique( - it->second, location_spec.issuer_checker_, param_key)); + it.value(), location_spec.issuer_checker_, param_key)); } } } diff --git a/source/extensions/filters/http/jwt_authn/filter.cc b/source/extensions/filters/http/jwt_authn/filter.cc index 93fee62e2df5..8d773b88da1b 100644 --- a/source/extensions/filters/http/jwt_authn/filter.cc +++ b/source/extensions/filters/http/jwt_authn/filter.cc @@ -104,6 +104,8 @@ void Filter::setExtractedData(const ProtobufWkt::Struct& extracted_data) { extracted_data); } +void Filter::clearRouteCache() { decoder_callbacks_->downstreamCallbacks()->clearRouteCache(); } + void Filter::onComplete(const Status& status) { ENVOY_LOG(debug, "Jwt authentication completed with: {}", ::google::jwt_verify::getStatusString(status)); diff --git a/source/extensions/filters/http/jwt_authn/filter.h b/source/extensions/filters/http/jwt_authn/filter.h index 9330d07be612..461f4bc9deb7 100644 --- a/source/extensions/filters/http/jwt_authn/filter.h +++ b/source/extensions/filters/http/jwt_authn/filter.h @@ -33,6 +33,7 @@ class Filter : public Http::StreamDecoderFilter, // Following two functions are for Verifier::Callbacks interface. // Pass the extracted data from a verified JWT as an opaque ProtobufWkt::Struct. void setExtractedData(const ProtobufWkt::Struct& extracted_data) override; + void clearRouteCache() override; // It will be called when its verify() call is completed. void onComplete(const ::google::jwt_verify::Status& status) override; diff --git a/source/extensions/filters/http/jwt_authn/filter_config.cc b/source/extensions/filters/http/jwt_authn/filter_config.cc index c9ce824e6359..9dec0cf1f58e 100644 --- a/source/extensions/filters/http/jwt_authn/filter_config.cc +++ b/source/extensions/filters/http/jwt_authn/filter_config.cc @@ -15,7 +15,8 @@ FilterConfigImpl::FilterConfigImpl( envoy::extensions::filters::http::jwt_authn::v3::JwtAuthentication proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) : proto_config_(std::move(proto_config)), stats_(generateStats(stats_prefix, context.scope())), - cm_(context.clusterManager()), time_source_(context.mainThreadDispatcher().timeSource()) { + cm_(context.serverFactoryContext().clusterManager()), + time_source_(context.serverFactoryContext().mainThreadDispatcher().timeSource()) { ENVOY_LOG(debug, "Loaded JwtAuthConfig: {}", proto_config_.DebugString()); diff --git a/source/extensions/filters/http/jwt_authn/filter_factory.cc b/source/extensions/filters/http/jwt_authn/filter_factory.cc index ee1243dc9a45..5bb39e47477e 100644 --- a/source/extensions/filters/http/jwt_authn/filter_factory.cc +++ b/source/extensions/filters/http/jwt_authn/filter_factory.cc @@ -42,7 +42,7 @@ Http::FilterFactoryCb FilterFactory::createFilterFactoryFromProtoTyped(const JwtAuthentication& proto_config, const std::string& prefix, Server::Configuration::FactoryContext& context) { - validateJwtConfig(proto_config, context.api()); + validateJwtConfig(proto_config, context.serverFactoryContext().api()); auto filter_config = std::make_shared(proto_config, prefix, context); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(filter_config)); diff --git a/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc b/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc index cccc674d05db..af2e567b6252 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc +++ b/source/extensions/filters/http/jwt_authn/jwks_async_fetcher.cc @@ -57,7 +57,8 @@ JwksAsyncFetcher::JwksAsyncFetcher(const RemoteJwks& remote_jwks, } failed_refetch_duration_ = getFailedRefetchDuration(remote_jwks.async_fetch()); - refetch_timer_ = context_.mainThreadDispatcher().createTimer([this]() -> void { fetch(); }); + refetch_timer_ = context_.serverFactoryContext().mainThreadDispatcher().createTimer( + [this]() -> void { fetch(); }); // For fast_listener, just trigger a fetch, not register with init_manager. if (remote_jwks_.async_fetch().fast_listener()) { @@ -83,7 +84,7 @@ void JwksAsyncFetcher::fetch() { } ENVOY_LOG(debug, "{}: started", debug_name_); - fetcher_ = create_fetcher_fn_(context_.clusterManager(), remote_jwks_); + fetcher_ = create_fetcher_fn_(context_.serverFactoryContext().clusterManager(), remote_jwks_); fetcher_->fetch(Tracing::NullSpan::instance(), *this); } diff --git a/source/extensions/filters/http/jwt_authn/jwks_cache.cc b/source/extensions/filters/http/jwt_authn/jwks_cache.cc index cc71a8f72c46..818108701001 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_cache.cc +++ b/source/extensions/filters/http/jwt_authn/jwks_cache.cc @@ -29,8 +29,8 @@ class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable(enable_jwt_cache, config, dispatcher.timeSource()); }); - const auto inline_jwks = - Config::DataSource::read(jwt_provider_.local_jwks(), true, context.api()); + const auto inline_jwks = Config::DataSource::read(jwt_provider_.local_jwks(), true, + context.serverFactoryContext().api()); if (!inline_jwks.empty()) { auto jwks = ::google::jwt_verify::Jwks::createFrom(inline_jwks, ::google::jwt_verify::Jwks::JWKS); diff --git a/source/extensions/filters/http/jwt_authn/matcher.cc b/source/extensions/filters/http/jwt_authn/matcher.cc index 8f1c6ca70993..a95474ca5e09 100644 --- a/source/extensions/filters/http/jwt_authn/matcher.cc +++ b/source/extensions/filters/http/jwt_authn/matcher.cc @@ -42,8 +42,8 @@ class BaseMatcherImpl : public Matcher, public Logger::Loggable matches &= Http::HeaderUtility::matchHeaders(headers, config_headers_); if (!config_query_parameters_.empty()) { - Http::Utility::QueryParams query_parameters = - Http::Utility::parseQueryString(headers.getPathValue()); + Http::Utility::QueryParamsMulti query_parameters = + Http::Utility::QueryParamsMulti::parseQueryString(headers.getPathValue()); matches &= ConfigUtility::matchQueryParams(query_parameters, config_query_parameters_); } return matches; diff --git a/source/extensions/filters/http/jwt_authn/verifier.cc b/source/extensions/filters/http/jwt_authn/verifier.cc index a9e4c55d8d21..92d1f5182406 100644 --- a/source/extensions/filters/http/jwt_authn/verifier.cc +++ b/source/extensions/filters/http/jwt_authn/verifier.cc @@ -127,7 +127,8 @@ class ProviderVerifierImpl : public BaseVerifierImpl { [&ctximpl](const std::string& name, const ProtobufWkt::Struct& extracted_data) { ctximpl.addExtractedData(name, extracted_data); }, - [this, &ctximpl](const Status& status) { onComplete(status, ctximpl); }); + [this, &ctximpl](const Status& status) { onComplete(status, ctximpl); }, + [&ctximpl]() { ctximpl.callback()->clearRouteCache(); }); if (!ctximpl.getCompletionState(this).is_completed_) { ctximpl.storeAuth(std::move(auth)); } else { @@ -176,7 +177,8 @@ class AllowFailedVerifierImpl : public BaseVerifierImpl { [&ctximpl](const std::string& name, const ProtobufWkt::Struct& extracted_data) { ctximpl.addExtractedData(name, extracted_data); }, - [this, &ctximpl](const Status& status) { onComplete(status, ctximpl); }); + [this, &ctximpl](const Status& status) { onComplete(status, ctximpl); }, + [&ctximpl]() { ctximpl.callback()->clearRouteCache(); }); if (!ctximpl.getCompletionState(this).is_completed_) { ctximpl.storeAuth(std::move(auth)); } else { @@ -208,7 +210,8 @@ class AllowMissingVerifierImpl : public BaseVerifierImpl { [&ctximpl](const std::string& name, const ProtobufWkt::Struct& extracted_data) { ctximpl.addExtractedData(name, extracted_data); }, - [this, &ctximpl](const Status& status) { onComplete(status, ctximpl); }); + [this, &ctximpl](const Status& status) { onComplete(status, ctximpl); }, + [&ctximpl]() { ctximpl.callback()->clearRouteCache(); }); if (!ctximpl.getCompletionState(this).is_completed_) { ctximpl.storeAuth(std::move(auth)); } else { diff --git a/source/extensions/filters/http/jwt_authn/verifier.h b/source/extensions/filters/http/jwt_authn/verifier.h index 7d20e709660a..dd775caf1d6f 100644 --- a/source/extensions/filters/http/jwt_authn/verifier.h +++ b/source/extensions/filters/http/jwt_authn/verifier.h @@ -34,6 +34,11 @@ class Verifier { */ virtual void setExtractedData(const ProtobufWkt::Struct& payload) PURE; + /** + * JWT payloads added to headers may require clearing the cached route. + */ + virtual void clearRouteCache() PURE; + /** * Called on completion of request. * diff --git a/source/extensions/filters/http/kill_request/kill_request_config.cc b/source/extensions/filters/http/kill_request/kill_request_config.cc index ba74b8c16da9..50b01bb28eb2 100644 --- a/source/extensions/filters/http/kill_request/kill_request_config.cc +++ b/source/extensions/filters/http/kill_request/kill_request_config.cc @@ -15,8 +15,8 @@ Http::FilterFactoryCb KillRequestFilterFactory::createFilterFactoryFromProtoType const envoy::extensions::filters::http::kill_request::v3::KillRequest& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { return [proto_config, &context](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter( - std::make_shared(proto_config, context.api().randomGenerator())); + callbacks.addStreamFilter(std::make_shared( + proto_config, context.serverFactoryContext().api().randomGenerator())); }; } diff --git a/source/extensions/filters/http/local_ratelimit/config.cc b/source/extensions/filters/http/local_ratelimit/config.cc index 551f66c9d9d6..cbf719cae489 100644 --- a/source/extensions/filters/http/local_ratelimit/config.cc +++ b/source/extensions/filters/http/local_ratelimit/config.cc @@ -15,9 +15,11 @@ namespace LocalRateLimitFilter { Http::FilterFactoryCb LocalRateLimitFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::local_ratelimit::v3::LocalRateLimit& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + FilterConfigSharedPtr filter_config = std::make_shared( - proto_config, context.localInfo(), context.mainThreadDispatcher(), context.scope(), - context.runtime()); + proto_config, server_context.localInfo(), server_context.mainThreadDispatcher(), + context.scope(), server_context.runtime()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc index 97cae22f1304..ac0ba0bed89d 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.cc @@ -33,8 +33,13 @@ FilterConfig::FilterConfig( tokens_per_fill_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.token_bucket(), tokens_per_fill, 1)), descriptors_(config.descriptors()), rate_limit_per_connection_(config.local_rate_limit_per_downstream_connection()), + always_consume_default_token_bucket_( + config.has_always_consume_default_token_bucket() + ? config.always_consume_default_token_bucket().value() + : true), rate_limiter_(new Filters::Common::LocalRateLimit::LocalRateLimiterImpl( - fill_interval_, max_tokens_, tokens_per_fill_, dispatcher, descriptors_)), + fill_interval_, max_tokens_, tokens_per_fill_, dispatcher, descriptors_, + always_consume_default_token_bucket_)), local_info_(local_info), runtime_(runtime), filter_enabled_( config.has_filter_enabled() @@ -54,7 +59,11 @@ FilterConfig::FilterConfig( has_descriptors_(!config.descriptors().empty()), enable_x_rate_limit_headers_(config.enable_x_ratelimit_headers() == envoy::extensions::common::ratelimit::v3::DRAFT_VERSION_03), - vh_rate_limits_(config.vh_rate_limits()) { + vh_rate_limits_(config.vh_rate_limits()), + rate_limited_grpc_status_( + config.rate_limited_as_resource_exhausted() + ? absl::make_optional(Grpc::Status::WellKnownGrpcStatus::ResourceExhausted) + : absl::nullopt) { // Note: no token bucket is fine for the global config, which would be the case for enabling // the filter globally but disabled and then applying limits at the virtual host or // route level. At the virtual or route level, it makes no sense to have an no token @@ -142,7 +151,7 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, [this, config](Http::HeaderMap& headers) { config->responseHeadersParser().evaluateHeaders(headers, decoder_callbacks_->streamInfo()); }, - absl::nullopt, "local_rate_limited"); + config->rateLimitedGrpcStatus(), "local_rate_limited"); decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::RateLimited); return Http::FilterHeadersStatus::StopIteration; @@ -208,7 +217,8 @@ const Filters::Common::LocalRateLimit::LocalRateLimiterImpl& Filter::getPerConne if (typed_state == nullptr) { auto limiter = std::make_shared( config->fillInterval(), config->maxTokens(), config->tokensPerFill(), - decoder_callbacks_->dispatcher(), config->descriptors()); + decoder_callbacks_->dispatcher(), config->descriptors(), + config->consumeDefaultTokenBucket()); decoder_callbacks_->streamInfo().filterState()->setData( PerConnectionRateLimiter::key(), limiter, StreamInfo::FilterState::StateType::ReadOnly, diff --git a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h index 3030ebea0005..e816da64e37f 100644 --- a/source/extensions/filters/http/local_ratelimit/local_ratelimit.h +++ b/source/extensions/filters/http/local_ratelimit/local_ratelimit.h @@ -53,8 +53,10 @@ class PerConnectionRateLimiter : public StreamInfo::FilterState::Object { const std::chrono::milliseconds& fill_interval, uint32_t max_tokens, uint32_t tokens_per_fill, Envoy::Event::Dispatcher& dispatcher, const Protobuf::RepeatedPtrField< - envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptor) - : rate_limiter_(fill_interval, max_tokens, tokens_per_fill, dispatcher, descriptor) {} + envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor>& descriptor, + bool always_consume_default_token_bucket) + : rate_limiter_(fill_interval, max_tokens, tokens_per_fill, dispatcher, descriptor, + always_consume_default_token_bucket) {} static const std::string& key(); const Filters::Common::LocalRateLimit::LocalRateLimiterImpl& value() const { return rate_limiter_; @@ -108,6 +110,10 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { envoy::extensions::common::ratelimit::v3::VhRateLimitsOptions virtualHostRateLimits() const { return vh_rate_limits_; } + bool consumeDefaultTokenBucket() const { return always_consume_default_token_bucket_; } + const absl::optional rateLimitedGrpcStatus() const { + return rate_limited_grpc_status_; + } private: friend class FilterTest; @@ -132,6 +138,7 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { envoy::extensions::common::ratelimit::v3::LocalRateLimitDescriptor> descriptors_; const bool rate_limit_per_connection_; + const bool always_consume_default_token_bucket_{}; std::unique_ptr rate_limiter_; const LocalInfo::LocalInfo& local_info_; Runtime::Loader& runtime_; @@ -143,6 +150,7 @@ class FilterConfig : public Router::RouteSpecificFilterConfig { const bool has_descriptors_; const bool enable_x_rate_limit_headers_; const envoy::extensions::common::ratelimit::v3::VhRateLimitsOptions vh_rate_limits_; + const absl::optional rate_limited_grpc_status_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/lua/config.cc b/source/extensions/filters/http/lua/config.cc index 12562c97fb31..73cc5db9e7a3 100644 --- a/source/extensions/filters/http/lua/config.cc +++ b/source/extensions/filters/http/lua/config.cc @@ -14,10 +14,12 @@ namespace Lua { Http::FilterFactoryCb LuaFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::lua::v3::Lua& proto_config, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { - FilterConfigConstSharedPtr filter_config(new FilterConfig{proto_config, context.threadLocal(), - context.clusterManager(), context.api(), - context.scope(), stat_prefix}); - auto& time_source = context.mainThreadDispatcher().timeSource(); + auto& server_context = context.serverFactoryContext(); + + FilterConfigConstSharedPtr filter_config( + new FilterConfig{proto_config, server_context.threadLocal(), server_context.clusterManager(), + server_context.api(), context.scope(), stat_prefix}); + auto& time_source = server_context.mainThreadDispatcher().timeSource(); return [filter_config, &time_source](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config, time_source)); }; diff --git a/source/extensions/filters/http/lua/wrappers.cc b/source/extensions/filters/http/lua/wrappers.cc index 78e33443f6b7..e8f45539dc2e 100644 --- a/source/extensions/filters/http/lua/wrappers.cc +++ b/source/extensions/filters/http/lua/wrappers.cc @@ -199,6 +199,13 @@ int StreamInfoWrapper::luaDownstreamDirectRemoteAddress(lua_State* state) { return 1; } +int StreamInfoWrapper::luaDownstreamRemoteAddress(lua_State* state) { + const std::string& remote_address = + stream_info_.downstreamAddressProvider().remoteAddress()->asString(); + lua_pushlstring(state, remote_address.data(), remote_address.size()); + return 1; +} + int StreamInfoWrapper::luaRequestedServerName(lua_State* state) { absl::string_view requested_serve_name = stream_info_.downstreamAddressProvider().requestedServerName(); diff --git a/source/extensions/filters/http/lua/wrappers.h b/source/extensions/filters/http/lua/wrappers.h index f61ea91237a6..bf7a842156fc 100644 --- a/source/extensions/filters/http/lua/wrappers.h +++ b/source/extensions/filters/http/lua/wrappers.h @@ -209,6 +209,7 @@ class StreamInfoWrapper : public Filters::Common::Lua::BaseLuaObject( - secret_provider_token_secret, secret_provider_hmac_secret, context.api()); + auto secret_reader = + std::make_shared(secret_provider_token_secret, secret_provider_hmac_secret, + context.serverFactoryContext().api()); auto config = std::make_shared(proto_config, cluster_manager, secret_reader, context.scope(), stats_prefix); @@ -73,8 +74,8 @@ Http::FilterFactoryCb OAuth2Config::createFilterFactoryFromProtoTyped( [&context, config, &cluster_manager](Http::FilterChainFactoryCallbacks& callbacks) -> void { std::unique_ptr oauth_client = std::make_unique(cluster_manager, config->oauthTokenEndpoint()); - callbacks.addStreamDecoderFilter( - std::make_shared(config, std::move(oauth_client), context.timeSource())); + callbacks.addStreamFilter(std::make_shared( + config, std::move(oauth_client), context.serverFactoryContext().timeSource())); }; } diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index 57984a648a36..33b509d6d620 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -126,16 +126,17 @@ getAuthType(envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType } } -Http::Utility::QueryParams buildAutorizationQueryParams( +Http::Utility::QueryParamsMulti buildAutorizationQueryParams( const envoy::extensions::filters::http::oauth2::v3::OAuth2Config& proto_config) { - auto query_params = Http::Utility::parseQueryString(proto_config.authorization_endpoint()); - query_params["client_id"] = proto_config.credentials().client_id(); - query_params["response_type"] = "code"; + auto query_params = + Http::Utility::QueryParamsMulti::parseQueryString(proto_config.authorization_endpoint()); + query_params.overwrite("client_id", proto_config.credentials().client_id()); + query_params.overwrite("response_type", "code"); std::string scopes_list = absl::StrJoin(authScopesList(proto_config.auth_scopes()), " "); - query_params["scope"] = - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_use_url_encoding") - ? Http::Utility::PercentEncoding::urlEncodeQueryParameter(scopes_list) - : Http::Utility::PercentEncoding::encode(scopes_list, ":/=&? "); + query_params.overwrite( + "scope", Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_use_url_encoding") + ? Http::Utility::PercentEncoding::urlEncodeQueryParameter(scopes_list) + : Http::Utility::PercentEncoding::encode(scopes_list, ":/=&? ")); return query_params; } @@ -194,7 +195,8 @@ FilterConfig::FilterConfig( forward_bearer_token_(proto_config.forward_bearer_token()), pass_through_header_matchers_(headerMatchers(proto_config.pass_through_matcher())), cookie_names_(proto_config.credentials().cookie_names()), - auth_type_(getAuthType(proto_config.auth_type())) { + auth_type_(getAuthType(proto_config.auth_type())), + use_refresh_token_(proto_config.use_refresh_token().value()) { if (!cluster_manager.clusters().hasCluster(oauth_token_endpoint_.cluster())) { throw EnvoyException(fmt::format("OAuth2 filter: unknown cluster '{}' in config. Please " "specify which cluster to direct OAuth requests to.", @@ -230,6 +232,10 @@ void OAuth2CookieValidator::setParams(const Http::RequestHeaderMap& headers, secret_.assign(secret.begin(), secret.end()); } +bool OAuth2CookieValidator::canUpdateTokenByRefreshToken() const { + return (!token_.empty() && !refresh_token_.empty()); +} + bool OAuth2CookieValidator::hmacIsValid() const { return ( (encodeHmacBase64(secret_, host_, expires_, token_, id_token_, refresh_token_) == hmac_) || @@ -251,8 +257,8 @@ bool OAuth2CookieValidator::isValid() const { return hmacIsValid() && timestampI OAuth2Filter::OAuth2Filter(FilterConfigSharedPtr config, std::unique_ptr&& oauth_client, TimeSource& time_source) : validator_(std::make_shared(time_source, config->cookieNames())), - oauth_client_(std::move(oauth_client)), config_(std::move(config)), - time_source_(time_source) { + was_refresh_token_flow_(false), oauth_client_(std::move(oauth_client)), + config_(std::move(config)), time_source_(time_source) { oauth_client_->setCallbacks(*this); } @@ -304,20 +310,21 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he // to the callback path. if (config_->redirectPathMatcher().match(path_str)) { - Http::Utility::QueryParams query_parameters = Http::Utility::parseQueryString(path_str); + Http::Utility::QueryParamsMulti query_parameters = + Http::Utility::QueryParamsMulti::parseQueryString(path_str); - if (query_parameters.find(queryParamsState()) == query_parameters.end()) { - ENVOY_LOG(debug, "state query param does not exist: \n{}", query_parameters); + auto stateVal = query_parameters.getFirstValue(queryParamsState()); + if (!stateVal.has_value()) { + ENVOY_LOG(error, "state query param does not exist: \n{}", query_parameters.data()); sendUnauthorizedResponse(); return Http::FilterHeadersStatus::StopIteration; } std::string state; if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_use_url_encoding")) { - state = Http::Utility::PercentEncoding::urlDecodeQueryParameter( - query_parameters.at(queryParamsState())); + state = Http::Utility::PercentEncoding::urlDecodeQueryParameter(stateVal.value()); } else { - state = Http::Utility::PercentEncoding::decode(query_parameters.at(queryParamsState())); + state = Http::Utility::PercentEncoding::decode(stateVal.value()); } Http::Utility::Url state_url; if (!state_url.initialize(state, false)) { @@ -351,6 +358,16 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he // The following conditional could be replaced with a regex pattern-match, // if we're concerned about strict matching against the callback path. if (!config_->redirectPathMatcher().match(path_str)) { + + // Check if we can update the access token via a refresh token. + if (config_->useRefreshToken() && validator_->canUpdateTokenByRefreshToken()) { + // try to update access token by refresh token + oauth_client_->asyncRefreshAccessToken(validator_->refreshToken(), config_->clientId(), + config_->clientSecret(), config_->authType()); + // pause while we await the next step from the OAuth server + return Http::FilterHeadersStatus::StopAllIterationAndWatermark; + } + ENVOY_LOG(debug, "path {} does not match with redirect matcher. redirecting to OAuth server.", path_str); redirectToOAuthServer(headers); @@ -360,25 +377,25 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he // At this point, we *are* on /_oauth. We believe this request comes from the authorization // server and we expect the query strings to contain the information required to get the access // token - const auto query_parameters = Http::Utility::parseQueryString(path_str); - if (query_parameters.find(queryParamsError()) != query_parameters.end()) { + const auto query_parameters = Http::Utility::QueryParamsMulti::parseQueryString(path_str); + if (query_parameters.getFirstValue(queryParamsError()).has_value()) { sendUnauthorizedResponse(); return Http::FilterHeadersStatus::StopIteration; } // if the data we need is not present on the URL, stop execution - if (query_parameters.find(queryParamsCode()) == query_parameters.end() || - query_parameters.find(queryParamsState()) == query_parameters.end()) { + auto codeVal = query_parameters.getFirstValue(queryParamsCode()); + auto stateVal = query_parameters.getFirstValue(queryParamsState()); + if (!codeVal.has_value() || !stateVal.has_value()) { sendUnauthorizedResponse(); return Http::FilterHeadersStatus::StopIteration; } - auth_code_ = query_parameters.at(queryParamsCode()); + auth_code_ = codeVal.value(); if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.oauth_use_url_encoding")) { - state_ = Http::Utility::PercentEncoding::urlDecodeQueryParameter( - query_parameters.at(queryParamsState())); + state_ = Http::Utility::PercentEncoding::urlDecodeQueryParameter(stateVal.value()); } else { - state_ = Http::Utility::PercentEncoding::decode(query_parameters.at(queryParamsState())); + state_ = Http::Utility::PercentEncoding::decode(stateVal.value()); } Http::Utility::Url state_url; @@ -397,6 +414,15 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he return Http::FilterHeadersStatus::StopAllIterationAndBuffer; } +Http::FilterHeadersStatus OAuth2Filter::encodeHeaders(Http::ResponseHeaderMap& headers, bool) { + if (was_refresh_token_flow_) { + addResponseCookies(headers, getEncodedToken()); + was_refresh_token_flow_ = false; + } + + return Http::FilterHeadersStatus::Continue; +} + // Defines a sequence of checks determining whether we should initiate a new OAuth flow or skip to // the next filter in the chain. bool OAuth2Filter::canSkipOAuth(Http::RequestHeaderMap& headers) const { @@ -444,12 +470,12 @@ void OAuth2Filter::redirectToOAuthServer(Http::RequestHeaderMap& headers) const : Http::Utility::PercentEncoding::encode(redirect_uri, ":/=&?"); auto query_params = config_->authorizationQueryParams(); - query_params["redirect_uri"] = escaped_redirect_uri; - query_params["state"] = escaped_state; + query_params.overwrite("redirect_uri", escaped_redirect_uri); + query_params.overwrite("state", escaped_state); // Copy the authorization endpoint URL to replace its query params. auto authorization_endpoint_url = config_->authorizationEndpointUrl(); - const std::string path_and_query_params = Http::Utility::replaceQueryString( - Http::HeaderString(authorization_endpoint_url.pathAndQueryParams()), query_params); + const std::string path_and_query_params = query_params.replaceQueryString( + Http::HeaderString(authorization_endpoint_url.pathAndQueryParams())); authorization_endpoint_url.setPathAndQueryParams(path_and_query_params); const std::string new_url = authorization_endpoint_url.toString(); @@ -518,6 +544,15 @@ void OAuth2Filter::onGetAccessTokenSuccess(const std::string& access_code, finishGetAccessTokenFlow(); } +void OAuth2Filter::onRefreshAccessTokenSuccess(const std::string& access_code, + const std::string& id_token, + const std::string& refresh_token, + std::chrono::seconds expires_in) { + ASSERT(config_->useRefreshToken()); + updateTokens(access_code, id_token, refresh_token, expires_in); + finishRefreshAccessTokenFlow(); +} + void OAuth2Filter::finishGetAccessTokenFlow() { // At this point we have all of the pieces needed to authorize a user. // Now, we construct a redirect request to return the user to their @@ -533,6 +568,48 @@ void OAuth2Filter::finishGetAccessTokenFlow() { config_->stats().oauth_success_.inc(); } +void OAuth2Filter::finishRefreshAccessTokenFlow() { + ASSERT(config_->useRefreshToken()); + // At this point we have updated all of the pieces need to authorize a user + // We need to actualize keys in the cookie header of the current request related + // with authorization. So, the upstream can use updated cookies for itself purpose + const CookieNames& cookie_names = config_->cookieNames(); + + absl::flat_hash_map cookies = + Http::Utility::parseCookies(*request_headers_); + + cookies.insert_or_assign(cookie_names.oauth_hmac_, getEncodedToken()); + cookies.insert_or_assign(cookie_names.oauth_expires_, new_expires_); + + if (config_->forwardBearerToken()) { + cookies.insert_or_assign(cookie_names.bearer_token_, access_token_); + if (!id_token_.empty()) { + cookies.insert_or_assign(cookie_names.id_token_, id_token_); + } + if (!refresh_token_.empty()) { + cookies.insert_or_assign(cookie_names.refresh_token_, refresh_token_); + } + } + + std::string new_cookies(absl::StrJoin(cookies, "; ", absl::PairFormatter("="))); + request_headers_->addReferenceKey(Http::Headers::get().Cookie, new_cookies); + if (config_->forwardBearerToken() && !access_token_.empty()) { + setBearerToken(*request_headers_, access_token_); + } + + was_refresh_token_flow_ = true; + + config_->stats().oauth_refreshtoken_success_.inc(); + config_->stats().oauth_success_.inc(); + decoder_callbacks_->continueDecoding(); +} + +void OAuth2Filter::onRefreshAccessTokenFailure() { + config_->stats().oauth_refreshtoken_failure_.inc(); + // We failed to get an access token via the refresh token, so send the user to the oauth endpoint. + redirectToOAuthServer(*request_headers_); +} + void OAuth2Filter::addResponseCookies(Http::ResponseHeaderMap& headers, const std::string& encoded_token) const { std::string max_age; @@ -565,13 +642,13 @@ void OAuth2Filter::addResponseCookies(Http::ResponseHeaderMap& headers, headers.addReferenceKey( Http::Headers::get().SetCookie, absl::StrCat(cookie_names.bearer_token_, "=", access_token_, cookie_attribute_httponly)); - if (id_token_ != EMPTY_STRING) { + if (!id_token_.empty()) { headers.addReferenceKey( Http::Headers::get().SetCookie, absl::StrCat(cookie_names.id_token_, "=", id_token_, cookie_attribute_httponly)); } - if (refresh_token_ != EMPTY_STRING) { + if (!refresh_token_.empty()) { headers.addReferenceKey(Http::Headers::get().SetCookie, absl::StrCat(cookie_names.refresh_token_, "=", refresh_token_, cookie_attribute_httponly)); diff --git a/source/extensions/filters/http/oauth2/filter.h b/source/extensions/filters/http/oauth2/filter.h index 64e4ee002f11..b47d0c2ce873 100644 --- a/source/extensions/filters/http/oauth2/filter.h +++ b/source/extensions/filters/http/oauth2/filter.h @@ -85,7 +85,9 @@ class SDSSecretReader : public SecretReader { COUNTER(oauth_unauthorized_rq) \ COUNTER(oauth_failure) \ COUNTER(oauth_passthrough) \ - COUNTER(oauth_success) + COUNTER(oauth_success) \ + COUNTER(oauth_refreshtoken_success) \ + COUNTER(oauth_refreshtoken_failure) /** * Wrapper struct filter stats. @see stats_macros.h @@ -145,7 +147,7 @@ class FilterConfig { return oauth_token_endpoint_; } const Http::Utility::Url& authorizationEndpointUrl() const { return authorization_endpoint_url_; } - const Http::Utility::QueryParams& authorizationQueryParams() const { + const Http::Utility::QueryParamsMulti& authorizationQueryParams() const { return authorization_query_params_; } const std::string& redirectUri() const { return redirect_uri_; } @@ -157,6 +159,7 @@ class FilterConfig { const std::string& encodedResourceQueryParams() const { return encoded_resource_query_params_; } const CookieNames& cookieNames() const { return cookie_names_; } const AuthType& authType() const { return auth_type_; } + bool useRefreshToken() const { return use_refresh_token_; } private: static FilterStats generateStats(const std::string& prefix, Stats::Scope& scope); @@ -165,7 +168,7 @@ class FilterConfig { // Owns the data exposed by authorization_endpoint_url_. const std::string authorization_endpoint_; Http::Utility::Url authorization_endpoint_url_; - const Http::Utility::QueryParams authorization_query_params_; + const Http::Utility::QueryParamsMulti authorization_query_params_; const std::string client_id_; const std::string redirect_uri_; const Matchers::PathMatcher redirect_matcher_; @@ -178,6 +181,7 @@ class FilterConfig { const std::vector pass_through_header_matchers_; const CookieNames cookie_names_; const AuthType auth_type_; + const bool use_refresh_token_{}; }; using FilterConfigSharedPtr = std::shared_ptr; @@ -200,6 +204,7 @@ class CookieValidator { virtual const std::string& refreshToken() const PURE; virtual void setParams(const Http::RequestHeaderMap& headers, const std::string& secret) PURE; virtual bool isValid() const PURE; + virtual bool canUpdateTokenByRefreshToken() const PURE; }; class OAuth2CookieValidator : public CookieValidator { @@ -214,6 +219,7 @@ class OAuth2CookieValidator : public CookieValidator { bool isValid() const override; bool hmacIsValid() const; bool timestampIsValid() const; + bool canUpdateTokenByRefreshToken() const override; private: std::string token_; @@ -232,25 +238,36 @@ class OAuth2CookieValidator : public CookieValidator { * receive incoming requests and decide at what state of the OAuth workflow they are in. Logic * beyond that is broken into component classes. */ -class OAuth2Filter : public Http::PassThroughDecoderFilter, +class OAuth2Filter : public Http::PassThroughFilter, FilterCallbacks, Logger::Loggable { public: OAuth2Filter(FilterConfigSharedPtr config, std::unique_ptr&& oauth_client, TimeSource& time_source); - // Http::PassThroughDecoderFilter + // Http::PassThroughFilter Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, bool) override; + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap& headers, bool) override; // FilterCallbacks void onGetAccessTokenSuccess(const std::string& access_code, const std::string& id_token, const std::string& refresh_token, std::chrono::seconds expires_in) override; + + void onRefreshAccessTokenSuccess(const std::string& access_code, const std::string& id_token, + const std::string& refresh_token, + std::chrono::seconds expires_in) override; + + void onRefreshAccessTokenFailure() override; + // a catch-all function used for request failures. we don't retry, as a user can simply refresh // the page in the case of a network blip. void sendUnauthorizedResponse() override; void finishGetAccessTokenFlow(); + void finishRefreshAccessTokenFlow(); + void updateTokens(const std::string& access_token, const std::string& id_token, + const std::string& refresh_token, std::chrono::seconds expires_in); private: friend class OAuth2Test; @@ -267,6 +284,7 @@ class OAuth2Filter : public Http::PassThroughDecoderFilter, absl::string_view host_; std::string state_; Http::RequestHeaderMap* request_headers_{nullptr}; + bool was_refresh_token_flow_; std::unique_ptr oauth_client_; FilterConfigSharedPtr config_; @@ -276,8 +294,6 @@ class OAuth2Filter : public Http::PassThroughDecoderFilter, // connection is mTLS, etc.) bool canSkipOAuth(Http::RequestHeaderMap& headers) const; void redirectToOAuthServer(Http::RequestHeaderMap& headers) const; - void updateTokens(const std::string& access_token, const std::string& id_token, - const std::string& refresh_token, std::chrono::seconds expires_in); Http::FilterHeadersStatus signOutUser(const Http::RequestHeaderMap& headers); diff --git a/source/extensions/filters/http/oauth2/oauth.h b/source/extensions/filters/http/oauth2/oauth.h index 261050ed7c1c..f7850d35fd99 100644 --- a/source/extensions/filters/http/oauth2/oauth.h +++ b/source/extensions/filters/http/oauth2/oauth.h @@ -23,6 +23,13 @@ class FilterCallbacks { const std::string& refresh_token, std::chrono::seconds expires_in) PURE; + virtual void onRefreshAccessTokenSuccess(const std::string& access_token, + const std::string& id_token, + const std::string& refresh_token, + std::chrono::seconds expires_in) PURE; + + virtual void onRefreshAccessTokenFailure() PURE; + virtual void sendUnauthorizedResponse() PURE; }; diff --git a/source/extensions/filters/http/oauth2/oauth_client.cc b/source/extensions/filters/http/oauth2/oauth_client.cc index 21fd343beb09..b7be673c761d 100644 --- a/source/extensions/filters/http/oauth2/oauth_client.cc +++ b/source/extensions/filters/http/oauth2/oauth_client.cc @@ -22,12 +22,18 @@ namespace HttpFilters { namespace Oauth2 { namespace { -constexpr const char* UrlBodyTemplateWithCredentials = +constexpr const char* UrlBodyTemplateWithCredentialsForAuthCode = "grant_type=authorization_code&code={0}&client_id={1}&client_secret={2}&redirect_uri={3}"; -constexpr const char* UrlBodyTemplateWithoutCredentials = +constexpr const char* UrlBodyTemplateWithoutCredentialsForAuthCode = "grant_type=authorization_code&code={0}&redirect_uri={1}"; +constexpr const char* UrlBodyTemplateWithCredentialsForRefreshToken = + "grant_type=refresh_token&refresh_token={0}&client_id={1}&client_secret={2}"; + +constexpr const char* UrlBodyTemplateWithoutCredentialsForRefreshToken = + "grant_type=refresh_token&refresh_token={0}"; + } // namespace void OAuth2ClientImpl::asyncGetAccessToken(const std::string& auth_code, @@ -39,7 +45,7 @@ void OAuth2ClientImpl::asyncGetAccessToken(const std::string& auth_code, switch (auth_type) { case AuthType::UrlEncodedBody: - body = fmt::format(UrlBodyTemplateWithCredentials, auth_code, + body = fmt::format(UrlBodyTemplateWithCredentialsForAuthCode, auth_code, Http::Utility::PercentEncoding::encode(client_id, ":/=&?"), Http::Utility::PercentEncoding::encode(secret, ":/=&?"), encoded_cb_url); break; @@ -49,7 +55,7 @@ void OAuth2ClientImpl::asyncGetAccessToken(const std::string& auth_code, const auto basic_auth_header_value = absl::StrCat("Basic ", encoded_token); request->headers().appendCopy(Http::CustomHeaders::get().Authorization, basic_auth_header_value); - body = fmt::format(UrlBodyTemplateWithoutCredentials, auth_code, encoded_cb_url); + body = fmt::format(UrlBodyTemplateWithoutCredentialsForAuthCode, auth_code, encoded_cb_url); break; } @@ -62,6 +68,39 @@ void OAuth2ClientImpl::asyncGetAccessToken(const std::string& auth_code, state_ = OAuthState::PendingAccessToken; } +void OAuth2ClientImpl::asyncRefreshAccessToken(const std::string& refresh_token, + const std::string& client_id, + const std::string& secret, AuthType auth_type) { + Http::RequestMessagePtr request = createPostRequest(); + std::string body; + + switch (auth_type) { + case AuthType::UrlEncodedBody: + body = fmt::format(UrlBodyTemplateWithCredentialsForRefreshToken, + Http::Utility::PercentEncoding::encode(refresh_token, ":/=&?"), + Http::Utility::PercentEncoding::encode(client_id, ":/=&?"), + Http::Utility::PercentEncoding::encode(secret, ":/=&?")); + break; + case AuthType::BasicAuth: + const auto basic_auth_token = absl::StrCat(client_id, ":", secret); + const auto encoded_token = Base64::encode(basic_auth_token.data(), basic_auth_token.size()); + const auto basic_auth_header_value = absl::StrCat("Basic ", encoded_token); + request->headers().appendCopy(Http::CustomHeaders::get().Authorization, + basic_auth_header_value); + body = fmt::format(UrlBodyTemplateWithoutCredentialsForRefreshToken, + Http::Utility::PercentEncoding::encode(refresh_token)); + break; + } + + request->body().add(body); + request->headers().setContentLength(body.length()); + ENVOY_LOG(debug, "Dispatching OAuth request for update access token by refresh token."); + dispatchRequest(std::move(request)); + + ASSERT(state_ == OAuthState::Idle); + state_ = OAuthState::PendingAccessTokenByRefreshToken; +} + void OAuth2ClientImpl::dispatchRequest(Http::RequestMessagePtr&& msg) { const auto thread_local_cluster = cm_.getThreadLocalCluster(uri_.cluster()); if (thread_local_cluster != nullptr) { @@ -78,15 +117,27 @@ void OAuth2ClientImpl::onSuccess(const Http::AsyncClient::Request&, Http::ResponseMessagePtr&& message) { in_flight_request_ = nullptr; - ASSERT(state_ == OAuthState::PendingAccessToken); + ASSERT(state_ == OAuthState::PendingAccessToken || + state_ == OAuthState::PendingAccessTokenByRefreshToken); + const OAuthState oldState = state_; state_ = OAuthState::Idle; // Check that the auth cluster returned a happy response. const auto response_code = message->headers().Status()->value().getStringView(); + if (response_code != "200") { ENVOY_LOG(debug, "Oauth response code: {}", response_code); ENVOY_LOG(debug, "Oauth response body: {}", message->bodyAsString()); - parent_->sendUnauthorizedResponse(); + switch (oldState) { + case OAuthState::PendingAccessToken: + parent_->sendUnauthorizedResponse(); + break; + case OAuthState::PendingAccessTokenByRefreshToken: + parent_->onRefreshAccessTokenFailure(); + break; + default: + PANIC("Malformed oauth client state"); + } return; } @@ -117,14 +168,35 @@ void OAuth2ClientImpl::onSuccess(const Http::AsyncClient::Request&, PROTOBUF_GET_WRAPPED_OR_DEFAULT(response, refresh_token, EMPTY_STRING)}; const std::chrono::seconds expires_in{PROTOBUF_GET_WRAPPED_REQUIRED(response, expires_in)}; - parent_->onGetAccessTokenSuccess(access_token, id_token, refresh_token, expires_in); + switch (oldState) { + case OAuthState::PendingAccessToken: + parent_->onGetAccessTokenSuccess(access_token, id_token, refresh_token, expires_in); + break; + case OAuthState::PendingAccessTokenByRefreshToken: + parent_->onRefreshAccessTokenSuccess(access_token, id_token, refresh_token, expires_in); + break; + default: + PANIC("Malformed oauth client state"); + } } void OAuth2ClientImpl::onFailure(const Http::AsyncClient::Request&, Http::AsyncClient::FailureReason) { ENVOY_LOG(debug, "OAuth request failed."); in_flight_request_ = nullptr; - parent_->sendUnauthorizedResponse(); + const OAuthState oldState = state_; + state_ = OAuthState::Idle; + + switch (oldState) { + case OAuthState::PendingAccessToken: + parent_->sendUnauthorizedResponse(); + break; + case OAuthState::PendingAccessTokenByRefreshToken: + parent_->onRefreshAccessTokenFailure(); + break; + default: + PANIC("Malformed oauth client state"); + } } } // namespace Oauth2 diff --git a/source/extensions/filters/http/oauth2/oauth_client.h b/source/extensions/filters/http/oauth2/oauth_client.h index a659714699ea..d18823c0ccbf 100644 --- a/source/extensions/filters/http/oauth2/oauth_client.h +++ b/source/extensions/filters/http/oauth2/oauth_client.h @@ -28,6 +28,11 @@ class OAuth2Client : public Http::AsyncClient::Callbacks { virtual void asyncGetAccessToken(const std::string& auth_code, const std::string& client_id, const std::string& secret, const std::string& cb_url, AuthType auth_type = AuthType::UrlEncodedBody) PURE; + + virtual void asyncRefreshAccessToken(const std::string& refresh_token, + const std::string& client_id, const std::string& secret, + AuthType auth_type = AuthType::UrlEncodedBody) PURE; + virtual void setCallbacks(FilterCallbacks& callbacks) PURE; // Http::AsyncClient::Callbacks @@ -55,11 +60,15 @@ class OAuth2ClientImpl : public OAuth2Client, Logger::Loggable( - proto_config, context.clusterManager(), context.messageValidationVisitor()); + proto_config, context.serverFactoryContext().clusterManager(), + context.messageValidationVisitor()); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(config)); }; diff --git a/source/extensions/filters/http/rate_limit_quota/BUILD b/source/extensions/filters/http/rate_limit_quota/BUILD index c4421278e1f0..ecadb2459154 100644 --- a/source/extensions/filters/http/rate_limit_quota/BUILD +++ b/source/extensions/filters/http/rate_limit_quota/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( ":client_lib", ":matcher_lib", ":quota_bucket_cache", + "//envoy/grpc:async_client_manager_interface", "//envoy/registry", "//source/common/http:headers_lib", "//source/common/http:message_lib", @@ -39,6 +40,7 @@ envoy_cc_extension( deps = [ ":client_interface", ":rate_limit_quota", + "//envoy/grpc:async_client_manager_interface", "//envoy/registry", "//source/extensions/filters/http/common:factory_base_lib", "@envoy_api//envoy/extensions/filters/http/rate_limit_quota/v3:pkg_cc_proto", @@ -98,6 +100,7 @@ envoy_cc_library( deps = [ ":client_interface", "//envoy/common:time_interface", + "//source/common/common:token_bucket_impl_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", "//source/common/protobuf:utility_lib", diff --git a/source/extensions/filters/http/rate_limit_quota/client.h b/source/extensions/filters/http/rate_limit_quota/client.h index a612636858a7..a8d31c39b06e 100644 --- a/source/extensions/filters/http/rate_limit_quota/client.h +++ b/source/extensions/filters/http/rate_limit_quota/client.h @@ -4,6 +4,7 @@ #include "envoy/common/pure.h" #include "envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.pb.h" +#include "envoy/grpc/async_client_manager.h" #include "envoy/service/rate_limit_quota/v3/rlqs.pb.h" #include "envoy/stream_info/stream_info.h" diff --git a/source/extensions/filters/http/rate_limit_quota/client_impl.cc b/source/extensions/filters/http/rate_limit_quota/client_impl.cc index 9e6894829ccb..fbda6884bcab 100644 --- a/source/extensions/filters/http/rate_limit_quota/client_impl.cc +++ b/source/extensions/filters/http/rate_limit_quota/client_impl.cc @@ -60,6 +60,25 @@ void RateLimitClientImpl::onReceiveMessage(RateLimitQuotaResponsePtr&& response) response->ShortDebugString()); } else { quota_buckets_[bucket_id]->bucket_action = action; + // TODO(tyxia) Handle expired assignment via `assignment_time_to_live`. + if (quota_buckets_[bucket_id]->bucket_action.has_quota_assignment_action()) { + auto rate_limit_strategy = quota_buckets_[bucket_id] + ->bucket_action.quota_assignment_action() + .rate_limit_strategy(); + + if (rate_limit_strategy.has_token_bucket()) { + const auto& interval_proto = rate_limit_strategy.token_bucket().fill_interval(); + // Convert absl::duration to int64_t seconds + int64_t fill_interval_sec = absl::ToInt64Seconds( + absl::Seconds(interval_proto.seconds()) + absl::Nanoseconds(interval_proto.nanos())); + double fill_rate_per_sec = + static_cast(rate_limit_strategy.token_bucket().tokens_per_fill().value()) / + fill_interval_sec; + + quota_buckets_[bucket_id]->token_bucket_limiter = std::make_unique( + rate_limit_strategy.token_bucket().max_tokens(), time_source_, fill_rate_per_sec); + } + } } } diff --git a/source/extensions/filters/http/rate_limit_quota/client_impl.h b/source/extensions/filters/http/rate_limit_quota/client_impl.h index 1f3dd1b47336..eb7c0f6a6685 100644 --- a/source/extensions/filters/http/rate_limit_quota/client_impl.h +++ b/source/extensions/filters/http/rate_limit_quota/client_impl.h @@ -32,14 +32,17 @@ class RateLimitClientImpl : public RateLimitClient, envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse>, public Logger::Loggable { public: - RateLimitClientImpl(const envoy::config::core::v3::GrpcService& grpc_service, + RateLimitClientImpl(const Grpc::GrpcServiceConfigWithHashKey& config_with_hash_key, Server::Configuration::FactoryContext& context, absl::string_view domain_name, RateLimitQuotaCallbacks* callbacks, BucketsCache& quota_buckets) : domain_name_(domain_name), - aync_client_(context.clusterManager().grpcAsyncClientManager().getOrCreateRawAsyncClient( - grpc_service, context.scope(), true)), + aync_client_( + context.serverFactoryContext() + .clusterManager() + .grpcAsyncClientManager() + .getOrCreateRawAsyncClientWithHashKey(config_with_hash_key, context.scope(), true)), rlqs_callback_(callbacks), quota_buckets_(quota_buckets), - time_source_(context.mainThreadDispatcher().timeSource()) {} + time_source_(context.serverFactoryContext().mainThreadDispatcher().timeSource()) {} void onReceiveMessage(RateLimitQuotaResponsePtr&& response) override; @@ -83,11 +86,11 @@ using RateLimitClientPtr = std::unique_ptr; */ inline RateLimitClientPtr createRateLimitClient(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::GrpcService& grpc_service, RateLimitQuotaCallbacks* callbacks, BucketsCache& quota_buckets, - absl::string_view domain_name) { - return std::make_unique(grpc_service, context, domain_name, callbacks, - quota_buckets); + absl::string_view domain_name, + Grpc::GrpcServiceConfigWithHashKey& config_with_hash_key) { + return std::make_unique(config_with_hash_key, context, domain_name, + callbacks, quota_buckets); } } // namespace RateLimitQuota diff --git a/source/extensions/filters/http/rate_limit_quota/config.cc b/source/extensions/filters/http/rate_limit_quota/config.cc index 5cc4940224a5..361bac4bce2a 100644 --- a/source/extensions/filters/http/rate_limit_quota/config.cc +++ b/source/extensions/filters/http/rate_limit_quota/config.cc @@ -24,12 +24,13 @@ Http::FilterFactoryCb RateLimitQuotaFilterFactory::createFilterFactoryFromProtoT // Quota bucket TLS object is created on the main thread and shared between worker threads. std::shared_ptr bucket_cache = std::make_shared(context); - - return [config = std::move(config), &context, bucket_cache = std::move(bucket_cache)]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key = + Grpc::GrpcServiceConfigWithHashKey(config->rlqs_server()); + return [config = std::move(config), &context, bucket_cache = std::move(bucket_cache), + config_with_hash_key](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared( config, context, bucket_cache->tls.get()->quotaBuckets(), - bucket_cache->tls.get()->rateLimitClient())); + bucket_cache->tls.get()->rateLimitClient(), config_with_hash_key)); }; } diff --git a/source/extensions/filters/http/rate_limit_quota/filter.cc b/source/extensions/filters/http/rate_limit_quota/filter.cc index 3624e68765e2..916e0d84b290 100644 --- a/source/extensions/filters/http/rate_limit_quota/filter.cc +++ b/source/extensions/filters/http/rate_limit_quota/filter.cc @@ -17,7 +17,8 @@ Http::FilterHeadersStatus RateLimitQuotaFilter::decodeHeaders(Http::RequestHeade if (!match_result.ok()) { // When the request is not matched by any matchers, it is ALLOWED by default (i.e., fail-open) // and its quota usage will not be reported to RLQS server. - // TODO(tyxia) Add stats here and other places throughout the filter (if needed). + // TODO(tyxia) Add stats here and other places throughout the filter. e.g. request + // allowed/denied, matching succeed/fail and so on. ENVOY_LOG(debug, "The request is not matched by any matchers: ", match_result.status().message()); return Envoy::Http::FilterHeadersStatus::Continue; @@ -44,25 +45,14 @@ Http::FilterHeadersStatus RateLimitQuotaFilter::decodeHeaders(Http::RequestHeade return sendImmediateReport(bucket_id, match_action); } else { // Found the cached bucket entry. - // First, get the quota assignment (if exists) from the cached bucket action. - // TODO(tyxia) Implement other assignment type besides ALLOW ALL. - if (quota_buckets_[bucket_id]->bucket_action.has_quota_assignment_action()) { - auto rate_limit_strategy = - quota_buckets_[bucket_id]->bucket_action.quota_assignment_action().rate_limit_strategy(); - - if (rate_limit_strategy.has_blanket_rule() && - rate_limit_strategy.blanket_rule() == envoy::type::v3::RateLimitStrategy::ALLOW_ALL) { - quota_buckets_[bucket_id]->quota_usage.num_requests_allowed += 1; - } - } + return processCachedBucket(bucket_id); } - return Envoy::Http::FilterHeadersStatus::Continue; } void RateLimitQuotaFilter::createMatcher() { RateLimitOnMatchActionContext context; Matcher::MatchTreeFactory factory( - context, factory_context_.getServerFactoryContext(), visitor_); + context, factory_context_.serverFactoryContext(), visitor_); if (config_->has_bucket_matchers()) { matcher_ = factory.create(config_->bucket_matchers())(); } @@ -119,8 +109,8 @@ void RateLimitQuotaFilter::onDestroy() { void RateLimitQuotaFilter::createNewBucket(const BucketId& bucket_id, size_t id) { // The first matched request doesn't have quota assignment from the RLQS server yet, so the // action is performed based on pre-configured strategy from no assignment behavior config. - // TODO(tyxia) Check no assignment logic for new bucket (i.e., first matched request). Default is - // allow all. + // TODO(tyxia) Check no assignment logic for new bucket (i.e., first matched request). It + // should not wait for RLQS server response. QuotaUsage quota_usage; quota_usage.num_requests_allowed = 1; quota_usage.num_requests_denied = 0; @@ -141,8 +131,8 @@ RateLimitQuotaFilter::sendImmediateReport(const size_t bucket_id, // Create the gRPC client if it has not been created. if (client_.rate_limit_client == nullptr) { - client_.rate_limit_client = createRateLimitClient(factory_context_, config_->rlqs_server(), - this, quota_buckets_, config_->domain()); + client_.rate_limit_client = createRateLimitClient(factory_context_, this, quota_buckets_, + config_->domain(), config_with_hash_key_); } else { // Callback has been reset to nullptr when filter was destroyed last time. // Reset it here when new filter has been created. @@ -156,6 +146,7 @@ RateLimitQuotaFilter::sendImmediateReport(const size_t bucket_id, auto status = client_.rate_limit_client->startStream(callbacks_->streamInfo()); if (!status.ok()) { ENVOY_LOG(error, "Failed to start the gRPC stream: ", status.message()); + // TODO(tyxia) Check `NoAssignmentBehavior` behavior instead of fail-open here. return Envoy::Http::FilterHeadersStatus::Continue; } @@ -177,6 +168,39 @@ RateLimitQuotaFilter::sendImmediateReport(const size_t bucket_id, return Http::FilterHeadersStatus::StopAllIterationAndWatermark; } +Http::FilterHeadersStatus RateLimitQuotaFilter::processCachedBucket(size_t bucket_id) { + // First, get the quota assignment (if exists) from the cached bucket action. + if (quota_buckets_[bucket_id]->bucket_action.has_quota_assignment_action()) { + auto rate_limit_strategy = + quota_buckets_[bucket_id]->bucket_action.quota_assignment_action().rate_limit_strategy(); + + // TODO(tyxia) Currently only ALLOW_ALL and token bucket strategies are implemented. + // Change to switch case when more strategies are implemented. + if (rate_limit_strategy.has_blanket_rule() && + rate_limit_strategy.blanket_rule() == envoy::type::v3::RateLimitStrategy::ALLOW_ALL) { + quota_buckets_[bucket_id]->quota_usage.num_requests_allowed += 1; + } else if (rate_limit_strategy.has_token_bucket()) { + ASSERT(quota_buckets_[bucket_id]->token_bucket_limiter != nullptr); + TokenBucket* limiter = quota_buckets_[bucket_id]->token_bucket_limiter.get(); + // Try to consume 1 token from the bucket. + if (limiter->consume(1, /*allow_partial=*/false)) { + // Request is allowed. + quota_buckets_[bucket_id]->quota_usage.num_requests_allowed += 1; + } else { + // Request is throttled. + quota_buckets_[bucket_id]->quota_usage.num_requests_denied += 1; + // TODO(tyxia) Build the customized response based on `DenyResponseSettings` if it is + // configured. + callbacks_->sendLocalReply(Envoy::Http::Code::TooManyRequests, "", nullptr, absl::nullopt, + ""); + callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::RateLimited); + return Envoy::Http::FilterHeadersStatus::StopIteration; + } + } + } + return Envoy::Http::FilterHeadersStatus::Continue; +} + } // namespace RateLimitQuota } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/rate_limit_quota/filter.h b/source/extensions/filters/http/rate_limit_quota/filter.h index 4b1141213e87..e8e6fa1ff22e 100644 --- a/source/extensions/filters/http/rate_limit_quota/filter.h +++ b/source/extensions/filters/http/rate_limit_quota/filter.h @@ -3,6 +3,7 @@ #include "envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.pb.h" #include "envoy/extensions/filters/http/rate_limit_quota/v3/rate_limit_quota.pb.validate.h" +#include "envoy/grpc/async_client_manager.h" #include "envoy/registry/registry.h" #include "envoy/service/rate_limit_quota/v3/rlqs.pb.h" #include "envoy/service/rate_limit_quota/v3/rlqs.pb.validate.h" @@ -48,10 +49,11 @@ class RateLimitQuotaFilter : public Http::PassThroughFilter, public: RateLimitQuotaFilter(FilterConfigConstSharedPtr config, Server::Configuration::FactoryContext& factory_context, - BucketsCache& quota_buckets, ThreadLocalClient& client) - : config_(std::move(config)), factory_context_(factory_context), - quota_buckets_(quota_buckets), client_(client), - time_source_(factory_context.mainThreadDispatcher().timeSource()) { + BucketsCache& quota_buckets, ThreadLocalClient& client, + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key) + : config_(std::move(config)), config_with_hash_key_(config_with_hash_key), + factory_context_(factory_context), quota_buckets_(quota_buckets), client_(client), + time_source_(factory_context.serverFactoryContext().mainThreadDispatcher().timeSource()) { createMatcher(); } @@ -90,7 +92,9 @@ class RateLimitQuotaFilter : public Http::PassThroughFilter, Http::FilterHeadersStatus sendImmediateReport(const size_t bucket_id, const RateLimitOnMatchAction& match_action); + Http::FilterHeadersStatus processCachedBucket(size_t bucket_id); FilterConfigConstSharedPtr config_; + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key_; Server::Configuration::FactoryContext& factory_context_; Http::StreamDecoderFilterCallbacks* callbacks_ = nullptr; RateLimitQuotaValidationVisitor visitor_ = {}; diff --git a/source/extensions/filters/http/rate_limit_quota/quota_bucket_cache.h b/source/extensions/filters/http/rate_limit_quota/quota_bucket_cache.h index ab069959a301..e5ecdadc49b1 100644 --- a/source/extensions/filters/http/rate_limit_quota/quota_bucket_cache.h +++ b/source/extensions/filters/http/rate_limit_quota/quota_bucket_cache.h @@ -7,6 +7,7 @@ #include "envoy/service/rate_limit_quota/v3/rlqs.pb.h" #include "envoy/service/rate_limit_quota/v3/rlqs.pb.validate.h" +#include "source/common/common/token_bucket_impl.h" #include "source/common/protobuf/utility.h" #include "source/extensions/filters/http/common/factory_base.h" #include "source/extensions/filters/http/rate_limit_quota/client.h" @@ -43,6 +44,8 @@ struct Bucket { BucketAction bucket_action; // Cache quota usage. QuotaUsage quota_usage; + // Rate limiter based on token bucket algorithm. + TokenBucketPtr token_bucket_limiter; }; using BucketsCache = absl::flat_hash_map>; @@ -96,7 +99,8 @@ class ThreadLocalBucket : public Envoy::ThreadLocal::ThreadLocalObject { }; struct QuotaBucket { - QuotaBucket(Envoy::Server::Configuration::FactoryContext& context) : tls(context.threadLocal()) { + QuotaBucket(Envoy::Server::Configuration::FactoryContext& context) + : tls(context.serverFactoryContext().threadLocal()) { tls.set([](Envoy::Event::Dispatcher& dispatcher) { return std::make_shared(dispatcher); }); diff --git a/source/extensions/filters/http/ratelimit/config.cc b/source/extensions/filters/http/ratelimit/config.cc index c116b5fa048b..ceebc3641ca2 100644 --- a/source/extensions/filters/http/ratelimit/config.cc +++ b/source/extensions/filters/http/ratelimit/config.cc @@ -20,19 +20,23 @@ namespace RateLimitFilter { Http::FilterFactoryCb RateLimitFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::ratelimit::v3::RateLimit& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + ASSERT(!proto_config.domain().empty()); - FilterConfigSharedPtr filter_config(new FilterConfig(proto_config, context.localInfo(), - context.scope(), context.runtime(), - context.httpContext())); + FilterConfigSharedPtr filter_config(new FilterConfig(proto_config, server_context.localInfo(), + context.scope(), server_context.runtime(), + server_context.httpContext())); const std::chrono::milliseconds timeout = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(proto_config, timeout, 20)); THROW_IF_NOT_OK(Config::Utility::checkTransportVersion(proto_config.rate_limit_service())); - return [proto_config, &context, timeout, + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key = + Grpc::GrpcServiceConfigWithHashKey(proto_config.rate_limit_service().grpc_service()); + return [config_with_hash_key, &context, timeout, filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared( - filter_config, Filters::Common::RateLimit::rateLimitClient( - context, proto_config.rate_limit_service().grpc_service(), timeout))); + filter_config, + Filters::Common::RateLimit::rateLimitClient(context, config_with_hash_key, timeout))); }; } diff --git a/source/extensions/filters/http/ratelimit/ratelimit.cc b/source/extensions/filters/http/ratelimit/ratelimit.cc index 9872bbaf9295..a8576331a16f 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.cc +++ b/source/extensions/filters/http/ratelimit/ratelimit.cc @@ -205,7 +205,7 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, [this](Http::HeaderMap& headers) { populateResponseHeaders(headers, /*from_local_reply=*/true); config_->responseHeadersParser().evaluateHeaders( - headers, *request_headers_, dynamic_cast(headers), + headers, {request_headers_, dynamic_cast(&headers)}, callbacks_->streamInfo()); }, config_->rateLimitedGrpcStatus(), RcDetails::get().RateLimited); @@ -219,8 +219,8 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, } else { state_ = State::Responded; callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError); - callbacks_->sendLocalReply(Http::Code::InternalServerError, response_body, nullptr, - absl::nullopt, RcDetails::get().RateLimitError); + callbacks_->sendLocalReply(config_->statusOnError(), response_body, nullptr, absl::nullopt, + RcDetails::get().RateLimitError); } } else if (!initiating_call_) { appendRequestHeaders(req_headers_to_add); diff --git a/source/extensions/filters/http/ratelimit/ratelimit.h b/source/extensions/filters/http/ratelimit/ratelimit.h index 893a7fc7b7a6..fb35a2380241 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.h +++ b/source/extensions/filters/http/ratelimit/ratelimit.h @@ -56,10 +56,11 @@ class FilterConfig { config.rate_limited_as_resource_exhausted() ? absl::make_optional(Grpc::Status::WellKnownGrpcStatus::ResourceExhausted) : absl::nullopt), - http_context_(http_context), stat_names_(scope.symbolTable()), + http_context_(http_context), stat_names_(scope.symbolTable(), config.stat_prefix()), rate_limited_status_(toErrorCode(config.rate_limited_status().code())), response_headers_parser_( - Envoy::Router::HeaderParser::configure(config.response_headers_to_add())) {} + Envoy::Router::HeaderParser::configure(config.response_headers_to_add())), + status_on_error_(toRatelimitServerErrorCode(config.status_on_error().code())) {} const std::string& domain() const { return domain_; } const LocalInfo::LocalInfo& localInfo() const { return local_info_; } uint64_t stage() const { return stage_; } @@ -76,6 +77,7 @@ class FilterConfig { Filters::Common::RateLimit::StatNames& statNames() { return stat_names_; } Http::Code rateLimitedStatus() { return rate_limited_status_; } const Router::HeaderParser& responseHeadersParser() const { return *response_headers_parser_; } + Http::Code statusOnError() const { return status_on_error_; } private: static FilterRequestType stringToType(const std::string& request_type) { @@ -97,6 +99,14 @@ class FilterConfig { return Http::Code::TooManyRequests; } + static Http::Code toRatelimitServerErrorCode(uint64_t status) { + const auto code = static_cast(status); + if (code >= Http::Code::Continue && code <= Http::Code::NetworkAuthenticationRequired) { + return code; + } + return Http::Code::InternalServerError; + } + const std::string domain_; const uint64_t stage_; const FilterRequestType request_type_; @@ -111,6 +121,7 @@ class FilterConfig { Filters::Common::RateLimit::StatNames stat_names_; const Http::Code rate_limited_status_; Router::HeaderParserPtr response_headers_parser_; + const Http::Code status_on_error_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/rbac/config.cc b/source/extensions/filters/http/rbac/config.cc index fefe21e1ba1c..0ee0d231a28d 100644 --- a/source/extensions/filters/http/rbac/config.cc +++ b/source/extensions/filters/http/rbac/config.cc @@ -16,7 +16,7 @@ Http::FilterFactoryCb RoleBasedAccessControlFilterConfigFactory::createFilterFac const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { auto config = std::make_shared( - proto_config, stats_prefix, context.scope(), context.getServerFactoryContext(), + proto_config, stats_prefix, context.scope(), context.serverFactoryContext(), context.messageValidationVisitor()); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { diff --git a/source/extensions/filters/http/router/config.cc b/source/extensions/filters/http/router/config.cc index c3f899311477..06ac8f5fec75 100644 --- a/source/extensions/filters/http/router/config.cc +++ b/source/extensions/filters/http/router/config.cc @@ -17,7 +17,8 @@ Http::FilterFactoryCb RouterFilterConfig::createFilterFactoryFromProtoTyped( Stats::StatNameManagedStorage prefix(stat_prefix, context.scope().symbolTable()); Router::FilterConfigSharedPtr filter_config(new Router::FilterConfig( prefix.statName(), context, - std::make_unique(context.clusterManager()), proto_config)); + std::make_unique(context.serverFactoryContext().clusterManager()), + proto_config)); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter( diff --git a/source/extensions/filters/http/set_filter_state/BUILD b/source/extensions/filters/http/set_filter_state/BUILD new file mode 100644 index 000000000000..12d41eb821ba --- /dev/null +++ b/source/extensions/filters/http/set_filter_state/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//envoy/formatter:substitution_formatter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/extensions/filters/common/set_filter_state:filter_config_lib", + "//source/extensions/filters/http/common:factory_base_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//source/server:generic_factory_context_lib", + "@envoy_api//envoy/extensions/filters/http/set_filter_state/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/http/set_filter_state/config.cc b/source/extensions/filters/http/set_filter_state/config.cc new file mode 100644 index 000000000000..d0edcb486fe6 --- /dev/null +++ b/source/extensions/filters/http/set_filter_state/config.cc @@ -0,0 +1,61 @@ +#include "source/extensions/filters/http/set_filter_state/config.h" + +#include + +#include "envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.pb.h" +#include "envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.pb.validate.h" +#include "envoy/formatter/substitution_formatter.h" +#include "envoy/registry/registry.h" + +#include "source/common/protobuf/utility.h" +#include "source/server/generic_factory_context.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetFilterState { + +SetFilterState::SetFilterState(const Filters::Common::SetFilterState::ConfigSharedPtr config) + : config_(config) {} + +Http::FilterHeadersStatus SetFilterState::decodeHeaders(Http::RequestHeaderMap& headers, bool) { + config_->updateFilterState({&headers}, decoder_callbacks_->streamInfo()); + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterFactoryCb SetFilterStateConfig::createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::set_filter_state::v3::Config& proto_config, + const std::string&, Server::Configuration::FactoryContext& context) { + + const auto filter_config = std::make_shared( + proto_config.on_request_headers(), StreamInfo::FilterState::LifeSpan::FilterChain, context); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter( + Http::StreamDecoderFilterSharedPtr{new SetFilterState(filter_config)}); + }; +} + +Http::FilterFactoryCb SetFilterStateConfig::createFilterFactoryFromProtoWithServerContextTyped( + const envoy::extensions::filters::http::set_filter_state::v3::Config& proto_config, + const std::string&, Server::Configuration::ServerFactoryContext& context) { + + // TODO(wbpcode): these is a potential bug of message validation. The validation visitor + // of server context should not be used here directly. But this is bug of + // 'createFilterFactoryFromProtoWithServerContext' and will be fixed in the future. + Server::GenericFactoryContextImpl generic_context(context, context.messageValidationVisitor()); + + const auto filter_config = std::make_shared( + proto_config.on_request_headers(), StreamInfo::FilterState::LifeSpan::FilterChain, + generic_context); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter( + Http::StreamDecoderFilterSharedPtr{new SetFilterState(filter_config)}); + }; +} + +REGISTER_FACTORY(SetFilterStateConfig, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace SetFilterState +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/set_filter_state/config.h b/source/extensions/filters/http/set_filter_state/config.h new file mode 100644 index 000000000000..be0d72a6098f --- /dev/null +++ b/source/extensions/filters/http/set_filter_state/config.h @@ -0,0 +1,50 @@ +#pragma once + +#include "envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.pb.h" +#include "envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.pb.validate.h" + +#include "source/common/common/logger.h" +#include "source/extensions/filters/common/set_filter_state/filter_config.h" +#include "source/extensions/filters/http/common/factory_base.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetFilterState { + +class SetFilterState : public Http::PassThroughDecoderFilter, + public Logger::Loggable { +public: + explicit SetFilterState(const Filters::Common::SetFilterState::ConfigSharedPtr config); + + // StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, bool) override; + +private: + const Filters::Common::SetFilterState::ConfigSharedPtr config_; +}; + +/** + * Config registration. @see NamedHttpFilterConfigFactory. + */ +class SetFilterStateConfig + : public Common::FactoryBase { +public: + SetFilterStateConfig() : FactoryBase("envoy.filters.http.set_filter_state") {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::http::set_filter_state::v3::Config& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; + + Http::FilterFactoryCb createFilterFactoryFromProtoWithServerContextTyped( + const envoy::extensions::filters::http::set_filter_state::v3::Config& proto_config, + const std::string& stats_prefix, + Server::Configuration::ServerFactoryContext& server_context) override; +}; + +} // namespace SetFilterState +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/set_metadata/config.cc b/source/extensions/filters/http/set_metadata/config.cc index 4989e41b2c18..2c17079e280d 100644 --- a/source/extensions/filters/http/set_metadata/config.cc +++ b/source/extensions/filters/http/set_metadata/config.cc @@ -16,8 +16,9 @@ namespace SetMetadataFilter { Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config, - const std::string&, Server::Configuration::FactoryContext&) { - ConfigSharedPtr filter_config(std::make_shared(proto_config)); + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + ConfigSharedPtr filter_config( + std::make_shared(proto_config, context.scope(), stats_prefix)); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter( @@ -27,8 +28,9 @@ Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoTyped( Http::FilterFactoryCb SetMetadataConfig::createFilterFactoryFromProtoWithServerContextTyped( const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config, - const std::string&, Server::Configuration::ServerFactoryContext&) { - ConfigSharedPtr filter_config(std::make_shared(proto_config)); + const std::string& stats_prefix, Server::Configuration::ServerFactoryContext& server_context) { + ConfigSharedPtr filter_config( + std::make_shared(proto_config, server_context.scope(), stats_prefix)); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter( diff --git a/source/extensions/filters/http/set_metadata/set_metadata_filter.cc b/source/extensions/filters/http/set_metadata/set_metadata_filter.cc index 8d824a3190e7..129fba5af7d2 100644 --- a/source/extensions/filters/http/set_metadata/set_metadata_filter.cc +++ b/source/extensions/filters/http/set_metadata/set_metadata_filter.cc @@ -13,9 +13,34 @@ namespace Extensions { namespace HttpFilters { namespace SetMetadataFilter { -Config::Config(const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config) { - namespace_ = proto_config.metadata_namespace(); - value_ = proto_config.value(); +Config::Config(const envoy::extensions::filters::http::set_metadata::v3::Config& proto_config, + Stats::Scope& scope, const std::string& stats_prefix) + : stats_(generateStats(stats_prefix, scope)) { + if (proto_config.has_value() && !proto_config.metadata_namespace().empty()) { + UntypedMetadataEntry deprecated_api_val{true, proto_config.metadata_namespace(), + proto_config.value()}; + untyped_.emplace_back(deprecated_api_val); + } + + for (const auto& metadata : proto_config.metadata()) { + if (metadata.has_value()) { + UntypedMetadataEntry untyped_entry{metadata.allow_overwrite(), metadata.metadata_namespace(), + metadata.value()}; + untyped_.emplace_back(untyped_entry); + } else if (metadata.has_typed_value()) { + TypedMetadataEntry typed_entry{metadata.allow_overwrite(), metadata.metadata_namespace(), + metadata.typed_value()}; + typed_.emplace_back(typed_entry); + } else { + ENVOY_LOG(warn, "set_metadata filter configuration contains metadata entries without value " + "or typed_value"); + } + } +} + +FilterStats Config::generateStats(const std::string& prefix, Stats::Scope& scope) { + std::string final_prefix = prefix + "set_metadata."; + return {ALL_SET_METADATA_FILTER_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; } SetMetadataFilter::SetMetadataFilter(const ConfigSharedPtr config) : config_(config) {} @@ -23,13 +48,48 @@ SetMetadataFilter::SetMetadataFilter(const ConfigSharedPtr config) : config_(con SetMetadataFilter::~SetMetadataFilter() = default; Http::FilterHeadersStatus SetMetadataFilter::decodeHeaders(Http::RequestHeaderMap&, bool) { - const absl::string_view metadata_namespace = config_->metadataNamespace(); - auto& metadata = *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_filter_metadata(); - ProtobufWkt::Struct& org_fields = - metadata[toStdStringView(metadata_namespace)]; // NOLINT(std::string_view) - const ProtobufWkt::Struct& to_merge = config_->value(); - StructUtil::update(org_fields, to_merge); + // Add configured untyped metadata. + if (!config_->untyped().empty()) { + auto& mut_untyped_metadata = + *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_filter_metadata(); + + for (const auto& entry : config_->untyped()) { + if (!mut_untyped_metadata.contains(entry.metadata_namespace)) { + // Insert the new entry. + mut_untyped_metadata[entry.metadata_namespace] = entry.value; + } else if (entry.allow_overwrite) { + // Get the existing metadata at this key for merging. + ProtobufWkt::Struct& orig_fields = mut_untyped_metadata[entry.metadata_namespace]; + const auto& to_merge = entry.value; + + // Merge the new metadata into the existing metadata. + StructUtil::update(orig_fields, to_merge); + } else { + // The entry exists, and we are not allowed to overwrite -- emit a stat. + config_->stats().overwrite_denied_.inc(); + } + } + } + + // Add configured typed metadata. + if (!config_->typed().empty()) { + auto& mut_typed_metadata = + *decoder_callbacks_->streamInfo().dynamicMetadata().mutable_typed_filter_metadata(); + + for (const auto& entry : config_->typed()) { + if (!mut_typed_metadata.contains(entry.metadata_namespace)) { + // Insert the new entry. + mut_typed_metadata[entry.metadata_namespace] = entry.value; + } else if (entry.allow_overwrite) { + // Overwrite the existing typed metadata at this key. + mut_typed_metadata[entry.metadata_namespace] = entry.value; + } else { + // The entry exists, and we are not allowed to overwrite -- emit a stat. + config_->stats().overwrite_denied_.inc(); + } + } + } return Http::FilterHeadersStatus::Continue; } diff --git a/source/extensions/filters/http/set_metadata/set_metadata_filter.h b/source/extensions/filters/http/set_metadata/set_metadata_filter.h index 8611980117f2..e293690a32ab 100644 --- a/source/extensions/filters/http/set_metadata/set_metadata_filter.h +++ b/source/extensions/filters/http/set_metadata/set_metadata_filter.h @@ -5,28 +5,51 @@ #include #include "envoy/extensions/filters/http/set_metadata/v3/set_metadata.pb.h" +#include "envoy/stats/stats_macros.h" #include "source/common/common/logger.h" #include "source/extensions/filters/http/common/pass_through_filter.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" namespace Envoy { namespace Extensions { namespace HttpFilters { namespace SetMetadataFilter { +#define ALL_SET_METADATA_FILTER_STATS(COUNTER) COUNTER(overwrite_denied) + +struct FilterStats { + ALL_SET_METADATA_FILTER_STATS(GENERATE_COUNTER_STRUCT) +}; + +struct UntypedMetadataEntry { + bool allow_overwrite{}; + std::string metadata_namespace; + ProtobufWkt::Struct value; +}; +struct TypedMetadataEntry { + bool allow_overwrite{}; + std::string metadata_namespace; + ProtobufWkt::Any value; +}; class Config : public ::Envoy::Router::RouteSpecificFilterConfig, public Logger::Loggable { public: - Config(const envoy::extensions::filters::http::set_metadata::v3::Config& config); + Config(const envoy::extensions::filters::http::set_metadata::v3::Config& config, + Stats::Scope& scope, const std::string& stats_prefix); - absl::string_view metadataNamespace() const { return namespace_; } - const ProtobufWkt::Struct& value() { return value_; } + const std::vector& untyped() { return untyped_; } + const std::vector& typed() { return typed_; } + const FilterStats& stats() const { return stats_; } private: - std::string namespace_; - ProtobufWkt::Struct value_; + static FilterStats generateStats(const std::string& prefix, Stats::Scope& scope); + + std::vector untyped_; + std::vector typed_; + FilterStats stats_; }; using ConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/stateful_session/BUILD b/source/extensions/filters/http/stateful_session/BUILD index 7ae11125ae57..4c158154d75c 100644 --- a/source/extensions/filters/http/stateful_session/BUILD +++ b/source/extensions/filters/http/stateful_session/BUILD @@ -36,6 +36,7 @@ envoy_cc_extension( "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", "//source/extensions/filters/http/stateful_session:stateful_session_lib", + "//source/server:generic_factory_context_lib", "@envoy_api//envoy/extensions/filters/http/stateful_session/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/stateful_session/config.cc b/source/extensions/filters/http/stateful_session/config.cc index aa5469d9f816..3883171e0b40 100644 --- a/source/extensions/filters/http/stateful_session/config.cc +++ b/source/extensions/filters/http/stateful_session/config.cc @@ -4,6 +4,8 @@ #include "envoy/registry/registry.h" +#include "source/server/generic_factory_context.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { @@ -12,8 +14,8 @@ namespace StatefulSession { Http::FilterFactoryCb StatefulSessionFactoryConfig::createFilterFactoryFromProtoTyped( const ProtoConfig& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { - auto filter_config(std::make_shared(proto_config, context)); + auto filter_config(std::make_shared(proto_config, context)); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(Http::StreamFilterSharedPtr{new StatefulSession(filter_config)}); }; @@ -22,8 +24,10 @@ Http::FilterFactoryCb StatefulSessionFactoryConfig::createFilterFactoryFromProto Router::RouteSpecificFilterConfigConstSharedPtr StatefulSessionFactoryConfig::createRouteSpecificFilterConfigTyped( const PerRouteProtoConfig& proto_config, Server::Configuration::ServerFactoryContext& context, - ProtobufMessage::ValidationVisitor&) { - return std::make_shared(proto_config, context); + ProtobufMessage::ValidationVisitor& visitor) { + Server::GenericFactoryContextImpl generic_context(context, visitor); + + return std::make_shared(proto_config, generic_context); } REGISTER_FACTORY(StatefulSessionFactoryConfig, Server::Configuration::NamedHttpFilterConfigFactory); diff --git a/source/extensions/filters/http/stateful_session/stateful_session.cc b/source/extensions/filters/http/stateful_session/stateful_session.cc index f39921080e71..ff6bc670d27b 100644 --- a/source/extensions/filters/http/stateful_session/stateful_session.cc +++ b/source/extensions/filters/http/stateful_session/stateful_session.cc @@ -24,7 +24,8 @@ class EmptySessionStateFactory : public Envoy::Http::SessionStateFactory { } // namespace StatefulSessionConfig::StatefulSessionConfig(const ProtoConfig& config, - Server::Configuration::CommonFactoryContext& context) { + Server::Configuration::GenericFactoryContext& context) + : strict_(config.strict()) { if (!config.has_session_state()) { factory_ = std::make_shared(); return; @@ -41,7 +42,7 @@ StatefulSessionConfig::StatefulSessionConfig(const ProtoConfig& config, } PerRouteStatefulSession::PerRouteStatefulSession( - const PerRouteProtoConfig& config, Server::Configuration::CommonFactoryContext& context) { + const PerRouteProtoConfig& config, Server::Configuration::GenericFactoryContext& context) { if (config.override_case() == PerRouteProtoConfig::kDisabled) { disabled_ = true; return; @@ -66,7 +67,8 @@ Http::FilterHeadersStatus StatefulSession::decodeHeaders(Http::RequestHeaderMap& } if (auto upstream_address = session_state_->upstreamAddress(); upstream_address.has_value()) { - decoder_callbacks_->setUpstreamOverrideHost(upstream_address.value()); + decoder_callbacks_->setUpstreamOverrideHost( + std::make_pair(upstream_address.value(), config->isStrict())); } return Http::FilterHeadersStatus::Continue; } diff --git a/source/extensions/filters/http/stateful_session/stateful_session.h b/source/extensions/filters/http/stateful_session/stateful_session.h index ede031a2d862..f4ad02de2678 100644 --- a/source/extensions/filters/http/stateful_session/stateful_session.h +++ b/source/extensions/filters/http/stateful_session/stateful_session.h @@ -26,22 +26,25 @@ using PerRouteProtoConfig = class StatefulSessionConfig { public: StatefulSessionConfig(const ProtoConfig& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::GenericFactoryContext& context); Http::SessionStatePtr createSessionState(const Http::RequestHeaderMap& headers) const { ASSERT(factory_ != nullptr); return factory_->create(headers); } + bool isStrict() const { return strict_; } + private: Http::SessionStateFactorySharedPtr factory_; + bool strict_{false}; }; using StatefulSessionConfigSharedPtr = std::shared_ptr; class PerRouteStatefulSession : public Router::RouteSpecificFilterConfig { public: PerRouteStatefulSession(const PerRouteProtoConfig& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::GenericFactoryContext& context); bool disabled() const { return disabled_; } StatefulSessionConfig* statefuleSessionConfig() const { return config_.get(); } diff --git a/source/extensions/filters/http/tap/config.cc b/source/extensions/filters/http/tap/config.cc index c9051d7c68b6..81e64ee372d0 100644 --- a/source/extensions/filters/http/tap/config.cc +++ b/source/extensions/filters/http/tap/config.cc @@ -32,10 +32,12 @@ class HttpTapConfigFactoryImpl : public Extensions::Common::Tap::TapConfigFactor Http::FilterFactoryCb TapFilterFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::tap::v3::Tap& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + FilterConfigSharedPtr filter_config(new FilterConfigImpl( proto_config, stats_prefix, std::make_unique(context), - context.scope(), context.admin(), context.singletonManager(), context.threadLocal(), - context.mainThreadDispatcher())); + context.scope(), server_context.admin(), server_context.singletonManager(), + server_context.threadLocal(), server_context.mainThreadDispatcher())); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { auto filter = std::make_shared(filter_config); callbacks.addStreamFilter(filter); diff --git a/source/extensions/filters/http/tap/tap_config.h b/source/extensions/filters/http/tap/tap_config.h index cfa0f728eebe..d318686b608e 100644 --- a/source/extensions/filters/http/tap/tap_config.h +++ b/source/extensions/filters/http/tap/tap_config.h @@ -71,7 +71,7 @@ class HttpTapConfig : public virtual Extensions::Common::Tap::TapConfig { */ virtual HttpPerRequestTapperPtr createPerRequestTapper(const envoy::extensions::filters::http::tap::v3::Tap& tap_config, - uint64_t stream_id) PURE; + uint64_t stream_id, OptRef connection) PURE; /** * @return time source to use for timestamp diff --git a/source/extensions/filters/http/tap/tap_config_impl.cc b/source/extensions/filters/http/tap/tap_config_impl.cc index ecc920726a87..d26d5ebe1485 100644 --- a/source/extensions/filters/http/tap/tap_config_impl.cc +++ b/source/extensions/filters/http/tap/tap_config_impl.cc @@ -5,6 +5,7 @@ #include "envoy/data/tap/v3/http.pb.h" #include "source/common/common/assert.h" +#include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/runtime/runtime_features.h" @@ -32,11 +33,13 @@ HttpTapConfigImpl::HttpTapConfigImpl(const envoy::config::tap::v3::TapConfig& pr Common::Tap::Sink* admin_streamer, Server::Configuration::FactoryContext& context) : TapCommon::TapConfigBaseImpl(std::move(proto_config), admin_streamer, context), - time_source_(context.mainThreadDispatcher().timeSource()) {} + time_source_(context.serverFactoryContext().mainThreadDispatcher().timeSource()) {} HttpPerRequestTapperPtr HttpTapConfigImpl::createPerRequestTapper( - const envoy::extensions::filters::http::tap::v3::Tap& tap_config, uint64_t stream_id) { - return std::make_unique(shared_from_this(), tap_config, stream_id); + const envoy::extensions::filters::http::tap::v3::Tap& tap_config, uint64_t stream_id, + OptRef connection) { + return std::make_unique(shared_from_this(), tap_config, stream_id, + connection); } void HttpPerRequestTapperImpl::streamRequestHeaders() { @@ -188,6 +191,22 @@ bool HttpPerRequestTapperImpl::onDestroyLog() { response_trailers_->iterate(fillHeaderList(http_trace.mutable_response()->mutable_trailers())); } + if (should_record_downstream_connection_ && connection_.has_value()) { + + envoy::config::core::v3::Address downstream_local_address; + envoy::config::core::v3::Address downstream_remote_address; + + Envoy::Network::Utility::addressToProtobufAddress( + *connection_->connectionInfoProvider().localAddress(), downstream_local_address); + Envoy::Network::Utility::addressToProtobufAddress( + *connection_->connectionInfoProvider().remoteAddress(), downstream_remote_address); + + http_trace.mutable_downstream_connection()->mutable_local_address()->MergeFrom( + downstream_local_address); + http_trace.mutable_downstream_connection()->mutable_remote_address()->MergeFrom( + downstream_remote_address); + } + ENVOY_LOG(debug, "submitting buffered trace sink"); // move is safe as onDestroyLog is the last method called. sink_handle_->submitTrace(std::move(buffered_full_trace_)); @@ -234,7 +253,6 @@ void HttpPerRequestTapperImpl::onBody( data, 0, data.length()); } } - } // namespace TapFilter } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/tap/tap_config_impl.h b/source/extensions/filters/http/tap/tap_config_impl.h index 62042b449982..042b6bbe91de 100644 --- a/source/extensions/filters/http/tap/tap_config_impl.h +++ b/source/extensions/filters/http/tap/tap_config_impl.h @@ -26,7 +26,7 @@ class HttpTapConfigImpl : public Extensions::Common::Tap::TapConfigBaseImpl, // TapFilter::HttpTapConfig HttpPerRequestTapperPtr createPerRequestTapper(const envoy::extensions::filters::http::tap::v3::Tap& tap_config, - uint64_t stream_id) override; + uint64_t stream_id, OptRef connection) override; TimeSource& timeSource() const override { return time_source_; } @@ -38,11 +38,12 @@ class HttpPerRequestTapperImpl : public HttpPerRequestTapper, Logger::Loggable connection) : config_(std::move(config)), should_record_headers_received_time_(tap_config.record_headers_received_time()), + should_record_downstream_connection_(tap_config.record_downstream_connection()), stream_id_(stream_id), sink_handle_(config_->createPerTapSinkHandleManager(stream_id)), - statuses_(config_->createMatchStatusVector()) { + statuses_(config_->createMatchStatusVector()), connection_(connection) { config_->rootMatcher().onNewStream(statuses_); } @@ -85,17 +86,19 @@ class HttpPerRequestTapperImpl : public HttpPerRequestTapper, Logger::Loggable( - config_->timeSource().systemTime().time_since_epoch()) - .count(); + void setTimeStamp(long& timestamp) { + timestamp = std::chrono::duration_cast( + config_->timeSource().systemTime().time_since_epoch()) + .count(); } HttpTapConfigSharedPtr config_; const bool should_record_headers_received_time_; + const bool should_record_downstream_connection_; const uint64_t stream_id_; Extensions::Common::Tap::PerTapSinkHandleManagerPtr sink_handle_; Extensions::Common::Tap::Matcher::MatchStatusVector statuses_; + OptRef connection_; bool started_streaming_trace_{}; long request_headers_received_time_; long response_headers_received_time_; diff --git a/source/extensions/filters/http/tap/tap_filter.cc b/source/extensions/filters/http/tap/tap_filter.cc index d701a882873c..51719e068368 100644 --- a/source/extensions/filters/http/tap/tap_filter.cc +++ b/source/extensions/filters/http/tap/tap_filter.cc @@ -69,9 +69,7 @@ Http::FilterTrailersStatus Filter::encodeTrailers(Http::ResponseTrailerMap& trai return Http::FilterTrailersStatus::Continue; } -void Filter::log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&, - AccessLog::AccessLogType) { +void Filter::log(const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&) { if (tapper_ != nullptr && tapper_->onDestroyLog()) { config_->stats().rq_tapped_.inc(); } diff --git a/source/extensions/filters/http/tap/tap_filter.h b/source/extensions/filters/http/tap/tap_filter.h index 8fff05d93262..b779e6f8426b 100644 --- a/source/extensions/filters/http/tap/tap_filter.h +++ b/source/extensions/filters/http/tap/tap_filter.h @@ -99,8 +99,13 @@ class Filter : public Http::StreamFilter, public AccessLog::Instance { Http::FilterTrailersStatus decodeTrailers(Http::RequestTrailerMap& trailers) override; void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { HttpTapConfigSharedPtr config = config_->currentConfig(); - tapper_ = config ? config->createPerRequestTapper(config_->getTapConfig(), callbacks.streamId()) - : nullptr; + if (config != nullptr) { + auto streamId = callbacks.streamId(); + auto connection = callbacks.connection(); + tapper_ = config->createPerRequestTapper(config_->getTapConfig(), streamId, connection); + } else { + tapper_ = nullptr; + } } // Http::StreamEncoderFilter @@ -117,11 +122,7 @@ class Filter : public Http::StreamFilter, public AccessLog::Instance { void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks&) override {} // AccessLog::Instance - void log(const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) override; + void log(const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&) override; private: FilterConfigSharedPtr config_; diff --git a/source/extensions/filters/http/wasm/config.cc b/source/extensions/filters/http/wasm/config.cc index e50184df677a..b949606f30bc 100644 --- a/source/extensions/filters/http/wasm/config.cc +++ b/source/extensions/filters/http/wasm/config.cc @@ -16,7 +16,7 @@ namespace Wasm { Http::FilterFactoryCb WasmFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::wasm::v3::Wasm& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { - context.api().customStatNamespaces().registerStatNamespace( + context.serverFactoryContext().api().customStatNamespaces().registerStatNamespace( Extensions::Common::Wasm::CustomStatNamespace); auto filter_config = std::make_shared(proto_config, context); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { diff --git a/source/extensions/filters/http/wasm/wasm_filter.cc b/source/extensions/filters/http/wasm/wasm_filter.cc index 75e06e69b735..c9f106ccaa57 100644 --- a/source/extensions/filters/http/wasm/wasm_filter.cc +++ b/source/extensions/filters/http/wasm/wasm_filter.cc @@ -6,11 +6,14 @@ namespace HttpFilters { namespace Wasm { FilterConfig::FilterConfig(const envoy::extensions::filters::http::wasm::v3::Wasm& config, - Server::Configuration::FactoryContext& context) - : tls_slot_(ThreadLocal::TypedSlot::makeUnique( - context.threadLocal())) { + Server::Configuration::FactoryContext& context) { + auto& server = context.serverFactoryContext(); + tls_slot_ = ThreadLocal::TypedSlot::makeUnique( + server.threadLocal()); + const auto plugin = std::make_shared( - config.config(), context.direction(), context.localInfo(), &context.listenerMetadata()); + config.config(), context.listenerInfo().direction(), server.localInfo(), + &context.listenerInfo().metadata()); auto callback = [plugin, this](const Common::Wasm::WasmHandleSharedPtr& base_wasm) { // NB: the Slot set() call doesn't complete inline, so all arguments must outlive this call. @@ -20,9 +23,9 @@ FilterConfig::FilterConfig(const envoy::extensions::filters::http::wasm::v3::Was }); }; - if (!Common::Wasm::createWasm(plugin, context.scope().createScope(""), context.clusterManager(), - context.initManager(), context.mainThreadDispatcher(), - context.api(), context.lifecycleNotifier(), remote_data_provider_, + if (!Common::Wasm::createWasm(plugin, context.scope().createScope(""), server.clusterManager(), + context.initManager(), server.mainThreadDispatcher(), server.api(), + server.lifecycleNotifier(), remote_data_provider_, std::move(callback))) { throw Common::Wasm::WasmException( fmt::format("Unable to create Wasm HTTP filter {}", plugin->name_)); diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index 0049279f0300..8e33a0699c0a 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -16,6 +16,8 @@ class HttpFilterNameValues { const std::string Buffer = "envoy.filters.http.buffer"; // Bandwidth limit filter const std::string BandwidthLimit = "envoy.filters.http.bandwidth_limit"; + // Basic Auth filter + const std::string BasicAuth = "envoy.filters.http.basic_auth"; // Cache filter const std::string Cache = "envoy.filters.http.cache"; // CDN Loop filter @@ -44,6 +46,8 @@ class HttpFilterNameValues { const std::string GrpcJsonTranscoder = "envoy.filters.http.grpc_json_transcoder"; // GRPC web filter const std::string GrpcWeb = "envoy.filters.http.grpc_web"; + // GRPC Field Extraction filter + const std::string GrpcFieldExtraction = "envoy.filters.http.grpc_field_extraction"; // GRPC http1 reverse bridge filter const std::string GrpcHttp1ReverseBridge = "envoy.filters.http.grpc_http1_reverse_bridge"; // GRPC telemetry diff --git a/source/extensions/filters/listener/local_ratelimit/config.cc b/source/extensions/filters/listener/local_ratelimit/config.cc index aa6a5f6b6c17..5a2dbde8252c 100644 --- a/source/extensions/filters/listener/local_ratelimit/config.cc +++ b/source/extensions/filters/listener/local_ratelimit/config.cc @@ -29,7 +29,8 @@ class LocalRateLimitConfigFactory : public Server::Configuration::NamedListenerF message, context.messageValidationVisitor()); FilterConfigSharedPtr config = std::make_shared( - proto_config, context.mainThreadDispatcher(), context.scope(), context.runtime()); + proto_config, context.serverFactoryContext().mainThreadDispatcher(), context.scope(), + context.serverFactoryContext().runtime()); return [listener_filter_matcher, config](Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter(listener_filter_matcher, std::make_unique(config)); diff --git a/source/extensions/filters/listener/original_dst/BUILD b/source/extensions/filters/listener/original_dst/BUILD index a7954c898977..0479242f70d5 100644 --- a/source/extensions/filters/listener/original_dst/BUILD +++ b/source/extensions/filters/listener/original_dst/BUILD @@ -17,22 +17,31 @@ envoy_cc_library( srcs = ["original_dst.cc"], hdrs = ["original_dst.h"], deps = [ + "//envoy/common:hashable_interface", "//envoy/network:filter_interface", "//envoy/network:listen_socket_interface", + "//envoy/stream_info:filter_state_interface", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", + "//source/common/config:metadata_lib", + "//source/common/network:filter_state_dst_address_lib", "//source/common/network:upstream_socket_options_filter_state_lib", "//source/common/network:utility_lib", + "//source/common/singleton:const_singleton", ], ) envoy_cc_extension( name = "config", srcs = ["config.cc"], + extra_visibility = [ + "//test/common/listener_manager:__subpackages__", + ], deps = [ ":original_dst_lib", "//envoy/registry", "//envoy/server:filter_config_interface", + "//source/common/network:filter_state_dst_address_lib", "@envoy_api//envoy/extensions/filters/listener/original_dst/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/listener/original_dst/config.cc b/source/extensions/filters/listener/original_dst/config.cc index 3fe3f714430a..974febbb99c1 100644 --- a/source/extensions/filters/listener/original_dst/config.cc +++ b/source/extensions/filters/listener/original_dst/config.cc @@ -5,6 +5,7 @@ #include "envoy/registry/registry.h" #include "envoy/server/filter_config.h" +#include "source/common/network/filter_state_dst_address.h" #include "source/extensions/filters/listener/original_dst/original_dst.h" namespace Envoy { @@ -28,7 +29,7 @@ class OriginalDstConfigFactory : public Server::Configuration::NamedListenerFilt // 1. The platform supports the original destination feature // 2. The `traffic_direction` property is set on the listener. This is required to redirect the // traffic. - if (context.listenerConfig().direction() == envoy::config::core::v3::UNSPECIFIED) { + if (context.listenerInfo().direction() == envoy::config::core::v3::UNSPECIFIED) { throw EnvoyException("[Windows] Setting original destination filter on a listener without " "specifying the traffic_direction." "Configure the traffic_direction listener option"); @@ -39,7 +40,7 @@ class OriginalDstConfigFactory : public Server::Configuration::NamedListenerFilt } #endif - return [listener_filter_matcher, traffic_direction = context.listenerConfig().direction()]( + return [listener_filter_matcher, traffic_direction = context.listenerInfo().direction()]( Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter(listener_filter_matcher, std::make_unique(traffic_direction)); @@ -50,7 +51,7 @@ class OriginalDstConfigFactory : public Server::Configuration::NamedListenerFilt return std::make_unique(); } - std::string name() const override { return "envoy.filters.listener.original_dst"; } + std::string name() const override { return FilterNames::get().Name; } }; /** @@ -59,6 +60,20 @@ class OriginalDstConfigFactory : public Server::Configuration::NamedListenerFilt REGISTER_FACTORY(OriginalDstConfigFactory, Server::Configuration::NamedListenerFilterConfigFactory){ "envoy.listener.original_dst"}; +class OriginalDstLocalFilterStateFactory : public Network::BaseAddressObjectFactory { +public: + std::string name() const override { return FilterNames::get().LocalFilterStateKey; } +}; + +REGISTER_FACTORY(OriginalDstLocalFilterStateFactory, StreamInfo::FilterState::ObjectFactory); + +class OriginalDstRemoteFilterStateFactory : public Network::BaseAddressObjectFactory { +public: + std::string name() const override { return FilterNames::get().RemoteFilterStateKey; } +}; + +REGISTER_FACTORY(OriginalDstRemoteFilterStateFactory, StreamInfo::FilterState::ObjectFactory); + } // namespace OriginalDst } // namespace ListenerFilters } // namespace Extensions diff --git a/source/extensions/filters/listener/original_dst/original_dst.cc b/source/extensions/filters/listener/original_dst/original_dst.cc index 7f8c63f5f271..1a60b7e6eb51 100644 --- a/source/extensions/filters/listener/original_dst/original_dst.cc +++ b/source/extensions/filters/listener/original_dst/original_dst.cc @@ -3,6 +3,8 @@ #include "envoy/network/listen_socket.h" #include "source/common/common/assert.h" +#include "source/common/config/metadata.h" +#include "source/common/network/filter_state_dst_address.h" #include "source/common/network/socket_option_factory.h" #include "source/common/network/upstream_socket_options_filter_state.h" #include "source/common/network/utility.h" @@ -20,7 +22,8 @@ Network::FilterStatus OriginalDstFilter::onAccept(Network::ListenerFilterCallbac ENVOY_LOG(trace, "original_dst: new connection accepted"); Network::ConnectionSocket& socket = cb.socket(); - if (socket.addressType() == Network::Address::Type::Ip) { + switch (socket.addressType()) { + case Network::Address::Type::Ip: { Network::Address::InstanceConstSharedPtr original_local_address = getOriginalDst(socket); // A listener that has the use_original_dst flag set to true can still receive // connections that are NOT redirected using iptables. If a connection was not redirected, @@ -68,6 +71,42 @@ Network::FilterStatus OriginalDstFilter::onAccept(Network::ListenerFilterCallbac // Restore the local address to the original one. socket.connectionInfoProvider().restoreLocalAddress(original_local_address); } + break; + } + case Network::Address::Type::EnvoyInternal: { + const auto& local_value = Config::Metadata::metadataValue( + &cb.dynamicMetadata(), FilterNames::get().Name, FilterNames::get().LocalField); + if (local_value.has_string_value()) { + const auto local_address = Envoy::Network::Utility::parseInternetAddressAndPortNoThrow( + local_value.string_value(), /*v6only=*/false); + if (local_address) { + ENVOY_LOG_MISC(debug, "original_dst: set destination from metadata to {}", + local_address->asString()); + socket.connectionInfoProvider().restoreLocalAddress(local_address); + } else { + ENVOY_LOG_MISC(debug, "original_dst: failed to parse address: {}", + local_value.DebugString()); + } + } else { + const auto* local_object = cb.filterState().getDataReadOnly( + FilterNames::get().LocalFilterStateKey); + if (local_object) { + ENVOY_LOG_MISC(debug, "original_dst: set destination from filter state to {}", + local_object->address()->asString()); + socket.connectionInfoProvider().restoreLocalAddress(local_object->address()); + } + } + const auto* remote_object = cb.filterState().getDataReadOnly( + FilterNames::get().RemoteFilterStateKey); + if (remote_object) { + ENVOY_LOG_MISC(debug, "original_dst: set source from filter state to {}", + remote_object->address()->asString()); + socket.connectionInfoProvider().setRemoteAddress(remote_object->address()); + } + break; + } + default: + break; } return Network::FilterStatus::Continue; diff --git a/source/extensions/filters/listener/original_dst/original_dst.h b/source/extensions/filters/listener/original_dst/original_dst.h index a0de9d2024fb..d7340cafc95f 100644 --- a/source/extensions/filters/listener/original_dst/original_dst.h +++ b/source/extensions/filters/listener/original_dst/original_dst.h @@ -1,14 +1,27 @@ #pragma once +#include "envoy/common/hashable.h" #include "envoy/network/filter.h" +#include "envoy/stream_info/filter_state.h" #include "source/common/common/logger.h" +#include "source/common/singleton/const_singleton.h" namespace Envoy { namespace Extensions { namespace ListenerFilters { namespace OriginalDst { +class FilterNameValues { +public: + const std::string Name = "envoy.filters.listener.original_dst"; + const std::string LocalField = "local"; + const std::string LocalFilterStateKey = "envoy.filters.listener.original_dst.local_ip"; + const std::string RemoteFilterStateKey = "envoy.filters.listener.original_dst.remote_ip"; +}; + +using FilterNames = ConstSingleton; + /** * Implementation of an original destination listener filter. */ diff --git a/source/extensions/filters/listener/proxy_protocol/BUILD b/source/extensions/filters/listener/proxy_protocol/BUILD index 7ddeb6e56d9c..7401b9156501 100644 --- a/source/extensions/filters/listener/proxy_protocol/BUILD +++ b/source/extensions/filters/listener/proxy_protocol/BUILD @@ -42,6 +42,9 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], + extra_visibility = [ + "//test/common/listener_manager:__subpackages__", + ], deps = [ "//envoy/registry", "//envoy/server:filter_config_interface", diff --git a/source/extensions/filters/listener/tls_inspector/BUILD b/source/extensions/filters/listener/tls_inspector/BUILD index 1ddd9ce98f2c..edbdc7df6153 100644 --- a/source/extensions/filters/listener/tls_inspector/BUILD +++ b/source/extensions/filters/listener/tls_inspector/BUILD @@ -35,6 +35,9 @@ envoy_cc_library( envoy_cc_extension( name = "config", srcs = ["config.cc"], + extra_visibility = [ + "//test/common/listener_manager:__subpackages__", + ], deps = [ "//envoy/registry", "//envoy/server:filter_config_interface", diff --git a/source/extensions/filters/network/common/redis/supported_commands.h b/source/extensions/filters/network/common/redis/supported_commands.h index c6b3d81887e7..560689eba715 100644 --- a/source/extensions/filters/network/common/redis/supported_commands.h +++ b/source/extensions/filters/network/common/redis/supported_commands.h @@ -22,14 +22,14 @@ struct SupportedCommands { CONSTRUCT_ON_FIRST_USE( absl::flat_hash_set, "append", "bitcount", "bitfield", "bitpos", "decr", "decrby", "dump", "expire", "expireat", "geoadd", "geodist", "geohash", "geopos", - "georadius_ro", "georadiusbymember_ro", "get", "getbit", "getrange", "getset", "hdel", - "hexists", "hget", "hgetall", "hincrby", "hincrbyfloat", "hkeys", "hlen", "hmget", "hmset", - "hscan", "hset", "hsetnx", "hstrlen", "hvals", "incr", "incrby", "incrbyfloat", "lindex", - "linsert", "llen", "lmove", "lpop", "lpush", "lpushx", "lrange", "lrem", "lset", "ltrim", - "persist", "pexpire", "pexpireat", "pfadd", "pfcount", "psetex", "pttl", "restore", "rpop", - "rpush", "rpushx", "sadd", "scard", "set", "setbit", "setex", "setnx", "setrange", + "georadius_ro", "georadiusbymember_ro", "get", "getbit", "getdel", "getrange", "getset", + "hdel", "hexists", "hget", "hgetall", "hincrby", "hincrbyfloat", "hkeys", "hlen", "hmget", + "hmset", "hscan", "hset", "hsetnx", "hstrlen", "hvals", "incr", "incrby", "incrbyfloat", + "lindex", "linsert", "llen", "lmove", "lpop", "lpush", "lpushx", "lrange", "lrem", "lset", + "ltrim", "persist", "pexpire", "pexpireat", "pfadd", "pfcount", "psetex", "pttl", "restore", + "rpop", "rpush", "rpushx", "sadd", "scard", "set", "setbit", "setex", "setnx", "setrange", "sismember", "smembers", "spop", "srandmember", "srem", "sscan", "strlen", "ttl", "type", - "zadd", "zcard", "zcount", "zincrby", "zlexcount", "zpopmin", "zpopmax", "zrange", + "watch", "zadd", "zcard", "zcount", "zincrby", "zlexcount", "zpopmin", "zpopmax", "zrange", "zrangebylex", "zrangebyscore", "zrank", "zrem", "zremrangebylex", "zremrangebyrank", "zremrangebyscore", "zrevrange", "zrevrangebylex", "zrevrangebyscore", "zrevrank", "zscan", "zscore"); @@ -92,12 +92,12 @@ struct SupportedCommands { static const absl::flat_hash_set& writeCommands() { CONSTRUCT_ON_FIRST_USE(absl::flat_hash_set, "append", "bitfield", "decr", "decrby", "del", "discard", "exec", "expire", "expireat", "eval", "evalsha", - "geoadd", "hdel", "hincrby", "hincrbyfloat", "hmset", "hset", "hsetnx", - "incr", "incrby", "incrbyfloat", "linsert", "lmove", "lpop", "lpush", - "lpushx", "lrem", "lset", "ltrim", "mset", "multi", "persist", "pexpire", - "pexpireat", "pfadd", "psetex", "restore", "rpop", "rpush", "rpushx", - "sadd", "set", "setbit", "setex", "setnx", "setrange", "spop", "srem", - "zadd", "zincrby", "touch", "zpopmin", "zpopmax", "zrem", + "geoadd", "getdel", "hdel", "hincrby", "hincrbyfloat", "hmset", "hset", + "hsetnx", "incr", "incrby", "incrbyfloat", "linsert", "lmove", "lpop", + "lpush", "lpushx", "lrem", "lset", "ltrim", "mset", "multi", "persist", + "pexpire", "pexpireat", "pfadd", "psetex", "restore", "rpop", "rpush", + "rpushx", "sadd", "set", "setbit", "setex", "setnx", "setrange", "spop", + "srem", "zadd", "zincrby", "touch", "zpopmin", "zpopmax", "zrem", "zremrangebylex", "zremrangebyrank", "zremrangebyscore", "unlink"); } diff --git a/source/extensions/filters/network/connection_limit/config.cc b/source/extensions/filters/network/connection_limit/config.cc index 4fbc51d28953..62860b03524e 100644 --- a/source/extensions/filters/network/connection_limit/config.cc +++ b/source/extensions/filters/network/connection_limit/config.cc @@ -13,7 +13,8 @@ namespace ConnectionLimitFilter { Network::FilterFactoryCb ConnectionLimitConfigFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::connection_limit::v3::ConnectionLimit& proto_config, Server::Configuration::FactoryContext& context) { - ConfigSharedPtr filter_config(new Config(proto_config, context.scope(), context.runtime())); + ConfigSharedPtr filter_config( + new Config(proto_config, context.scope(), context.serverFactoryContext().runtime())); return [filter_config](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/network/direct_response/config.cc b/source/extensions/filters/network/direct_response/config.cc index 943de3ee791b..5d39492f590a 100644 --- a/source/extensions/filters/network/direct_response/config.cc +++ b/source/extensions/filters/network/direct_response/config.cc @@ -25,8 +25,10 @@ class DirectResponseConfigFactory Network::FilterFactoryCb createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::direct_response::v3::Config& config, Server::Configuration::FactoryContext& context) override { - return [config, &context](Network::FilterManager& filter_manager) -> void { - auto content = Config::DataSource::read(config.response(), true, context.api()); + auto content = + Config::DataSource::read(config.response(), true, context.serverFactoryContext().api()); + + return [content](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(content)); }; } diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.cc b/source/extensions/filters/network/dubbo_proxy/active_message.cc index 9d019f747f19..841b3fd82f86 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.cc +++ b/source/extensions/filters/network/dubbo_proxy/active_message.cc @@ -15,8 +15,7 @@ ActiveResponseDecoder::ActiveResponseDecoder(ActiveMessage& parent, DubboFilterS ProtocolPtr&& protocol) : parent_(parent), stats_(stats), response_connection_(connection), protocol_(std::move(protocol)), - decoder_(std::make_unique(*protocol_, *this)), complete_(false), - response_status_(DubboFilters::UpstreamResponseStatus::MoreData) {} + decoder_(std::make_unique(*protocol_, *this)), complete_(false) {} DubboFilters::UpstreamResponseStatus ActiveResponseDecoder::onData(Buffer::Instance& data) { ENVOY_LOG(debug, "dubbo response: the received reply data length is {}", data.length()); diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.h b/source/extensions/filters/network/dubbo_proxy/active_message.h index 6287be7f77b4..54ec81f17efb 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.h +++ b/source/extensions/filters/network/dubbo_proxy/active_message.h @@ -55,7 +55,8 @@ class ActiveResponseDecoder : public ResponseDecoderCallbacks, ResponseDecoderPtr decoder_; MessageMetadataSharedPtr metadata_; bool complete_ : 1; - DubboFilters::UpstreamResponseStatus response_status_; + DubboFilters::UpstreamResponseStatus response_status_{ + DubboFilters::UpstreamResponseStatus::MoreData}; }; using ActiveResponseDecoderPtr = std::unique_ptr; diff --git a/source/extensions/filters/network/dubbo_proxy/config.cc b/source/extensions/filters/network/dubbo_proxy/config.cc index 4684854555da..a62ba64b56ec 100644 --- a/source/extensions/filters/network/dubbo_proxy/config.cc +++ b/source/extensions/filters/network/dubbo_proxy/config.cc @@ -23,20 +23,24 @@ SINGLETON_MANAGER_REGISTRATION(dubbo_route_config_provider_manager); Network::FilterFactoryCb DubboProxyFilterConfigFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::dubbo_proxy::v3::DubboProxy& proto_config, Server::Configuration::FactoryContext& context) { + + auto& server_context = context.serverFactoryContext(); + std::shared_ptr route_config_provider_manager = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(dubbo_route_config_provider_manager), [&context] { - return std::make_shared(context.admin()); + server_context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(dubbo_route_config_provider_manager), + [&server_context] { + return std::make_shared(server_context.admin()); }); std::shared_ptr filter_config( std::make_shared(proto_config, context, *route_config_provider_manager)); return [route_config_provider_manager, filter_config, - &context](Network::FilterManager& filter_manager) -> void { + &server_context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter( - std::make_shared(*filter_config, context.api().randomGenerator(), - context.mainThreadDispatcher().timeSource())); + std::make_shared(*filter_config, server_context.api().randomGenerator(), + server_context.mainThreadDispatcher().timeSource())); }; } @@ -109,20 +113,20 @@ ConfigImpl::ConfigImpl(const DubboProxyConfig& config, } } route_config_provider_ = route_config_provider_manager.createRdsRouteConfigProvider( - config.drds(), context_.getServerFactoryContext(), stats_prefix_, context_.initManager()); + config.drds(), context_.serverFactoryContext(), stats_prefix_, context_.initManager()); } else if (config.has_multiple_route_config()) { if (config.route_config_size() > 0) { throw EnvoyException("both mutiple_route_config and route_config is present in DubboProxy"); } route_config_provider_ = route_config_provider_manager.createStaticRouteConfigProvider( - config.multiple_route_config(), context_.getServerFactoryContext()); + config.multiple_route_config(), context_.serverFactoryContext()); } else { envoy::extensions::filters::network::dubbo_proxy::v3::MultipleRouteConfiguration multiple_route_config; *multiple_route_config.mutable_route_config() = config.route_config(); route_config_provider_ = route_config_provider_manager.createStaticRouteConfigProvider( - multiple_route_config, context_.getServerFactoryContext()); + multiple_route_config, context_.serverFactoryContext()); } if (config.dubbo_filters().empty()) { diff --git a/source/extensions/filters/network/dubbo_proxy/router/config.cc b/source/extensions/filters/network/dubbo_proxy/router/config.cc index aa42bf55d378..b7b0c648db79 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/config.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/config.cc @@ -16,7 +16,7 @@ DubboFilters::FilterFactoryCb RouterFilterConfig::createFilterFactoryFromProtoTy const envoy::extensions::filters::network::dubbo_proxy::router::v3::Router&, const std::string&, Server::Configuration::FactoryContext& context) { return [&context](DubboFilters::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addFilter(std::make_shared(context.clusterManager())); + callbacks.addFilter(std::make_shared(context.serverFactoryContext().clusterManager())); }; } diff --git a/source/extensions/filters/network/ext_authz/config.cc b/source/extensions/filters/network/ext_authz/config.cc index e874dfdebaca..fe385b77c24a 100644 --- a/source/extensions/filters/network/ext_authz/config.cc +++ b/source/extensions/filters/network/ext_authz/config.cc @@ -24,15 +24,16 @@ Network::FilterFactoryCb ExtAuthzConfigFactory::createFilterFactoryFromProtoType const envoy::extensions::filters::network::ext_authz::v3::ExtAuthz& proto_config, Server::Configuration::FactoryContext& context) { ConfigSharedPtr ext_authz_config = std::make_shared( - proto_config, context.scope(), context.getServerFactoryContext().bootstrap()); + proto_config, context.scope(), context.serverFactoryContext().bootstrap()); const uint32_t timeout_ms = PROTOBUF_GET_MS_OR_DEFAULT(proto_config.grpc_service(), timeout, 200); THROW_IF_NOT_OK(Envoy::Config::Utility::checkTransportVersion(proto_config)); return [grpc_service = proto_config.grpc_service(), &context, ext_authz_config, timeout_ms](Network::FilterManager& filter_manager) -> void { - auto async_client_factory = - context.clusterManager().grpcAsyncClientManager().factoryForGrpcService( - grpc_service, context.scope(), true); + auto async_client_factory = context.serverFactoryContext() + .clusterManager() + .grpcAsyncClientManager() + .factoryForGrpcService(grpc_service, context.scope(), true); auto client = std::make_unique( async_client_factory->createUncachedRawAsyncClient(), diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index bf9e54ad3393..2dca9a73952c 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -25,7 +25,6 @@ #include "source/common/http/conn_manager_config.h" #include "source/common/http/conn_manager_utility.h" #include "source/common/http/default_server_string.h" -#include "source/common/http/filter_chain_helper.h" #include "source/common/http/http1/codec_impl.h" #include "source/common/http/http1/settings.h" #include "source/common/http/http2/codec_impl.h" @@ -85,7 +84,7 @@ envoy::extensions::filters::network::http_connection_manager::v3::HttpConnection PathWithEscapedSlashesAction getPathWithEscapedSlashesActionRuntimeOverride(Server::Configuration::FactoryContext& context) { // The default behavior is to leave escaped slashes unchanged. - uint64_t runtime_override = context.runtime().snapshot().getInteger( + uint64_t runtime_override = context.serverFactoryContext().runtime().snapshot().getInteger( "http_connection_manager.path_with_escaped_slashes_action", 0); switch (runtime_override) { default: @@ -112,7 +111,7 @@ envoy::extensions::filters::network::http_connection_manager::v3::HttpConnection envoy::type::v3::FractionalPercent default_fraction; default_fraction.set_numerator(100); default_fraction.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED); - if (context.runtime().snapshot().featureEnabled( + if (context.serverFactoryContext().runtime().snapshot().featureEnabled( "http_connection_manager.path_with_escaped_slashes_action_enabled", default_fraction)) { return config.path_with_escaped_slashes_action() == envoy::extensions::filters::network::http_connection_manager::v3:: @@ -173,26 +172,26 @@ createHeaderValidatorFactory([[maybe_unused]] const envoy::extensions::filters:: auto* factory = Envoy::Config::Utility::getFactory( header_validator_config); if (!factory) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Header validator extension not found: '{}'", header_validator_config.name())); } header_validator_factory = factory->createFromProto(header_validator_config.typed_config(), - context.getServerFactoryContext()); + context.serverFactoryContext()); if (!header_validator_factory) { - throw EnvoyException(fmt::format("Header validator extension could not be created: '{}'", - header_validator_config.name())); + throwEnvoyExceptionOrPanic(fmt::format("Header validator extension could not be created: '{}'", + header_validator_config.name())); } #else if (config.has_typed_header_validation_config()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("This Envoy binary does not support header validator extensions.: '{}'", config.typed_header_validation_config().name())); } if (Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.enable_universal_header_validator")) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Header validator can not be enabled since this Envoy binary does not support it."); } #endif @@ -207,32 +206,34 @@ SINGLETON_MANAGER_REGISTRATION(route_config_provider_manager); SINGLETON_MANAGER_REGISTRATION(scoped_routes_config_provider_manager); Utility::Singletons Utility::createSingletons(Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + std::shared_ptr date_provider = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(date_provider), [&context] { + server_context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(date_provider), [&server_context] { return std::make_shared( - context.mainThreadDispatcher(), context.threadLocal()); + server_context.mainThreadDispatcher(), server_context.threadLocal()); }); Router::RouteConfigProviderManagerSharedPtr route_config_provider_manager = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(route_config_provider_manager), [&context] { - return std::make_shared(context.admin()); + server_context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(route_config_provider_manager), [&server_context] { + return std::make_shared(server_context.admin()); }); Router::ScopedRoutesConfigProviderManagerSharedPtr scoped_routes_config_provider_manager = - context.singletonManager().getTyped( + server_context.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), - [&context, route_config_provider_manager] { + [&server_context, route_config_provider_manager] { return std::make_shared( - context.admin(), *route_config_provider_manager); + server_context.admin(), *route_config_provider_manager); }); auto tracer_manager = Tracing::TracerManagerImpl::singleton(context); std::shared_ptr filter_config_provider_manager = Http::FilterChainUtility::createSingletonDownstreamFilterConfigProviderManager( - context.getServerFactoryContext()); + server_context); return {date_provider, route_config_provider_manager, scoped_routes_config_provider_manager, tracer_manager, filter_config_provider_manager}; @@ -277,10 +278,13 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoAndHopByHo // as these captured objects are also global singletons. return [singletons, filter_config, &context, clear_hop_by_hop_headers](Network::FilterManager& filter_manager) -> void { + auto& server_context = context.serverFactoryContext(); + auto hcm = std::make_shared( - *filter_config, context.drainDecision(), context.api().randomGenerator(), - context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), - context.overloadManager(), context.mainThreadDispatcher().timeSource()); + *filter_config, context.drainDecision(), server_context.api().randomGenerator(), + server_context.httpContext(), server_context.runtime(), server_context.localInfo(), + server_context.clusterManager(), server_context.overloadManager(), + server_context.mainThreadDispatcher().timeSource()); if (!clear_hop_by_hop_headers) { hcm->setClearHopByHopResponseHeaders(false); } @@ -307,7 +311,11 @@ LEGACY_REGISTER_FACTORY(HttpConnectionManagerFilterConfigFactory, InternalAddressConfig::InternalAddressConfig( const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: InternalAddressConfig& config) - : unix_sockets_(config.unix_sockets()), cidr_ranges_(config.cidr_ranges()) {} + : unix_sockets_(config.unix_sockets()) { + auto list_or_error = Network::Address::IpList::create(config.cidr_ranges()); + THROW_IF_STATUS_NOT_OK(list_or_error, throw); + cidr_ranges_ = std::move(list_or_error.value()); +} HttpConnectionManagerConfig::HttpConnectionManagerConfig( const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& @@ -340,12 +348,12 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( xff_num_trusted_hops_ == 0 && use_remote_address_)), max_request_headers_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config, max_request_headers_kb, - context.runtime().snapshot().getInteger(Http::MaxRequestHeadersSizeOverrideKey, - Http::DEFAULT_MAX_REQUEST_HEADERS_KB))), + context.serverFactoryContext().runtime().snapshot().getInteger( + Http::MaxRequestHeadersSizeOverrideKey, Http::DEFAULT_MAX_REQUEST_HEADERS_KB))), max_request_headers_count_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config.common_http_protocol_options(), max_headers_count, - context.runtime().snapshot().getInteger(Http::MaxRequestHeadersCountOverrideKey, - Http::DEFAULT_MAX_HEADERS_COUNT))), + context.serverFactoryContext().runtime().snapshot().getInteger( + Http::MaxRequestHeadersCountOverrideKey, Http::DEFAULT_MAX_HEADERS_COUNT))), idle_timeout_(PROTOBUF_GET_OPTIONAL_MS(config.common_http_protocol_options(), idle_timeout)), max_connection_duration_( PROTOBUF_GET_OPTIONAL_MS(config.common_http_protocol_options(), max_connection_duration)), @@ -371,14 +379,14 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( normalize_path_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config, normalize_path, // TODO(htuch): we should have a boolean variant of featureEnabled() here. - context.runtime().snapshot().featureEnabled("http_connection_manager.normalize_path", - 100))), + context.serverFactoryContext().runtime().snapshot().featureEnabled( + "http_connection_manager.normalize_path", 100))), #else normalize_path_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config, normalize_path, // TODO(htuch): we should have a boolean variant of featureEnabled() here. - context.runtime().snapshot().featureEnabled("http_connection_manager.normalize_path", - 0))), + context.serverFactoryContext().runtime().snapshot().featureEnabled( + "http_connection_manager.normalize_path", 0))), #endif merge_slashes_(config.merge_slashes()), headers_with_underscores_action_( @@ -403,7 +411,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( } if (config.strip_any_host_port() && config.strip_matching_host_port()) { - throw EnvoyException(fmt::format( + throwEnvoyExceptionOrPanic(fmt::format( "Error: Only one of `strip_matching_host_port` or `strip_any_host_port` can be set.")); } @@ -440,12 +448,12 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( extension->mutable_typed_config()->PackFrom(xff_config); } else { if (use_remote_address_) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Original IP detection extensions and use_remote_address may not be mixed"); } if (xff_num_trusted_hops_ > 0) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Original IP detection extensions and xff_num_trusted_hops may not be mixed"); } } @@ -455,14 +463,14 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( auto* factory = Envoy::Config::Utility::getFactory(extension_config); if (!factory) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Original IP detection extension not found: '{}'", extension_config.name())); } auto extension = factory->createExtension(extension_config.typed_config(), context_); if (!extension) { - throw EnvoyException(fmt::format("Original IP detection extension could not be created: '{}'", - extension_config.name())); + throwEnvoyExceptionOrPanic(fmt::format( + "Original IP detection extension could not be created: '{}'", extension_config.name())); } original_ip_detection_extensions_.push_back(extension); } @@ -473,14 +481,14 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( auto* factory = Envoy::Config::Utility::getFactory(extension_config); if (!factory) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Early header mutation extension not found: '{}'", extension_config.name())); } auto extension = factory->createExtension(extension_config.typed_config(), context_); if (!extension) { - throw EnvoyException(fmt::format("Early header mutation extension could not be created: '{}'", - extension_config.name())); + throwEnvoyExceptionOrPanic(fmt::format( + "Early header mutation extension could not be created: '{}'", extension_config.name())); } early_header_mutation_extensions_.push_back(std::move(extension)); } @@ -493,13 +501,13 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::kRouteConfig: route_config_provider_ = Router::RouteConfigProviderUtil::create( - config, context_.getServerFactoryContext(), context_.messageValidationVisitor(), + config, context_.serverFactoryContext(), context_.messageValidationVisitor(), context_.initManager(), stats_prefix_, route_config_provider_manager_); break; case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::kScopedRoutes: scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::create( - config, context_.getServerFactoryContext(), context_.initManager(), stats_prefix_, + config, context_.serverFactoryContext(), context_.initManager(), stats_prefix_, scoped_routes_config_provider_manager_); scope_key_builder_ = Router::ScopedRoutesConfigProviderUtil::createScopeKeyBuilder(config); break; @@ -550,13 +558,13 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( } if (config.has_add_user_agent() && config.add_user_agent().value()) { - user_agent_ = context_.localInfo().clusterName(); + user_agent_ = context_.serverFactoryContext().localInfo().clusterName(); } if (config.has_tracing()) { tracer_ = tracer_manager.getOrCreateTracer(getPerFilterTracerConfig(config)); - tracing_config_ = std::make_unique(context.direction(), - config.tracing()); + tracing_config_ = std::make_unique( + context.listenerInfo().direction(), config.tracing()); } for (const auto& access_log : config.access_log()) { @@ -567,12 +575,12 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( if (config.has_access_log_options()) { if (config.flush_access_log_on_new_request() /* deprecated */) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Only one of flush_access_log_on_new_request or access_log_options can be specified."); } if (config.has_access_log_flush_interval()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Only one of access_log_flush_interval or access_log_options can be specified."); } @@ -624,22 +632,22 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( HTTP3: #ifdef ENVOY_ENABLE_QUIC codec_type_ = CodecType::HTTP3; - if (!context_.isQuicListener()) { - throw EnvoyException("HTTP/3 codec configured on non-QUIC listener."); + if (!context_.listenerInfo().isQuic()) { + throwEnvoyExceptionOrPanic("HTTP/3 codec configured on non-QUIC listener."); } #else - throw EnvoyException("HTTP3 configured but not enabled in the build."); + throwEnvoyExceptionOrPanic("HTTP3 configured but not enabled in the build."); #endif break; } - if (codec_type_ != CodecType::HTTP3 && context_.isQuicListener()) { - throw EnvoyException("Non-HTTP/3 codec configured on QUIC listener."); + if (codec_type_ != CodecType::HTTP3 && context_.listenerInfo().isQuic()) { + throwEnvoyExceptionOrPanic("Non-HTTP/3 codec configured on QUIC listener."); } Http::FilterChainHelper - helper(filter_config_provider_manager_, context_.getServerFactoryContext(), - context_.clusterManager(), context_, stats_prefix_); + helper(filter_config_provider_manager_, context_.serverFactoryContext(), + context_.serverFactoryContext().clusterManager(), context_, stats_prefix_); THROW_IF_NOT_OK(helper.processFilters(config.http_filters(), "http", "http", filter_factories_)); for (const auto& upgrade_config : config.upgrade_configs()) { @@ -647,7 +655,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( const bool enabled = upgrade_config.has_enabled() ? upgrade_config.enabled().value() : true; if (findUpgradeCaseInsensitive(upgrade_filter_factories_, name) != upgrade_filter_factories_.end()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Error: multiple upgrade configs with the same name: '{}'", name)); } if (!upgrade_config.filters().empty()) { @@ -658,7 +666,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( // TODO(auni53): Validate encode dependencies too. auto status = upgrade_dependency_manager.validDecodeDependencies(); if (!status.ok()) { - throw EnvoyException(std::string(status.message())); + throwEnvoyExceptionOrPanic(std::string(status.message())); } upgrade_filter_factories_.emplace( @@ -684,8 +692,9 @@ Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec( return std::make_unique( connection, callbacks, Http::Http2::CodecStats::atomicGet(http2_codec_stats_, context_.scope()), - context_.api().randomGenerator(), http2_options_, maxRequestHeadersKb(), - maxRequestHeadersCount(), headersWithUnderscoresAction(), overload_manager); + context_.serverFactoryContext().api().randomGenerator(), http2_options_, + maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction(), + overload_manager); case CodecType::HTTP3: return Config::Utility::getAndCheckFactoryByName( "quic.http_server_connection.default") @@ -696,10 +705,10 @@ Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec( headersWithUnderscoresAction()); case CodecType::AUTO: return Http::ConnectionManagerUtility::autoCreateCodec( - connection, data, callbacks, context_.scope(), context_.api().randomGenerator(), - http1_codec_stats_, http2_codec_stats_, http1_settings_, http2_options_, - maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction(), - overload_manager); + connection, data, callbacks, context_.scope(), + context_.serverFactoryContext().api().randomGenerator(), http1_codec_stats_, + http2_codec_stats_, http1_settings_, http2_options_, maxRequestHeadersKb(), + maxRequestHeadersCount(), headersWithUnderscoresAction(), overload_manager); } PANIC_DUE_TO_CORRUPT_ENUM; } @@ -744,7 +753,7 @@ bool HttpConnectionManagerConfig::createUpgradeFilterChain( } const Network::Address::Instance& HttpConnectionManagerConfig::localAddress() { - return *context_.localInfo().address(); + return *context_.serverFactoryContext().localInfo().address(); } /** @@ -761,8 +770,8 @@ const envoy::config::trace::v3::Tracing_Http* HttpConnectionManagerConfig::getPe } // Otherwise, for the sake of backwards compatibility, fall back to using tracing provider // configuration defined in the bootstrap config. - if (context_.httpContext().defaultTracingConfig().has_http()) { - return &context_.httpContext().defaultTracingConfig().http(); + if (context_.serverFactoryContext().httpContext().defaultTracingConfig().has_http()) { + return &context_.serverFactoryContext().httpContext().defaultTracingConfig().http(); } return nullptr; } @@ -801,10 +810,13 @@ HttpConnectionManagerFactory::createHttpConnectionManagerFactoryFromProto( // as these captured objects are also global singletons. return [singletons, filter_config, &context, clear_hop_by_hop_headers]( Network::ReadFilterCallbacks& read_callbacks) -> Http::ApiListenerPtr { + auto& server_context = context.serverFactoryContext(); + auto conn_manager = std::make_unique( - *filter_config, context.drainDecision(), context.api().randomGenerator(), - context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), - context.overloadManager(), context.mainThreadDispatcher().timeSource()); + *filter_config, context.drainDecision(), server_context.api().randomGenerator(), + server_context.httpContext(), server_context.runtime(), server_context.localInfo(), + server_context.clusterManager(), server_context.overloadManager(), + server_context.mainThreadDispatcher().timeSource()); if (!clear_hop_by_hop_headers) { conn_manager->setClearHopByHopResponseHeaders(false); } diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index d2c43c2fe142..b0a29a0f009b 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -27,6 +27,7 @@ #include "source/common/http/conn_manager_impl.h" #include "source/common/http/date_provider_impl.h" #include "source/common/http/dependency_manager.h" +#include "source/common/http/filter_chain_helper.h" #include "source/common/http/http1/codec_stats.h" #include "source/common/http/http2/codec_stats.h" #include "source/common/http/http3/codec_stats.h" @@ -108,15 +109,15 @@ class InternalAddressConfig : public Http::InternalAddressConfig { // TODO: cleanup isInternalAddress and default to initializing cidr_ranges_ // based on RFC1918 / RFC4193, if config is unset. - if (cidr_ranges_.getIpListSize() != 0 && address.type() == Network::Address::Type::Ip) { - return cidr_ranges_.contains(address); + if (cidr_ranges_->getIpListSize() != 0 && address.type() == Network::Address::Type::Ip) { + return cidr_ranges_->contains(address); } return Network::Utility::isInternalAddress(address); } private: const bool unix_sockets_; - const Network::Address::IpList cidr_ranges_; + std::unique_ptr cidr_ranges_; }; /** @@ -139,8 +140,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, bool createFilterChain( Http::FilterChainManager& manager, bool = false, const Http::FilterChainOptions& = Http::EmptyFilterChainOptions{}) const override; - using FilterFactoriesList = - std::list>; + using FilterFactoriesList = Envoy::Http::FilterChainUtility::FilterFactoriesList; struct FilterConfig { std::unique_ptr filter_factories; bool allow_upgrade; diff --git a/source/extensions/filters/network/local_ratelimit/config.cc b/source/extensions/filters/network/local_ratelimit/config.cc index 6fabf6b2f3e5..f2b935479e9b 100644 --- a/source/extensions/filters/network/local_ratelimit/config.cc +++ b/source/extensions/filters/network/local_ratelimit/config.cc @@ -13,9 +13,9 @@ namespace LocalRateLimitFilter { Network::FilterFactoryCb LocalRateLimitConfigFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::local_ratelimit::v3::LocalRateLimit& proto_config, Server::Configuration::FactoryContext& context) { - ConfigSharedPtr filter_config( - std::make_shared(proto_config, context.mainThreadDispatcher(), context.scope(), - context.runtime(), context.singletonManager())); + ConfigSharedPtr filter_config(std::make_shared( + proto_config, context.serverFactoryContext().mainThreadDispatcher(), context.scope(), + context.serverFactoryContext().runtime(), context.serverFactoryContext().singletonManager())); return [filter_config](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(filter_config)); }; diff --git a/source/extensions/filters/network/mongo_proxy/config.cc b/source/extensions/filters/network/mongo_proxy/config.cc index c9a28c6d1e53..67becffb161c 100644 --- a/source/extensions/filters/network/mongo_proxy/config.cc +++ b/source/extensions/filters/network/mongo_proxy/config.cc @@ -24,8 +24,9 @@ Network::FilterFactoryCb MongoProxyFilterConfigFactory::createFilterFactoryFromP const std::string stat_prefix = fmt::format("mongo.{}", proto_config.stat_prefix()); AccessLogSharedPtr access_log; if (!proto_config.access_log().empty()) { - access_log = std::make_shared(proto_config.access_log(), context.accessLogManager(), - context.mainThreadDispatcher().timeSource()); + access_log = std::make_shared( + proto_config.access_log(), context.serverFactoryContext().accessLogManager(), + context.serverFactoryContext().mainThreadDispatcher().timeSource()); } Filters::Common::Fault::FaultDelayConfigSharedPtr fault_config; @@ -44,8 +45,9 @@ Network::FilterFactoryCb MongoProxyFilterConfigFactory::createFilterFactoryFromP return [stat_prefix, &context, access_log, fault_config, emit_dynamic_metadata, stats](Network::FilterManager& filter_manager) -> void { filter_manager.addFilter(std::make_shared( - stat_prefix, context.scope(), context.runtime(), access_log, fault_config, - context.drainDecision(), context.mainThreadDispatcher().timeSource(), emit_dynamic_metadata, + stat_prefix, context.scope(), context.serverFactoryContext().runtime(), access_log, + fault_config, context.drainDecision(), + context.serverFactoryContext().mainThreadDispatcher().timeSource(), emit_dynamic_metadata, stats)); }; } diff --git a/source/extensions/filters/network/mongo_proxy/utility.cc b/source/extensions/filters/network/mongo_proxy/utility.cc index ecdff4da9074..e0f7db7a3115 100644 --- a/source/extensions/filters/network/mongo_proxy/utility.cc +++ b/source/extensions/filters/network/mongo_proxy/utility.cc @@ -12,8 +12,7 @@ namespace Extensions { namespace NetworkFilters { namespace MongoProxy { -QueryMessageInfo::QueryMessageInfo(const QueryMessage& query) - : request_id_{query.requestId()}, max_time_{0} { +QueryMessageInfo::QueryMessageInfo(const QueryMessage& query) : request_id_{query.requestId()} { // First see if this is a command, if so we are done. const Bson::Document* command = parseCommand(query); if (command) { diff --git a/source/extensions/filters/network/mongo_proxy/utility.h b/source/extensions/filters/network/mongo_proxy/utility.h index 4394640c9e36..e9980c61fd5e 100644 --- a/source/extensions/filters/network/mongo_proxy/utility.h +++ b/source/extensions/filters/network/mongo_proxy/utility.h @@ -65,7 +65,7 @@ class QueryMessageInfo { int32_t request_id_; std::string collection_; std::string callsite_; - int32_t max_time_; + int32_t max_time_{0}; QueryType type_{QueryType::ScatterGet}; std::string command_; }; diff --git a/source/extensions/filters/network/ratelimit/config.cc b/source/extensions/filters/network/ratelimit/config.cc index 799003c0ae12..40bb96398fd0 100644 --- a/source/extensions/filters/network/ratelimit/config.cc +++ b/source/extensions/filters/network/ratelimit/config.cc @@ -25,15 +25,19 @@ Network::FilterFactoryCb RateLimitConfigFactory::createFilterFactoryFromProtoTyp ASSERT(!proto_config.domain().empty()); ASSERT(proto_config.descriptors_size() > 0); - ConfigSharedPtr filter_config(new Config(proto_config, context.scope(), context.runtime())); + ConfigSharedPtr filter_config( + new Config(proto_config, context.scope(), context.serverFactoryContext().runtime())); const std::chrono::milliseconds timeout = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(proto_config, timeout, 20)); + THROW_IF_NOT_OK(Envoy::Config::Utility::checkTransportVersion(proto_config.rate_limit_service())); - return [proto_config, &context, timeout, + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key = + Grpc::GrpcServiceConfigWithHashKey(proto_config.rate_limit_service().grpc_service()); + return [config_with_hash_key, &context, timeout, filter_config](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared( - filter_config, Filters::Common::RateLimit::rateLimitClient( - context, proto_config.rate_limit_service().grpc_service(), timeout))); + filter_config, + Filters::Common::RateLimit::rateLimitClient(context, config_with_hash_key, timeout))); }; } diff --git a/source/extensions/filters/network/rbac/config.cc b/source/extensions/filters/network/rbac/config.cc index e7071c87d4ad..f6fc33a0a326 100644 --- a/source/extensions/filters/network/rbac/config.cc +++ b/source/extensions/filters/network/rbac/config.cc @@ -86,7 +86,7 @@ RoleBasedAccessControlNetworkFilterConfigFactory::createFilterFactoryFromProtoTy } RoleBasedAccessControlFilterConfigSharedPtr config( std::make_shared(proto_config, context.scope(), - context.getServerFactoryContext(), + context.serverFactoryContext(), context.messageValidationVisitor())); return [config](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(config)); diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc index b2bb3af5c90d..692b5e073243 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc @@ -32,8 +32,8 @@ Common::Redis::Client::PoolRequest* makeSingleServerRequest( // If a transaction is active, clients_[0] is the primary connection to the cluster. // The subsequent clients in the array are used for mirroring. transaction.current_client_idx_ = 0; - auto handler = route->upstream()->makeRequest(key, ConnPool::RespVariant(incoming_request), - callbacks, transaction); + auto handler = route->upstream(command)->makeRequest(key, ConnPool::RespVariant(incoming_request), + callbacks, transaction); if (handler) { for (auto& mirror_policy : route->mirrorPolicies()) { transaction.current_client_idx_++; @@ -62,8 +62,8 @@ makeFragmentedRequest(const RouteSharedPtr& route, const std::string& command, const std::string& key, const Common::Redis::RespValue& incoming_request, ConnPool::PoolCallbacks& callbacks, Common::Redis::Client::Transaction& transaction) { - auto handler = route->upstream()->makeRequest(key, ConnPool::RespVariant(incoming_request), - callbacks, transaction); + auto handler = route->upstream(command)->makeRequest(key, ConnPool::RespVariant(incoming_request), + callbacks, transaction); if (handler) { for (auto& mirror_policy : route->mirrorPolicies()) { if (mirror_policy->shouldMirror(command)) { diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index d31ba0573c74..4cf0ac6c42b4 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -27,6 +27,9 @@ inline void addUniqueClusters( for (auto& mirror : route.request_mirror_policy()) { clusters.emplace(mirror.cluster()); } + if (route.has_read_command_policy()) { + clusters.emplace(route.read_command_policy().cluster()); + } } } // namespace @@ -34,19 +37,22 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP const envoy::extensions::filters::network::redis_proxy::v3::RedisProxy& proto_config, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + ASSERT(!proto_config.stat_prefix().empty()); ASSERT(proto_config.has_settings()); Extensions::Common::Redis::ClusterRefreshManagerSharedPtr refresh_manager = Extensions::Common::Redis::getClusterRefreshManager( - context.singletonManager(), context.mainThreadDispatcher(), context.clusterManager(), - context.timeSource()); + server_context.singletonManager(), server_context.mainThreadDispatcher(), + server_context.clusterManager(), server_context.timeSource()); Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( context); - auto filter_config = - std::make_shared(proto_config, context.scope(), context.drainDecision(), - context.runtime(), context.api(), cache_manager_factory); + + auto filter_config = std::make_shared( + proto_config, context.scope(), context.drainDecision(), server_context.runtime(), + server_context.api(), cache_manager_factory); envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::PrefixRoutes prefix_routes( proto_config.prefix_routes()); @@ -70,23 +76,24 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP Stats::ScopeSharedPtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); auto conn_pool_ptr = std::make_shared( - cluster, context.clusterManager(), Common::Redis::Client::ClientFactoryImpl::instance_, - context.threadLocal(), proto_config.settings(), context.api(), std::move(stats_scope), - redis_command_stats, refresh_manager, filter_config->dns_cache_); + cluster, server_context.clusterManager(), + Common::Redis::Client::ClientFactoryImpl::instance_, server_context.threadLocal(), + proto_config.settings(), server_context.api(), std::move(stats_scope), redis_command_stats, + refresh_manager, filter_config->dns_cache_); conn_pool_ptr->init(); upstreams.emplace(cluster, conn_pool_ptr); } auto router = - std::make_unique(prefix_routes, std::move(upstreams), context.runtime()); + std::make_unique(prefix_routes, std::move(upstreams), server_context.runtime()); auto fault_manager = std::make_unique( - context.api().randomGenerator(), context.runtime(), proto_config.faults()); + server_context.api().randomGenerator(), server_context.runtime(), proto_config.faults()); std::shared_ptr splitter = std::make_shared( - std::move(router), context.scope(), filter_config->stat_prefix_, context.timeSource(), - proto_config.latency_in_micros(), std::move(fault_manager)); + std::move(router), context.scope(), filter_config->stat_prefix_, + server_context.timeSource(), proto_config.latency_in_micros(), std::move(fault_manager)); return [splitter, filter_config](Network::FilterManager& filter_manager) -> void { Common::Redis::DecoderFactoryImpl factory; filter_manager.addReadFilter(std::make_shared( diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index e745658cf5af..c8adf10386db 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -35,6 +35,24 @@ const Common::Redis::RespValue& getRequest(const RespVariant& request) { } static uint16_t default_port = 6379; + +bool isClusterProvidedLb(const Upstream::ClusterInfo& info) { + const auto lb_type = info.lbType(); + bool cluster_provided_lb = lb_type == Upstream::LoadBalancerType::ClusterProvided; + if (lb_type == Upstream::LoadBalancerType::LoadBalancingPolicyConfig) { + auto* typed_lb_factory = info.loadBalancerFactory(); + if (typed_lb_factory == nullptr) { + // This should never happen because if there is no valid factory, the cluster should + // have been rejected during config load and this code should never be reached. + IS_ENVOY_BUG("ClusterInfo should contain a valid factory"); + return false; + } + cluster_provided_lb = + typed_lb_factory->name() == "envoy.load_balancing_policies.cluster_provided"; + } + return cluster_provided_lb; +} + } // namespace InstanceImpl::InstanceImpl( @@ -92,7 +110,7 @@ InstanceImpl::ThreadLocalPool::ThreadLocalPool( : parent_(parent), dispatcher_(dispatcher), cluster_name_(std::move(cluster_name)), dns_cache_(dns_cache), drain_timer_(dispatcher.createTimer([this]() -> void { drainClients(); })), - is_redis_cluster_(false), client_factory_(parent->client_factory_), config_(parent->config_), + client_factory_(parent->client_factory_), config_(parent->config_), stats_scope_(parent->stats_scope_), redis_command_stats_(parent->redis_command_stats_), redis_cluster_stats_(parent->redis_cluster_stats_), refresh_manager_(parent->refresh_manager_) { @@ -162,8 +180,8 @@ void InstanceImpl::ThreadLocalPool::onClusterAddOrUpdateNonVirtual( Upstream::ClusterInfoConstSharedPtr info = cluster_->info(); OptRef cluster_type = info->clusterType(); - is_redis_cluster_ = info->lbType() == Upstream::LoadBalancerType::ClusterProvided && - cluster_type.has_value() && cluster_type->name() == "envoy.clusters.redis"; + is_redis_cluster_ = isClusterProvidedLb(*info) && cluster_type.has_value() && + cluster_type->name() == "envoy.clusters.redis"; } void InstanceImpl::ThreadLocalPool::onClusterRemoval(const std::string& cluster_name) { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index fe33f44ae5a0..7068744e520c 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -197,7 +197,7 @@ class InstanceImpl : public Instance, public std::enable_shared_from_thisgetCache(config.settings().dns_cache_config()) - : nullptr; + if (config.settings().has_dns_cache_config()) { + auto cache_or_error = dns_cache_manager_->getCache(config.settings().dns_cache_config()); + if (cache_or_error.status().ok()) { + return cache_or_error.value(); + } + } + return nullptr; } ProxyStats ProxyFilterConfig::generateStats(const std::string& prefix, Stats::Scope& scope) { diff --git a/source/extensions/filters/network/redis_proxy/router.h b/source/extensions/filters/network/redis_proxy/router.h index d9bd44bf058e..45e992a80cbf 100644 --- a/source/extensions/filters/network/redis_proxy/router.h +++ b/source/extensions/filters/network/redis_proxy/router.h @@ -46,7 +46,7 @@ class Route { public: virtual ~Route() = default; - virtual ConnPool::InstanceSharedPtr upstream() const PURE; + virtual ConnPool::InstanceSharedPtr upstream(const std::string& command) const PURE; virtual const MirrorPolicies& mirrorPolicies() const PURE; }; diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc index 14724fa15717..4aae1dad7dfa 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.cc +++ b/source/extensions/filters/network/redis_proxy/router_impl.cc @@ -53,6 +53,21 @@ Prefix::Prefix( mirror_policies_.emplace_back(std::make_shared( mirror_policy, upstreams.at(mirror_policy.cluster()), runtime)); } + if (route.has_read_command_policy()) { + read_upstream_ = upstreams.at(route.read_command_policy().cluster()); + } +} + +ConnPool::InstanceSharedPtr Prefix::upstream(const std::string& command) const { + + if (read_upstream_) { + std::string to_lower_string = absl::AsciiStrToLower(command); + if (Common::Redis::SupportedCommands::isReadCommand(to_lower_string)) { + return read_upstream_; + } + } + + return upstream_; } PrefixRoutes::PrefixRoutes( @@ -91,6 +106,13 @@ RouteSharedPtr PrefixRoutes::upstreamPool(std::string& key, if (value == nullptr) { // prefix route not found, default to catch all route. value = catch_all_route_; + // prefix route not found, check if catch_all_route is defined to fallback to. + if (catch_all_route_ != nullptr) { + value = catch_all_route_; + } else { + // no route found. + return value; + } } if (value->removePrefix()) { diff --git a/source/extensions/filters/network/redis_proxy/router_impl.h b/source/extensions/filters/network/redis_proxy/router_impl.h index 7d7bb8429605..2afb8c155eb5 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.h +++ b/source/extensions/filters/network/redis_proxy/router_impl.h @@ -50,7 +50,7 @@ class Prefix : public Route { route, Upstreams& upstreams, Runtime::Loader& runtime); - ConnPool::InstanceSharedPtr upstream() const override { return upstream_; } + ConnPool::InstanceSharedPtr upstream(const std::string& command) const override; const MirrorPolicies& mirrorPolicies() const override { return mirror_policies_; }; const std::string& prefix() const { return prefix_; } bool removePrefix() const { return remove_prefix_; } @@ -62,6 +62,7 @@ class Prefix : public Route { const bool remove_prefix_; const ConnPool::InstanceSharedPtr upstream_; MirrorPolicies mirror_policies_; + ConnPool::InstanceSharedPtr read_upstream_; }; using PrefixSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/set_filter_state/BUILD b/source/extensions/filters/network/set_filter_state/BUILD new file mode 100644 index 000000000000..90b9ea589e5b --- /dev/null +++ b/source/extensions/filters/network/set_filter_state/BUILD @@ -0,0 +1,24 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//envoy/formatter:substitution_formatter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/extensions/filters/common/set_filter_state:filter_config_lib", + "//source/extensions/filters/network/common:factory_base_lib", + "//source/server:generic_factory_context_lib", + "@envoy_api//envoy/extensions/filters/network/set_filter_state/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/network/set_filter_state/config.cc b/source/extensions/filters/network/set_filter_state/config.cc new file mode 100644 index 000000000000..d9f2fe58840b --- /dev/null +++ b/source/extensions/filters/network/set_filter_state/config.cc @@ -0,0 +1,57 @@ +#include "source/extensions/filters/network/set_filter_state/config.h" + +#include + +#include "envoy/extensions/filters/network/set_filter_state/v3/set_filter_state.pb.h" +#include "envoy/extensions/filters/network/set_filter_state/v3/set_filter_state.pb.validate.h" +#include "envoy/formatter/substitution_formatter.h" +#include "envoy/registry/registry.h" + +#include "source/common/protobuf/utility.h" +#include "source/extensions/filters/network/common/factory_base.h" +#include "source/server/generic_factory_context.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SetFilterState { + +Network::FilterStatus SetFilterState::onNewConnection() { + config_->updateFilterState({}, read_callbacks_->connection().streamInfo()); + return Network::FilterStatus::Continue; +} + +/** + * Config registration for the filter. @see NamedNetworkFilterConfigFactory. + */ +class SetFilterStateConfigFactory + : public Common::FactoryBase< + envoy::extensions::filters::network::set_filter_state::v3::Config> { +public: + SetFilterStateConfigFactory() : FactoryBase("envoy.filters.network.set_filter_state") {} + +private: + Network::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::filters::network::set_filter_state::v3::Config& proto_config, + Server::Configuration::FactoryContext& context) override { + auto filter_config(std::make_shared( + proto_config.on_new_connection(), StreamInfo::FilterState::LifeSpan::Connection, context)); + return [filter_config](Network::FilterManager& filter_manager) -> void { + filter_manager.addReadFilter(std::make_shared(filter_config)); + }; + } + + bool isTerminalFilterByProtoTyped( + const envoy::extensions::filters::network::set_filter_state::v3::Config&, + Server::Configuration::ServerFactoryContext&) override { + return false; + } +}; + +REGISTER_FACTORY(SetFilterStateConfigFactory, + Server::Configuration::NamedNetworkFilterConfigFactory); + +} // namespace SetFilterState +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/set_filter_state/config.h b/source/extensions/filters/network/set_filter_state/config.h new file mode 100644 index 000000000000..e554c3bf0a0c --- /dev/null +++ b/source/extensions/filters/network/set_filter_state/config.h @@ -0,0 +1,34 @@ +#pragma once + +#include "envoy/server/filter_config.h" + +#include "source/extensions/filters/common/set_filter_state/filter_config.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SetFilterState { + +class SetFilterState : public Network::ReadFilter, Logger::Loggable { +public: + explicit SetFilterState(const Filters::Common::SetFilterState::ConfigSharedPtr config) + : config_(config) {} + + // Network::ReadFilter + Network::FilterStatus onData(Buffer::Instance&, bool) override { + return Network::FilterStatus::Continue; + } + Network::FilterStatus onNewConnection() override; + void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + +private: + const Filters::Common::SetFilterState::ConfigSharedPtr config_; + Network::ReadFilterCallbacks* read_callbacks_{}; +}; + +} // namespace SetFilterState +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc b/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc index 14c2396cdf45..991b22dc7b9d 100644 --- a/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc @@ -21,7 +21,7 @@ SniDynamicForwardProxyNetworkFilterConfigFactory::createFilterFactoryFromProtoTy Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( context); ProxyFilterConfigSharedPtr filter_config(std::make_shared( - proto_config, cache_manager_factory, context.clusterManager())); + proto_config, cache_manager_factory, context.serverFactoryContext().clusterManager())); return [filter_config](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared(filter_config)); diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc index 7481f6e767d6..af3407c3d727 100644 --- a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc @@ -20,8 +20,11 @@ ProxyFilterConfig::ProxyFilterConfig( Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, Upstream::ClusterManager&) : port_(static_cast(proto_config.port_value())), - dns_cache_manager_(cache_manager_factory.get()), - dns_cache_(dns_cache_manager_->getCache(proto_config.dns_cache_config())) {} + dns_cache_manager_(cache_manager_factory.get()) { + auto cache_or_error = dns_cache_manager_->getCache(proto_config.dns_cache_config()); + THROW_IF_STATUS_NOT_OK(cache_or_error, throw); + dns_cache_ = std::move(cache_or_error.value()); +} ProxyFilter::ProxyFilter(ProxyFilterConfigSharedPtr config) : config_(std::move(config)) {} diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h index 23785a275090..ccba394462ce 100644 --- a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h @@ -28,7 +28,7 @@ class ProxyFilterConfig { private: const uint32_t port_; const Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr dns_cache_manager_; - const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; + Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; }; using ProxyFilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/tcp_proxy/config.cc b/source/extensions/filters/network/tcp_proxy/config.cc index 5572ff13ac14..56387eb78fe6 100644 --- a/source/extensions/filters/network/tcp_proxy/config.cc +++ b/source/extensions/filters/network/tcp_proxy/config.cc @@ -19,8 +19,8 @@ Network::FilterFactoryCb ConfigFactory::createFilterFactoryFromProtoTyped( Envoy::TcpProxy::ConfigSharedPtr filter_config( std::make_shared(proto_config, context)); return [filter_config, &context](Network::FilterManager& filter_manager) -> void { - filter_manager.addReadFilter( - std::make_shared(filter_config, context.clusterManager())); + filter_manager.addReadFilter(std::make_shared( + filter_config, context.serverFactoryContext().clusterManager())); }; } diff --git a/source/extensions/filters/network/thrift_proxy/BUILD b/source/extensions/filters/network/thrift_proxy/BUILD index 6e6faafb15d0..2d2b0448d640 100644 --- a/source/extensions/filters/network/thrift_proxy/BUILD +++ b/source/extensions/filters/network/thrift_proxy/BUILD @@ -52,7 +52,6 @@ envoy_cc_extension( ":unframed_transport_lib", "//envoy/access_log:access_log_interface", "//envoy/registry", - "//source/common/access_log:access_log_lib", "//source/common/config:utility_lib", "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", @@ -85,6 +84,7 @@ envoy_cc_library( "//envoy/network:filter_interface", "//envoy/stats:stats_interface", "//envoy/stats:timespan_interface", + "//source/common/access_log:access_log_lib", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:linked_object", diff --git a/source/extensions/filters/network/thrift_proxy/config.cc b/source/extensions/filters/network/thrift_proxy/config.cc index e3cf3cf62366..e845d2b8b98f 100644 --- a/source/extensions/filters/network/thrift_proxy/config.cc +++ b/source/extensions/filters/network/thrift_proxy/config.cc @@ -46,10 +46,13 @@ SINGLETON_MANAGER_REGISTRATION(thrift_route_config_provider_manager); Network::FilterFactoryCb ThriftProxyFilterConfigFactory::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::thrift_proxy::v3::ThriftProxy& proto_config, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + std::shared_ptr route_config_provider_manager = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(thrift_route_config_provider_manager), [&context] { - return std::make_shared(context.admin()); + server_context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(thrift_route_config_provider_manager), + [&server_context] { + return std::make_shared(server_context.admin()); }); std::shared_ptr filter_config( @@ -60,8 +63,9 @@ Network::FilterFactoryCb ThriftProxyFilterConfigFactory::createFilterFactoryFrom return [route_config_provider_manager, filter_config, &context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(std::make_shared( - *filter_config, context.api().randomGenerator(), - context.mainThreadDispatcher().timeSource(), context.drainDecision())); + *filter_config, context.serverFactoryContext().api().randomGenerator(), + context.serverFactoryContext().mainThreadDispatcher().timeSource(), + context.drainDecision())); }; } @@ -110,10 +114,10 @@ ConfigImpl::ConfigImpl( } } route_config_provider_ = route_config_provider_manager.createRdsRouteConfigProvider( - config.trds(), context_.getServerFactoryContext(), stats_prefix_, context_.initManager()); + config.trds(), context_.serverFactoryContext(), stats_prefix_, context_.initManager()); } else { route_config_provider_ = route_config_provider_manager.createStaticRouteConfigProvider( - config.route_config(), context_.getServerFactoryContext()); + config.route_config(), context_.serverFactoryContext()); } for (const envoy::config::accesslog::v3::AccessLog& log_config : config.access_log()) { diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc index 77bc7f02da24..b98471fe8720 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc @@ -54,9 +54,10 @@ Network::FilterStatus ConnectionManager::onData(Buffer::Instance& data, bool end void ConnectionManager::emitLogEntry(const Http::RequestHeaderMap* request_headers, const Http::ResponseHeaderMap* response_headers, const StreamInfo::StreamInfo& stream_info) { + const Formatter::HttpFormatterContext log_context{request_headers, response_headers}; + for (const auto& access_log : config_.accessLogs()) { - access_log->log(request_headers, response_headers, nullptr, stream_info, - AccessLog::AccessLogType::NotSet); + access_log->log(log_context, stream_info); } } diff --git a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/config.cc b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/config.cc index 8666a75148b1..91d767de4e46 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/config.cc +++ b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/config.cc @@ -24,18 +24,22 @@ RateLimitFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::thrift_proxy::filters::ratelimit::v3::RateLimit& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + ASSERT(!proto_config.domain().empty()); - ConfigSharedPtr config(new Config(proto_config, context.localInfo(), context.scope(), - context.runtime(), context.clusterManager())); + ConfigSharedPtr config(new Config(proto_config, server_context.localInfo(), context.scope(), + server_context.runtime(), server_context.clusterManager())); const std::chrono::milliseconds timeout = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(proto_config, timeout, 20)); THROW_IF_NOT_OK(Envoy::Config::Utility::checkTransportVersion(proto_config.rate_limit_service())); - return [proto_config, &context, timeout, + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key = + Grpc::GrpcServiceConfigWithHashKey(proto_config.rate_limit_service().grpc_service()); + return [config_with_hash_key, &context, timeout, config](ThriftProxy::ThriftFilters::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addDecoderFilter(std::make_shared( - config, Filters::Common::RateLimit::rateLimitClient( - context, proto_config.rate_limit_service().grpc_service(), timeout))); + config, + Filters::Common::RateLimit::rateLimitClient(context, config_with_hash_key, timeout))); }; } diff --git a/source/extensions/filters/network/thrift_proxy/router/config.cc b/source/extensions/filters/network/thrift_proxy/router/config.cc index 1a5fdfd779e9..908f60a03deb 100644 --- a/source/extensions/filters/network/thrift_proxy/router/config.cc +++ b/source/extensions/filters/network/thrift_proxy/router/config.cc @@ -16,18 +16,21 @@ namespace Router { ThriftFilters::FilterFactoryCb RouterFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::thrift_proxy::router::v3::Router& proto_config, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { + auto& server_context = context.serverFactoryContext(); + auto stats = - std::make_shared(stat_prefix, context.scope(), context.localInfo()); - auto shadow_writer = std::make_shared( - context.clusterManager(), *stats, context.mainThreadDispatcher(), context.threadLocal()); + std::make_shared(stat_prefix, context.scope(), server_context.localInfo()); + auto shadow_writer = std::make_shared(server_context.clusterManager(), *stats, + server_context.mainThreadDispatcher(), + server_context.threadLocal()); bool close_downstream_on_error = PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, close_downstream_on_upstream_error, true); return [&context, stats, shadow_writer, close_downstream_on_error]( ThriftFilters::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addDecoderFilter(std::make_shared(context.clusterManager(), *stats, - context.runtime(), *shadow_writer, - close_downstream_on_error)); + callbacks.addDecoderFilter(std::make_shared( + context.serverFactoryContext().clusterManager(), *stats, + context.serverFactoryContext().runtime(), *shadow_writer, close_downstream_on_error)); }; } diff --git a/source/extensions/filters/network/wasm/config.cc b/source/extensions/filters/network/wasm/config.cc index 43045f7a53a1..0fdde8db7a79 100644 --- a/source/extensions/filters/network/wasm/config.cc +++ b/source/extensions/filters/network/wasm/config.cc @@ -16,7 +16,7 @@ namespace Wasm { Network::FilterFactoryCb WasmFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::network::wasm::v3::Wasm& proto_config, Server::Configuration::FactoryContext& context) { - context.api().customStatNamespaces().registerStatNamespace( + context.serverFactoryContext().api().customStatNamespaces().registerStatNamespace( Extensions::Common::Wasm::CustomStatNamespace); auto filter_config = std::make_shared(proto_config, context); return [filter_config](Network::FilterManager& filter_manager) -> void { diff --git a/source/extensions/filters/network/wasm/wasm_filter.cc b/source/extensions/filters/network/wasm/wasm_filter.cc index d5f1a6a3ba38..55fe126d8253 100644 --- a/source/extensions/filters/network/wasm/wasm_filter.cc +++ b/source/extensions/filters/network/wasm/wasm_filter.cc @@ -8,9 +8,10 @@ namespace Wasm { FilterConfig::FilterConfig(const envoy::extensions::filters::network::wasm::v3::Wasm& config, Server::Configuration::FactoryContext& context) : tls_slot_(ThreadLocal::TypedSlot::makeUnique( - context.threadLocal())) { + context.serverFactoryContext().threadLocal())) { const auto plugin = std::make_shared( - config.config(), context.direction(), context.localInfo(), &context.listenerMetadata()); + config.config(), context.listenerInfo().direction(), + context.serverFactoryContext().localInfo(), &context.listenerInfo().metadata()); auto callback = [plugin, this](Common::Wasm::WasmHandleSharedPtr base_wasm) { // NB: the Slot set() call doesn't complete inline, so all arguments must outlive this call. @@ -20,10 +21,11 @@ FilterConfig::FilterConfig(const envoy::extensions::filters::network::wasm::v3:: }); }; - if (!Common::Wasm::createWasm(plugin, context.scope().createScope(""), context.clusterManager(), - context.initManager(), context.mainThreadDispatcher(), - context.api(), context.lifecycleNotifier(), remote_data_provider_, - std::move(callback))) { + if (!Common::Wasm::createWasm( + plugin, context.scope().createScope(""), context.serverFactoryContext().clusterManager(), + context.initManager(), context.serverFactoryContext().mainThreadDispatcher(), + context.serverFactoryContext().api(), context.serverFactoryContext().lifecycleNotifier(), + remote_data_provider_, std::move(callback))) { throw Common::Wasm::WasmException( fmt::format("Unable to create Wasm network filter {}", plugin->name_)); } diff --git a/source/extensions/filters/network/zookeeper_proxy/config.cc b/source/extensions/filters/network/zookeeper_proxy/config.cc index 691bf9cafef9..f789a884d9ca 100644 --- a/source/extensions/filters/network/zookeeper_proxy/config.cc +++ b/source/extensions/filters/network/zookeeper_proxy/config.cc @@ -27,6 +27,10 @@ Network::FilterFactoryCb ZooKeeperConfigFactory::createFilterFactoryFromProtoTyp const std::string stat_prefix = fmt::format("{}.zookeeper", proto_config.stat_prefix()); const uint32_t max_packet_bytes = PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, max_packet_bytes, 1024 * 1024); + const bool enable_per_opcode_request_bytes_metrics = + proto_config.enable_per_opcode_request_bytes_metrics(); + const bool enable_per_opcode_response_bytes_metrics = + proto_config.enable_per_opcode_response_bytes_metrics(); const bool enable_latency_threshold_metrics = proto_config.enable_latency_threshold_metrics(); const std::chrono::milliseconds default_latency_threshold( PROTOBUF_GET_MS_OR_DEFAULT(proto_config, default_latency_threshold, 100)); @@ -44,9 +48,10 @@ Network::FilterFactoryCb ZooKeeperConfigFactory::createFilterFactoryFromProtoTyp } ZooKeeperFilterConfigSharedPtr filter_config(std::make_shared( - stat_prefix, max_packet_bytes, enable_latency_threshold_metrics, default_latency_threshold, - latency_threshold_overrides, context.scope())); - auto& time_source = context.mainThreadDispatcher().timeSource(); + stat_prefix, max_packet_bytes, enable_per_opcode_request_bytes_metrics, + enable_per_opcode_response_bytes_metrics, enable_latency_threshold_metrics, + default_latency_threshold, latency_threshold_overrides, context.scope())); + auto& time_source = context.serverFactoryContext().mainThreadDispatcher().timeSource(); return [filter_config, &time_source](Network::FilterManager& filter_manager) -> void { filter_manager.addFilter(std::make_shared(filter_config, time_source)); diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc index b54a81724d41..02d7d0b7074b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc @@ -42,15 +42,26 @@ const char* createFlagsToString(CreateFlags flags) { return "unknown"; } -void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { +absl::StatusOr> DecoderImpl::decodeOnData(Buffer::Instance& data, + uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding request with {} bytes at offset {}", data.length(), offset); // Check message length. - const int32_t len = helper_.peekInt32(data, offset); - ENVOY_LOG(trace, "zookeeper_proxy: decoding request with len {} at offset {}", len, offset); - ensureMinLength(len, XID_LENGTH + INT_LENGTH); // xid + opcode - ensureMaxLength(len); + const absl::StatusOr len = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + len, fmt::format("peekInt32 for len: {}", len.status().message())); + + ENVOY_LOG(trace, "zookeeper_proxy: decoding request with len {} at offset {}", len.value(), + offset); + + absl::Status status = ensureMinLength(len.value(), XID_LENGTH + INT_LENGTH); // xid + opcode + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("ensureMinLength: {}", status.message())); + + status = ensureMaxLength(len.value()); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("ensureMaxLength: {}", status.message())); auto start_time = time_source_.monotonicTime(); @@ -64,27 +75,41 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { // ZooKeeper server to the next. Thus, the special xid. // However, some client implementations might expose setWatches // as a regular data request, so we support that as well. - const int32_t xid = helper_.peekInt32(data, offset); - ENVOY_LOG(trace, "zookeeper_proxy: decoding request with xid {} at offset {}", xid, offset); - switch (static_cast(xid)) { + const absl::StatusOr xid = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + xid, fmt::format("peerInt32 for xid: {}", xid.status().message())); + + ENVOY_LOG(trace, "zookeeper_proxy: decoding request with xid {} at offset {}", xid.value(), + offset); + + switch (static_cast(xid.value())) { case XidCodes::ConnectXid: - parseConnect(data, offset, len); - control_requests_by_xid_[xid].push({OpCodes::Connect, std::move(start_time)}); - return; + status = parseConnect(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("parseConnect: {}", status.message())); + + control_requests_by_xid_[xid.value()].push({OpCodes::Connect, std::move(start_time)}); + return OpCodes::Connect; case XidCodes::PingXid: offset += OPCODE_LENGTH; callbacks_.onPing(); - control_requests_by_xid_[xid].push({OpCodes::Ping, std::move(start_time)}); - return; + control_requests_by_xid_[xid.value()].push({OpCodes::Ping, std::move(start_time)}); + return OpCodes::Ping; case XidCodes::AuthXid: - parseAuthRequest(data, offset, len); - control_requests_by_xid_[xid].push({OpCodes::SetAuth, std::move(start_time)}); - return; + status = parseAuthRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("parseAuthRequest: {}", status.message())); + + control_requests_by_xid_[xid.value()].push({OpCodes::SetAuth, std::move(start_time)}); + return OpCodes::SetAuth; case XidCodes::SetWatchesXid: offset += OPCODE_LENGTH; - parseSetWatchesRequest(data, offset, len); - control_requests_by_xid_[xid].push({OpCodes::SetWatches, std::move(start_time)}); - return; + status = parseSetWatchesRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseSetWatchesRequest: {}", status.message())); + + control_requests_by_xid_[xid.value()].push({OpCodes::SetWatches, std::move(start_time)}); + return OpCodes::SetWatches; default: // WATCH_XID is generated by the server, so that and everything // else can be ignored here. @@ -97,98 +122,164 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { // for two cases: auth requests can happen at any time and ping requests // must happen every 1/3 of the negotiated session timeout, to keep // the session alive. - const int32_t oc = helper_.peekInt32(data, offset); - ENVOY_LOG(trace, "zookeeper_proxy: decoding request with opcode {} at offset {}", oc, offset); - const auto opcode = static_cast(oc); + const absl::StatusOr oc = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + oc, fmt::format("peekInt32 for opcode: {}", oc.status().message())); + + ENVOY_LOG(trace, "zookeeper_proxy: decoding request with opcode {} at offset {}", oc.value(), + offset); + + const auto opcode = static_cast(oc.value()); switch (opcode) { case OpCodes::GetData: - parseGetDataRequest(data, offset, len); + status = parseGetDataRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseGetDataRequest: {}", status.message())); break; case OpCodes::Create: case OpCodes::Create2: case OpCodes::CreateContainer: case OpCodes::CreateTtl: - parseCreateRequest(data, offset, len, static_cast(opcode)); + status = parseCreateRequest(data, offset, len.value(), static_cast(opcode)); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseCreateRequest: {}", status.message())); break; case OpCodes::SetData: - parseSetRequest(data, offset, len); + status = parseSetRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("parseSetRequest: {}", status.message())); break; case OpCodes::GetChildren: - parseGetChildrenRequest(data, offset, len, false); + status = parseGetChildrenRequest(data, offset, len.value(), false); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseGetChildrenRequest: {}", status.message())); break; case OpCodes::GetChildren2: - parseGetChildrenRequest(data, offset, len, true); + status = parseGetChildrenRequest(data, offset, len.value(), true); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseGetChildrenRequest: {}", status.message())); break; case OpCodes::Delete: - parseDeleteRequest(data, offset, len); + status = parseDeleteRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseDeleteRequest: {}", status.message())); break; case OpCodes::Exists: - parseExistsRequest(data, offset, len); + status = parseExistsRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseExistsRequest: {}", status.message())); break; case OpCodes::GetAcl: - parseGetAclRequest(data, offset, len); + status = parseGetAclRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseGetAclRequest: {}", status.message())); break; case OpCodes::SetAcl: - parseSetAclRequest(data, offset, len); + status = parseSetAclRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseSetAclRequest: {}", status.message())); break; case OpCodes::Sync: - callbacks_.onSyncRequest(pathOnlyRequest(data, offset, len)); + status = callbacks_.onSyncRequest(pathOnlyRequest(data, offset, len.value())); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("onSyncRequest: {}", status.message())); break; case OpCodes::Check: - parseCheckRequest(data, offset, len); + status = parseCheckRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("parseCheckRequest: {}", status.message())); break; case OpCodes::Multi: - parseMultiRequest(data, offset, len); + status = parseMultiRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("parseMultiRequest: {}", status.message())); break; case OpCodes::Reconfig: - parseReconfigRequest(data, offset, len); + status = parseReconfigRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseReconfigRequest: {}", status.message())); break; case OpCodes::SetWatches: - parseSetWatchesRequest(data, offset, len); + status = parseSetWatchesRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseSetWatchesRequest: {}", status.message())); break; case OpCodes::SetWatches2: - parseSetWatches2Request(data, offset, len); + status = parseSetWatches2Request(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseSetWatches2Request: {}", status.message())); break; case OpCodes::AddWatch: - parseAddWatchRequest(data, offset, len); + status = parseAddWatchRequest(data, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseAddWatchRequest: {}", status.message())); break; case OpCodes::CheckWatches: - parseXWatchesRequest(data, offset, len, OpCodes::CheckWatches); + status = parseXWatchesRequest(data, offset, len.value(), OpCodes::CheckWatches); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseXWatchesRequest (check watches): {}", status.message())); break; case OpCodes::RemoveWatches: - parseXWatchesRequest(data, offset, len, OpCodes::RemoveWatches); + status = parseXWatchesRequest(data, offset, len.value(), OpCodes::RemoveWatches); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseXWatchesRequest (remove watches): {}", status.message())); break; case OpCodes::GetEphemerals: - callbacks_.onGetEphemeralsRequest(pathOnlyRequest(data, offset, len)); + status = callbacks_.onGetEphemeralsRequest(pathOnlyRequest(data, offset, len.value())); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("onGetEphemeralsRequest: {}", status.message())); break; case OpCodes::GetAllChildrenNumber: - callbacks_.onGetAllChildrenNumberRequest(pathOnlyRequest(data, offset, len)); + status = callbacks_.onGetAllChildrenNumberRequest(pathOnlyRequest(data, offset, len.value())); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("onGetAllChildrenNumberRequest: {}", status.message())); break; case OpCodes::Close: callbacks_.onCloseRequest(); break; default: - throw EnvoyException(fmt::format("Unknown opcode: {}", enumToSignedInt(opcode))); + ENVOY_LOG(debug, "zookeeper_proxy: decodeOnData failed: unknown opcode {}", + enumToSignedInt(opcode)); + callbacks_.onDecodeError(); + return absl::nullopt; } - requests_by_xid_[xid] = {opcode, std::move(start_time)}; + requests_by_xid_[xid.value()] = {opcode, std::move(start_time)}; + + return opcode; } -void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { +absl::StatusOr> DecoderImpl::decodeOnWrite(Buffer::Instance& data, + uint64_t& offset) { ENVOY_LOG(trace, "zookeeper_proxy: decoding response with {} bytes at offset {}", data.length(), offset); // Check message length. - const int32_t len = helper_.peekInt32(data, offset); - ENVOY_LOG(trace, "zookeeper_proxy: decoding response with len {} at offset {}", len, offset); - ensureMinLength(len, XID_LENGTH + ZXID_LENGTH + INT_LENGTH); // xid + zxid + err - ensureMaxLength(len); + const absl::StatusOr len = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + len, fmt::format("peekInt32 for len: {}", len.status().message())); + + ENVOY_LOG(trace, "zookeeper_proxy: decoding response with len.value() {} at offset {}", + len.value(), offset); + + absl::Status status = + ensureMinLength(len.value(), XID_LENGTH + ZXID_LENGTH + INT_LENGTH); // xid + zxid + err + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("ensureMinLength: {}", status.message())); + + status = ensureMaxLength(len.value()); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("ensureMaxLength: {}", status.message())); - const auto xid = helper_.peekInt32(data, offset); - ENVOY_LOG(trace, "zookeeper_proxy: decoding response with xid {} at offset {}", xid, offset); - const auto xid_code = static_cast(xid); + const absl::StatusOr xid = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + xid, fmt::format("peekInt32 for xid: {}", xid.status().message())); - std::chrono::milliseconds latency; + ENVOY_LOG(trace, "zookeeper_proxy: decoding response with xid {} at offset {}", xid.value(), + offset); + const auto xid_code = static_cast(xid.value()); + + absl::StatusOr latency; OpCodes opcode; switch (xid_code) { @@ -199,319 +290,516 @@ void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { case XidCodes::AuthXid: ABSL_FALLTHROUGH_INTENDED; case XidCodes::SetWatchesXid: - latency = fetchControlRequestData(xid, opcode); + latency = fetchControlRequestData(xid.value(), opcode); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + latency, fmt::format("fetchControlRequestData: {}", latency.status().message())); break; case XidCodes::WatchXid: // WATCH_XID is generated by the server, no need to fetch opcode and latency here. break; default: - latency = fetchDataRequestData(xid, opcode); + latency = fetchDataRequestData(xid.value(), opcode); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + latency, fmt::format("fetchDataRequestData: {}", latency.status().message())); } // Connect responses are special, they have no full reply header // but just an XID with no zxid nor error fields like the ones // available for all other server generated messages. if (xid_code == XidCodes::ConnectXid) { - parseConnectResponse(data, offset, len, latency); - return; + status = parseConnectResponse(data, offset, len.value(), latency.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("parseConnectResponse: {}", status.message())) + return opcode; } // Control responses that aren't connect, with XIDs <= 0. - const auto zxid = helper_.peekInt64(data, offset); - const auto error = helper_.peekInt32(data, offset); - ENVOY_LOG(trace, "zookeeper_proxy: decoding response with zxid {} and error {} at offset {}", - zxid, error, offset); + const absl::StatusOr zxid = helper_.peekInt64(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + zxid, fmt::format("peekInt64 for zxid: {}", zxid.status().message())); + + const absl::StatusOr error = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + error, fmt::format("peekInt32 for error: {}", error.status().message())); + + ENVOY_LOG(trace, + "zookeeper_proxy: decoding response with zxid.value() {} and error {} at offset {}", + zxid.value(), error.value(), offset); + switch (xid_code) { case XidCodes::PingXid: - callbacks_.onResponse(OpCodes::Ping, xid, zxid, error, latency); - return; + callbacks_.onResponse(OpCodes::Ping, xid.value(), zxid.value(), error.value(), latency.value()); + return opcode; case XidCodes::AuthXid: - callbacks_.onResponse(OpCodes::SetAuth, xid, zxid, error, latency); - return; + callbacks_.onResponse(OpCodes::SetAuth, xid.value(), zxid.value(), error.value(), + latency.value()); + return opcode; case XidCodes::SetWatchesXid: - callbacks_.onResponse(OpCodes::SetWatches, xid, zxid, error, latency); - return; + callbacks_.onResponse(OpCodes::SetWatches, xid.value(), zxid.value(), error.value(), + latency.value()); + return opcode; case XidCodes::WatchXid: - parseWatchEvent(data, offset, len, zxid, error); - return; + status = parseWatchEvent(data, offset, len.value(), zxid.value(), error.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, + fmt::format("parseWatchEvent: {}", status.message())); + + return absl::nullopt; // WATCH_XID is generated by the server, it has no corresponding opcode. default: break; } - callbacks_.onResponse(opcode, xid, zxid, error, latency); - offset += (len - (XID_LENGTH + ZXID_LENGTH + INT_LENGTH)); + callbacks_.onResponse(opcode, xid.value(), zxid.value(), error.value(), latency.value()); + offset += (len.value() - (XID_LENGTH + ZXID_LENGTH + INT_LENGTH)); + + return opcode; } -void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const { +absl::Status DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const { if (len < minlen) { - throw EnvoyException("Packet is too small"); + return absl::InvalidArgumentError("packet is too small"); } + return absl::OkStatus(); } -void DecoderImpl::ensureMaxLength(const int32_t len) const { +absl::Status DecoderImpl::ensureMaxLength(const int32_t len) const { if (static_cast(len) > max_packet_bytes_) { - throw EnvoyException("Packet is too big"); + return absl::InvalidArgumentError("packet is too big"); } + return absl::OkStatus(); } -void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); +absl::Status DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + absl::Status status = + ensureMinLength(len, XID_LENGTH + ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); // Skip zxid, timeout, and session id. offset += ZXID_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH; // Skip password. - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const bool readonly = maybeReadBool(data, offset); + const absl::StatusOr readonly = maybeReadBool(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(readonly, + readonly.status().message()); - callbacks_.onConnect(readonly); -} + callbacks_.onConnect(readonly.value()); -void DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + INT_LENGTH + INT_LENGTH); + return absl::OkStatus(); +} +absl::Status DecoderImpl::parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + absl::Status status = + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + INT_LENGTH + INT_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); // Skip opcode + type. offset += OPCODE_LENGTH + INT_LENGTH; - const std::string scheme = helper_.peekString(data, offset); + + const absl::StatusOr scheme = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(scheme, scheme.status().message()); + // Skip credential. - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - callbacks_.onAuthRequest(scheme); + callbacks_.onAuthRequest(scheme.value()); + + return absl::OkStatus(); } -void DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); +absl::Status DecoderImpl::parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + + const absl::StatusOr watch = helper_.peekBool(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(watch, watch.status().message()); - const std::string path = helper_.peekString(data, offset); - const bool watch = helper_.peekBool(data, offset); + callbacks_.onGetDataRequest(path.value(), watch.value()); - callbacks_.onGetDataRequest(path, watch); + return absl::OkStatus(); } -void DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) { - const int32_t count = helper_.peekInt32(data, offset); +absl::Status DecoderImpl::skipAcls(Buffer::Instance& data, uint64_t& offset) { + const absl::StatusOr count = helper_.peekInt32(data, offset); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(count, + fmt::format("skipAcls: {}", count.status().message())); - for (int i = 0; i < count; ++i) { + for (int i = 0; i < count.value(); ++i) { // Perms. - helper_.peekInt32(data, offset); + absl::StatusOr perms = helper_.peekInt32(data, offset); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(perms, + fmt::format("skipAcls: {}", perms.status().message())); // Skip scheme. - skipString(data, offset); + absl::Status status = skipString(data, offset); + ABSL_STATUS_RETURN_IF_STATUS_NOT_OK(status); // Skip cred. - skipString(data, offset); + status = skipString(data, offset); + ABSL_STATUS_RETURN_IF_STATUS_NOT_OK(status); } + + return absl::OkStatus(); } -void DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, - OpCodes opcode) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (4 * INT_LENGTH)); +absl::Status DecoderImpl::parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + OpCodes opcode) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (4 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const std::string path = helper_.peekString(data, offset); + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); // Skip data. - skipString(data, offset); - skipAcls(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + status = skipAcls(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const CreateFlags flags = static_cast(helper_.peekInt32(data, offset)); - callbacks_.onCreateRequest(path, flags, opcode); + absl::StatusOr flag_data = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(flag_data, + flag_data.status().message()); + + const CreateFlags flags = static_cast(flag_data.value()); + status = callbacks_.onCreateRequest(path.value(), flags, opcode); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + return absl::OkStatus(); } -void DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); +absl::Status DecoderImpl::parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); - const std::string path = helper_.peekString(data, offset); // Skip data. - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Ignore version. - helper_.peekInt32(data, offset); + absl::StatusOr version = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(version, version.status().message()); + + callbacks_.onSetRequest(path.value()); - callbacks_.onSetRequest(path); + return absl::OkStatus(); } -void DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const bool two) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); +absl::Status DecoderImpl::parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len, const bool two) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); - const std::string path = helper_.peekString(data, offset); - const bool watch = helper_.peekBool(data, offset); + const absl::StatusOr watch = helper_.peekBool(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(watch, watch.status().message()); - callbacks_.onGetChildrenRequest(path, watch, two); + callbacks_.onGetChildrenRequest(path.value(), watch.value(), two); + + return absl::OkStatus(); } -void DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); +absl::Status DecoderImpl::parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + + const absl::StatusOr version = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(version, version.status().message()); - const std::string path = helper_.peekString(data, offset); - const int32_t version = helper_.peekInt32(data, offset); + callbacks_.onDeleteRequest(path.value(), version.value()); - callbacks_.onDeleteRequest(path, version); + return absl::OkStatus(); } -void DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); +absl::Status DecoderImpl::parseExistsRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH + BOOL_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + + const absl::StatusOr watch = helper_.peekBool(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(watch, watch.status().message()); - const std::string path = helper_.peekString(data, offset); - const bool watch = helper_.peekBool(data, offset); + callbacks_.onExistsRequest(path.value(), watch.value()); - callbacks_.onExistsRequest(path, watch); + return absl::OkStatus(); } -void DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); +absl::Status DecoderImpl::parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const std::string path = helper_.peekString(data, offset); + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); - callbacks_.onGetAclRequest(path); + callbacks_.onGetAclRequest(path.value()); + + return absl::OkStatus(); } -void DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); +absl::Status DecoderImpl::parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + + status = skipAcls(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const std::string path = helper_.peekString(data, offset); - skipAcls(data, offset); - const int32_t version = helper_.peekInt32(data, offset); + const absl::StatusOr version = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(version, version.status().message()); - callbacks_.onSetAclRequest(path, version); + callbacks_.onSetAclRequest(path.value(), version.value()); + + return absl::OkStatus(); } -std::string DecoderImpl::pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); +absl::StatusOr DecoderImpl::pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + INT_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + status, fmt::format("zookeeper_proxy: pathOnlyRequest failed: {}", status.message())); + return helper_.peekString(data, offset); } -void DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); +absl::Status DecoderImpl::parseCheckRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); - const std::string path = helper_.peekString(data, offset); - const int32_t version = helper_.peekInt32(data, offset); + const absl::StatusOr version = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(version, version.status().message()); - callbacks_.onCheckRequest(path, version); + callbacks_.onCheckRequest(path.value(), version.value()); + + return absl::OkStatus(); } -void DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { +absl::Status DecoderImpl::parseMultiRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { // Treat empty transactions as a decoding error, there should be at least 1 header. - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + MULTI_HEADER_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); while (true) { - const int32_t opcode = helper_.peekInt32(data, offset); - const bool done = helper_.peekBool(data, offset); + const absl::StatusOr opcode = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(opcode, opcode.status().message()); + + const absl::StatusOr done = helper_.peekBool(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(done, done.status().message()); + // Ignore error field. - helper_.peekInt32(data, offset); + const absl::StatusOr error = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(error, error.status().message()); - if (done) { + if (done.value()) { break; } - switch (static_cast(opcode)) { + switch (static_cast(opcode.value())) { case OpCodes::Create: - parseCreateRequest(data, offset, len, OpCodes::Create); + status = parseCreateRequest(data, offset, len, OpCodes::Create); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); break; case OpCodes::SetData: - parseSetRequest(data, offset, len); + status = parseSetRequest(data, offset, len); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); break; case OpCodes::Check: - parseCheckRequest(data, offset, len); + status = parseCheckRequest(data, offset, len); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); break; case OpCodes::Delete: - parseDeleteRequest(data, offset, len); + status = parseDeleteRequest(data, offset, len); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); break; default: - throw EnvoyException(fmt::format("Unknown opcode within a transaction: {}", opcode)); + callbacks_.onDecodeError(); + return absl::InvalidArgumentError( + fmt::format("unknown opcode within a transaction: {}", opcode.value())); } } callbacks_.onMultiRequest(); + + return absl::OkStatus(); } -void DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH) + LONG_LENGTH); +absl::Status DecoderImpl::parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (3 * INT_LENGTH) + LONG_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); // Skip joining. - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Skip leaving. - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); // Skip new members. - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Read config id. - helper_.peekInt64(data, offset); + absl::StatusOr config_id = helper_.peekInt64(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(config_id, + config_id.status().message()); callbacks_.onReconfigRequest(); + + return absl::OkStatus(); } -void DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + LONG_LENGTH + (3 * INT_LENGTH)); +absl::Status DecoderImpl::parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + LONG_LENGTH + (3 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); // Ignore relative Zxid. - helper_.peekInt64(data, offset); + absl::StatusOr zxid = helper_.peekInt64(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(zxid, zxid.status().message()); + // Data watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Exist watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Child watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); callbacks_.onSetWatchesRequest(); + + return absl::OkStatus(); } -void DecoderImpl::parseSetWatches2Request(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + LONG_LENGTH + (5 * INT_LENGTH)); +absl::Status DecoderImpl::parseSetWatches2Request(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = + ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + LONG_LENGTH + (5 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); // Ignore relative Zxid. - helper_.peekInt64(data, offset); + absl::StatusOr zxid = helper_.peekInt64(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(zxid, zxid.status().message()); + // Data watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Exist watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Child watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Persistent watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + // Persistent recursive watches. - skipStrings(data, offset); + status = skipStrings(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); callbacks_.onSetWatches2Request(); + + return absl::OkStatus(); } -void DecoderImpl::parseAddWatchRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); +absl::Status DecoderImpl::parseAddWatchRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + + const absl::StatusOr mode = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(mode, mode.status().message()); - const std::string path = helper_.peekString(data, offset); - const int32_t mode = helper_.peekInt32(data, offset); + callbacks_.onAddWatchRequest(path.value(), mode.value()); - callbacks_.onAddWatchRequest(path, mode); + return absl::OkStatus(); } -void DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, - OpCodes opcode) { - ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); +absl::Status DecoderImpl::parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len, OpCodes opcode) { + absl::Status status = ensureMinLength(len, XID_LENGTH + OPCODE_LENGTH + (2 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const std::string path = helper_.peekString(data, offset); - const int32_t type = helper_.peekInt32(data, offset); + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + + const absl::StatusOr watch_type = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(watch_type, + watch_type.status().message()); if (opcode == OpCodes::CheckWatches) { - callbacks_.onCheckWatchesRequest(path, type); + callbacks_.onCheckWatchesRequest(path.value(), watch_type.value()); } else { - callbacks_.onRemoveWatchesRequest(path, type); + callbacks_.onRemoveWatchesRequest(path.value(), watch_type.value()); } + + return absl::OkStatus(); } -void DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) { - const int32_t slen = helper_.peekInt32(data, offset); - if (slen < 0) { +absl::Status DecoderImpl::skipString(Buffer::Instance& data, uint64_t& offset) { + const absl::StatusOr slen = helper_.peekInt32(data, offset); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(slen, + fmt::format("skipString: {}", slen.status().message())); + + if (slen.value() < 0) { ENVOY_LOG(trace, "zookeeper_proxy: decoding response with negative string length {} at offset {}", - slen, offset); - return; + slen.value(), offset); + return absl::OkStatus(); } - helper_.skip(slen, offset); + + helper_.skip(slen.value(), offset); + + return absl::OkStatus(); } -void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) { - const int32_t count = helper_.peekInt32(data, offset); +absl::Status DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) { + const absl::StatusOr count = helper_.peekInt32(data, offset); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(count, + fmt::format("skipStrings: {}", count.status().message())); - for (int i = 0; i < count; ++i) { - skipString(data, offset); + for (int i = 0; i < count.value(); ++i) { + absl::Status status = skipString(data, offset); + ABSL_STATUS_RETURN_IF_STATUS_NOT_OK(status); } + + return absl::OkStatus(); } Network::FilterStatus DecoderImpl::onData(Buffer::Instance& data) { @@ -525,9 +813,14 @@ Network::FilterStatus DecoderImpl::onWrite(Buffer::Instance& data) { Network::FilterStatus DecoderImpl::decodeAndBuffer(Buffer::Instance& data, DecodeType dtype, Buffer::OwnedImpl& zk_filter_buffer) { const uint32_t zk_filter_buffer_len = zk_filter_buffer.length(); + absl::Status status; if (zk_filter_buffer_len == 0) { - decodeAndBufferHelper(data, dtype, zk_filter_buffer); + status = decodeAndBufferHelper(data, dtype, zk_filter_buffer); + if (!status.ok()) { + ENVOY_LOG(debug, "zookeeper_proxy: decodeAndBufferHelper failed: {}", status.message()); + } + return Network::FilterStatus::Continue; } @@ -535,19 +828,25 @@ Network::FilterStatus DecoderImpl::decodeAndBuffer(Buffer::Instance& data, Decod // Prepending ZooKeeper filter buffer to the current network filter buffer can help to generate // full packets. data.prepend(zk_filter_buffer); - decodeAndBufferHelper(data, dtype, zk_filter_buffer); + + status = decodeAndBufferHelper(data, dtype, zk_filter_buffer); + if (!status.ok()) { + ENVOY_LOG(debug, "zookeeper_proxy: decodeAndBufferHelper failed: {}", status.message()); + } + // Drain the prepended ZooKeeper filter buffer. data.drain(zk_filter_buffer_len); return Network::FilterStatus::Continue; } -void DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype, - Buffer::OwnedImpl& zk_filter_buffer) { +absl::Status DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype, + Buffer::OwnedImpl& zk_filter_buffer) { ASSERT(dtype == DecodeType::READ || dtype == DecodeType::WRITE); const uint32_t data_len = data.length(); uint64_t offset = 0; - uint32_t len = 0; + absl::StatusOr len = 0; + absl::Status status; // Boolean to check whether there is at least one full packet in the network filter buffer (to // which the ZooKeeper filter buffer is prepended). bool has_full_packets = false; @@ -556,31 +855,39 @@ void DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype TRY_NEEDS_AUDIT { // Peek packet length. len = helper_.peekInt32(data, offset); - ensureMinLength(len, dtype == DecodeType::READ ? XID_LENGTH + INT_LENGTH - : XID_LENGTH + ZXID_LENGTH + INT_LENGTH); - ensureMaxLength(len); - offset += len; + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK( + len, fmt::format("peekInt32 for len: {}", len.status().message())); + + status = ensureMinLength(len.value(), dtype == DecodeType::READ + ? XID_LENGTH + INT_LENGTH + : XID_LENGTH + ZXID_LENGTH + INT_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + status = ensureMaxLength(len.value()); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + offset += len.value(); if (offset <= data_len) { has_full_packets = true; } } END_TRY catch (const EnvoyException& e) { - ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what()); + IS_ENVOY_BUG(fmt::format("zookeeper_proxy: decodeAndBufferHelper failed: {}", e.what())); callbacks_.onDecodeError(); - return; + return absl::OkStatus(); } } if (offset == data_len) { decode(data, dtype, offset); - return; + return absl::OkStatus(); } ASSERT(offset > data_len); std::string temp_data; if (has_full_packets) { - offset -= INT_LENGTH + len; + offset -= INT_LENGTH + len.value(); ASSERT(offset < data_len); // Decode full packets. // offset here represents the length of all full packets. @@ -597,6 +904,8 @@ void DecoderImpl::decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype data.copyOut(0, data_len, temp_data.data()); zk_filter_buffer.add(temp_data.data(), temp_data.length()); } + + return absl::OkStatus(); } void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype, uint64_t full_packets_len) { @@ -614,69 +923,99 @@ void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype, uint64_t full helper_.reset(); const uint64_t current = offset; + absl::StatusOr> opcode; switch (dtype) { case DecodeType::READ: - decodeOnData(data, offset); - callbacks_.onRequestBytes(offset - current); - break; + opcode = decodeOnData(data, offset); + if (opcode.ok()) { + callbacks_.onRequestBytes(opcode.value(), offset - current); + break; + } + ENVOY_LOG(debug, "zookeeper_proxy: decodeOnData failed: {}", opcode.status().message()); + return; case DecodeType::WRITE: - decodeOnWrite(data, offset); - callbacks_.onResponseBytes(offset - current); - break; + opcode = decodeOnWrite(data, offset); + if (opcode.ok()) { + callbacks_.onResponseBytes(opcode.value(), offset - current); + break; + } + ENVOY_LOG(debug, "zookeeper_proxy: decodeOnWrite failed: {}", opcode.status().message()); + return; } } } END_TRY catch (const EnvoyException& e) { - ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what()); + IS_ENVOY_BUG(fmt::format("zookeeper_proxy: decode failed: {}", e.what())); callbacks_.onDecodeError(); } } -void DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const std::chrono::milliseconds latency) { - ensureMinLength(len, PROTOCOL_VERSION_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); +absl::Status DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, + uint32_t len, + const std::chrono::milliseconds latency) { + absl::Status status = + ensureMinLength(len, PROTOCOL_VERSION_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); - const auto timeout = helper_.peekInt32(data, offset); + const absl::StatusOr timeout = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(timeout, timeout.status().message()); // Skip session id + password. offset += SESSION_LENGTH; - skipString(data, offset); + status = skipString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr readonly = maybeReadBool(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(readonly, + readonly.status().message()); - const bool readonly = maybeReadBool(data, offset); + callbacks_.onConnectResponse(0, timeout.value(), readonly.value(), latency); - callbacks_.onConnectResponse(0, timeout, readonly, latency); + return absl::OkStatus(); } -void DecoderImpl::parseWatchEvent(Buffer::Instance& data, uint64_t& offset, const uint32_t len, - const int64_t zxid, const int32_t error) { - ensureMinLength(len, SERVER_HEADER_LENGTH + (3 * INT_LENGTH)); +absl::Status DecoderImpl::parseWatchEvent(Buffer::Instance& data, uint64_t& offset, + const uint32_t len, const int64_t zxid, + const int32_t error) { + absl::Status status = ensureMinLength(len, SERVER_HEADER_LENGTH + (3 * INT_LENGTH)); + EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status); + + const absl::StatusOr event_type = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(event_type, + event_type.status().message()); + + const absl::StatusOr client_state = helper_.peekInt32(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(client_state, + client_state.status().message()); + + const absl::StatusOr path = helper_.peekString(data, offset); + EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); - const auto event_type = helper_.peekInt32(data, offset); - const auto client_state = helper_.peekInt32(data, offset); - const auto path = helper_.peekString(data, offset); + callbacks_.onWatchEvent(event_type.value(), client_state.value(), path.value(), zxid, error); - callbacks_.onWatchEvent(event_type, client_state, path, zxid, error); + return absl::OkStatus(); } -bool DecoderImpl::maybeReadBool(Buffer::Instance& data, uint64_t& offset) { +absl::StatusOr DecoderImpl::maybeReadBool(Buffer::Instance& data, uint64_t& offset) { if (data.length() >= offset + 1) { return helper_.peekBool(data, offset); } return false; } -std::chrono::milliseconds DecoderImpl::fetchControlRequestData(const int32_t xid, OpCodes& opcode) { +absl::StatusOr DecoderImpl::fetchControlRequestData(const int32_t xid, + OpCodes& opcode) { // Find the corresponding request queue for this XID. const auto it = control_requests_by_xid_.find(xid); // If this fails, it's either a server-side bug or a malformed packet. if (it == control_requests_by_xid_.end()) { - throw EnvoyException(fmt::format("control request xid {} not found", xid)); + return absl::InvalidArgumentError(fmt::format("control request xid {} not found", xid)); } std::queue& rq_queue = it->second; if (rq_queue.empty()) { - throw EnvoyException(fmt::format("control request queue for {} is empty", xid)); + return absl::InvalidArgumentError(fmt::format("control request queue for {} is empty", xid)); } std::chrono::milliseconds latency = std::chrono::duration_cast( @@ -687,13 +1026,14 @@ std::chrono::milliseconds DecoderImpl::fetchControlRequestData(const int32_t xid return latency; } -std::chrono::milliseconds DecoderImpl::fetchDataRequestData(const int32_t xid, OpCodes& opcode) { +absl::StatusOr DecoderImpl::fetchDataRequestData(const int32_t xid, + OpCodes& opcode) { // Find the corresponding request for this XID. const auto it = requests_by_xid_.find(xid); // If this fails, it's either a server-side bug or a malformed packet. if (it == requests_by_xid_.end()) { - throw EnvoyException(fmt::format("xid {} not found", xid)); + return absl::InvalidArgumentError(fmt::format("data request xid {} not found", xid)); } std::chrono::milliseconds latency = std::chrono::duration_cast( diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h index 8d788481ac07..8718a8d9d6b1 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h @@ -12,6 +12,7 @@ #include "source/extensions/filters/network/zookeeper_proxy/utils.h" #include "absl/container/node_hash_map.h" +#include "absl/status/statusor.h" namespace Envoy { namespace Extensions { @@ -80,21 +81,22 @@ class DecoderCallbacks { virtual ~DecoderCallbacks() = default; virtual void onDecodeError() PURE; - virtual void onRequestBytes(uint64_t bytes) PURE; + virtual void onRequestBytes(const absl::optional opcode, const uint64_t bytes) PURE; virtual void onConnect(bool readonly) PURE; virtual void onPing() PURE; virtual void onAuthRequest(const std::string& scheme) PURE; virtual void onGetDataRequest(const std::string& path, bool watch) PURE; - virtual void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) PURE; + virtual absl::Status onCreateRequest(const std::string& path, CreateFlags flags, + OpCodes opcode) PURE; virtual void onSetRequest(const std::string& path) PURE; virtual void onGetChildrenRequest(const std::string& path, bool watch, bool v2) PURE; - virtual void onGetEphemeralsRequest(const std::string& path) PURE; - virtual void onGetAllChildrenNumberRequest(const std::string& path) PURE; + virtual absl::Status onGetEphemeralsRequest(const absl::StatusOr& path) PURE; + virtual absl::Status onGetAllChildrenNumberRequest(const absl::StatusOr& path) PURE; virtual void onDeleteRequest(const std::string& path, int32_t version) PURE; virtual void onExistsRequest(const std::string& path, bool watch) PURE; virtual void onGetAclRequest(const std::string& path) PURE; virtual void onSetAclRequest(const std::string& path, int32_t version) PURE; - virtual void onSyncRequest(const std::string& path) PURE; + virtual absl::Status onSyncRequest(const absl::StatusOr& path) PURE; virtual void onCheckRequest(const std::string& path, int32_t version) PURE; virtual void onMultiRequest() PURE; virtual void onReconfigRequest() PURE; @@ -104,7 +106,7 @@ class DecoderCallbacks { virtual void onCheckWatchesRequest(const std::string& path, int32_t type) PURE; virtual void onRemoveWatchesRequest(const std::string& path, int32_t type) PURE; virtual void onCloseRequest() PURE; - virtual void onResponseBytes(uint64_t bytes) PURE; + virtual void onResponseBytes(const absl::optional opcode, const uint64_t bytes) PURE; virtual void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly, const std::chrono::milliseconds latency) PURE; virtual void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error, @@ -151,41 +153,50 @@ class DecoderImpl : public Decoder, Logger::Loggable { // (4) removes the prepended data. Network::FilterStatus decodeAndBuffer(Buffer::Instance& data, DecodeType dtype, Buffer::OwnedImpl& zk_filter_buffer); - void decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype, - Buffer::OwnedImpl& zk_filter_buffer); + absl::Status decodeAndBufferHelper(Buffer::Instance& data, DecodeType dtype, + Buffer::OwnedImpl& zk_filter_buffer); void decode(Buffer::Instance& data, DecodeType dtype, uint64_t full_packets_len); - void decodeOnData(Buffer::Instance& data, uint64_t& offset); - void decodeOnWrite(Buffer::Instance& data, uint64_t& offset); - void parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); - void skipAcls(Buffer::Instance& data, uint64_t& offset); - void parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, bool two); - void parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseSetWatches2Request(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseAddWatchRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, OpCodes opcode); - void skipString(Buffer::Instance& data, uint64_t& offset); - void skipStrings(Buffer::Instance& data, uint64_t& offset); - void ensureMinLength(int32_t len, int32_t minlen) const; - void ensureMaxLength(int32_t len) const; - std::string pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, - const std::chrono::milliseconds latency); - void parseWatchEvent(Buffer::Instance& data, uint64_t& offset, uint32_t len, int64_t zxid, - int32_t error); - bool maybeReadBool(Buffer::Instance& data, uint64_t& offset); - std::chrono::milliseconds fetchControlRequestData(const int32_t xid, OpCodes& opcode); - std::chrono::milliseconds fetchDataRequestData(const int32_t xid, OpCodes& opcode); + // decodeOnData and decodeOnWrite return ZooKeeper opcode or absl::nullopt. + // absl::nullopt indicates WATCH_XID, which is generated by the server and has no corresponding + // opcode. + absl::StatusOr> decodeOnData(Buffer::Instance& data, uint64_t& offset); + absl::StatusOr> decodeOnWrite(Buffer::Instance& data, uint64_t& offset); + absl::Status parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseCreateRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + OpCodes opcode); + absl::Status skipAcls(Buffer::Instance& data, uint64_t& offset); + absl::Status parseSetRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseGetChildrenRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + bool two); + absl::Status parseDeleteRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseExistsRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseGetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseSetAclRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseCheckRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseMultiRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseReconfigRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseSetWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseSetWatches2Request(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseAddWatchRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + absl::Status parseXWatchesRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len, + OpCodes opcode); + absl::Status skipString(Buffer::Instance& data, uint64_t& offset); + absl::Status skipStrings(Buffer::Instance& data, uint64_t& offset); + absl::Status ensureMinLength(int32_t len, int32_t minlen) const; + absl::Status ensureMaxLength(int32_t len) const; + absl::StatusOr pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, + uint32_t len); + absl::Status parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, + const std::chrono::milliseconds latency); + absl::Status parseWatchEvent(Buffer::Instance& data, uint64_t& offset, uint32_t len, int64_t zxid, + int32_t error); + absl::StatusOr maybeReadBool(Buffer::Instance& data, uint64_t& offset); + absl::StatusOr fetchControlRequestData(const int32_t xid, + OpCodes& opcode); + absl::StatusOr fetchDataRequestData(const int32_t xid, + OpCodes& opcode); DecoderCallbacks& callbacks_; const uint32_t max_packet_bytes_; diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index 03d2202a7395..817fa14077d4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -22,6 +22,8 @@ namespace ZooKeeperProxy { ZooKeeperFilterConfig::ZooKeeperFilterConfig( const std::string& stat_prefix, const uint32_t max_packet_bytes, + const bool enable_per_opcode_request_bytes_metrics, + const bool enable_per_opcode_response_bytes_metrics, const bool enable_latency_threshold_metrics, const std::chrono::milliseconds default_latency_threshold, const LatencyThresholdOverrideList& latency_threshold_overrides, Stats::Scope& scope) @@ -31,6 +33,8 @@ ZooKeeperFilterConfig::ZooKeeperFilterConfig( connect_latency_(stat_name_set_->add("connect_response_latency")), unknown_scheme_rq_(stat_name_set_->add("unknown_scheme_rq")), unknown_opcode_latency_(stat_name_set_->add("unknown_opcode_latency")), + enable_per_opcode_request_bytes_metrics_(enable_per_opcode_request_bytes_metrics), + enable_per_opcode_response_bytes_metrics_(enable_per_opcode_response_bytes_metrics), enable_latency_threshold_metrics_(enable_latency_threshold_metrics), default_latency_threshold_(default_latency_threshold), latency_threshold_override_map_(parseLatencyThresholdOverrides(latency_threshold_overrides)) { @@ -41,59 +45,79 @@ ZooKeeperFilterConfig::ZooKeeperFilterConfig( {"auth_rq", "digest_rq", "host_rq", "ip_rq", "ping_response_rq", "world_rq", "x509_rq"}); initOpCode(OpCodes::Ping, stats_.ping_resp_, stats_.ping_resp_fast_, stats_.ping_resp_slow_, - "ping_response"); + stats_.ping_rq_bytes_, stats_.ping_resp_bytes_, "ping_response"); initOpCode(OpCodes::SetAuth, stats_.auth_resp_, stats_.auth_resp_fast_, stats_.auth_resp_slow_, - "auth_response"); + stats_.auth_rq_bytes_, stats_.auth_resp_bytes_, "auth_response"); initOpCode(OpCodes::GetData, stats_.getdata_resp_, stats_.getdata_resp_fast_, - stats_.getdata_resp_slow_, "getdata_resp"); + stats_.getdata_resp_slow_, stats_.getdata_rq_bytes_, stats_.getdata_resp_bytes_, + "getdata_resp"); initOpCode(OpCodes::Create, stats_.create_resp_, stats_.create_resp_fast_, - stats_.create_resp_slow_, "create_resp"); + stats_.create_resp_slow_, stats_.create_rq_bytes_, stats_.create_resp_bytes_, + "create_resp"); initOpCode(OpCodes::Create2, stats_.create2_resp_, stats_.create2_resp_fast_, - stats_.create2_resp_slow_, "create2_resp"); + stats_.create2_resp_slow_, stats_.create2_rq_bytes_, stats_.create2_resp_bytes_, + "create2_resp"); initOpCode(OpCodes::CreateContainer, stats_.createcontainer_resp_, stats_.createcontainer_resp_fast_, stats_.createcontainer_resp_slow_, + stats_.createcontainer_rq_bytes_, stats_.createcontainer_resp_bytes_, "createcontainer_resp"); initOpCode(OpCodes::CreateTtl, stats_.createttl_resp_, stats_.createttl_resp_fast_, - stats_.createttl_resp_slow_, "createttl_resp"); + stats_.createttl_resp_slow_, stats_.createttl_rq_bytes_, stats_.createttl_resp_bytes_, + "createttl_resp"); initOpCode(OpCodes::SetData, stats_.setdata_resp_, stats_.setdata_resp_fast_, - stats_.setdata_resp_slow_, "setdata_resp"); + stats_.setdata_resp_slow_, stats_.setdata_rq_bytes_, stats_.setdata_resp_bytes_, + "setdata_resp"); initOpCode(OpCodes::GetChildren, stats_.getchildren_resp_, stats_.getchildren_resp_fast_, - stats_.getchildren_resp_slow_, "getchildren_resp"); + stats_.getchildren_resp_slow_, stats_.getchildren_rq_bytes_, + stats_.getchildren_resp_bytes_, "getchildren_resp"); initOpCode(OpCodes::GetChildren2, stats_.getchildren2_resp_, stats_.getchildren2_resp_fast_, - stats_.getchildren2_resp_slow_, "getchildren2_resp"); + stats_.getchildren2_resp_slow_, stats_.getchildren2_rq_bytes_, + stats_.getchildren2_resp_bytes_, "getchildren2_resp"); initOpCode(OpCodes::Delete, stats_.delete_resp_, stats_.delete_resp_fast_, - stats_.delete_resp_slow_, "delete_resp"); + stats_.delete_resp_slow_, stats_.delete_rq_bytes_, stats_.delete_resp_bytes_, + "delete_resp"); initOpCode(OpCodes::Exists, stats_.exists_resp_, stats_.exists_resp_fast_, - stats_.exists_resp_slow_, "exists_resp"); + stats_.exists_resp_slow_, stats_.exists_rq_bytes_, stats_.exists_resp_bytes_, + "exists_resp"); initOpCode(OpCodes::GetAcl, stats_.getacl_resp_, stats_.getacl_resp_fast_, - stats_.getacl_resp_slow_, "getacl_resp"); + stats_.getacl_resp_slow_, stats_.getacl_rq_bytes_, stats_.getacl_resp_bytes_, + "getacl_resp"); initOpCode(OpCodes::SetAcl, stats_.setacl_resp_, stats_.setacl_resp_fast_, - stats_.setacl_resp_slow_, "setacl_resp"); + stats_.setacl_resp_slow_, stats_.setacl_rq_bytes_, stats_.setacl_resp_bytes_, + "setacl_resp"); initOpCode(OpCodes::Sync, stats_.sync_resp_, stats_.sync_resp_fast_, stats_.sync_resp_slow_, - "sync_resp"); + stats_.sync_rq_bytes_, stats_.sync_resp_bytes_, "sync_resp"); initOpCode(OpCodes::Check, stats_.check_resp_, stats_.check_resp_fast_, stats_.check_resp_slow_, - "check_resp"); + stats_.check_rq_bytes_, stats_.check_resp_bytes_, "check_resp"); initOpCode(OpCodes::Multi, stats_.multi_resp_, stats_.multi_resp_fast_, stats_.multi_resp_slow_, - "multi_resp"); + stats_.multi_rq_bytes_, stats_.multi_resp_bytes_, "multi_resp"); initOpCode(OpCodes::Reconfig, stats_.reconfig_resp_, stats_.reconfig_resp_fast_, - stats_.reconfig_resp_slow_, "reconfig_resp"); + stats_.reconfig_resp_slow_, stats_.reconfig_rq_bytes_, stats_.reconfig_resp_bytes_, + "reconfig_resp"); initOpCode(OpCodes::SetWatches, stats_.setwatches_resp_, stats_.setwatches_resp_fast_, - stats_.setwatches_resp_slow_, "setwatches_resp"); + stats_.setwatches_resp_slow_, stats_.setwatches_rq_bytes_, + stats_.setwatches_resp_bytes_, "setwatches_resp"); initOpCode(OpCodes::SetWatches2, stats_.setwatches2_resp_, stats_.setwatches2_resp_fast_, - stats_.setwatches2_resp_slow_, "setwatches2_resp"); + stats_.setwatches2_resp_slow_, stats_.setwatches2_rq_bytes_, + stats_.setwatches2_resp_bytes_, "setwatches2_resp"); initOpCode(OpCodes::AddWatch, stats_.addwatch_resp_, stats_.addwatch_resp_fast_, - stats_.addwatch_resp_slow_, "addwatch_resp"); + stats_.addwatch_resp_slow_, stats_.addwatch_rq_bytes_, stats_.addwatch_resp_bytes_, + "addwatch_resp"); initOpCode(OpCodes::CheckWatches, stats_.checkwatches_resp_, stats_.checkwatches_resp_fast_, - stats_.checkwatches_resp_slow_, "checkwatches_resp"); + stats_.checkwatches_resp_slow_, stats_.checkwatches_rq_bytes_, + stats_.checkwatches_resp_bytes_, "checkwatches_resp"); initOpCode(OpCodes::RemoveWatches, stats_.removewatches_resp_, stats_.removewatches_resp_fast_, - stats_.removewatches_resp_slow_, "removewatches_resp"); + stats_.removewatches_resp_slow_, stats_.removewatches_rq_bytes_, + stats_.removewatches_resp_bytes_, "removewatches_resp"); initOpCode(OpCodes::GetEphemerals, stats_.getephemerals_resp_, stats_.getephemerals_resp_fast_, - stats_.getephemerals_resp_slow_, "getephemerals_resp"); + stats_.getephemerals_resp_slow_, stats_.getephemerals_rq_bytes_, + stats_.getephemerals_resp_bytes_, "getephemerals_resp"); initOpCode(OpCodes::GetAllChildrenNumber, stats_.getallchildrennumber_resp_, stats_.getallchildrennumber_resp_fast_, stats_.getallchildrennumber_resp_slow_, + stats_.getallchildrennumber_rq_bytes_, stats_.getallchildrennumber_resp_bytes_, "getallchildrennumber_resp"); initOpCode(OpCodes::Close, stats_.close_resp_, stats_.close_resp_fast_, stats_.close_resp_slow_, - "close_resp"); + stats_.close_rq_bytes_, stats_.close_resp_bytes_, "close_resp"); } ErrorBudgetResponseType @@ -120,11 +144,15 @@ ZooKeeperFilterConfig::errorBudgetDecision(const OpCodes opcode, void ZooKeeperFilterConfig::initOpCode(OpCodes opcode, Stats::Counter& resp_counter, Stats::Counter& resp_fast_counter, - Stats::Counter& resp_slow_counter, absl::string_view name) { + Stats::Counter& resp_slow_counter, + Stats::Counter& rq_bytes_counter, + Stats::Counter& resp_bytes_counter, absl::string_view name) { OpCodeInfo& opcode_info = op_code_map_[opcode]; opcode_info.resp_counter_ = &resp_counter; opcode_info.resp_fast_counter_ = &resp_fast_counter; opcode_info.resp_slow_counter_ = &resp_slow_counter; + opcode_info.rq_bytes_counter_ = &rq_bytes_counter; + opcode_info.resp_bytes_counter_ = &resp_bytes_counter; opcode_info.opname_ = std::string(name); opcode_info.latency_name_ = stat_name_set_->add(absl::StrCat(name, "_latency")); } @@ -216,13 +244,33 @@ void ZooKeeperFilter::onDecodeError() { setDynamicMetadata("opname", "error"); } -void ZooKeeperFilter::onRequestBytes(const uint64_t bytes) { +void ZooKeeperFilter::onRequestBytes(const absl::optional opcode, const uint64_t bytes) { config_->stats_.request_bytes_.add(bytes); + + if (config_->enable_per_opcode_request_bytes_metrics_ && opcode.has_value()) { + if (*opcode == OpCodes::Connect) { + config_->stats_.connect_rq_bytes_.add(bytes); + } else { + ASSERT(config_->op_code_map_.contains(*opcode)); + config_->op_code_map_[*opcode].rq_bytes_counter_->add(bytes); + } + } + setDynamicMetadata("bytes", std::to_string(bytes)); } -void ZooKeeperFilter::onResponseBytes(const uint64_t bytes) { +void ZooKeeperFilter::onResponseBytes(const absl::optional opcode, const uint64_t bytes) { config_->stats_.response_bytes_.add(bytes); + + if (config_->enable_per_opcode_response_bytes_metrics_ && opcode.has_value()) { + if (*opcode == OpCodes::Connect) { + config_->stats_.connect_resp_bytes_.add(bytes); + } else { + ASSERT(config_->op_code_map_.contains(*opcode)); + config_->op_code_map_[*opcode].resp_bytes_counter_->add(bytes); + } + } + setDynamicMetadata("bytes", std::to_string(bytes)); } @@ -245,8 +293,8 @@ void ZooKeeperFilter::onGetDataRequest(const std::string& path, const bool watch setDynamicMetadata({{"opname", "getdata"}, {"path", path}, {"watch", watch ? "true" : "false"}}); } -void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags flags, - const OpCodes opcode) { +absl::Status ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags flags, + const OpCodes opcode) { std::string opname; switch (opcode) { @@ -267,12 +315,14 @@ void ZooKeeperFilter::onCreateRequest(const std::string& path, const CreateFlags config_->stats_.createttl_rq_.inc(); break; default: - throw EnvoyException(fmt::format("Unknown opcode: {}", enumToSignedInt(opcode))); + return absl::InvalidArgumentError(fmt::format("unknown opcode: {}", enumToSignedInt(opcode))); break; } setDynamicMetadata( {{"opname", opname}, {"path", path}, {"create_type", createFlagsToString(flags)}}); + + return absl::OkStatus(); } void ZooKeeperFilter::onSetRequest(const std::string& path) { @@ -314,9 +364,13 @@ void ZooKeeperFilter::onSetAclRequest(const std::string& path, const int32_t ver setDynamicMetadata({{"opname", "setacl"}, {"path", path}, {"version", std::to_string(version)}}); } -void ZooKeeperFilter::onSyncRequest(const std::string& path) { +absl::Status ZooKeeperFilter::onSyncRequest(const absl::StatusOr& path) { + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + config_->stats_.sync_rq_.inc(); - setDynamicMetadata({{"opname", "sync"}, {"path", path}}); + setDynamicMetadata({{"opname", "sync"}, {"path", path.value()}}); + + return absl::OkStatus(); } void ZooKeeperFilter::onCheckRequest(const std::string&, const int32_t) { @@ -358,14 +412,23 @@ void ZooKeeperFilter::onAddWatchRequest(const std::string& path, const int32_t m setDynamicMetadata({{"opname", "addwatch"}, {"path", path}, {"mode", std::to_string(mode)}}); } -void ZooKeeperFilter::onGetEphemeralsRequest(const std::string& path) { +absl::Status ZooKeeperFilter::onGetEphemeralsRequest(const absl::StatusOr& path) { + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + config_->stats_.getephemerals_rq_.inc(); - setDynamicMetadata({{"opname", "getephemerals"}, {"path", path}}); + setDynamicMetadata({{"opname", "getephemerals"}, {"path", path.value()}}); + + return absl::OkStatus(); } -void ZooKeeperFilter::onGetAllChildrenNumberRequest(const std::string& path) { +absl::Status +ZooKeeperFilter::onGetAllChildrenNumberRequest(const absl::StatusOr& path) { + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(path, path.status().message()); + config_->stats_.getallchildrennumber_rq_.inc(); - setDynamicMetadata({{"opname", "getallchildrennumber"}, {"path", path}}); + setDynamicMetadata({{"opname", "getallchildrennumber"}, {"path", path.value()}}); + + return absl::OkStatus(); } void ZooKeeperFilter::onCloseRequest() { diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index c152fc59f399..94f47506b211 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -17,6 +17,8 @@ #include "source/common/stats/symbol_table.h" #include "source/extensions/filters/network/zookeeper_proxy/decoder.h" +#include "absl/status/statusor.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -28,6 +30,35 @@ namespace ZooKeeperProxy { #define ALL_ZOOKEEPER_PROXY_STATS(COUNTER) \ COUNTER(decoder_error) \ COUNTER(request_bytes) \ + COUNTER(connect_rq_bytes) \ + COUNTER(connect_readonly_rq_bytes) \ + COUNTER(ping_rq_bytes) \ + COUNTER(auth_rq_bytes) \ + COUNTER(getdata_rq_bytes) \ + COUNTER(create_rq_bytes) \ + COUNTER(create2_rq_bytes) \ + COUNTER(createcontainer_rq_bytes) \ + COUNTER(createttl_rq_bytes) \ + COUNTER(setdata_rq_bytes) \ + COUNTER(getchildren_rq_bytes) \ + COUNTER(getchildren2_rq_bytes) \ + COUNTER(getephemerals_rq_bytes) \ + COUNTER(getallchildrennumber_rq_bytes) \ + COUNTER(delete_rq_bytes) \ + COUNTER(exists_rq_bytes) \ + COUNTER(getacl_rq_bytes) \ + COUNTER(setacl_rq_bytes) \ + COUNTER(sync_rq_bytes) \ + COUNTER(multi_rq_bytes) \ + COUNTER(reconfig_rq_bytes) \ + COUNTER(close_rq_bytes) \ + COUNTER(setauth_rq_bytes) \ + COUNTER(setwatches_rq_bytes) \ + COUNTER(setwatches2_rq_bytes) \ + COUNTER(addwatch_rq_bytes) \ + COUNTER(checkwatches_rq_bytes) \ + COUNTER(removewatches_rq_bytes) \ + COUNTER(check_rq_bytes) \ COUNTER(connect_rq) \ COUNTER(connect_readonly_rq) \ COUNTER(getdata_rq) \ @@ -57,6 +88,34 @@ namespace ZooKeeperProxy { COUNTER(removewatches_rq) \ COUNTER(check_rq) \ COUNTER(response_bytes) \ + COUNTER(connect_resp_bytes) \ + COUNTER(ping_resp_bytes) \ + COUNTER(auth_resp_bytes) \ + COUNTER(getdata_resp_bytes) \ + COUNTER(create_resp_bytes) \ + COUNTER(create2_resp_bytes) \ + COUNTER(createcontainer_resp_bytes) \ + COUNTER(createttl_resp_bytes) \ + COUNTER(setdata_resp_bytes) \ + COUNTER(getchildren_resp_bytes) \ + COUNTER(getchildren2_resp_bytes) \ + COUNTER(getephemerals_resp_bytes) \ + COUNTER(getallchildrennumber_resp_bytes) \ + COUNTER(delete_resp_bytes) \ + COUNTER(exists_resp_bytes) \ + COUNTER(getacl_resp_bytes) \ + COUNTER(setacl_resp_bytes) \ + COUNTER(sync_resp_bytes) \ + COUNTER(multi_resp_bytes) \ + COUNTER(reconfig_resp_bytes) \ + COUNTER(close_resp_bytes) \ + COUNTER(setauth_resp_bytes) \ + COUNTER(setwatches_resp_bytes) \ + COUNTER(setwatches2_resp_bytes) \ + COUNTER(addwatch_resp_bytes) \ + COUNTER(checkwatches_resp_bytes) \ + COUNTER(removewatches_resp_bytes) \ + COUNTER(check_resp_bytes) \ COUNTER(connect_resp) \ COUNTER(ping_resp) \ COUNTER(auth_resp) \ @@ -164,6 +223,8 @@ using OpcodeMap = absl::flat_hash_map; class ZooKeeperFilterConfig { public: ZooKeeperFilterConfig(const std::string& stat_prefix, const uint32_t max_packet_bytes, + const bool enable_per_opcode_request_bytes_metrics, + const bool enable_per_opcode_response_bytes_metrics, const bool enable_latency_threshold_metrics, const std::chrono::milliseconds default_latency_threshold, const LatencyThresholdOverrideList& latency_threshold_overrides, @@ -184,6 +245,8 @@ class ZooKeeperFilterConfig { Stats::Counter* resp_counter_; Stats::Counter* resp_fast_counter_; Stats::Counter* resp_slow_counter_; + Stats::Counter* rq_bytes_counter_; + Stats::Counter* resp_bytes_counter_; std::string opname_; Stats::StatName latency_name_; }; @@ -198,13 +261,16 @@ class ZooKeeperFilterConfig { const Stats::StatName connect_latency_; const Stats::StatName unknown_scheme_rq_; const Stats::StatName unknown_opcode_latency_; + const bool enable_per_opcode_request_bytes_metrics_; + const bool enable_per_opcode_response_bytes_metrics_; ErrorBudgetResponseType errorBudgetDecision(const OpCodes opcode, const std::chrono::milliseconds latency) const; private: void initOpCode(OpCodes opcode, Stats::Counter& resp_counter, Stats::Counter& resp_fast_counter, - Stats::Counter& resp_slow_counter, absl::string_view name); + Stats::Counter& resp_slow_counter, Stats::Counter& rq_bytes_counter, + Stats::Counter& resp_bytes_counter, absl::string_view name); ZooKeeperProxyStats generateStats(const std::string& prefix, Stats::Scope& scope) { return ZooKeeperProxyStats{ALL_ZOOKEEPER_PROXY_STATS(POOL_COUNTER_PREFIX(scope, prefix))}; @@ -271,19 +337,19 @@ class ZooKeeperFilter : public Network::Filter, // ZooKeeperProxy::DecoderCallback void onDecodeError() override; - void onRequestBytes(uint64_t bytes) override; + void onRequestBytes(const absl::optional opcode, const uint64_t bytes) override; void onConnect(bool readonly) override; void onPing() override; void onAuthRequest(const std::string& scheme) override; void onGetDataRequest(const std::string& path, bool watch) override; - void onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) override; + absl::Status onCreateRequest(const std::string& path, CreateFlags flags, OpCodes opcode) override; void onSetRequest(const std::string& path) override; void onGetChildrenRequest(const std::string& path, bool watch, bool v2) override; void onDeleteRequest(const std::string& path, int32_t version) override; void onExistsRequest(const std::string& path, bool watch) override; void onGetAclRequest(const std::string& path) override; void onSetAclRequest(const std::string& path, int32_t version) override; - void onSyncRequest(const std::string& path) override; + absl::Status onSyncRequest(const absl::StatusOr& path) override; void onCheckRequest(const std::string& path, int32_t version) override; void onMultiRequest() override; void onReconfigRequest() override; @@ -292,10 +358,10 @@ class ZooKeeperFilter : public Network::Filter, void onAddWatchRequest(const std::string& path, const int32_t mode) override; void onCheckWatchesRequest(const std::string& path, int32_t type) override; void onRemoveWatchesRequest(const std::string& path, int32_t type) override; - void onGetEphemeralsRequest(const std::string& path) override; - void onGetAllChildrenNumberRequest(const std::string& path) override; + absl::Status onGetEphemeralsRequest(const absl::StatusOr& path) override; + absl::Status onGetAllChildrenNumberRequest(const absl::StatusOr& path) override; void onCloseRequest() override; - void onResponseBytes(uint64_t bytes) override; + void onResponseBytes(const absl::optional opcode, const uint64_t bytes) override; void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly, const std::chrono::milliseconds latency) override; void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error, diff --git a/source/extensions/filters/network/zookeeper_proxy/utils.cc b/source/extensions/filters/network/zookeeper_proxy/utils.cc index cf1c77a8bf19..8d6fe89a8dd4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/utils.cc @@ -7,24 +7,36 @@ namespace Extensions { namespace NetworkFilters { namespace ZooKeeperProxy { -int32_t BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset) { - ensureMaxLen(sizeof(int32_t)); +absl::StatusOr BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset) { + absl::Status status = ensureMinLen(buffer, offset, sizeof(int32_t)); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekInt32: {}", status.message())); + + status = ensureMaxLen(sizeof(int32_t)); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekInt32: {}", status.message())); const int32_t val = buffer.peekBEInt(offset); offset += sizeof(int32_t); return val; } -int64_t BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { - ensureMaxLen(sizeof(int64_t)); +absl::StatusOr BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { + absl::Status status = ensureMinLen(buffer, offset, sizeof(int64_t)); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekInt64: {}", status.message())); + + status = ensureMaxLen(sizeof(int64_t)); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekInt64: {}", status.message())); const int64_t val = buffer.peekBEInt(offset); offset += sizeof(int64_t); return val; } -bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { - ensureMaxLen(1); +absl::StatusOr BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { + absl::Status status = ensureMinLen(buffer, offset, 1); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekBool: {}", status.message())); + + status = ensureMaxLen(1); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekBool: {}", status.message())); const char byte = buffer.peekInt(offset); const bool val = static_cast(byte); @@ -32,24 +44,26 @@ bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { return val; } -std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) { +absl::StatusOr BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) { std::string val; - const uint32_t len = peekInt32(buffer, offset); + const absl::StatusOr len = peekInt32(buffer, offset); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(len, + fmt::format("peekString: {}", len.status().message())); - if (len == 0) { + if (len.value() == 0) { return val; } - if (buffer.length() < (offset + len)) { - throw EnvoyException("peekString: buffer is smaller than string length"); - } + absl::Status status = ensureMinLen(buffer, offset, len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekString: {}", status.message())); - ensureMaxLen(len); + status = ensureMaxLen(len.value()); + RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, fmt::format("peekString: {}", status.message())); - std::unique_ptr data(new char[len]); - buffer.copyOut(offset, len, data.get()); - val.assign(data.get(), len); - offset += len; + std::unique_ptr data(new char[len.value()]); + buffer.copyOut(offset, len.value(), data.get()); + val.assign(data.get(), len.value()); + offset += len.value(); return val; } @@ -59,12 +73,23 @@ void BufferHelper::skip(const uint32_t len, uint64_t& offset) { current_ += len; } -void BufferHelper::ensureMaxLen(const uint32_t size) { +absl::Status BufferHelper::ensureMaxLen(const uint32_t size) { current_ += size; if (current_ > max_len_) { - throw EnvoyException("read beyond max length"); + return absl::InvalidArgumentError("read beyond max length"); } + + return absl::OkStatus(); +} + +absl::Status BufferHelper::ensureMinLen(const Buffer::Instance& buffer, const uint64_t offset, + const uint32_t size) { + if (buffer.length() < (offset + size)) { + return absl::InvalidArgumentError("read beyond buffer size"); + } + + return absl::OkStatus(); } } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/utils.h b/source/extensions/filters/network/zookeeper_proxy/utils.h index dc699afaf548..e851d8ab49ec 100644 --- a/source/extensions/filters/network/zookeeper_proxy/utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/utils.h @@ -9,6 +9,8 @@ #include "source/common/common/byte_order.h" #include "source/common/common/logger.h" +#include "absl/status/statusor.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -17,8 +19,8 @@ namespace ZooKeeperProxy { /** * Helper for extracting ZooKeeper data from a buffer. * - * If at any point a peek is tried beyond max_len, an EnvoyException - * will be thrown. This is important to protect Envoy against malformed + * If at any point a peek is tried beyond max_len, an invalid argument error + * will be returned. This is important to protect Envoy against malformed * requests (e.g.: when the declared and actual length don't match). * * Note: ZooKeeper's protocol uses network byte ordering (big-endian). @@ -27,20 +29,43 @@ class BufferHelper : public Logger::Loggable { public: BufferHelper(uint32_t max_len) : max_len_(max_len) {} - int32_t peekInt32(Buffer::Instance& buffer, uint64_t& offset); - int64_t peekInt64(Buffer::Instance& buffer, uint64_t& offset); - std::string peekString(Buffer::Instance& buffer, uint64_t& offset); - bool peekBool(Buffer::Instance& buffer, uint64_t& offset); + absl::StatusOr peekInt32(Buffer::Instance& buffer, uint64_t& offset); + absl::StatusOr peekInt64(Buffer::Instance& buffer, uint64_t& offset); + absl::StatusOr peekString(Buffer::Instance& buffer, uint64_t& offset); + absl::StatusOr peekBool(Buffer::Instance& buffer, uint64_t& offset); void skip(uint32_t len, uint64_t& offset); void reset() { current_ = 0; } private: - void ensureMaxLen(uint32_t size); + absl::Status ensureMaxLen(const uint32_t size); + absl::Status ensureMinLen(const Buffer::Instance& buffer, const uint64_t offset, + const uint32_t size); const uint32_t max_len_; uint32_t current_{}; }; +#define ABSL_STATUS_RETURN_IF_STATUS_NOT_OK(status) \ + if (!status.ok()) { \ + return status; \ + } + +#define EMIT_DECODER_ERR_AND_RETURN_IF_STATUS_NOT_OK(status) \ + if (!status.ok()) { \ + callbacks_.onDecodeError(); \ + return status; \ + } + +#define RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, message) \ + if (!status.ok()) { \ + return absl::InvalidArgumentError(message); \ + } + +#define EMIT_DECODER_ERR_AND_RETURN_INVALID_ARG_ERR_IF_STATUS_NOT_OK(status, message) \ + if (!status.ok()) { \ + callbacks_.onDecodeError(); \ + return absl::InvalidArgumentError(message); \ + } } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/udp/dns_filter/dns_filter.cc b/source/extensions/filters/udp/dns_filter/dns_filter.cc index c56a16770ac2..211f9d95a13c 100644 --- a/source/extensions/filters/udp/dns_filter/dns_filter.cc +++ b/source/extensions/filters/udp/dns_filter/dns_filter.cc @@ -20,9 +20,12 @@ static constexpr std::chrono::seconds DEFAULT_RESOLVER_TTL{300}; DnsFilterEnvoyConfig::DnsFilterEnvoyConfig( Server::Configuration::ListenerFactoryContext& context, const envoy::extensions::filters::udp::dns_filter::v3::DnsFilterConfig& config) - : root_scope_(context.scope()), cluster_manager_(context.clusterManager()), api_(context.api()), + : root_scope_(context.scope()), + cluster_manager_(context.serverFactoryContext().clusterManager()), + api_(context.serverFactoryContext().api()), stats_(generateStats(config.stat_prefix(), root_scope_)), - resolver_timeout_(DEFAULT_RESOLVER_TIMEOUT), random_(context.api().randomGenerator()) { + resolver_timeout_(DEFAULT_RESOLVER_TIMEOUT), + random_(context.serverFactoryContext().api().randomGenerator()) { using envoy::extensions::filters::udp::dns_filter::v3::DnsFilterConfig; const auto& server_config = config.server_config(); @@ -499,7 +502,7 @@ bool DnsFilter::resolveClusterHost(DnsQueryContextPtr& context, const DnsQueryRe size_t cluster_endpoints = 0; Upstream::ThreadLocalCluster* cluster = cluster_manager_.getThreadLocalCluster(lookup_name); if (cluster != nullptr) { - // TODO(abaptiste): consider using host weights when returning answer addresses + // TODO(suniltheta): consider using host weights when returning answer addresses const std::chrono::seconds ttl = getDomainTTL(lookup_name); for (const auto& hostsets : cluster->prioritySet().hostSetsPerPriority()) { diff --git a/source/extensions/filters/udp/udp_proxy/BUILD b/source/extensions/filters/udp/udp_proxy/BUILD index f0f070f85836..7e0c5561690c 100644 --- a/source/extensions/filters/udp/udp_proxy/BUILD +++ b/source/extensions/filters/udp/udp_proxy/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( srcs = ["hash_policy_impl.cc"], hdrs = ["hash_policy_impl.h"], deps = [ + "//envoy/common:exception_lib", "//envoy/udp:hash_policy_interface", "//source/common/common:assert_lib", "//source/common/common:hash_lib", @@ -30,8 +31,10 @@ envoy_cc_library( "//envoy/access_log:access_log_interface", "//envoy/event:file_event_interface", "//envoy/event:timer_interface", + "//envoy/http:header_evaluator", "//envoy/network:filter_interface", "//envoy/network:listener_interface", + "//envoy/stream_info:uint32_accessor_interface", "//envoy/upstream:cluster_manager_interface", "//source/common/access_log:access_log_lib", "//source/common/api:os_sys_calls_lib", @@ -41,6 +44,7 @@ envoy_cc_library( "//source/common/network:socket_lib", "//source/common/network:socket_option_factory_lib", "//source/common/network:utility_lib", + "//source/common/router:header_parser_lib", "//source/common/stream_info:stream_info_lib", "//source/common/upstream:load_balancer_lib", "//source/extensions/filters/udp/udp_proxy/router:router_lib", @@ -55,10 +59,12 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + visibility = ["//visibility:public"], deps = [ ":udp_proxy_filter_lib", "//envoy/registry", "//envoy/server:filter_config_interface", + "//source/common/formatter:substitution_format_string_lib", "@envoy_api//envoy/extensions/filters/udp/udp_proxy/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/udp/udp_proxy/config.cc b/source/extensions/filters/udp/udp_proxy/config.cc index 49f4f8d68683..3f8e76230464 100644 --- a/source/extensions/filters/udp/udp_proxy/config.cc +++ b/source/extensions/filters/udp/udp_proxy/config.cc @@ -1,22 +1,108 @@ #include "source/extensions/filters/udp/udp_proxy/config.h" +#include "source/common/formatter/substitution_format_string.h" + namespace Envoy { namespace Extensions { namespace UdpFilters { namespace UdpProxy { +constexpr uint32_t DefaultMaxConnectAttempts = 1; +constexpr uint32_t DefaultMaxBufferedDatagrams = 1024; +constexpr uint64_t DefaultMaxBufferedBytes = 16384; + +ProtobufTypes::MessagePtr TunnelResponseHeadersOrTrailers::serializeAsProto() const { + auto proto_out = std::make_unique(); + value().iterate([&proto_out](const Http::HeaderEntry& e) -> Http::HeaderMap::Iterate { + auto* new_header = proto_out->add_headers(); + new_header->set_key(std::string(e.key().getStringView())); + new_header->set_value(std::string(e.value().getStringView())); + return Http::HeaderMap::Iterate::Continue; + }); + return proto_out; +} + +const std::string& TunnelResponseHeaders::key() { + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.udp_proxy.propagate_response_headers"); +} + +const std::string& TunnelResponseTrailers::key() { + CONSTRUCT_ON_FIRST_USE(std::string, "envoy.udp_proxy.propagate_response_trailers"); +} + +TunnelingConfigImpl::TunnelingConfigImpl(const TunnelingConfig& config, + Server::Configuration::FactoryContext& context) + : header_parser_(Envoy::Router::HeaderParser::configure(config.headers_to_add())), + proxy_port_(), target_port_(config.default_target_port()), use_post_(config.use_post()), + post_path_(config.post_path()), + max_connect_attempts_(config.has_retry_options() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.retry_options(), + max_connect_attempts, + DefaultMaxConnectAttempts) + : DefaultMaxConnectAttempts), + buffer_enabled_(config.has_buffer_options()), + max_buffered_datagrams_(config.has_buffer_options() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.buffer_options(), + max_buffered_datagrams, + DefaultMaxBufferedDatagrams) + : DefaultMaxBufferedDatagrams), + max_buffered_bytes_(config.has_buffer_options() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.buffer_options(), + max_buffered_bytes, + DefaultMaxBufferedBytes) + : DefaultMaxBufferedBytes), + propagate_response_headers_(config.propagate_response_headers()), + propagate_response_trailers_(config.propagate_response_trailers()) { + if (!post_path_.empty() && !use_post_) { + throw EnvoyException("Can't set a post path when POST method isn't used"); + } + + if (post_path_.empty()) { + post_path_ = "/"; + } else if (post_path_.rfind("/", 0) != 0) { + throw EnvoyException("Path must start with '/'"); + } + + envoy::config::core::v3::SubstitutionFormatString proxy_substitution_format_config; + proxy_substitution_format_config.mutable_text_format_source()->set_inline_string( + config.proxy_host()); + proxy_host_formatter_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + proxy_substitution_format_config, context); + + if (config.has_proxy_port()) { + uint32_t port = config.proxy_port().value(); + if (port == 0 || port > 65535) { + throw EnvoyException("Port value not in range"); + } + + proxy_port_ = port; + } + + envoy::config::core::v3::SubstitutionFormatString target_substitution_format_config; + target_substitution_format_config.mutable_text_format_source()->set_inline_string( + config.target_host()); + target_host_formatter_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + target_substitution_format_config, context); +} + UdpProxyFilterConfigImpl::UdpProxyFilterConfigImpl( Server::Configuration::ListenerFactoryContext& context, const envoy::extensions::filters::udp::udp_proxy::v3::UdpProxyConfig& config) - : cluster_manager_(context.clusterManager()), time_source_(context.timeSource()), - router_(std::make_shared(config, context.getServerFactoryContext())), + : cluster_manager_(context.serverFactoryContext().clusterManager()), + time_source_(context.serverFactoryContext().timeSource()), + router_(std::make_shared(config, context.serverFactoryContext())), session_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, idle_timeout, 60 * 1000)), use_original_src_ip_(config.use_original_src_ip()), use_per_packet_load_balancing_(config.use_per_packet_load_balancing()), stats_(generateStats(config.stat_prefix(), context.scope())), // Default prefer_gro to true for upstream client traffic. upstream_socket_config_(config.upstream_socket_config(), true), - random_(context.api().randomGenerator()) { + random_generator_(context.serverFactoryContext().api().randomGenerator()) { + if (use_per_packet_load_balancing_ && config.has_tunneling_config()) { + throw EnvoyException( + "Only one of use_per_packet_load_balancing or tunneling_config can be used."); + } + if (use_per_packet_load_balancing_ && !config.session_filters().empty()) { throw EnvoyException( "Only one of use_per_packet_load_balancing or session_filters can be used."); @@ -24,7 +110,7 @@ UdpProxyFilterConfigImpl::UdpProxyFilterConfigImpl( if (use_original_src_ip_ && !Api::OsSysCallsSingleton::get().supportsIpTransparent( - context.getServerFactoryContext().options().localAddressIpVersion())) { + context.serverFactoryContext().options().localAddressIpVersion())) { ExceptionUtil::throwEnvoyException( "The platform does not support either IP_TRANSPARENT or IPV6_TRANSPARENT. Or the envoy " "is not running with the CAP_NET_ADMIN capability."); @@ -44,6 +130,23 @@ UdpProxyFilterConfigImpl::UdpProxyFilterConfigImpl( hash_policy_ = std::make_unique(config.hash_policies()); } + if (config.has_tunneling_config()) { + tunneling_config_ = std::make_unique(config.tunneling_config(), context); + } + + if (config.has_access_log_options()) { + flush_access_log_on_tunnel_connected_ = + config.access_log_options().flush_access_log_on_tunnel_connected(); + + if (config.access_log_options().has_access_log_flush_interval()) { + const uint64_t flush_interval = DurationUtil::durationToMilliseconds( + config.access_log_options().access_log_flush_interval()); + access_log_flush_interval_ = std::chrono::milliseconds(flush_interval); + } + } else { + flush_access_log_on_tunnel_connected_ = false; + } + for (const auto& filter : config.session_filters()) { ENVOY_LOG(debug, " UDP session filter #{}", filter_factories_.size()); ENVOY_LOG(debug, " name: {}", filter.name()); diff --git a/source/extensions/filters/udp/udp_proxy/config.h b/source/extensions/filters/udp/udp_proxy/config.h index d0f6f943d1c4..71dfa4728795 100644 --- a/source/extensions/filters/udp/udp_proxy/config.h +++ b/source/extensions/filters/udp/udp_proxy/config.h @@ -11,6 +11,111 @@ namespace Extensions { namespace UdpFilters { namespace UdpProxy { +using TunnelingConfig = + envoy::extensions::filters::udp::udp_proxy::v3::UdpProxyConfig::UdpTunnelingConfig; + +/** + * Base class for both tunnel response headers and trailers. + */ +class TunnelResponseHeadersOrTrailers : public StreamInfo::FilterState::Object { +public: + ProtobufTypes::MessagePtr serializeAsProto() const override; + virtual const Http::HeaderMap& value() const PURE; +}; + +/** + * Response headers for the tunneling connections. + */ +class TunnelResponseHeaders : public TunnelResponseHeadersOrTrailers { +public: + TunnelResponseHeaders(Http::ResponseHeaderMapPtr&& response_headers) + : response_headers_(std::move(response_headers)) {} + const Http::HeaderMap& value() const override { return *response_headers_; } + static const std::string& key(); + +private: + const Http::ResponseHeaderMapPtr response_headers_; +}; + +/** + * Response trailers for the tunneling connections. + */ +class TunnelResponseTrailers : public TunnelResponseHeadersOrTrailers { +public: + TunnelResponseTrailers(Http::ResponseTrailerMapPtr&& response_trailers) + : response_trailers_(std::move(response_trailers)) {} + const Http::HeaderMap& value() const override { return *response_trailers_; } + static const std::string& key(); + +private: + const Http::ResponseTrailerMapPtr response_trailers_; +}; + +class TunnelingConfigImpl : public UdpTunnelingConfig { +public: + TunnelingConfigImpl(const TunnelingConfig& config, + Server::Configuration::FactoryContext& context); + + const std::string proxyHost(const StreamInfo::StreamInfo& stream_info) const override { + return proxy_host_formatter_->formatWithContext({}, stream_info); + } + + const std::string targetHost(const StreamInfo::StreamInfo& stream_info) const override { + return target_host_formatter_->formatWithContext({}, stream_info); + } + + const absl::optional& proxyPort() const override { return proxy_port_; }; + uint32_t defaultTargetPort() const override { return target_port_; }; + bool usePost() const override { return use_post_; }; + const std::string& postPath() const override { return post_path_; } + Http::HeaderEvaluator& headerEvaluator() const override { return *header_parser_; }; + uint32_t maxConnectAttempts() const override { return max_connect_attempts_; }; + bool bufferEnabled() const override { return buffer_enabled_; }; + uint32_t maxBufferedDatagrams() const override { return max_buffered_datagrams_; }; + uint64_t maxBufferedBytes() const override { return max_buffered_bytes_; }; + + void + propagateResponseHeaders(Http::ResponseHeaderMapPtr&& headers, + const StreamInfo::FilterStateSharedPtr& filter_state) const override { + if (!propagate_response_headers_) { + return; + } + + filter_state->setData(TunnelResponseHeaders::key(), + std::make_shared(std::move(headers)), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::Connection); + } + + void + propagateResponseTrailers(Http::ResponseTrailerMapPtr&& trailers, + const StreamInfo::FilterStateSharedPtr& filter_state) const override { + if (!propagate_response_trailers_) { + return; + } + + filter_state->setData(TunnelResponseTrailers::key(), + std::make_shared(std::move(trailers)), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::Connection); + } + +private: + std::unique_ptr header_parser_; + Formatter::FormatterPtr proxy_host_formatter_; + absl::optional proxy_port_; + Formatter::FormatterPtr target_host_formatter_; + const uint32_t target_port_; + bool use_post_; + std::string post_path_; + const uint32_t max_connect_attempts_; + bool buffer_enabled_; + uint32_t max_buffered_datagrams_; + uint64_t max_buffered_bytes_; + bool propagate_response_headers_; + bool propagate_response_trailers_; +}; + class UdpProxyFilterConfigImpl : public UdpProxyFilterConfig, public FilterChainFactory, Logger::Loggable { @@ -34,7 +139,6 @@ class UdpProxyFilterConfigImpl : public UdpProxyFilterConfig, const Udp::HashPolicy* hashPolicy() const override { return hash_policy_.get(); } UdpProxyDownstreamStats& stats() const override { return stats_; } TimeSource& timeSource() const override { return time_source_; } - Random::RandomGenerator& randomGenerator() const override { return random_; } const Network::ResolvedUdpSocketConfig& upstreamSocketConfig() const override { return upstream_socket_config_; } @@ -46,6 +150,14 @@ class UdpProxyFilterConfigImpl : public UdpProxyFilterConfig, } const FilterChainFactory& sessionFilterFactory() const override { return *this; }; bool hasSessionFilters() const override { return !filter_factories_.empty(); } + const UdpTunnelingConfigPtr& tunnelingConfig() const override { return tunneling_config_; }; + bool flushAccessLogOnTunnelConnected() const override { + return flush_access_log_on_tunnel_connected_; + } + const absl::optional& accessLogFlushInterval() const override { + return access_log_flush_interval_; + } + Random::RandomGenerator& randomGenerator() const override { return random_generator_; } // FilterChainFactory void createFilterChain(FilterChainFactoryCallbacks& callbacks) const override { @@ -68,13 +180,16 @@ class UdpProxyFilterConfigImpl : public UdpProxyFilterConfig, const std::chrono::milliseconds session_timeout_; const bool use_original_src_ip_; const bool use_per_packet_load_balancing_; + bool flush_access_log_on_tunnel_connected_; + absl::optional access_log_flush_interval_; std::unique_ptr hash_policy_; mutable UdpProxyDownstreamStats stats_; const Network::ResolvedUdpSocketConfig upstream_socket_config_; std::vector session_access_logs_; std::vector proxy_access_logs_; - Random::RandomGenerator& random_; + UdpTunnelingConfigPtr tunneling_config_; std::list filter_factories_; + Random::RandomGenerator& random_generator_; }; /** diff --git a/source/extensions/filters/udp/udp_proxy/hash_policy_impl.cc b/source/extensions/filters/udp/udp_proxy/hash_policy_impl.cc index a4d5701d4452..c72bbeb29fec 100644 --- a/source/extensions/filters/udp/udp_proxy/hash_policy_impl.cc +++ b/source/extensions/filters/udp/udp_proxy/hash_policy_impl.cc @@ -1,5 +1,7 @@ #include "source/extensions/filters/udp/udp_proxy/hash_policy_impl.h" +#include "envoy/common/exception.h" + #include "source/common/common/assert.h" #include "source/common/common/macros.h" diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/BUILD b/source/extensions/filters/udp/udp_proxy/session_filters/BUILD index 9adf6995d55d..4cbdb01a5796 100644 --- a/source/extensions/filters/udp/udp_proxy/session_filters/BUILD +++ b/source/extensions/filters/udp/udp_proxy/session_filters/BUILD @@ -23,6 +23,7 @@ envoy_cc_library( envoy_cc_library( name = "factory_base_lib", hdrs = ["factory_base.h"], + visibility = ["//visibility:public"], deps = [ ":filter_config_interface", "//source/common/protobuf:utility_lib", @@ -32,6 +33,7 @@ envoy_cc_library( envoy_cc_library( name = "filter_interface", hdrs = ["filter.h"], + visibility = ["//visibility:public"], deps = [ "//envoy/network:listener_interface", "//envoy/stream_info:stream_info_interface", diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/BUILD b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..2ddede80431e --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "proxy_filter_lib", + srcs = ["proxy_filter.cc"], + hdrs = ["proxy_filter.h"], + deps = [ + "//envoy/router:string_accessor_interface", + "//source/common/common:assert_lib", + "//source/common/http:header_utility_lib", + "//source/common/stream_info:uint32_accessor_lib", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", + "//source/extensions/filters/udp/udp_proxy/session_filters:filter_interface", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":proxy_filter_lib", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", + "//source/extensions/filters/udp/udp_proxy/session_filters:factory_base_lib", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.cc b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.cc new file mode 100644 index 000000000000..0a5062b4a2a0 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.cc @@ -0,0 +1,42 @@ +#include "source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.h" + +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { + +DynamicForwardProxyNetworkFilterConfigFactory::DynamicForwardProxyNetworkFilterConfigFactory() + : FactoryBase("envoy.filters.udp.session.dynamic_forward_proxy") {} + +FilterFactoryCb DynamicForwardProxyNetworkFilterConfigFactory::createFilterFactoryFromProtoTyped( + const FilterConfig& proto_config, Server::Configuration::FactoryContext& context) { + + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( + context); + ProxyFilterConfigSharedPtr filter_config( + std::make_shared(proto_config, cache_manager_factory, context)); + + return [filter_config](FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addReadFilter(std::make_shared(filter_config)); + }; +} + +/** + * Static registration for the dynamic_forward_proxy filter. @see RegisterFactory. + */ +REGISTER_FACTORY(DynamicForwardProxyNetworkFilterConfigFactory, NamedUdpSessionFilterConfigFactory); + +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.h b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.h new file mode 100644 index 000000000000..11d98c0eee97 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.h @@ -0,0 +1,37 @@ +#pragma once + +#include "envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" +#include "envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.validate.h" + +#include "source/extensions/filters/udp/udp_proxy/session_filters/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { + +using FilterConfig = + envoy::extensions::filters::udp::udp_proxy::session::dynamic_forward_proxy::v3::FilterConfig; + +/** + * Config registration for the dynamic_forward_proxy filter. @see + * NamedNetworkFilterConfigFactory. + */ +class DynamicForwardProxyNetworkFilterConfigFactory : public FactoryBase { +public: + DynamicForwardProxyNetworkFilterConfigFactory(); + +private: + FilterFactoryCb + createFilterFactoryFromProtoTyped(const FilterConfig& proto_config, + Server::Configuration::FactoryContext& context) override; +}; + +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.cc new file mode 100644 index 000000000000..00cd4298aa4a --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.cc @@ -0,0 +1,163 @@ +#include "source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h" + +#include "envoy/router/string_accessor.h" +#include "envoy/stream_info/uint32_accessor.h" +#include "envoy/upstream/thread_local_cluster.h" + +#include "source/common/common/assert.h" +#include "source/common/stream_info/uint32_accessor_impl.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { + +constexpr uint32_t DefaultMaxBufferedDatagrams = 1024; +constexpr uint64_t DefaultMaxBufferedBytes = 16384; + +ProxyFilterConfig::ProxyFilterConfig( + const FilterConfig& config, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Server::Configuration::FactoryContext& context) + : dns_cache_manager_(cache_manager_factory.get()), + stats_scope_(context.scope().createScope( + absl::StrCat("udp.session.dynamic_forward_proxy.", config.stat_prefix(), "."))), + filter_stats_(generateStats(*stats_scope_)), buffer_enabled_(config.has_buffer_options()), + max_buffered_datagrams_(config.has_buffer_options() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.buffer_options(), + max_buffered_datagrams, + DefaultMaxBufferedDatagrams) + : DefaultMaxBufferedDatagrams), + max_buffered_bytes_(config.has_buffer_options() + ? PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.buffer_options(), + max_buffered_bytes, + DefaultMaxBufferedBytes) + : DefaultMaxBufferedBytes) { + auto cache_or_error = dns_cache_manager_->getCache(config.dns_cache_config()); + THROW_IF_STATUS_NOT_OK(cache_or_error, throw); + dns_cache_ = std::move(cache_or_error.value()); +} + +ReadFilterStatus ProxyFilter::onNewSession() { + absl::string_view host; + const auto* host_filter_state = + read_callbacks_->streamInfo().filterState()->getDataReadOnly( + "envoy.upstream.dynamic_host"); + if (host_filter_state != nullptr) { + host = host_filter_state->asString(); + } + + absl::optional port; + const auto* port_filter_state = + read_callbacks_->streamInfo().filterState()->getDataReadOnly( + "envoy.upstream.dynamic_port"); + if (port_filter_state != nullptr && port_filter_state->value() > 0 && + port_filter_state->value() <= 65535) { + port = port_filter_state->value(); + } + + if (host.empty() || !port.has_value()) { + ENVOY_LOG(trace, "new session missing host or port"); + // TODO(ohadvano): add callback to remove session. + return ReadFilterStatus::StopIteration; + } + + ENVOY_LOG(trace, "new session with host '{}', port '{}'", host, port.value()); + + circuit_breaker_ = config_->cache().canCreateDnsRequest(); + if (circuit_breaker_ == nullptr) { + ENVOY_LOG(debug, "pending request overflow"); + // TODO(ohadvano): add callback to remove session. + return ReadFilterStatus::StopIteration; + } + + auto result = config_->cache().loadDnsCacheEntry(host, port.value(), false, *this); + + cache_load_handle_ = std::move(result.handle_); + if (cache_load_handle_ == nullptr) { + circuit_breaker_.reset(); + } + + switch (result.status_) { + case LoadDnsCacheEntryStatus::InCache: + ASSERT(cache_load_handle_ == nullptr); + ENVOY_LOG(debug, "DNS cache entry already loaded, continuing"); + load_dns_cache_completed_ = true; + return ReadFilterStatus::Continue; + case LoadDnsCacheEntryStatus::Loading: + ASSERT(cache_load_handle_ != nullptr); + ENVOY_LOG(debug, "waiting to load DNS cache entry"); + return ReadFilterStatus::StopIteration; + case LoadDnsCacheEntryStatus::Overflow: + ASSERT(cache_load_handle_ == nullptr); + ENVOY_LOG(debug, "DNS cache overflow"); + // TODO(ohadvano): add callback to remove session. + return ReadFilterStatus::StopIteration; + } + + PANIC_DUE_TO_CORRUPT_ENUM; +} + +ReadFilterStatus ProxyFilter::onData(Network::UdpRecvData& data) { + if (load_dns_cache_completed_) { + return ReadFilterStatus::Continue; + } + + maybeBufferDatagram(data); + return ReadFilterStatus::StopIteration; +} + +void ProxyFilter::onLoadDnsCacheComplete( + const Common::DynamicForwardProxy::DnsHostInfoSharedPtr& host_info) { + ENVOY_LOG(debug, "load DNS cache complete, continuing"); + if (!host_info || !host_info->address()) { + ENVOY_LOG(debug, "empty DNS respose received"); + } + + ASSERT(circuit_breaker_ != nullptr); + circuit_breaker_.reset(); + + load_dns_cache_completed_ = true; + + if (!read_callbacks_->continueFilterChain()) { + return; + } + + while (!datagrams_buffer_.empty()) { + BufferedDatagramPtr buffered_datagram = std::move(datagrams_buffer_.front()); + datagrams_buffer_.pop(); + read_callbacks_->injectDatagramToFilterChain(*buffered_datagram); + } + + config_->disableBuffer(); + buffered_bytes_ = 0; +} + +void ProxyFilter::maybeBufferDatagram(Network::UdpRecvData& data) { + if (!config_->bufferEnabled()) { + return; + } + + if (datagrams_buffer_.size() == config_->maxBufferedDatagrams() || + buffered_bytes_ + data.buffer_->length() > config_->maxBufferedBytes()) { + config_->filterStats().buffer_overflow_.inc(); + return; + } + + auto buffered_datagram = std::make_unique(); + buffered_datagram->addresses_ = {std::move(data.addresses_.local_), + std::move(data.addresses_.peer_)}; + buffered_datagram->buffer_ = std::move(data.buffer_); + buffered_datagram->receive_time_ = data.receive_time_; + buffered_bytes_ += buffered_datagram->buffer_->length(); + datagrams_buffer_.push(std::move(buffered_datagram)); +} + +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h new file mode 100644 index 000000000000..64516fb9ba5a --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h @@ -0,0 +1,102 @@ +#pragma once + +#include + +#include "envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" + +#include "source/common/common/logger.h" +#include "source/common/http/header_utility.h" +#include "source/extensions/common/dynamic_forward_proxy/dns_cache.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/filter.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { + +using FilterConfig = + envoy::extensions::filters::udp::udp_proxy::session::dynamic_forward_proxy::v3::FilterConfig; + +/** + * All filter state dynamic forward proxy stats. @see stats_macros.h + */ +#define ALL_DYNAMIC_FORWARD_PROXY_STATS(COUNTER) COUNTER(buffer_overflow) + +/** + * Struct definition for all filter state dynamic forward proxy stats. @see stats_macros.h + */ +struct DynamicForwardProxyStats { + ALL_DYNAMIC_FORWARD_PROXY_STATS(GENERATE_COUNTER_STRUCT) +}; + +class ProxyFilterConfig { +public: + ProxyFilterConfig( + const FilterConfig& config, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Server::Configuration::FactoryContext& context); + + Extensions::Common::DynamicForwardProxy::DnsCache& cache() { return *dns_cache_; } + DynamicForwardProxyStats& filterStats() { return filter_stats_; } + bool bufferEnabled() const { return buffer_enabled_; }; + void disableBuffer() { buffer_enabled_ = false; } + uint32_t maxBufferedDatagrams() const { return max_buffered_datagrams_; }; + uint64_t maxBufferedBytes() const { return max_buffered_bytes_; }; + +private: + static DynamicForwardProxyStats generateStats(Stats::Scope& scope) { + return {ALL_DYNAMIC_FORWARD_PROXY_STATS(POOL_COUNTER(scope))}; + } + + const Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr dns_cache_manager_; + Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; + const Stats::ScopeSharedPtr stats_scope_; + DynamicForwardProxyStats filter_stats_; + bool buffer_enabled_; + uint32_t max_buffered_datagrams_; + uint64_t max_buffered_bytes_; +}; + +using ProxyFilterConfigSharedPtr = std::shared_ptr; +using BufferedDatagramPtr = std::unique_ptr; +using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryStatus; + +class ProxyFilter + : public ReadFilter, + public Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks, + Logger::Loggable { +public: + ProxyFilter(ProxyFilterConfigSharedPtr config) : config_(std::move(config)){}; + + // Network::ReadFilter + ReadFilterStatus onNewSession() override; + ReadFilterStatus onData(Network::UdpRecvData& data) override; + + void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + + // Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks + void onLoadDnsCacheComplete( + const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr&) override; + +private: + void maybeBufferDatagram(Network::UdpRecvData& data); + + const ProxyFilterConfigSharedPtr config_; + Upstream::ResourceAutoIncDecPtr circuit_breaker_; + Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryHandlePtr cache_load_handle_; + ReadFilterCallbacks* read_callbacks_{}; + bool load_dns_cache_completed_{false}; + uint64_t buffered_bytes_{0}; + std::queue datagrams_buffer_; +}; + +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/filter.h b/source/extensions/filters/udp/udp_proxy/session_filters/filter.h index 44ed8ab08790..b0daa46b27ba 100644 --- a/source/extensions/filters/udp/udp_proxy/session_filters/filter.h +++ b/source/extensions/filters/udp/udp_proxy/session_filters/filter.h @@ -43,8 +43,9 @@ class ReadFilterCallbacks : public FilterCallbacks { /** * If a read filter stopped filter iteration, continueFilterChain() can be called to continue the * filter chain. It will have onNewSession() called if it was not previously called. + * @return false if the session is removed and no longer valid, otherwise returns true. */ - virtual void continueFilterChain() PURE; + virtual bool continueFilterChain() PURE; }; class WriteFilterCallbacks : public FilterCallbacks {}; @@ -59,10 +60,35 @@ enum class ReadFilterStatus { StopIteration, }; +class FilterBase { +public: + virtual ~FilterBase() = default; + + /** + * This routine is called before the access log handlers' final log() is called. Filters can use + * this callback to enrich the data passed in to the log handlers. + */ + void onSessionComplete() { + if (!on_session_complete_already_called_) { + onSessionCompleteInternal(); + on_session_complete_already_called_ = true; + } + } + +protected: + /** + * This routine is called by onSessionComplete to enrich the data passed in to the log handlers. + */ + virtual void onSessionCompleteInternal() { ASSERT(!on_session_complete_already_called_); } + +private: + bool on_session_complete_already_called_{false}; +}; + /** * Session read filter interface. */ -class ReadFilter { +class ReadFilter : public FilterBase { public: virtual ~ReadFilter() = default; @@ -108,7 +134,7 @@ enum class WriteFilterStatus { /** * Session write filter interface. */ -class WriteFilter { +class WriteFilter : public FilterBase { public: virtual ~WriteFilter() = default; diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/filter_config.h b/source/extensions/filters/udp/udp_proxy/session_filters/filter_config.h index e89be71de9ad..07b41415b554 100644 --- a/source/extensions/filters/udp/udp_proxy/session_filters/filter_config.h +++ b/source/extensions/filters/udp/udp_proxy/session_filters/filter_config.h @@ -33,7 +33,7 @@ class NamedUdpSessionFilterConfigFactory : public Envoy::Config::TypedFactory { createFilterFactoryFromProto(const Protobuf::Message& config, Server::Configuration::FactoryContext& context) PURE; - std::string category() const override { return "envoy.udp.session_filters"; } + std::string category() const override { return "envoy.filters.udp.session"; } }; } // namespace SessionFilters diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/BUILD b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/BUILD new file mode 100644 index 000000000000..264c6840f882 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "http_capsule_filter_lib", + srcs = ["http_capsule.cc"], + hdrs = ["http_capsule.h"], + deps = [ + "//source/common/buffer:buffer_lib", + "//source/common/common:assert_lib", + "//source/common/common:hex_lib", + "//source/extensions/filters/udp/udp_proxy/session_filters:filter_interface", + "@com_github_google_quiche//:quiche_common_capsule_lib", + "@com_github_google_quiche//:quiche_common_connect_udp_datagram_payload_lib", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + visibility = ["//visibility:public"], + deps = [ + ":http_capsule_filter_lib", + "//source/extensions/filters/udp/udp_proxy/session_filters:factory_base_lib", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.cc b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.cc new file mode 100644 index 000000000000..9623e87472b3 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.cc @@ -0,0 +1,33 @@ +#include "source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.h" + +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace HttpCapsule { + +FilterFactoryCb HttpCapsuleFilterConfigFactory::createFilterFactoryFromProtoTyped( + const FilterConfig&, Server::Configuration::FactoryContext& context) { + return [&context](FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addFilter( + std::make_shared(context.serverFactoryContext().timeSource())); + }; +} + +/** + * Static registration for the http_capsule filter. @see RegisterFactory. + */ +REGISTER_FACTORY(HttpCapsuleFilterConfigFactory, NamedUdpSessionFilterConfigFactory); + +} // namespace HttpCapsule +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.h b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.h new file mode 100644 index 000000000000..3ac7f37a5911 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.h @@ -0,0 +1,37 @@ +#pragma once + +#include "envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.pb.h" +#include "envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.pb.validate.h" + +#include "source/extensions/filters/udp/udp_proxy/session_filters/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace HttpCapsule { + +using FilterConfig = + envoy::extensions::filters::udp::udp_proxy::session::http_capsule::v3::FilterConfig; + +/** + * Config registration for the http_capsule filter. @see + * NamedNetworkFilterConfigFactory. + */ +class HttpCapsuleFilterConfigFactory : public FactoryBase { +public: + HttpCapsuleFilterConfigFactory() : FactoryBase("envoy.filters.udp.session.http_capsule"){}; + +private: + FilterFactoryCb + createFilterFactoryFromProtoTyped(const FilterConfig& proto_config, + Server::Configuration::FactoryContext& context) override; +}; + +} // namespace HttpCapsule +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.cc b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.cc new file mode 100644 index 000000000000..6a21371b0140 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.cc @@ -0,0 +1,87 @@ +#include "source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/common/hex.h" + +#include "absl/strings/escaping.h" +#include "quiche/common/masque/connect_udp_datagram_payload.h" +#include "quiche/common/simple_buffer_allocator.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace HttpCapsule { + +ReadFilterStatus HttpCapsuleFilter::onData(Network::UdpRecvData& data) { + std::string buffer = data.buffer_->toString(); + quiche::ConnectUdpDatagramUdpPacketPayload payload(buffer); + quiche::QuicheBuffer serialized_capsule = + SerializeCapsule(quiche::Capsule::Datagram(payload.Serialize()), &capsule_buffer_allocator_); + + data.buffer_->drain(data.buffer_->length()); + data.buffer_->add(serialized_capsule.AsStringView()); + return ReadFilterStatus::Continue; +} + +WriteFilterStatus HttpCapsuleFilter::onWrite(Network::UdpRecvData& data) { + // TODO(ohadvano): add filter callbacks to get addresses instead of saving them. + local_address_ = data.addresses_.local_; + peer_address_ = data.addresses_.peer_; + + for (const Buffer::RawSlice& slice : data.buffer_->getRawSlices()) { + absl::string_view mem_slice(reinterpret_cast(slice.mem_), slice.len_); + if (!capsule_parser_.IngestCapsuleFragment(mem_slice)) { + ENVOY_LOG(error, "Capsule ingestion error occured: slice length = {}", slice.len_); + break; + } + } + + // We always stop here as OnCapsule() callback will be responsible to inject + // datagrams to the filter chain once they are ready. + data.buffer_->drain(data.buffer_->length()); + return WriteFilterStatus::StopIteration; +} + +bool HttpCapsuleFilter::OnCapsule(const quiche::Capsule& capsule) { + quiche::CapsuleType capsule_type = capsule.capsule_type(); + if (capsule_type != quiche::CapsuleType::DATAGRAM) { + // Silently drops capsules with an unknown type. + return true; + } + + std::unique_ptr connect_udp_datagram_payload = + quiche::ConnectUdpDatagramPayload::Parse(capsule.datagram_capsule().http_datagram_payload); + if (!connect_udp_datagram_payload) { + // Indicates parsing failure to reset the data stream. + ENVOY_LOG(debug, "capsule parsing error"); + return false; + } + + if (connect_udp_datagram_payload->GetType() != + quiche::ConnectUdpDatagramPayload::Type::kUdpPacket) { + // Silently drops Datagrams with an unknown Context ID. + return true; + } + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(connect_udp_datagram_payload->GetUdpProxyingPayload()); + datagram.receive_time_ = time_source_.monotonicTime(); + datagram.addresses_ = {local_address_, peer_address_}; + + write_callbacks_->injectDatagramToFilterChain(datagram); + return true; +} + +void HttpCapsuleFilter::OnCapsuleParseFailure(absl::string_view reason) { + ENVOY_LOG(debug, "capsule parse failure: {}", reason); +} + +} // namespace HttpCapsule +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h new file mode 100644 index 000000000000..0b5dea6d60a6 --- /dev/null +++ b/source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h @@ -0,0 +1,56 @@ +#pragma once + +#include "envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.pb.h" + +#include "source/common/common/logger.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/filter.h" + +#include "quiche/common/capsule.h" +#include "quiche/common/simple_buffer_allocator.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace HttpCapsule { + +class HttpCapsuleFilter : public Filter, + public quiche::CapsuleParser::Visitor, + Logger::Loggable { +public: + HttpCapsuleFilter(TimeSource& time_source) : time_source_(time_source) {} + + // ReadFilter + ReadFilterStatus onNewSession() override { return ReadFilterStatus::Continue; } + ReadFilterStatus onData(Network::UdpRecvData& data) override; + void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + + // WriteFilter + WriteFilterStatus onWrite(Network::UdpRecvData& data) override; + void initializeWriteFilterCallbacks(WriteFilterCallbacks& callbacks) override { + write_callbacks_ = &callbacks; + } + + // quiche::CapsuleParser::Visitor + bool OnCapsule(const quiche::Capsule& capsule) override; + void OnCapsuleParseFailure(absl::string_view error_message) override; + +private: + ReadFilterCallbacks* read_callbacks_{}; + WriteFilterCallbacks* write_callbacks_{}; + quiche::CapsuleParser capsule_parser_{this}; + quiche::SimpleBufferAllocator capsule_buffer_allocator_; + Network::Address::InstanceConstSharedPtr local_address_; + Network::Address::InstanceConstSharedPtr peer_address_; + TimeSource& time_source_; +}; + +} // namespace HttpCapsule +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 0adb70307def..45c331540bee 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -2,6 +2,7 @@ #include "envoy/network/listener.h" +#include "source/common/buffer/buffer_impl.h" #include "source/common/network/socket_option_factory.h" namespace Envoy { @@ -33,8 +34,7 @@ UdpProxyFilter::~UdpProxyFilter() { if (!config_->proxyAccessLogs().empty()) { fillProxyStreamInfo(); for (const auto& access_log : config_->proxyAccessLogs()) { - access_log->log(nullptr, nullptr, nullptr, udp_proxy_stats_.value(), - AccessLog::AccessLogType::NotSet); + access_log->log({}, udp_proxy_stats_.value()); } } } @@ -94,7 +94,8 @@ UdpProxyFilter::ClusterInfo::ClusterInfo(UdpProxyFilter& filter, // left. It would be nice to unify the logic but that can be cleaned up later. auto host_sessions_it = host_to_sessions_.find(host.get()); if (host_sessions_it != host_to_sessions_.end()) { - for (const auto& session : host_sessions_it->second) { + for (auto& session : host_sessions_it->second) { + session->onSessionComplete(); ASSERT(sessions_.count(session) == 1); sessions_.erase(session); } @@ -112,7 +113,7 @@ UdpProxyFilter::ClusterInfo::~ClusterInfo() { ASSERT(host_to_sessions_.empty()); } -void UdpProxyFilter::ClusterInfo::removeSession(const ActiveSession* session) { +void UdpProxyFilter::ClusterInfo::removeSession(ActiveSession* session) { if (session->host().has_value()) { // First remove from the host to sessions map, in case the host was resolved. ASSERT(host_to_sessions_[&session->host().value().get()].count(session) == 1); @@ -123,6 +124,8 @@ void UdpProxyFilter::ClusterInfo::removeSession(const ActiveSession* session) { } } + session->onSessionComplete(); + // Now remove it from the primary map. ASSERT(sessions_.count(session) == 1); sessions_.erase(session); @@ -143,43 +146,49 @@ UdpProxyFilter::ClusterInfo::createSession(Network::UdpRecvData::LocalPeerAddres if (defer_socket_creation) { ASSERT(!optional_host); - return createSessionWithOptionalHost(std::move(addresses), nullptr, true); + return createSessionWithOptionalHost(std::move(addresses), nullptr); } if (optional_host) { - return createSessionWithOptionalHost(std::move(addresses), optional_host, false); + return createSessionWithOptionalHost(std::move(addresses), optional_host); } - auto host = chooseHost(addresses.peer_); + auto host = chooseHost(addresses.peer_, nullptr); if (host == nullptr) { ENVOY_LOG(debug, "cannot find any valid host."); cluster_.info()->trafficStats()->upstream_cx_none_healthy_.inc(); return nullptr; } - return createSessionWithOptionalHost(std::move(addresses), host, defer_socket_creation); + return createSessionWithOptionalHost(std::move(addresses), host); } UdpProxyFilter::ActiveSession* UdpProxyFilter::ClusterInfo::createSessionWithOptionalHost( - Network::UdpRecvData::LocalPeerAddresses&& addresses, const Upstream::HostConstSharedPtr& host, - bool defer_socket_creation) { - ASSERT((defer_socket_creation && !host) || (!defer_socket_creation && host)); - auto new_session = - std::make_unique(*this, std::move(addresses), host, defer_socket_creation); + Network::UdpRecvData::LocalPeerAddresses&& addresses, + const Upstream::HostConstSharedPtr& host) { + ActiveSessionPtr new_session; + if (filter_.config_->tunnelingConfig()) { + ASSERT(!host); + new_session = std::make_unique(*this, std::move(addresses)); + } else { + new_session = std::make_unique(*this, std::move(addresses), host); + } + new_session->createFilterChain(); - new_session->onNewSession(); - auto new_session_ptr = new_session.get(); - sessions_.emplace(std::move(new_session)); - if (!defer_socket_creation) { - host_to_sessions_[host.get()].emplace(new_session_ptr); + if (new_session->onNewSession()) { + auto new_session_ptr = new_session.get(); + sessions_.emplace(std::move(new_session)); + return new_session_ptr; } - return new_session_ptr; + new_session->onSessionComplete(); + return nullptr; } Upstream::HostConstSharedPtr UdpProxyFilter::ClusterInfo::chooseHost( - const Network::Address::InstanceConstSharedPtr& peer_address) const { - UdpLoadBalancerContext context(filter_.config_->hashPolicy(), peer_address); + const Network::Address::InstanceConstSharedPtr& peer_address, + const StreamInfo::StreamInfo* stream_info) const { + UdpLoadBalancerContext context(filter_.config_->hashPolicy(), peer_address, stream_info); Upstream::HostConstSharedPtr host = cluster_.loadBalancer().chooseHost(&context); return host; } @@ -191,7 +200,7 @@ UdpProxyFilter::StickySessionClusterInfo::StickySessionClusterInfo( HeterogeneousActiveSessionEqual(false))) {} Network::FilterStatus UdpProxyFilter::StickySessionClusterInfo::onData(Network::UdpRecvData& data) { - bool defer_socket = filter_.config_->hasSessionFilters(); + bool defer_socket = filter_.config_->hasSessionFilters() || filter_.config_->tunnelingConfig(); const auto active_session_it = sessions_.find(data.addresses_); ActiveSession* active_session; if (active_session_it == sessions_.end()) { @@ -211,7 +220,7 @@ Network::FilterStatus UdpProxyFilter::StickySessionClusterInfo::onData(Network:: // If a host becomes unhealthy, we optimally would like to replace it with a new session // to a healthy host. We may eventually want to make this behavior configurable, but for now // this will be the universal behavior. - auto host = chooseHost(data.addresses_.peer_); + auto host = chooseHost(data.addresses_.peer_, nullptr); if (host != nullptr && host->coarseHealth() != Upstream::Host::Health::Unhealthy && host.get() != &active_session->host().value().get()) { ENVOY_LOG(debug, "upstream session unhealthy, recreating the session"); @@ -237,7 +246,7 @@ UdpProxyFilter::PerPacketLoadBalancingClusterInfo::PerPacketLoadBalancingCluster Network::FilterStatus UdpProxyFilter::PerPacketLoadBalancingClusterInfo::onData(Network::UdpRecvData& data) { - auto host = chooseHost(data.addresses_.peer_); + auto host = chooseHost(data.addresses_.peer_, nullptr); if (host == nullptr) { ENVOY_LOG(debug, "cannot find any valid host."); cluster_.info()->trafficStats()->upstream_cx_none_healthy_.inc(); @@ -269,60 +278,35 @@ std::atomic UdpProxyFilter::ActiveSession::next_global_session_id_; UdpProxyFilter::ActiveSession::ActiveSession(ClusterInfo& cluster, Network::UdpRecvData::LocalPeerAddresses&& addresses, - const Upstream::HostConstSharedPtr& host, - bool defer_socket_creation) - : cluster_(cluster), use_original_src_ip_(cluster_.filter_.config_->usingOriginalSrcIp()), - addresses_(std::move(addresses)), host_(host), + const Upstream::HostConstSharedPtr& host) + : cluster_(cluster), addresses_(std::move(addresses)), host_(host), idle_timer_(cluster.filter_.read_callbacks_->udpListener().dispatcher().createTimer( [this] { onIdleTimer(); })), udp_session_info_( - StreamInfo::StreamInfoImpl(cluster_.filter_.config_->timeSource(), nullptr)), + StreamInfo::StreamInfoImpl(cluster_.filter_.config_->timeSource(), + std::make_shared( + addresses_.local_, addresses_.peer_))), session_id_(next_global_session_id_++) { - ASSERT((defer_socket_creation && !host) || (!defer_socket_creation && host)); + udp_session_info_.setUpstreamInfo(std::make_shared()); cluster_.filter_.config_->stats().downstream_sess_total_.inc(); cluster_.filter_.config_->stats().downstream_sess_active_.inc(); cluster_.cluster_.info() ->resourceManager(Upstream::ResourcePriority::Default) .connections() .inc(); - - if (!defer_socket_creation) { - createSocket(host); - } } -void UdpProxyFilter::ActiveSession::createSocket(const Upstream::HostConstSharedPtr& host) { - // NOTE: The socket call can only fail due to memory/fd exhaustion. No local ephemeral port - // is bound until the first packet is sent to the upstream host. - socket_ = cluster_.filter_.createSocket(host); - socket_->ioHandle().initializeFileEvent( - cluster_.filter_.read_callbacks_->udpListener().dispatcher(), - [this](uint32_t) { onReadReady(); }, Event::PlatformDefaultTriggerType, - Event::FileReadyType::Read); - - ENVOY_LOG(debug, "creating new session: downstream={} local={} upstream={}", - addresses_.peer_->asStringView(), addresses_.local_->asStringView(), - host->address()->asStringView()); - - if (use_original_src_ip_) { - const Network::Socket::OptionsSharedPtr socket_options = - Network::SocketOptionFactory::buildIpTransparentOptions(); - const bool ok = Network::Socket::applyOptions( - socket_options, *socket_, envoy::config::core::v3::SocketOption::STATE_PREBIND); - - RELEASE_ASSERT(ok, "Should never occur!"); - ENVOY_LOG(debug, "The original src is enabled for address {}.", - addresses_.peer_->asStringView()); - } +UdpProxyFilter::UdpActiveSession::UdpActiveSession( + ClusterInfo& cluster, Network::UdpRecvData::LocalPeerAddresses&& addresses, + const Upstream::HostConstSharedPtr& host) + : ActiveSession(cluster, std::move(addresses), std::move(host)), + use_original_src_ip_(cluster.filter_.config_->usingOriginalSrcIp()) {} - // TODO(mattklein123): Enable dropped packets socket option. In general the Socket abstraction - // does not work well right now for client sockets. It's too heavy weight and is aimed at listener - // sockets. We need to figure out how to either refactor Socket into something that works better - // for this use case or allow the socket option abstractions to work directly against an IO - // handle. +UdpProxyFilter::ActiveSession::~ActiveSession() { + ENVOY_BUG(on_session_complete_called_, "onSessionComplete() not called"); } -UdpProxyFilter::ActiveSession::~ActiveSession() { +void UdpProxyFilter::ActiveSession::onSessionComplete() { ENVOY_LOG(debug, "deleting the session: downstream={} local={} upstream={}", addresses_.peer_->asStringView(), addresses_.local_->asStringView(), host_ != nullptr ? host_->address()->asStringView() : "unknown"); @@ -333,13 +317,26 @@ UdpProxyFilter::ActiveSession::~ActiveSession() { .connections() .dec(); + disableAccessLogFlushTimer(); + + for (auto& active_read_filter : read_filters_) { + active_read_filter->read_filter_->onSessionComplete(); + } + + for (auto& active_write_filter : write_filters_) { + active_write_filter->write_filter_->onSessionComplete(); + } + if (!cluster_.filter_.config_->sessionAccessLogs().empty()) { fillSessionStreamInfo(); + const Formatter::HttpFormatterContext log_context{ + nullptr, nullptr, nullptr, {}, AccessLog::AccessLogType::UdpSessionEnd}; for (const auto& access_log : cluster_.filter_.config_->sessionAccessLogs()) { - access_log->log(nullptr, nullptr, nullptr, udp_session_info_, - AccessLog::AccessLogType::NotSet); + access_log->log(log_context, udp_session_info_); } } + + on_session_complete_called_ = true; } void UdpProxyFilter::ActiveSession::fillSessionStreamInfo() { @@ -381,24 +378,24 @@ void UdpProxyFilter::fillProxyStreamInfo() { udp_proxy_stats_.value().setDynamicMetadata("udp.proxy.proxy", stats_obj); } -void UdpProxyFilter::ActiveSession::onIdleTimer() { +void UdpProxyFilter::UdpActiveSession::onIdleTimer() { ENVOY_LOG(debug, "session idle timeout: downstream={} local={}", addresses_.peer_->asStringView(), addresses_.local_->asStringView()); cluster_.filter_.config_->stats().idle_timeout_.inc(); cluster_.removeSession(this); } -void UdpProxyFilter::ActiveSession::onReadReady() { - idle_timer_->enableTimer(cluster_.filter_.config_->sessionTimeout()); +void UdpProxyFilter::UdpActiveSession::onReadReady() { + resetIdleTimer(); // TODO(mattklein123): We should not be passing *addresses_.local_ to this function as we are // not trying to populate the local address for received packets. uint32_t packets_dropped = 0; const Api::IoErrorPtr result = Network::Utility::readPacketsFromSocket( - socket_->ioHandle(), *addresses_.local_, *this, cluster_.filter_.config_->timeSource(), + udp_socket_->ioHandle(), *addresses_.local_, *this, cluster_.filter_.config_->timeSource(), cluster_.filter_.config_->upstreamSocketConfig().prefer_gro_, packets_dropped); if (result == nullptr) { - socket_->ioHandle().activateFileEvents(Event::FileReadyType::Read); + udp_socket_->ioHandle().activateFileEvents(Event::FileReadyType::Read); return; } if (result->getErrorCode() != Api::IoError::IoErrorCode::Again) { @@ -408,7 +405,19 @@ void UdpProxyFilter::ActiveSession::onReadReady() { cluster_.filter_.read_callbacks_->udpListener().flush(); } -void UdpProxyFilter::ActiveSession::onNewSession() { +bool UdpProxyFilter::ActiveSession::onNewSession() { + if (cluster_.filter_.config_->accessLogFlushInterval().has_value() && + !cluster_.filter_.config_->sessionAccessLogs().empty()) { + access_log_flush_timer_ = + cluster_.filter_.read_callbacks_->udpListener().dispatcher().createTimer( + [this] { onAccessLogFlushInterval(); }); + rearmAccessLogFlushTimer(); + } + + // Set UUID for the session. This is used for logging and tracing. + udp_session_info_.setStreamIdProvider(std::make_shared( + cluster_.filter_.config_->randomGenerator().uuid())); + for (auto& active_read_filter : read_filters_) { if (active_read_filter->initialized_) { // The filter may call continueFilterChain() in onNewSession(), causing next @@ -419,25 +428,25 @@ void UdpProxyFilter::ActiveSession::onNewSession() { active_read_filter->initialized_ = true; auto status = active_read_filter->read_filter_->onNewSession(); if (status == ReadFilterStatus::StopIteration) { - return; + return true; } } - createSocketDeferred(); + return createUpstream(); } void UdpProxyFilter::ActiveSession::onData(Network::UdpRecvData& data) { - absl::string_view host = socket_ != nullptr ? host_->address()->asStringView() : "unknown"; ENVOY_LOG(trace, "received {} byte datagram from downstream: downstream={} local={} upstream={}", data.buffer_->length(), addresses_.peer_->asStringView(), - addresses_.local_->asStringView(), host); + addresses_.local_->asStringView(), + host_ != nullptr ? host_->address()->asStringView() : "unknown"); + const uint64_t rx_buffer_length = data.buffer_->length(); cluster_.filter_.config_->stats().downstream_sess_rx_bytes_.add(rx_buffer_length); session_stats_.downstream_sess_rx_bytes_ += rx_buffer_length; cluster_.filter_.config_->stats().downstream_sess_rx_datagrams_.inc(); ++session_stats_.downstream_sess_rx_datagrams_; - - idle_timer_->enableTimer(cluster_.filter_.config_->sessionTimeout()); + resetIdleTimer(); for (auto& active_read_filter : read_filters_) { auto status = active_read_filter->read_filter_->onData(data); @@ -449,8 +458,8 @@ void UdpProxyFilter::ActiveSession::onData(Network::UdpRecvData& data) { writeUpstream(data); } -void UdpProxyFilter::ActiveSession::writeUpstream(Network::UdpRecvData& data) { - if (!socket_) { +void UdpProxyFilter::UdpActiveSession::writeUpstream(Network::UdpRecvData& data) { + if (!udp_socket_) { ENVOY_LOG(debug, "cannot write upstream because the socket was not created."); return; } @@ -464,7 +473,7 @@ void UdpProxyFilter::ActiveSession::writeUpstream(Network::UdpRecvData& data) { // set. We allow the OS to select the right IP based on outbound routing rules if // use_original_src_ip_ is not set, else use downstream peer IP as local IP. if (!connected_ && !use_original_src_ip_) { - Api::SysCallIntResult rc = socket_->ioHandle().connect(host_->address()); + Api::SysCallIntResult rc = udp_socket_->ioHandle().connect(host_->address()); if (SOCKET_FAILURE(rc.return_value_)) { ENVOY_LOG(debug, "cannot connect: ({}) {}", rc.errno_, errorDetails(rc.errno_)); cluster_.cluster_stats_.sess_tx_errors_.inc(); @@ -474,7 +483,7 @@ void UdpProxyFilter::ActiveSession::writeUpstream(Network::UdpRecvData& data) { connected_ = true; } - ASSERT((connected_ || use_original_src_ip_) && socket_ && host_); + ASSERT((connected_ || use_original_src_ip_) && udp_socket_ && host_); const uint64_t tx_buffer_length = data.buffer_->length(); ENVOY_LOG(trace, "writing {} byte datagram upstream: downstream={} local={} upstream={}", @@ -482,8 +491,8 @@ void UdpProxyFilter::ActiveSession::writeUpstream(Network::UdpRecvData& data) { host_->address()->asStringView()); const Network::Address::Ip* local_ip = use_original_src_ip_ ? addresses_.peer_->ip() : nullptr; - Api::IoCallUint64Result rc = Network::Utility::writeToSocket(socket_->ioHandle(), *data.buffer_, - local_ip, *host_->address()); + Api::IoCallUint64Result rc = Network::Utility::writeToSocket( + udp_socket_->ioHandle(), *data.buffer_, local_ip, *host_->address()); if (!rc.ok()) { cluster_.cluster_stats_.sess_tx_errors_.inc(); @@ -493,7 +502,7 @@ void UdpProxyFilter::ActiveSession::writeUpstream(Network::UdpRecvData& data) { } } -void UdpProxyFilter::ActiveSession::onContinueFilterChain(ActiveReadFilter* filter) { +bool UdpProxyFilter::ActiveSession::onContinueFilterChain(ActiveReadFilter* filter) { ASSERT(filter != nullptr); std::list::iterator entry = std::next(filter->entry()); @@ -505,29 +514,69 @@ void UdpProxyFilter::ActiveSession::onContinueFilterChain(ActiveReadFilter* filt (*entry)->initialized_ = true; auto status = (*entry)->read_filter_->onNewSession(); if (status == ReadFilterStatus::StopIteration) { - break; + return true; } } - createSocketDeferred(); + if (!createUpstream()) { + cluster_.removeSession(this); + return false; + } + + return true; } -void UdpProxyFilter::ActiveSession::createSocketDeferred() { - if (socket_) { +bool UdpProxyFilter::UdpActiveSession::createUpstream() { + if (udp_socket_) { // A session filter may call on continueFilterChain(), after already creating the socket, // so we first check that the socket was not created already. - return; + return true; } - host_ = cluster_.chooseHost(addresses_.peer_); - if (host_ == nullptr) { - ENVOY_LOG(debug, "cannot find any valid host."); - cluster_.cluster_.info()->trafficStats()->upstream_cx_none_healthy_.inc(); - return; + if (!host_) { + host_ = cluster_.chooseHost(addresses_.peer_, &udp_session_info_); + if (host_ == nullptr) { + ENVOY_LOG(debug, "cannot find any valid host."); + cluster_.cluster_.info()->trafficStats()->upstream_cx_none_healthy_.inc(); + return false; + } } + udp_session_info_.upstreamInfo()->setUpstreamHost(host_); cluster_.addSession(host_.get(), this); - createSocket(host_); + createUdpSocket(host_); + return true; +} + +void UdpProxyFilter::UdpActiveSession::createUdpSocket(const Upstream::HostConstSharedPtr& host) { + // NOTE: The socket call can only fail due to memory/fd exhaustion. No local ephemeral port + // is bound until the first packet is sent to the upstream host. + udp_socket_ = cluster_.filter_.createUdpSocket(host); + udp_socket_->ioHandle().initializeFileEvent( + cluster_.filter_.read_callbacks_->udpListener().dispatcher(), + [this](uint32_t) { onReadReady(); }, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read); + + ENVOY_LOG(debug, "creating new session: downstream={} local={} upstream={}", + addresses_.peer_->asStringView(), addresses_.local_->asStringView(), + host->address()->asStringView()); + + if (use_original_src_ip_) { + const Network::Socket::OptionsSharedPtr socket_options = + Network::SocketOptionFactory::buildIpTransparentOptions(); + const bool ok = Network::Socket::applyOptions( + socket_options, *udp_socket_, envoy::config::core::v3::SocketOption::STATE_PREBIND); + + RELEASE_ASSERT(ok, "Should never occur!"); + ENVOY_LOG(debug, "The original src is enabled for address {}.", + addresses_.peer_->asStringView()); + } + + // TODO(mattklein123): Enable dropped packets socket option. In general the Socket abstraction + // does not work well right now for client sockets. It's too heavy weight and is aimed at listener + // sockets. We need to figure out how to either refactor Socket into something that works better + // for this use case or allow the socket option abstractions to work directly against an IO + // handle. } void UdpProxyFilter::ActiveSession::onInjectReadDatagramToFilterChain(ActiveReadFilter* filter, @@ -568,20 +617,32 @@ void UdpProxyFilter::ActiveSession::onInjectWriteDatagramToFilterChain(ActiveWri writeDownstream(data); } -void UdpProxyFilter::ActiveSession::processPacket( +void UdpProxyFilter::UdpActiveSession::processPacket( Network::Address::InstanceConstSharedPtr local_address, Network::Address::InstanceConstSharedPtr peer_address, Buffer::InstancePtr buffer, MonotonicTime receive_time) { const uint64_t rx_buffer_length = buffer->length(); ENVOY_LOG(trace, "received {} byte datagram from upstream: downstream={} local={} upstream={}", rx_buffer_length, addresses_.peer_->asStringView(), addresses_.local_->asStringView(), - host_->address()->asStringView()); + host_ != nullptr ? host_->address()->asStringView() : "unknown"); cluster_.cluster_stats_.sess_rx_datagrams_.inc(); cluster_.cluster_.info()->trafficStats()->upstream_cx_rx_bytes_total_.add(rx_buffer_length); Network::UdpRecvData recv_data{ {std::move(local_address), std::move(peer_address)}, std::move(buffer), receive_time}; + processUpstreamDatagram(recv_data); +} + +void UdpProxyFilter::ActiveSession::resetIdleTimer() { + if (idle_timer_ == nullptr) { + return; + } + + idle_timer_->enableTimer(cluster_.filter_.config_->sessionTimeout()); +} + +void UdpProxyFilter::ActiveSession::processUpstreamDatagram(Network::UdpRecvData& recv_data) { for (auto& active_write_filter : write_filters_) { auto status = active_write_filter->write_filter_->onWrite(recv_data); if (status == WriteFilterStatus::StopIteration) { @@ -596,7 +657,7 @@ void UdpProxyFilter::ActiveSession::writeDownstream(Network::UdpRecvData& recv_d const uint64_t tx_buffer_length = recv_data.buffer_->length(); ENVOY_LOG(trace, "writing {} byte datagram downstream: downstream={} local={} upstream={}", tx_buffer_length, addresses_.peer_->asStringView(), addresses_.local_->asStringView(), - host_->address()->asStringView()); + host_ != nullptr ? host_->address()->asStringView() : "unknown"); Network::UdpSendData data{addresses_.local_->ip(), *addresses_.peer_, *recv_data.buffer_}; const Api::IoCallUint64Result rc = cluster_.filter_.read_callbacks_->udpListener().send(data); @@ -611,6 +672,408 @@ void UdpProxyFilter::ActiveSession::writeDownstream(Network::UdpRecvData& recv_d } } +void UdpProxyFilter::ActiveSession::onAccessLogFlushInterval() { + fillSessionStreamInfo(); + const Formatter::HttpFormatterContext log_context{ + nullptr, nullptr, nullptr, {}, AccessLog::AccessLogType::UdpPeriodic}; + for (const auto& access_log : cluster_.filter_.config_->sessionAccessLogs()) { + access_log->log(log_context, udp_session_info_); + } + + rearmAccessLogFlushTimer(); +} + +void UdpProxyFilter::ActiveSession::rearmAccessLogFlushTimer() { + if (access_log_flush_timer_ != nullptr) { + ASSERT(cluster_.filter_.config_->accessLogFlushInterval().has_value()); + access_log_flush_timer_->enableTimer( + cluster_.filter_.config_->accessLogFlushInterval().value()); + } +} + +void UdpProxyFilter::ActiveSession::disableAccessLogFlushTimer() { + if (access_log_flush_timer_ != nullptr) { + access_log_flush_timer_->disableTimer(); + access_log_flush_timer_.reset(); + } +} + +void HttpUpstreamImpl::encodeData(Buffer::Instance& data) { + if (!request_encoder_) { + return; + } + + request_encoder_->encodeData(data, false); +} + +void HttpUpstreamImpl::setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl) { + request_encoder_ = &request_encoder; + request_encoder_->getStream().addCallbacks(*this); + + const std::string& scheme = + is_ssl ? Http::Headers::get().SchemeValues.Https : Http::Headers::get().SchemeValues.Http; + + std::string host = tunnel_config_.proxyHost(downstream_info_); + const auto* dynamic_port = + downstream_info_.filterState()->getDataReadOnly( + "udp.connect.proxy_port"); + if (dynamic_port != nullptr && dynamic_port->value() > 0 && dynamic_port->value() <= 65535) { + absl::StrAppend(&host, ":", std::to_string(dynamic_port->value())); + } else if (tunnel_config_.proxyPort().has_value()) { + absl::StrAppend(&host, ":", std::to_string(tunnel_config_.proxyPort().value())); + } + + auto headers = Http::RequestHeaderMapImpl::create(); + headers->addReferenceKey(Http::Headers::get().Scheme, scheme); + headers->addReferenceKey(Http::Headers::get().Host, host); + + if (tunnel_config_.usePost()) { + headers->addReferenceKey(Http::Headers::get().Method, "POST"); + headers->addReferenceKey(Http::Headers::get().Path, tunnel_config_.postPath()); + } else { + // The Envoy HTTP/2 and HTTP/3 clients expect the request header map to be in the form of HTTP/1 + // upgrade to issue an extended CONNECT request. + headers->addReferenceKey(Http::Headers::get().Method, "GET"); + headers->addReferenceKey(Http::Headers::get().Connection, "Upgrade"); + headers->addReferenceKey(Http::Headers::get().Upgrade, "connect-udp"); + headers->addReferenceKey(Http::Headers::get().CapsuleProtocol, "?1"); + const std::string target_tunnel_path = resolveTargetTunnelPath(); + headers->addReferenceKey(Http::Headers::get().Path, target_tunnel_path); + } + + tunnel_config_.headerEvaluator().evaluateHeaders(*headers, {downstream_info_.getRequestHeaders()}, + downstream_info_); + + const auto status = request_encoder_->encodeHeaders(*headers, false); + // Encoding can only fail on missing required request headers. + ASSERT(status.ok()); +} + +const std::string HttpUpstreamImpl::resolveTargetTunnelPath() { + std::string target_host = tunnel_config_.targetHost(downstream_info_); + target_host = Http::Utility::PercentEncoding::encode(target_host, ":"); + + const auto* dynamic_port = + downstream_info_.filterState()->getDataReadOnly( + "udp.connect.target_port"); + + std::string target_port; + if (dynamic_port != nullptr && dynamic_port->value() > 0 && dynamic_port->value() <= 65535) { + target_port = std::to_string(dynamic_port->value()); + } else { + target_port = std::to_string(tunnel_config_.defaultTargetPort()); + } + + // TODO(ohadvano): support configurable URI template. + return absl::StrCat("/.well-known/masque/udp/", target_host, "/", target_port, "/"); +} + +HttpUpstreamImpl::~HttpUpstreamImpl() { resetEncoder(Network::ConnectionEvent::LocalClose); } + +void HttpUpstreamImpl::resetEncoder(Network::ConnectionEvent event, bool by_downstream) { + if (!request_encoder_) { + return; + } + + request_encoder_->getStream().removeCallbacks(*this); + if (by_downstream) { + request_encoder_->getStream().resetStream(Http::StreamResetReason::LocalReset); + } + + request_encoder_ = nullptr; + + // If we did not receive a valid CONNECT response yet we treat this as a pool + // failure, otherwise we forward the event downstream. + if (tunnel_creation_callbacks_.has_value()) { + tunnel_creation_callbacks_.value().get().onStreamFailure(); + return; + } + + if (!by_downstream) { + upstream_callbacks_.onUpstreamEvent(event); + } +} + +TunnelingConnectionPoolImpl::TunnelingConnectionPoolImpl( + Upstream::ThreadLocalCluster& thread_local_cluster, Upstream::LoadBalancerContext* context, + const UdpTunnelingConfig& tunnel_config, UpstreamTunnelCallbacks& upstream_callbacks, + StreamInfo::StreamInfo& downstream_info, bool flush_access_log_on_tunnel_connected, + const std::vector& session_access_logs) + : upstream_callbacks_(upstream_callbacks), tunnel_config_(tunnel_config), + downstream_info_(downstream_info), + flush_access_log_on_tunnel_connected_(flush_access_log_on_tunnel_connected), + session_access_logs_(session_access_logs) { + // TODO(ohadvano): support upstream HTTP/3. + absl::optional protocol = Http::Protocol::Http2; + conn_pool_data_ = + thread_local_cluster.httpConnPool(Upstream::ResourcePriority::Default, protocol, context); +} + +void TunnelingConnectionPoolImpl::newStream(HttpStreamCallbacks& callbacks) { + callbacks_ = &callbacks; + upstream_ = + std::make_unique(upstream_callbacks_, tunnel_config_, downstream_info_); + Tcp::ConnectionPool::Cancellable* handle = + conn_pool_data_.value().newStream(upstream_->responseDecoder(), *this, + {/*can_send_early_data_=*/false, + /*can_use_http3_=*/false}); + + if (handle != nullptr) { + upstream_handle_ = handle; + } +} + +void TunnelingConnectionPoolImpl::onPoolFailure(Http::ConnectionPool::PoolFailureReason reason, + absl::string_view failure_reason, + Upstream::HostDescriptionConstSharedPtr host) { + upstream_handle_ = nullptr; + callbacks_->onStreamFailure(reason, failure_reason, host); + downstream_info_.upstreamInfo()->setUpstreamHost(host); + downstream_info_.upstreamInfo()->setUpstreamTransportFailureReason(failure_reason); +} + +void TunnelingConnectionPoolImpl::onPoolReady(Http::RequestEncoder& request_encoder, + Upstream::HostDescriptionConstSharedPtr upstream_host, + StreamInfo::StreamInfo& upstream_info, + absl::optional) { + ENVOY_LOG(debug, "Upstream connection [C{}] ready, creating tunnel stream", + upstream_info.downstreamAddressProvider().connectionID().value()); + + upstream_handle_ = nullptr; + upstream_host_ = upstream_host; + upstream_info_ = &upstream_info; + ssl_info_ = upstream_info.downstreamAddressProvider().sslConnection(); + + bool is_ssl = upstream_host->transportSocketFactory().implementsSecureTransport(); + upstream_->setRequestEncoder(request_encoder, is_ssl); + upstream_->setTunnelCreationCallbacks(*this); + downstream_info_.upstreamInfo()->setUpstreamHost(upstream_host); + + if (flush_access_log_on_tunnel_connected_) { + const Formatter::HttpFormatterContext log_context{ + nullptr, nullptr, nullptr, {}, AccessLog::AccessLogType::UdpTunnelUpstreamConnected}; + for (const auto& access_log : session_access_logs_) { + access_log->log(log_context, downstream_info_); + } + } +} + +TunnelingConnectionPoolPtr TunnelingConnectionPoolFactory::createConnPool( + Upstream::ThreadLocalCluster& thread_local_cluster, Upstream::LoadBalancerContext* context, + const UdpTunnelingConfig& tunnel_config, UpstreamTunnelCallbacks& upstream_callbacks, + StreamInfo::StreamInfo& downstream_info, bool flush_access_log_on_tunnel_connected, + const std::vector& session_access_logs) const { + auto pool = std::make_unique( + thread_local_cluster, context, tunnel_config, upstream_callbacks, downstream_info, + flush_access_log_on_tunnel_connected, session_access_logs); + return (pool->valid() ? std::move(pool) : nullptr); +} + +UdpProxyFilter::TunnelingActiveSession::TunnelingActiveSession( + ClusterInfo& cluster, Network::UdpRecvData::LocalPeerAddresses&& addresses) + : ActiveSession(cluster, std::move(addresses), nullptr) {} + +bool UdpProxyFilter::TunnelingActiveSession::createUpstream() { + if (conn_pool_factory_) { + // A session filter may call on continueFilterChain(), after already creating the upstream, + // so we first check that the factory was not created already. + return true; + } + + conn_pool_factory_ = std::make_unique(); + load_balancer_context_ = std::make_unique( + cluster_.filter_.config_->hashPolicy(), addresses_.peer_, &udp_session_info_); + + return establishUpstreamConnection(); +} + +bool UdpProxyFilter::TunnelingActiveSession::establishUpstreamConnection() { + if (!createConnectionPool()) { + ENVOY_LOG(debug, "failed to create upstream connection pool"); + cluster_.cluster_stats_.sess_tunnel_failure_.inc(); + return false; + } + + return true; +} + +bool UdpProxyFilter::TunnelingActiveSession::createConnectionPool() { + ASSERT(conn_pool_factory_); + + // Check this here because the TCP conn pool will queue our request waiting for a connection that + // will never be released. + if (!cluster_.cluster_.info() + ->resourceManager(Upstream::ResourcePriority::Default) + .connections() + .canCreate()) { + udp_session_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow); + cluster_.cluster_.info()->trafficStats()->upstream_cx_overflow_.inc(); + return false; + } + + if (connect_attempts_ >= cluster_.filter_.config_->tunnelingConfig()->maxConnectAttempts()) { + udp_session_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamRetryLimitExceeded); + cluster_.cluster_.info()->trafficStats()->upstream_cx_connect_attempts_exceeded_.inc(); + return false; + } else if (connect_attempts_ >= 1) { + cluster_.cluster_.info()->trafficStats()->upstream_rq_retry_.inc(); + } + + conn_pool_ = conn_pool_factory_->createConnPool( + cluster_.cluster_, load_balancer_context_.get(), *cluster_.filter_.config_->tunnelingConfig(), + *this, udp_session_info_, cluster_.filter_.config_->flushAccessLogOnTunnelConnected(), + cluster_.filter_.config_->sessionAccessLogs()); + + if (conn_pool_) { + connecting_ = true; + connect_attempts_++; + udp_session_info_.setAttemptCount(connect_attempts_); + conn_pool_->newStream(*this); + return true; + } + + udp_session_info_.setResponseFlag(StreamInfo::ResponseFlag::NoHealthyUpstream); + return false; +} + +void UdpProxyFilter::TunnelingActiveSession::onStreamFailure( + ConnectionPool::PoolFailureReason reason, absl::string_view failure_reason, + Upstream::HostDescriptionConstSharedPtr) { + ENVOY_LOG(debug, "Failed to create upstream stream: {}", failure_reason); + + conn_pool_.reset(); + upstream_.reset(); + + switch (reason) { + case ConnectionPool::PoolFailureReason::Overflow: + case ConnectionPool::PoolFailureReason::LocalConnectionFailure: + onUpstreamEvent(Network::ConnectionEvent::LocalClose); + break; + case ConnectionPool::PoolFailureReason::Timeout: + udp_session_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionFailure); + onUpstreamEvent(Network::ConnectionEvent::RemoteClose); + break; + case ConnectionPool::PoolFailureReason::RemoteConnectionFailure: + if (connecting_) { + udp_session_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionFailure); + } + onUpstreamEvent(Network::ConnectionEvent::RemoteClose); + break; + } +} + +void UdpProxyFilter::TunnelingActiveSession::onStreamReady(StreamInfo::StreamInfo* upstream_info, + std::unique_ptr&& upstream, + Upstream::HostDescriptionConstSharedPtr&, + const Network::ConnectionInfoProvider&, + Ssl::ConnectionInfoConstSharedPtr) { + // TODO(ohadvano): save the host description to host_ field. This requires refactoring because + // currently host_ is of type HostConstSharedPtr and not HostDescriptionConstSharedPtr. + ENVOY_LOG(debug, "Upstream connection [C{}] attached to session ID [S{}]", + upstream_info->downstreamAddressProvider().connectionID().value(), sessionId()); + + upstream_ = std::move(upstream); + conn_pool_.reset(); + connecting_ = false; + can_send_upstream_ = true; + cluster_.cluster_stats_.sess_tunnel_success_.inc(); + flushBuffer(); +} + +void UdpProxyFilter::TunnelingActiveSession::onUpstreamEvent(Network::ConnectionEvent event) { + if (event == Network::ConnectionEvent::Connected || + event == Network::ConnectionEvent::ConnectedZeroRtt) { + return; + } + + bool connecting = connecting_; + connecting_ = false; + + if (event == Network::ConnectionEvent::RemoteClose || + event == Network::ConnectionEvent::LocalClose) { + upstream_.reset(); + + if (!connecting || !establishUpstreamConnection()) { + cluster_.removeSession(this); + } + } +} + +void UdpProxyFilter::TunnelingActiveSession::onAboveWriteBufferHighWatermark() { + can_send_upstream_ = false; +} + +void UdpProxyFilter::TunnelingActiveSession::onBelowWriteBufferLowWatermark() { + can_send_upstream_ = true; + flushBuffer(); +} + +void UdpProxyFilter::TunnelingActiveSession::flushBuffer() { + while (!datagrams_buffer_.empty()) { + BufferedDatagramPtr buffered_datagram = std::move(datagrams_buffer_.front()); + datagrams_buffer_.pop(); + buffered_bytes_ -= buffered_datagram->buffer_->length(); + upstream_->encodeData(*buffered_datagram->buffer_); + } +} + +void UdpProxyFilter::TunnelingActiveSession::maybeBufferDatagram(Network::UdpRecvData& data) { + if (!cluster_.filter_.config_->tunnelingConfig()->bufferEnabled()) { + return; + } + + if (datagrams_buffer_.size() == + cluster_.filter_.config_->tunnelingConfig()->maxBufferedDatagrams() || + buffered_bytes_ + data.buffer_->length() > + cluster_.filter_.config_->tunnelingConfig()->maxBufferedBytes()) { + cluster_.cluster_stats_.sess_tunnel_buffer_overflow_.inc(); + return; + } + + auto buffered_datagram = std::make_unique(); + buffered_datagram->addresses_ = {std::move(data.addresses_.local_), + std::move(data.addresses_.peer_)}; + buffered_datagram->buffer_ = std::move(data.buffer_); + buffered_datagram->receive_time_ = data.receive_time_; + buffered_bytes_ += buffered_datagram->buffer_->length(); + datagrams_buffer_.push(std::move(buffered_datagram)); +} + +void UdpProxyFilter::TunnelingActiveSession::writeUpstream(Network::UdpRecvData& data) { + if (!upstream_ || !can_send_upstream_) { + maybeBufferDatagram(data); + return; + } + + if (upstream_) { + upstream_->encodeData(*data.buffer_); + } +} + +void UdpProxyFilter::TunnelingActiveSession::onUpstreamData(Buffer::Instance& data, bool) { + const uint64_t rx_buffer_length = data.length(); + ENVOY_LOG(trace, "received {} byte datagram from upstream: downstream={} local={} upstream={}", + rx_buffer_length, addresses_.peer_->asStringView(), addresses_.local_->asStringView(), + host_ != nullptr ? host_->address()->asStringView() : "unknown"); + + cluster_.cluster_stats_.sess_rx_datagrams_.inc(); + cluster_.cluster_.info()->trafficStats()->upstream_cx_rx_bytes_total_.add(rx_buffer_length); + resetIdleTimer(); + + Network::UdpRecvData recv_data{{addresses_.local_, addresses_.peer_}, + std::make_unique(data), + cluster_.filter_.config_->timeSource().monotonicTime()}; + processUpstreamDatagram(recv_data); +} + +void UdpProxyFilter::TunnelingActiveSession::onIdleTimer() { + ENVOY_LOG(debug, "session idle timeout: downstream={} local={}", addresses_.peer_->asStringView(), + addresses_.local_->asStringView()); + cluster_.filter_.config_->stats().idle_timeout_.inc(); + upstream_->onDownstreamEvent(Network::ConnectionEvent::LocalClose); + cluster_.removeSession(this); +} + } // namespace UdpProxy } // namespace UdpFilters } // namespace Extensions diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index f006bba45d02..cd66e5031178 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -5,8 +5,10 @@ #include "envoy/event/file_event.h" #include "envoy/event/timer.h" #include "envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.pb.h" +#include "envoy/http/header_evaluator.h" #include "envoy/network/filter.h" #include "envoy/stream_info/stream_info.h" +#include "envoy/stream_info/uint32_accessor.h" #include "envoy/upstream/cluster_manager.h" #include "source/common/access_log/access_log_impl.h" @@ -14,10 +16,15 @@ #include "source/common/common/empty_string.h" #include "source/common/common/linked_object.h" #include "source/common/common/random_generator.h" +#include "source/common/http/codes.h" +#include "source/common/http/header_map_impl.h" +#include "source/common/http/headers.h" +#include "source/common/http/utility.h" #include "source/common/network/socket_impl.h" #include "source/common/network/socket_interface.h" #include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" +#include "source/common/router/header_parser.h" #include "source/common/stream_info/stream_info_impl.h" #include "source/common/upstream/load_balancer_impl.h" #include "source/extensions/filters/udp/udp_proxy/hash_policy_impl.h" @@ -65,6 +72,9 @@ struct UdpProxyDownstreamStats { COUNTER(sess_rx_datagrams_dropped) \ COUNTER(sess_rx_errors) \ COUNTER(sess_tx_datagrams) \ + COUNTER(sess_tunnel_success) \ + COUNTER(sess_tunnel_failure) \ + COUNTER(sess_tunnel_buffer_overflow) \ COUNTER(sess_tx_errors) /** @@ -74,6 +84,36 @@ struct UdpProxyUpstreamStats { ALL_UDP_PROXY_UPSTREAM_STATS(GENERATE_COUNTER_STRUCT) }; +/** + * Configuration for raw UDP tunneling over HTTP streams. + */ +class UdpTunnelingConfig { +public: + virtual ~UdpTunnelingConfig() = default; + + virtual const std::string proxyHost(const StreamInfo::StreamInfo& stream_info) const PURE; + virtual const std::string targetHost(const StreamInfo::StreamInfo& stream_info) const PURE; + virtual const absl::optional& proxyPort() const PURE; + virtual uint32_t defaultTargetPort() const PURE; + virtual bool usePost() const PURE; + virtual const std::string& postPath() const PURE; + virtual Http::HeaderEvaluator& headerEvaluator() const PURE; + virtual uint32_t maxConnectAttempts() const PURE; + virtual bool bufferEnabled() const PURE; + virtual uint32_t maxBufferedDatagrams() const PURE; + virtual uint64_t maxBufferedBytes() const PURE; + + virtual void + propagateResponseHeaders(Http::ResponseHeaderMapPtr&& headers, + const StreamInfo::FilterStateSharedPtr& filter_state) const PURE; + + virtual void + propagateResponseTrailers(Http::ResponseTrailerMapPtr&& trailers, + const StreamInfo::FilterStateSharedPtr& filter_state) const PURE; +}; + +using UdpTunnelingConfigPtr = std::unique_ptr; + class UdpProxyFilterConfig { public: virtual ~UdpProxyFilterConfig() = default; @@ -88,12 +128,15 @@ class UdpProxyFilterConfig { virtual const Udp::HashPolicy* hashPolicy() const PURE; virtual UdpProxyDownstreamStats& stats() const PURE; virtual TimeSource& timeSource() const PURE; - virtual Random::RandomGenerator& randomGenerator() const PURE; virtual const Network::ResolvedUdpSocketConfig& upstreamSocketConfig() const PURE; virtual const std::vector& sessionAccessLogs() const PURE; virtual const std::vector& proxyAccessLogs() const PURE; virtual const FilterChainFactory& sessionFilterFactory() const PURE; virtual bool hasSessionFilters() const PURE; + virtual const UdpTunnelingConfigPtr& tunnelingConfig() const PURE; + virtual bool flushAccessLogOnTunnelConnected() const PURE; + virtual const absl::optional& accessLogFlushInterval() const PURE; + virtual Random::RandomGenerator& randomGenerator() const PURE; }; using UdpProxyFilterConfigSharedPtr = std::shared_ptr; @@ -104,18 +147,317 @@ using UdpProxyFilterConfigSharedPtr = std::shared_ptrgenerateHash(*peer_address); } } absl::optional computeHashKey() override { return hash_; } + const StreamInfo::StreamInfo* requestStreamInfo() const override { return stream_info_; } private: absl::optional hash_; + const StreamInfo::StreamInfo* stream_info_; +}; + +/** + * Represents an upstream HTTP stream handler, which is able to send data and signal + * the downstream about upstream events. + */ +class HttpUpstream { +public: + virtual ~HttpUpstream() = default; + + /** + * Used to encode data upstream using the HTTP stream. + * @param data supplies the data to encode upstream. + */ + virtual void encodeData(Buffer::Instance& data) PURE; + + /** + * Called when an event is received on the downstream session. + * @param event supplies the event which occurred. + */ + virtual void onDownstreamEvent(Network::ConnectionEvent event) PURE; +}; + +/** + * Callbacks to signal the status of upstream HTTP stream creation to the downstream. + */ +class HttpStreamCallbacks { +public: + virtual ~HttpStreamCallbacks() = default; + + /** + * Called when the stream is ready, after receiving successful header response from + * the upstream. + * @param info supplies the stream info object associated with the upstream connection. + * @param upstream supplies the HTTP upstream for the tunneling stream. + * @param host supplies the description of the upstream host that will carry the request. + * @param address_provider supplies the address provider of the upstream connection. + * @param ssl_info supplies the ssl information of the upstream connection. + */ + virtual void onStreamReady(StreamInfo::StreamInfo* info, std::unique_ptr&& upstream, + Upstream::HostDescriptionConstSharedPtr& host, + const Network::ConnectionInfoProvider& address_provider, + Ssl::ConnectionInfoConstSharedPtr ssl_info) PURE; + + /** + * Called to indicate a failure for when establishing the HTTP stream. + * @param reason supplies the failure reason. + * @param failure_reason failure reason string. + * @param host supplies the description of the host that caused the failure. This may be nullptr + * if no host was involved in the failure (for example overflow). + */ + virtual void onStreamFailure(ConnectionPool::PoolFailureReason reason, + absl::string_view failure_reason, + Upstream::HostDescriptionConstSharedPtr host) PURE; +}; + +/** + * Callbacks to signal the status of upstream HTTP stream creation to the connection pool. + */ +class TunnelCreationCallbacks { +public: + virtual ~TunnelCreationCallbacks() = default; + + /** + * Called when success response headers returned from the upstream. + * @param request_encoder the upstream request encoder associated with the stream. + */ + virtual void onStreamSuccess(Http::RequestEncoder& request_encoder) PURE; + + /** + * Called when failed to create the upstream HTTP stream. + */ + virtual void onStreamFailure() PURE; +}; + +/** + * Callbacks that allows upstream stream to signal events to the downstream. + */ +class UpstreamTunnelCallbacks { +public: + virtual ~UpstreamTunnelCallbacks() = default; + + /** + * Called when an event was raised by the upstream. + * @param event the event. + */ + virtual void onUpstreamEvent(Network::ConnectionEvent event) PURE; + + /** + * Called when the upstream buffer went above high watermark. + */ + virtual void onAboveWriteBufferHighWatermark() PURE; + + /** + * Called when the upstream buffer went below high watermark. + */ + virtual void onBelowWriteBufferLowWatermark() PURE; + + /** + * Called when an event was raised by the upstream. + * @param data the data received from the upstream. + * @param end_stream indicates if the received data is ending the stream. + */ + virtual void onUpstreamData(Buffer::Instance& data, bool end_stream) PURE; +}; + +class HttpUpstreamImpl : public HttpUpstream, protected Http::StreamCallbacks { +public: + HttpUpstreamImpl(UpstreamTunnelCallbacks& upstream_callbacks, const UdpTunnelingConfig& config, + StreamInfo::StreamInfo& downstream_info) + : response_decoder_(*this), upstream_callbacks_(upstream_callbacks), + downstream_info_(downstream_info), tunnel_config_(config) {} + ~HttpUpstreamImpl() override; + + Http::ResponseDecoder& responseDecoder() { return response_decoder_; } + void setRequestEncoder(Http::RequestEncoder& request_encoder, bool is_ssl); + void setTunnelCreationCallbacks(TunnelCreationCallbacks& callbacks) { + tunnel_creation_callbacks_.emplace(callbacks); + } + + // HttpUpstream + void encodeData(Buffer::Instance& data) override; + void onDownstreamEvent(Network::ConnectionEvent event) override { + if (event == Network::ConnectionEvent::LocalClose || + event == Network::ConnectionEvent::RemoteClose) { + resetEncoder(event, /*by_downstream=*/true); + } + }; + + // Http::StreamCallbacks + void onResetStream(Http::StreamResetReason, absl::string_view) override { + resetEncoder(Network::ConnectionEvent::LocalClose); + } + + void onAboveWriteBufferHighWatermark() override { + upstream_callbacks_.onAboveWriteBufferHighWatermark(); + } + + void onBelowWriteBufferLowWatermark() override { + upstream_callbacks_.onBelowWriteBufferLowWatermark(); + } + +private: + class ResponseDecoder : public Http::ResponseDecoder { + public: + ResponseDecoder(HttpUpstreamImpl& parent) : parent_(parent) {} + + // Http::ResponseDecoder + void decodeMetadata(Http::MetadataMapPtr&&) override {} + void decode1xxHeaders(Http::ResponseHeaderMapPtr&&) override {} + void dumpState(std::ostream& os, int indent_level) const override { + DUMP_STATE_UNIMPLEMENTED(ResponseDecoder); + } + + void decodeHeaders(Http::ResponseHeaderMapPtr&& headers, bool end_stream) override { + bool is_valid_response; + if (parent_.tunnel_config_.usePost()) { + auto status = Http::Utility::getResponseStatus(*headers); + is_valid_response = Http::CodeUtility::is2xx(status); + } else { + is_valid_response = Http::HeaderUtility::isConnectUdpResponse(*headers); + } + + parent_.tunnel_config_.propagateResponseHeaders(std::move(headers), + parent_.downstream_info_.filterState()); + + if (!is_valid_response || end_stream) { + parent_.resetEncoder(Network::ConnectionEvent::LocalClose); + } else if (parent_.tunnel_creation_callbacks_.has_value()) { + parent_.tunnel_creation_callbacks_.value().get().onStreamSuccess(*parent_.request_encoder_); + parent_.tunnel_creation_callbacks_.reset(); + } + } + + void decodeData(Buffer::Instance& data, bool end_stream) override { + parent_.upstream_callbacks_.onUpstreamData(data, end_stream); + if (end_stream) { + parent_.resetEncoder(Network::ConnectionEvent::LocalClose); + } + } + + void decodeTrailers(Http::ResponseTrailerMapPtr&& trailers) override { + parent_.tunnel_config_.propagateResponseTrailers(std::move(trailers), + parent_.downstream_info_.filterState()); + parent_.resetEncoder(Network::ConnectionEvent::LocalClose); + } + + private: + HttpUpstreamImpl& parent_; + }; + + const std::string resolveTargetTunnelPath(); + void resetEncoder(Network::ConnectionEvent event, bool by_downstream = false); + + ResponseDecoder response_decoder_; + Http::RequestEncoder* request_encoder_{}; + UpstreamTunnelCallbacks& upstream_callbacks_; + StreamInfo::StreamInfo& downstream_info_; + const UdpTunnelingConfig& tunnel_config_; + absl::optional> tunnel_creation_callbacks_; +}; + +/** + * Provide method to create upstream HTTP stream for UDP tunneling. + */ +class TunnelingConnectionPool { +public: + virtual ~TunnelingConnectionPool() = default; + + /** + * Called to create a TCP connection and HTTP stream. + * + * @param callbacks callbacks to communicate stream failure or creation on. + */ + virtual void newStream(HttpStreamCallbacks& callbacks) PURE; +}; + +using TunnelingConnectionPoolPtr = std::unique_ptr; + +class TunnelingConnectionPoolImpl : public TunnelingConnectionPool, + public TunnelCreationCallbacks, + public Http::ConnectionPool::Callbacks, + public Logger::Loggable { +public: + TunnelingConnectionPoolImpl(Upstream::ThreadLocalCluster& thread_local_cluster, + Upstream::LoadBalancerContext* context, + const UdpTunnelingConfig& tunnel_config, + UpstreamTunnelCallbacks& upstream_callbacks, + StreamInfo::StreamInfo& downstream_info, + bool flush_access_log_on_tunnel_connected, + const std::vector& session_access_logs); + ~TunnelingConnectionPoolImpl() override = default; + + bool valid() const { return conn_pool_data_.has_value(); } + + // TunnelingConnectionPool + void newStream(HttpStreamCallbacks& callbacks) override; + + // TunnelCreationCallbacks + void onStreamSuccess(Http::RequestEncoder& request_encoder) override { + callbacks_->onStreamReady(upstream_info_, std::move(upstream_), upstream_host_, + request_encoder.getStream().connectionInfoProvider(), ssl_info_); + } + + void onStreamFailure() override { + callbacks_->onStreamFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure, "", + upstream_host_); + } + + // Http::ConnectionPool::Callbacks + void onPoolFailure(Http::ConnectionPool::PoolFailureReason reason, + absl::string_view failure_reason, + Upstream::HostDescriptionConstSharedPtr host) override; + void onPoolReady(Http::RequestEncoder& request_encoder, + Upstream::HostDescriptionConstSharedPtr upstream_host, + StreamInfo::StreamInfo& upstream_info, absl::optional) override; + +private: + absl::optional conn_pool_data_{}; + HttpStreamCallbacks* callbacks_{}; + UpstreamTunnelCallbacks& upstream_callbacks_; + std::unique_ptr upstream_; + Http::ConnectionPool::Cancellable* upstream_handle_{}; + const UdpTunnelingConfig& tunnel_config_; + StreamInfo::StreamInfo& downstream_info_; + const bool flush_access_log_on_tunnel_connected_; + const std::vector& session_access_logs_; + Upstream::HostDescriptionConstSharedPtr upstream_host_; + Ssl::ConnectionInfoConstSharedPtr ssl_info_; + StreamInfo::StreamInfo* upstream_info_; +}; + +class TunnelingConnectionPoolFactory { +public: + /** + * Called to create a connection pool that can be used to create an upstream connection. + * + * @param thread_local_cluster the thread local cluster to use for conn pool creation. + * @param context the load balancing context for this connection. + * @param tunnel_config the tunneling config. + * @param upstream_callbacks the callbacks to provide to the connection if successfully created. + * @param stream_info is the downstream session stream info. + * @param flush_access_log_on_tunnel_connected indicates whether to flush access log on tunnel + * connected. + * @param session_access_logs is the list of access logs for the session. + * @return may be null if pool creation failed. + */ + TunnelingConnectionPoolPtr + createConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, + Upstream::LoadBalancerContext* context, const UdpTunnelingConfig& tunnel_config, + UpstreamTunnelCallbacks& upstream_callbacks, StreamInfo::StreamInfo& stream_info, + bool flush_access_log_on_tunnel_connected, + const std::vector& session_access_logs) const; }; +using TunnelingConnectionPoolFactoryPtr = std::unique_ptr; + class UdpProxyFilter : public Network::UdpListenerReadFilter, public Upstream::ClusterUpdateCallbacks, Logger::Loggable { @@ -139,7 +481,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, // SessionFilters::ReadFilterCallbacks uint64_t sessionId() const override { return parent_.sessionId(); }; StreamInfo::StreamInfo& streamInfo() override { return parent_.streamInfo(); }; - void continueFilterChain() override { parent_.onContinueFilterChain(this); } + bool continueFilterChain() override { return parent_.onContinueFilterChain(this); } void injectDatagramToFilterChain(Network::UdpRecvData& data) override { parent_.onInjectReadDatagramToFilterChain(this, data); } @@ -176,11 +518,12 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, * will be hashed to the same session and will be forwarded to the same upstream, using the same * local ephemeral IP/port. */ - class ActiveSession : public Network::UdpPacketProcessor, public FilterChainFactoryCallbacks { + class ActiveSession : public FilterChainFactoryCallbacks { public: ActiveSession(ClusterInfo& parent, Network::UdpRecvData::LocalPeerAddresses&& addresses, - const Upstream::HostConstSharedPtr& host, bool defer_socket_creation); + const Upstream::HostConstSharedPtr& host); ~ActiveSession() override; + const Network::UdpRecvData::LocalPeerAddresses& addresses() const { return addresses_; } absl::optional> host() const { if (host_) { @@ -189,12 +532,16 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, return absl::nullopt; } - void onNewSession(); + + bool onNewSession(); void onData(Network::UdpRecvData& data); - void writeUpstream(Network::UdpRecvData& data); + void processUpstreamDatagram(Network::UdpRecvData& data); void writeDownstream(Network::UdpRecvData& data); - void createSocket(const Upstream::HostConstSharedPtr& host); - void createSocketDeferred(); + void resetIdleTimer(); + + virtual bool createUpstream() PURE; + virtual void writeUpstream(Network::UdpRecvData& data) PURE; + virtual void onIdleTimer() PURE; void createFilterChain() { cluster_.filter_.config_->sessionFilterFactory().createFilterChain(*this); @@ -202,9 +549,10 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, uint64_t sessionId() const { return session_id_; }; StreamInfo::StreamInfo& streamInfo() { return udp_session_info_; }; - void onContinueFilterChain(ActiveReadFilter* filter); + bool onContinueFilterChain(ActiveReadFilter* filter); void onInjectReadDatagramToFilterChain(ActiveReadFilter* filter, Network::UdpRecvData& data); void onInjectWriteDatagramToFilterChain(ActiveWriteFilter* filter, Network::UdpRecvData& data); + void onSessionComplete(); // SessionFilters::FilterChainFactoryCallbacks void addReadFilter(ReadFilterSharedPtr filter) override { @@ -229,26 +577,9 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, LinkedList::moveIntoList(std::move(write_wrapper), write_filters_); }; - private: - void onIdleTimer(); - void onReadReady(); + protected: void fillSessionStreamInfo(); - // Network::UdpPacketProcessor - void processPacket(Network::Address::InstanceConstSharedPtr local_address, - Network::Address::InstanceConstSharedPtr peer_address, - Buffer::InstancePtr buffer, MonotonicTime receive_time) override; - uint64_t maxDatagramSize() const override { - return cluster_.filter_.config_->upstreamSocketConfig().max_rx_datagram_size_; - } - void onDatagramsDropped(uint32_t dropped) override { - cluster_.cluster_stats_.sess_rx_datagrams_dropped_.add(dropped); - } - size_t numPacketsExpectedPerEventLoop() const final { - // TODO(mattklein123) change this to a reasonable number if needed. - return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP; - } - /** * Struct definition for session access logging. */ @@ -264,7 +595,6 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, static std::atomic next_global_session_id_; ClusterInfo& cluster_; - const bool use_original_src_ip_; const Network::UdpRecvData::LocalPeerAddresses addresses_; Upstream::HostConstSharedPtr host_; // TODO(mattklein123): Consider replacing an idle timer for each session with a last used @@ -273,22 +603,118 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, // idle timeouts work so we should consider unifying the implementation if we move to a time // stamp and scan approach. const Event::TimerPtr idle_timer_; - // The socket is used for writing packets to the selected upstream host as well as receiving - // packets from the upstream host. Note that a a local ephemeral port is bound on the first - // write to the upstream host. - Network::SocketPtr socket_; - // The socket has been connected to avoid port exhaustion. - bool connected_{}; + Event::TimerPtr access_log_flush_timer_; UdpProxySessionStats session_stats_{}; StreamInfo::StreamInfoImpl udp_session_info_; uint64_t session_id_; std::list read_filters_; std::list write_filters_; + + private: + void onAccessLogFlushInterval(); + void rearmAccessLogFlushTimer(); + void disableAccessLogFlushTimer(); + + bool on_session_complete_called_{false}; }; using ActiveSessionPtr = std::unique_ptr; + class UdpActiveSession : public Network::UdpPacketProcessor, public ActiveSession { + public: + UdpActiveSession(ClusterInfo& parent, Network::UdpRecvData::LocalPeerAddresses&& addresses, + const Upstream::HostConstSharedPtr& host); + ~UdpActiveSession() override = default; + + // ActiveSession + bool createUpstream() override; + void writeUpstream(Network::UdpRecvData& data) override; + void onIdleTimer() override; + + // Network::UdpPacketProcessor + void processPacket(Network::Address::InstanceConstSharedPtr local_address, + Network::Address::InstanceConstSharedPtr peer_address, + Buffer::InstancePtr buffer, MonotonicTime receive_time) override; + + uint64_t maxDatagramSize() const override { + return cluster_.filter_.config_->upstreamSocketConfig().max_rx_datagram_size_; + } + + void onDatagramsDropped(uint32_t dropped) override { + cluster_.cluster_stats_.sess_rx_datagrams_dropped_.add(dropped); + } + + size_t numPacketsExpectedPerEventLoop() const final { + // TODO(mattklein123) change this to a reasonable number if needed. + return Network::MAX_NUM_PACKETS_PER_EVENT_LOOP; + } + + private: + void onReadReady(); + void createUdpSocket(const Upstream::HostConstSharedPtr& host); + + // The socket is used for writing packets to the selected upstream host as well as receiving + // packets from the upstream host. Note that a a local ephemeral port is bound on the first + // write to the upstream host. + Network::SocketPtr udp_socket_; + // The socket has been connected to avoid port exhaustion. + bool connected_{}; + const bool use_original_src_ip_; + }; + + /** + * This type of active session is used when tunneling is enabled by configuration. + * In this type of session, the upstream is HTTP stream, either a connect-udp request, + * or a POST request. + */ + class TunnelingActiveSession : public ActiveSession, + public UpstreamTunnelCallbacks, + public HttpStreamCallbacks { + public: + TunnelingActiveSession(ClusterInfo& parent, + Network::UdpRecvData::LocalPeerAddresses&& addresses); + ~TunnelingActiveSession() override = default; + + // ActiveSession + bool createUpstream() override; + void writeUpstream(Network::UdpRecvData& data) override; + void onIdleTimer() override; + + // UpstreamTunnelCallbacks + void onUpstreamEvent(Network::ConnectionEvent event) override; + void onAboveWriteBufferHighWatermark() override; + void onBelowWriteBufferLowWatermark() override; + void onUpstreamData(Buffer::Instance& data, bool end_stream) override; + + // HttpStreamCallbacks + void onStreamReady(StreamInfo::StreamInfo*, std::unique_ptr&&, + Upstream::HostDescriptionConstSharedPtr&, + const Network::ConnectionInfoProvider&, + Ssl::ConnectionInfoConstSharedPtr) override; + + void onStreamFailure(ConnectionPool::PoolFailureReason, absl::string_view, + Upstream::HostDescriptionConstSharedPtr) override; + + private: + using BufferedDatagramPtr = std::unique_ptr; + + bool establishUpstreamConnection(); + bool createConnectionPool(); + void maybeBufferDatagram(Network::UdpRecvData& data); + void flushBuffer(); + + TunnelingConnectionPoolFactoryPtr conn_pool_factory_; + std::unique_ptr load_balancer_context_; + TunnelingConnectionPoolPtr conn_pool_; + std::unique_ptr upstream_; + uint32_t connect_attempts_{}; + bool connecting_{}; + bool can_send_upstream_{}; + uint64_t buffered_bytes_{}; + std::queue datagrams_buffer_; + }; + struct LocalPeerHostAddresses { const Network::UdpRecvData::LocalPeerAddresses& local_peer_addresses_; absl::optional> host_; @@ -366,13 +792,14 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, SessionStorageType&& sessions); virtual ~ClusterInfo(); virtual Network::FilterStatus onData(Network::UdpRecvData& data) PURE; - void removeSession(const ActiveSession* session); - void addSession(const Upstream::Host* host, const ActiveSession* session) { + void removeSession(ActiveSession* session); + void addSession(const Upstream::Host* host, ActiveSession* session) { host_to_sessions_[host].emplace(session); } Upstream::HostConstSharedPtr - chooseHost(const Network::Address::InstanceConstSharedPtr& peer_address) const; + chooseHost(const Network::Address::InstanceConstSharedPtr& peer_address, + const StreamInfo::StreamInfo* stream_info) const; UdpProxyFilter& filter_; Upstream::ThreadLocalCluster& cluster_; @@ -390,13 +817,13 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, const auto final_prefix = "udp"; return {ALL_UDP_PROXY_UPSTREAM_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; } + ActiveSession* createSessionWithOptionalHost(Network::UdpRecvData::LocalPeerAddresses&& addresses, - const Upstream::HostConstSharedPtr& host, - bool defer_socket_creation); + const Upstream::HostConstSharedPtr& host); Envoy::Common::CallbackHandlePtr member_update_cb_handle_; - absl::flat_hash_map> + absl::flat_hash_map> host_to_sessions_; }; @@ -423,7 +850,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, Network::FilterStatus onData(Network::UdpRecvData& data) override; }; - virtual Network::SocketPtr createSocket(const Upstream::HostConstSharedPtr& host) { + virtual Network::SocketPtr createUdpSocket(const Upstream::HostConstSharedPtr& host) { // Virtual so this can be overridden in unit tests. return std::make_unique(Network::Socket::Type::Datagram, host->address(), nullptr, Network::SocketCreationOptions{}); diff --git a/source/extensions/formatter/cel/cel.cc b/source/extensions/formatter/cel/cel.cc index 74160a6651bb..ffee3e8c73ff 100644 --- a/source/extensions/formatter/cel/cel.cc +++ b/source/extensions/formatter/cel/cel.cc @@ -15,10 +15,12 @@ namespace Formatter { namespace Expr = Filters::Common::Expr; -CELFormatter::CELFormatter(Expr::BuilderInstanceSharedPtr expr_builder, +CELFormatter::CELFormatter(const ::Envoy::LocalInfo::LocalInfo& local_info, + Expr::BuilderInstanceSharedPtr expr_builder, const google::api::expr::v1alpha1::Expr& input_expr, absl::optional& max_length) - : expr_builder_(expr_builder), parsed_expr_(input_expr), max_length_(max_length) { + : local_info_(local_info), expr_builder_(expr_builder), parsed_expr_(input_expr), + max_length_(max_length) { compiled_expr_ = Expr::createExpression(expr_builder_->builder(), parsed_expr_); } @@ -26,8 +28,9 @@ absl::optional CELFormatter::formatWithContext(const Envoy::Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& stream_info) const { Protobuf::Arena arena; - auto eval_status = Expr::evaluate(*compiled_expr_, arena, stream_info, &context.requestHeaders(), - &context.responseHeaders(), &context.responseTrailers()); + auto eval_status = + Expr::evaluate(*compiled_expr_, arena, &local_info_, stream_info, &context.requestHeaders(), + &context.responseHeaders(), &context.responseTrailers()); if (!eval_status.has_value() || eval_status.value().IsError()) { return absl::nullopt; } @@ -60,7 +63,8 @@ CELFormatterCommandParser::parse(const std::string& command, const std::string& parse_status.status().ToString()); } - return std::make_unique(expr_builder_, parse_status.value().expr(), max_length); + return std::make_unique(local_info_, expr_builder_, parse_status.value().expr(), + max_length); } return nullptr; diff --git a/source/extensions/formatter/cel/cel.h b/source/extensions/formatter/cel/cel.h index b1135f5ffe03..2c5a600a3397 100644 --- a/source/extensions/formatter/cel/cel.h +++ b/source/extensions/formatter/cel/cel.h @@ -14,7 +14,8 @@ namespace Formatter { class CELFormatter : public ::Envoy::Formatter::FormatterProvider { public: - CELFormatter(Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr, + CELFormatter(const ::Envoy::LocalInfo::LocalInfo& local_info, + Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr, const google::api::expr::v1alpha1::Expr&, absl::optional&); absl::optional @@ -24,6 +25,7 @@ class CELFormatter : public ::Envoy::Formatter::FormatterProvider { const StreamInfo::StreamInfo&) const override; private: + const ::Envoy::LocalInfo::LocalInfo& local_info_; Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr expr_builder_; const google::api::expr::v1alpha1::Expr parsed_expr_; const absl::optional max_length_; @@ -33,12 +35,14 @@ class CELFormatter : public ::Envoy::Formatter::FormatterProvider { class CELFormatterCommandParser : public ::Envoy::Formatter::CommandParser { public: CELFormatterCommandParser(Server::Configuration::CommonFactoryContext& context) - : expr_builder_(Extensions::Filters::Common::Expr::getBuilder(context)){}; + : local_info_(context.localInfo()), + expr_builder_(Extensions::Filters::Common::Expr::getBuilder(context)){}; ::Envoy::Formatter::FormatterProviderPtr parse(const std::string& command, const std::string& subcommand, absl::optional& max_length) const override; private: + const ::Envoy::LocalInfo::LocalInfo& local_info_; Extensions::Filters::Common::Expr::BuilderInstanceSharedPtr expr_builder_; }; diff --git a/source/extensions/formatter/cel/config.cc b/source/extensions/formatter/cel/config.cc index 1c14da3fb3a3..6870f987abc3 100644 --- a/source/extensions/formatter/cel/config.cc +++ b/source/extensions/formatter/cel/config.cc @@ -9,9 +9,9 @@ namespace Extensions { namespace Formatter { ::Envoy::Formatter::CommandParserPtr CELFormatterFactory::createCommandParserFromProto( - const Protobuf::Message&, Server::Configuration::CommonFactoryContext& context) { + const Protobuf::Message&, Server::Configuration::GenericFactoryContext& context) { #if defined(USE_CEL_PARSER) - return std::make_unique(context); + return std::make_unique(context.serverFactoryContext()); #else UNREFERENCED_PARAMETER(context); throw EnvoyException("CEL is not available for use in this environment."); diff --git a/source/extensions/formatter/cel/config.h b/source/extensions/formatter/cel/config.h index ab2750dded5a..83ee642857a8 100644 --- a/source/extensions/formatter/cel/config.h +++ b/source/extensions/formatter/cel/config.h @@ -11,7 +11,7 @@ class CELFormatterFactory : public ::Envoy::Formatter::CommandParserFactory { public: ::Envoy::Formatter::CommandParserPtr createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) override; + Server::Configuration::GenericFactoryContext&) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override; }; diff --git a/source/extensions/formatter/metadata/config.cc b/source/extensions/formatter/metadata/config.cc index fa0db57fa540..1d823355565d 100644 --- a/source/extensions/formatter/metadata/config.cc +++ b/source/extensions/formatter/metadata/config.cc @@ -9,7 +9,7 @@ namespace Extensions { namespace Formatter { ::Envoy::Formatter::CommandParserPtr MetadataFormatterFactory::createCommandParserFromProto( - const Protobuf::Message&, Server::Configuration::CommonFactoryContext&) { + const Protobuf::Message&, Server::Configuration::GenericFactoryContext&) { return std::make_unique(); } diff --git a/source/extensions/formatter/metadata/config.h b/source/extensions/formatter/metadata/config.h index c615b5c10bd6..a664dacd7659 100644 --- a/source/extensions/formatter/metadata/config.h +++ b/source/extensions/formatter/metadata/config.h @@ -10,7 +10,7 @@ class MetadataFormatterFactory : public ::Envoy::Formatter::CommandParserFactory public: ::Envoy::Formatter::CommandParserPtr createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) override; + Server::Configuration::GenericFactoryContext&) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override; }; diff --git a/source/extensions/formatter/metadata/metadata.cc b/source/extensions/formatter/metadata/metadata.cc index 873912edadb4..4bf5b939c7c9 100644 --- a/source/extensions/formatter/metadata/metadata.cc +++ b/source/extensions/formatter/metadata/metadata.cc @@ -29,6 +29,23 @@ class RouteMetadataFormatter : public ::Envoy::Formatter::MetadataFormatter { }) {} }; +// Metadata formatter for listener metadata. +class ListenerMetadataFormatter : public ::Envoy::Formatter::MetadataFormatter { +public: + ListenerMetadataFormatter(const std::string& filter_namespace, + const std::vector& path, absl::optional max_length) + : ::Envoy::Formatter::MetadataFormatter( + filter_namespace, path, max_length, + [](const StreamInfo::StreamInfo& stream_info) + -> const envoy::config::core::v3::Metadata* { + const auto listener_info = stream_info.downstreamAddressProvider().listenerInfo(); + if (listener_info) { + return &listener_info->metadata(); + } + return nullptr; + }) {} +}; + // Constructor registers all types of supported metadata along with the // handlers accessing the required metadata type. MetadataFormatterCommandParser::MetadataFormatterCommandParser() { @@ -55,6 +72,12 @@ MetadataFormatterCommandParser::MetadataFormatterCommandParser() { return std::make_unique<::Envoy::Formatter::UpstreamHostMetadataFormatter>(filter_namespace, path, max_length); }; + + metadata_formatter_providers_["LISTENER"] = [](const std::string& filter_namespace, + const std::vector& path, + absl::optional max_length) { + return std::make_unique(filter_namespace, path, max_length); + }; } ::Envoy::Formatter::FormatterProviderPtr diff --git a/source/extensions/formatter/req_without_query/config.cc b/source/extensions/formatter/req_without_query/config.cc index 1dc562014f9e..a2ea8bf82d38 100644 --- a/source/extensions/formatter/req_without_query/config.cc +++ b/source/extensions/formatter/req_without_query/config.cc @@ -8,9 +8,8 @@ namespace Envoy { namespace Extensions { namespace Formatter { -::Envoy::Formatter::CommandParserPtr -ReqWithoutQueryFactory::createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) { +::Envoy::Formatter::CommandParserPtr ReqWithoutQueryFactory::createCommandParserFromProto( + const Protobuf::Message&, Server::Configuration::GenericFactoryContext&) { return std::make_unique(); } diff --git a/source/extensions/formatter/req_without_query/config.h b/source/extensions/formatter/req_without_query/config.h index a648197f392f..c414144c51b3 100644 --- a/source/extensions/formatter/req_without_query/config.h +++ b/source/extensions/formatter/req_without_query/config.h @@ -10,7 +10,7 @@ class ReqWithoutQueryFactory : public ::Envoy::Formatter::CommandParserFactory { public: ::Envoy::Formatter::CommandParserPtr createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) override; + Server::Configuration::GenericFactoryContext&) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override; }; diff --git a/source/extensions/geoip_providers/common/BUILD b/source/extensions/geoip_providers/common/BUILD new file mode 100644 index 000000000000..2f68534edc7a --- /dev/null +++ b/source/extensions/geoip_providers/common/BUILD @@ -0,0 +1,20 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "factory_base_lib", + hdrs = ["factory_base.h"], + tags = ["skip_on_windows"], + deps = [ + "//envoy/geoip:geoip_provider_driver_interface", + "//source/common/config:utility_lib", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/source/extensions/geoip_providers/common/factory_base.h b/source/extensions/geoip_providers/common/factory_base.h new file mode 100644 index 000000000000..9376192e8d66 --- /dev/null +++ b/source/extensions/geoip_providers/common/factory_base.h @@ -0,0 +1,46 @@ +#pragma once + +#include "envoy/geoip/geoip_provider_driver.h" + +#include "source/common/protobuf/protobuf.h" +#include "source/common/protobuf/utility.h" + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Common { + +/** + * Common base class for geoip provider factory registrations. Removes a substantial amount of + * boilerplate. + */ +template class FactoryBase : public Geolocation::GeoipProviderFactory { +public: + // GeoipProviderFactory + Geolocation::DriverSharedPtr + createGeoipProviderDriver(const Protobuf::Message& config, const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) override { + return createGeoipProviderDriverTyped(MessageUtil::downcastAndValidate( + config, context.messageValidationVisitor()), + stat_prefix, context); + } + + std::string name() const override { return name_; } + + std::string category() const override { return "envoy.geoip_providers"; } + +protected: + FactoryBase(const std::string& name) : name_(name) {} + +private: + virtual Geolocation::DriverSharedPtr + createGeoipProviderDriverTyped(const ConfigProto& proto_config, const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) PURE; + + const std::string name_; +}; + +} // namespace Common +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/geoip_providers/maxmind/BUILD b/source/extensions/geoip_providers/maxmind/BUILD new file mode 100644 index 000000000000..5d9f368bf64f --- /dev/null +++ b/source/extensions/geoip_providers/maxmind/BUILD @@ -0,0 +1,46 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +# HTTP L7 filter that decorates request with geolocation data +# Public docs: https://envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/geoip_filter + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = select({ + "//bazel:linux": ["config.cc"], + "//conditions:default": [], + }), + hdrs = ["config.h"], + tags = ["skip_on_windows"], + deps = [ + ":provider_impl", + "//envoy/geoip:geoip_provider_driver_interface", + "//source/common/config:utility_lib", + "//source/common/protobuf:utility_lib", + "//source/extensions/geoip_providers/common:factory_base_lib", + "@envoy_api//envoy/extensions/geoip_providers/maxmind/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "provider_impl", + srcs = select({ + "//bazel:linux": ["geoip_provider.cc"], + "//conditions:default": [], + }), + hdrs = ["geoip_provider.h"], + external_deps = ["maxmind"], + tags = ["skip_on_windows"], + deps = [ + "//envoy/geoip:geoip_provider_driver_interface", + "@envoy_api//envoy/extensions/geoip_providers/maxmind/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/geoip_providers/maxmind/config.cc b/source/extensions/geoip_providers/maxmind/config.cc new file mode 100644 index 000000000000..efe52b5ab1e9 --- /dev/null +++ b/source/extensions/geoip_providers/maxmind/config.cc @@ -0,0 +1,76 @@ +#include "source/extensions/geoip_providers/maxmind/config.h" + +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.h" +#include "envoy/registry/registry.h" + +#include "source/common/common/utility.h" +#include "source/common/protobuf/utility.h" + +#include "geoip_provider.h" + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Maxmind { + +using ConfigProto = envoy::extensions::geoip_providers::maxmind::v3::MaxMindConfig; + +/** + * A singleton that acts as a factory for generating and looking up GeoipProviders. + * When given equivalent provider configs, the singleton returns pointers to the same + * driver/provider. When given different configs, the singleton returns different provider + * instances. + */ +class DriverSingleton : public Envoy::Singleton::Instance { +public: + std::shared_ptr get(std::shared_ptr singleton, + const ConfigProto& proto_config, + const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) { + std::shared_ptr driver; + const uint64_t key = MessageUtil::hash(proto_config); + absl::MutexLock lock(&mu_); + auto it = drivers_.find(key); + if (it != drivers_.end()) { + driver = it->second.lock(); + } else { + const auto& provider_config = + std::make_shared(proto_config, stat_prefix, context.scope()); + driver = std::make_shared(singleton, provider_config); + drivers_[key] = driver; + } + return driver; + } + +private: + absl::Mutex mu_; + // We keep weak_ptr here so the providers can be destroyed if the config is updated to stop using + // that config of the provider. Each provider stores shared_ptrs to this singleton, which keeps + // the singleton from being destroyed unless it's no longer keeping track of any providers. (The + // singleton shared_ptr is *only* held by driver instances.) + absl::flat_hash_map> drivers_ ABSL_GUARDED_BY(mu_); +}; + +SINGLETON_MANAGER_REGISTRATION(maxmind_geolocation_provider_singleton); + +MaxmindProviderFactory::MaxmindProviderFactory() : FactoryBase("envoy.geoip_providers.maxmind") {} + +DriverSharedPtr MaxmindProviderFactory::createGeoipProviderDriverTyped( + const ConfigProto& proto_config, const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) { + std::shared_ptr drivers = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(maxmind_geolocation_provider_singleton), + [] { return std::make_shared(); }); + return drivers->get(drivers, proto_config, stat_prefix, context); +} + +/** + * Static registration for the Maxmind provider. @see RegisterFactory. + */ +REGISTER_FACTORY(MaxmindProviderFactory, Geolocation::GeoipProviderFactory); + +} // namespace Maxmind +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/geoip_providers/maxmind/config.h b/source/extensions/geoip_providers/maxmind/config.h new file mode 100644 index 000000000000..007bc90936ee --- /dev/null +++ b/source/extensions/geoip_providers/maxmind/config.h @@ -0,0 +1,37 @@ +#pragma once + +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.h" +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.validate.h" + +#include "source/common/protobuf/protobuf.h" +#include "source/extensions/geoip_providers/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Maxmind { + +using DriverSharedPtr = Envoy::Geolocation::DriverSharedPtr; + +class MaxmindProviderFactory + : public Common::FactoryBase { +public: + MaxmindProviderFactory(); + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + +private: + // FactoryBase + DriverSharedPtr createGeoipProviderDriverTyped( + const envoy::extensions::geoip_providers::maxmind::v3::MaxMindConfig& proto_config, + const std::string& stat_prefix, Server::Configuration::FactoryContext& context) override; +}; + +DECLARE_FACTORY(MaxmindProviderFactory); + +} // namespace Maxmind +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/geoip_providers/maxmind/geoip_provider.cc b/source/extensions/geoip_providers/maxmind/geoip_provider.cc new file mode 100644 index 000000000000..5af87388c318 --- /dev/null +++ b/source/extensions/geoip_providers/maxmind/geoip_provider.cc @@ -0,0 +1,265 @@ +#include "source/extensions/geoip_providers/maxmind/geoip_provider.h" + +#include "source/common/common/assert.h" +#include "source/common/protobuf/protobuf.h" + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Maxmind { + +namespace { +static constexpr const char* MMDB_CITY_LOOKUP_ARGS[] = {"city", "names", "en"}; +static constexpr const char* MMDB_REGION_LOOKUP_ARGS[] = {"subdivisions", "0", "iso_code"}; +static constexpr const char* MMDB_COUNTRY_LOOKUP_ARGS[] = {"country", "iso_code"}; +static constexpr const char* MMDB_ASN_LOOKUP_ARGS[] = {"autonomous_system_number"}; +static constexpr const char* MMDB_ANON_LOOKUP_ARGS[] = {"is_anonymous", "is_anonymous_vpn", + "is_hosting_provider", "is_tor_exit_node", + "is_public_proxy"}; +} // namespace + +GeoipProviderConfig::GeoipProviderConfig( + const envoy::extensions::geoip_providers::maxmind::v3::MaxMindConfig& config, + const std::string& stat_prefix, Stats::Scope& scope) + : city_db_path_(!config.city_db_path().empty() ? absl::make_optional(config.city_db_path()) + : absl::nullopt), + isp_db_path_(!config.isp_db_path().empty() ? absl::make_optional(config.isp_db_path()) + : absl::nullopt), + anon_db_path_(!config.anon_db_path().empty() ? absl::make_optional(config.anon_db_path()) + : absl::nullopt), + stats_scope_(scope.createScope(absl::StrCat(stat_prefix, "maxmind."))), + stat_name_set_(stats_scope_->symbolTable().makeSet("Maxmind")) { + auto geo_headers_to_add = config.common_provider_config().geo_headers_to_add(); + country_header_ = !geo_headers_to_add.country().empty() + ? absl::make_optional(geo_headers_to_add.country()) + : absl::nullopt; + city_header_ = !geo_headers_to_add.city().empty() ? absl::make_optional(geo_headers_to_add.city()) + : absl::nullopt; + region_header_ = !geo_headers_to_add.region().empty() + ? absl::make_optional(geo_headers_to_add.region()) + : absl::nullopt; + asn_header_ = !geo_headers_to_add.asn().empty() ? absl::make_optional(geo_headers_to_add.asn()) + : absl::nullopt; + anon_header_ = !geo_headers_to_add.is_anon().empty() + ? absl::make_optional(geo_headers_to_add.is_anon()) + : absl::nullopt; + anon_vpn_header_ = !geo_headers_to_add.anon_vpn().empty() + ? absl::make_optional(geo_headers_to_add.anon_vpn()) + : absl::nullopt; + anon_hosting_header_ = !geo_headers_to_add.anon_hosting().empty() + ? absl::make_optional(geo_headers_to_add.anon_hosting()) + : absl::nullopt; + anon_tor_header_ = !geo_headers_to_add.anon_tor().empty() + ? absl::make_optional(geo_headers_to_add.anon_tor()) + : absl::nullopt; + anon_proxy_header_ = !geo_headers_to_add.anon_proxy().empty() + ? absl::make_optional(geo_headers_to_add.anon_proxy()) + : absl::nullopt; + if (!city_db_path_ && !isp_db_path_ && !anon_db_path_) { + throw EnvoyException("At least one geolocation database path needs to be configured: " + "city_db_path, isp_db_path or anon_db_path"); + } + if (city_db_path_) { + registerGeoDbStats("city_db"); + } + if (isp_db_path_) { + registerGeoDbStats("isp_db"); + } + if (anon_db_path_) { + registerGeoDbStats("anon_db"); + } +}; + +void GeoipProviderConfig::registerGeoDbStats(const std::string& db_type) { + stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".total")); + stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".hit")); + stat_name_set_->rememberBuiltin(absl::StrCat(db_type, ".lookup_error")); +} + +bool GeoipProviderConfig::isLookupEnabledForHeader(const absl::optional& header) { + return (header && !header.value().empty()); +} + +void GeoipProviderConfig::incCounter(Stats::StatName name) { + stats_scope_->counterFromStatName(name).inc(); +} + +GeoipProvider::~GeoipProvider() { + ENVOY_LOG(debug, "Shutting down Maxmind geolocation provider"); + if (city_db_) { + MMDB_close(city_db_.get()); + } + if (isp_db_) { + MMDB_close(isp_db_.get()); + } + if (anon_db_) { + MMDB_close(anon_db_.get()); + } +} + +MaxmindDbPtr GeoipProvider::initMaxMindDb(const absl::optional& db_path) { + if (db_path) { + MMDB_s maxmind_db; + int result_code = MMDB_open(db_path.value().c_str(), MMDB_MODE_MMAP, &maxmind_db); + RELEASE_ASSERT(MMDB_SUCCESS == result_code, + fmt::format("Unable to open Maxmind database file {}. Error {}", db_path.value(), + std::string(MMDB_strerror(result_code)))); + return std::make_unique(maxmind_db); + } else { + ENVOY_LOG(debug, "Geolocation database path is empty, skipping database creation"); + return nullptr; + } +} + +void GeoipProvider::lookup(Geolocation::LookupRequest&& request, + Geolocation::LookupGeoHeadersCallback&& cb) const { + auto& remote_address = request.remoteAddress(); + auto lookup_result = absl::flat_hash_map{}; + lookupInCityDb(remote_address, lookup_result); + lookupInAsnDb(remote_address, lookup_result); + lookupInAnonDb(remote_address, lookup_result); + cb(std::move(lookup_result)); +} + +void GeoipProvider::lookupInCityDb( + const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const { + if (config_->isLookupEnabledForHeader(config_->cityHeader()) || + config_->isLookupEnabledForHeader(config_->regionHeader()) || + config_->isLookupEnabledForHeader(config_->countryHeader())) { + ASSERT(city_db_, "Maxmind city database is not initialised for performing lookups"); + int mmdb_error; + const uint32_t n_prev_hits = lookup_result.size(); + MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr( + city_db_.get(), reinterpret_cast(remote_address->sockAddr()), &mmdb_error); + if (!mmdb_error) { + MMDB_entry_data_list_s* entry_data_list; + int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list); + if (status == MMDB_SUCCESS) { + if (config_->isLookupEnabledForHeader(config_->cityHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, config_->cityHeader().value(), + MMDB_CITY_LOOKUP_ARGS[0], MMDB_CITY_LOOKUP_ARGS[1], + MMDB_CITY_LOOKUP_ARGS[2]); + } + if (config_->isLookupEnabledForHeader(config_->regionHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, + config_->regionHeader().value(), MMDB_REGION_LOOKUP_ARGS[0], + MMDB_REGION_LOOKUP_ARGS[1], MMDB_REGION_LOOKUP_ARGS[2]); + } + if (config_->isLookupEnabledForHeader(config_->countryHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, + config_->countryHeader().value(), MMDB_COUNTRY_LOOKUP_ARGS[0], + MMDB_COUNTRY_LOOKUP_ARGS[1]); + } + if (lookup_result.size() > n_prev_hits) { + config_->incHit("city_db"); + } + MMDB_free_entry_data_list(entry_data_list); + } + + } else { + config_->incLookupError("city_db"); + } + config_->incTotal("city_db"); + } +} + +void GeoipProvider::lookupInAsnDb( + const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const { + if (config_->isLookupEnabledForHeader(config_->asnHeader())) { + RELEASE_ASSERT(isp_db_, "Maxmind asn database is not initialized for performing lookups"); + int mmdb_error; + const uint32_t n_prev_hits = lookup_result.size(); + MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr( + isp_db_.get(), reinterpret_cast(remote_address->sockAddr()), &mmdb_error); + if (!mmdb_error) { + MMDB_entry_data_list_s* entry_data_list; + int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list); + if (status == MMDB_SUCCESS && entry_data_list) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, config_->asnHeader().value(), + MMDB_ASN_LOOKUP_ARGS[0]); + MMDB_free_entry_data_list(entry_data_list); + if (lookup_result.size() > n_prev_hits) { + config_->incHit("isp_db"); + } + } else { + config_->incLookupError("isp_db"); + } + } + config_->incTotal("isp_db"); + } +} + +void GeoipProvider::lookupInAnonDb( + const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const { + if (config_->isLookupEnabledForHeader(config_->anonHeader()) || config_->anonVpnHeader()) { + ASSERT(anon_db_, "Maxmind city database is not initialised for performing lookups"); + int mmdb_error; + const uint32_t n_prev_hits = lookup_result.size(); + MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr( + anon_db_.get(), reinterpret_cast(remote_address->sockAddr()), &mmdb_error); + if (!mmdb_error) { + MMDB_entry_data_list_s* entry_data_list; + int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list); + if (status == MMDB_SUCCESS) { + if (config_->isLookupEnabledForHeader(config_->anonHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, config_->anonHeader().value(), + MMDB_ANON_LOOKUP_ARGS[0]); + } + if (config_->isLookupEnabledForHeader(config_->anonVpnHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, + config_->anonVpnHeader().value(), MMDB_ANON_LOOKUP_ARGS[1]); + } + if (config_->isLookupEnabledForHeader(config_->anonHostingHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, + config_->anonHostingHeader().value(), MMDB_ANON_LOOKUP_ARGS[2]); + } + if (config_->isLookupEnabledForHeader(config_->anonTorHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, + config_->anonTorHeader().value(), MMDB_ANON_LOOKUP_ARGS[3]); + } + if (config_->isLookupEnabledForHeader(config_->anonProxyHeader())) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, + config_->anonProxyHeader().value(), MMDB_ANON_LOOKUP_ARGS[4]); + } + if (lookup_result.size() > n_prev_hits) { + config_->incHit("anon_db"); + } + MMDB_free_entry_data_list(entry_data_list); + } else { + config_->incLookupError("anon_db"); + } + } + config_->incTotal("anon_db"); + } +} + +template +void GeoipProvider::populateGeoLookupResult( + MMDB_lookup_result_s& mmdb_lookup_result, + absl::flat_hash_map& lookup_result, const std::string& result_key, + Params... lookup_params) const { + MMDB_entry_data_s entry_data; + if ((MMDB_get_value(&mmdb_lookup_result.entry, &entry_data, lookup_params..., NULL)) == + MMDB_SUCCESS) { + std::string result_value; + if (entry_data.has_data && entry_data.type == MMDB_DATA_TYPE_UTF8_STRING) { + result_value = std::string(entry_data.utf8_string, entry_data.data_size); + } else if (entry_data.has_data && entry_data.type == MMDB_DATA_TYPE_UINT32 && + entry_data.uint32 > 0) { + result_value = std::to_string(entry_data.uint32); + } else if (entry_data.has_data && entry_data.type == MMDB_DATA_TYPE_BOOLEAN) { + result_value = entry_data.boolean ? "true" : "false"; + } + if (!result_value.empty()) { + lookup_result.insert(std::make_pair(result_key, result_value)); + } + } +} + +} // namespace Maxmind +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/geoip_providers/maxmind/geoip_provider.h b/source/extensions/geoip_providers/maxmind/geoip_provider.h new file mode 100644 index 000000000000..db5ae83f72c9 --- /dev/null +++ b/source/extensions/geoip_providers/maxmind/geoip_provider.h @@ -0,0 +1,123 @@ +#pragma once + +#include "envoy/common/platform.h" +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.h" +#include "envoy/geoip/geoip_provider_driver.h" + +#include "source/common/common/logger.h" + +#include "maxminddb.h" + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Maxmind { + +class GeoipProviderConfig { +public: + GeoipProviderConfig(const envoy::extensions::geoip_providers::maxmind::v3::MaxMindConfig& config, + const std::string& stat_prefix, Stats::Scope& scope); + + const absl::optional& cityDbPath() const { return city_db_path_; } + const absl::optional& ispDbPath() const { return isp_db_path_; } + const absl::optional& anonDbPath() const { return anon_db_path_; } + + bool isLookupEnabledForHeader(const absl::optional& header); + + const absl::optional& countryHeader() const { return country_header_; } + const absl::optional& cityHeader() const { return city_header_; } + const absl::optional& regionHeader() const { return region_header_; } + const absl::optional& asnHeader() const { return asn_header_; } + + const absl::optional& anonHeader() const { return anon_header_; } + const absl::optional& anonVpnHeader() const { return anon_vpn_header_; } + const absl::optional& anonHostingHeader() const { return anon_hosting_header_; } + const absl::optional& anonTorHeader() const { return anon_tor_header_; } + const absl::optional& anonProxyHeader() const { return anon_proxy_header_; } + + void incLookupError(absl::string_view maxmind_db_type) { + incCounter( + stat_name_set_->getBuiltin(absl::StrCat(maxmind_db_type, ".lookup_error"), unknown_hit_)); + } + + void incTotal(absl::string_view maxmind_db_type) { + incCounter(stat_name_set_->getBuiltin(absl::StrCat(maxmind_db_type, ".total"), unknown_hit_)); + } + + void incHit(absl::string_view maxmind_db_type) { + incCounter(stat_name_set_->getBuiltin(absl::StrCat(maxmind_db_type, ".hit"), unknown_hit_)); + } + + void registerGeoDbStats(const std::string& db_type); + + Stats::Scope& getStatsScopeForTest() const { return *stats_scope_; } + +private: + absl::optional city_db_path_; + absl::optional isp_db_path_; + absl::optional anon_db_path_; + + absl::optional country_header_; + absl::optional city_header_; + absl::optional region_header_; + absl::optional asn_header_; + + absl::optional anon_header_; + absl::optional anon_vpn_header_; + absl::optional anon_hosting_header_; + absl::optional anon_tor_header_; + absl::optional anon_proxy_header_; + + Stats::ScopeSharedPtr stats_scope_; + Stats::StatNameSetPtr stat_name_set_; + const Stats::StatName unknown_hit_; + void incCounter(Stats::StatName name); +}; + +using GeoipProviderConfigSharedPtr = std::shared_ptr; + +using MaxmindDbPtr = std::unique_ptr; +class GeoipProvider : public Envoy::Geolocation::Driver, + public Logger::Loggable { + +public: + GeoipProvider(Singleton::InstanceSharedPtr owner, GeoipProviderConfigSharedPtr config) + : config_(config), owner_(owner) { + city_db_ = initMaxMindDb(config_->cityDbPath()); + isp_db_ = initMaxMindDb(config_->ispDbPath()); + anon_db_ = initMaxMindDb(config_->anonDbPath()); + }; + + ~GeoipProvider(); + + // Envoy::Geolocation::Driver + void lookup(Geolocation::LookupRequest&&, Geolocation::LookupGeoHeadersCallback&&) const override; + +private: + // Allow the unit test to have access to private members. + friend class GeoipProviderPeer; + GeoipProviderConfigSharedPtr config_; + MaxmindDbPtr city_db_; + MaxmindDbPtr isp_db_; + MaxmindDbPtr anon_db_; + MaxmindDbPtr initMaxMindDb(const absl::optional& db_path); + void lookupInCityDb(const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const; + void lookupInAsnDb(const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const; + void lookupInAnonDb(const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const; + template + void populateGeoLookupResult(MMDB_lookup_result_s& mmdb_lookup_result, + absl::flat_hash_map& lookup_result, + const std::string& result_key, Params... lookup_params) const; + // A shared_ptr to keep the provider singleton alive as long as any of its providers are in use. + const Singleton::InstanceSharedPtr owner_; +}; + +using GeoipProviderSharedPtr = std::shared_ptr; + +} // namespace Maxmind +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/grpc_credentials/aws_iam/config.cc b/source/extensions/grpc_credentials/aws_iam/config.cc index 0a8f0bab9782..9e3dab6454eb 100644 --- a/source/extensions/grpc_credentials/aws_iam/config.cc +++ b/source/extensions/grpc_credentials/aws_iam/config.cc @@ -44,10 +44,18 @@ std::shared_ptr AwsIamGrpcCredentialsFactory::getChann const auto& config = Envoy::MessageUtil::downcastAndValidate< const envoy::config::grpc_credential::v3::AwsIamConfig&>( *config_message, ProtobufMessage::getNullValidationVisitor()); + const auto region = getRegion(config); + // TODO(suniltheta): Due to the reasons explained in + // https://github.com/envoyproxy/envoy/issues/27586 this aws iam plugin is not able to + // utilize http async client to fetch AWS credentials. For time being this is still using + // libcurl to fetch the credentials. To fully get rid of curl, need to address the below + // usage of AWS credentials common utils. Until then we are setting nullopt for server + // factory context. auto credentials_provider = std::make_shared( - api, Common::Aws::Utility::fetchMetadata); + api, absl::nullopt /*Empty factory context*/, region, + Common::Aws::Utility::fetchMetadata); auto signer = std::make_unique( - config.service_name(), getRegion(config), credentials_provider, api.timeSource(), + config.service_name(), region, credentials_provider, api.timeSource(), // TODO: extend API to allow specifying header exclusion. ref: // https://github.com/envoyproxy/envoy/pull/18998 Common::Aws::AwsSigV4HeaderExclusionVector{}); @@ -101,7 +109,9 @@ AwsIamHeaderAuthenticator::GetMetadata(grpc::string_ref service_url, grpc::strin absl::string_view(method_name.data(), method_name.length())); TRY_NEEDS_AUDIT { signer_->sign(message, false); } - END_TRY catch (const EnvoyException& e) { return {grpc::StatusCode::INTERNAL, e.what()}; } + END_TRY catch (const EnvoyException& e) { + return grpc::Status(grpc::StatusCode::INTERNAL, e.what()); + } signedHeadersToMetadata(message.headers(), *metadata); diff --git a/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.cc b/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.cc index 75e96f55d0f0..8b3e43692f15 100644 --- a/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.cc +++ b/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.cc @@ -24,8 +24,7 @@ constexpr std::array ExpectedCacheVersionId = {'0', '0', '0', '0'}; } // namespace CacheFileFixedBlock::CacheFileFixedBlock() - : file_id_(ExpectedFileId), cache_version_id_(ExpectedCacheVersionId), header_size_(0), - trailer_size_(0), body_size_(0) {} + : file_id_(ExpectedFileId), cache_version_id_(ExpectedCacheVersionId) {} void CacheFileFixedBlock::populateFromStringView(absl::string_view s) { // The string view should be the size of the buffer, and diff --git a/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.h b/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.h index 873498bd472b..cc675dc536bb 100644 --- a/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.h +++ b/source/extensions/http/cache/file_system_http_cache/cache_file_fixed_block.h @@ -139,9 +139,9 @@ class CacheFileFixedBlock { private: std::array file_id_; std::array cache_version_id_; - uint32_t header_size_; - uint32_t trailer_size_; - uint64_t body_size_; + uint32_t header_size_{0}; + uint32_t trailer_size_{0}; + uint64_t body_size_{0}; }; } // namespace FileSystemHttpCache diff --git a/source/extensions/http/cache/file_system_http_cache/config.cc b/source/extensions/http/cache/file_system_http_cache/config.cc index 8bea632cb27e..bf08876588e7 100644 --- a/source/extensions/http/cache/file_system_http_cache/config.cc +++ b/source/extensions/http/cache/file_system_http_cache/config.cc @@ -100,12 +100,14 @@ class FileSystemHttpCacheFactory : public HttpCacheFactory { Server::Configuration::FactoryContext& context) override { ConfigProto config; MessageUtil::unpackTo(filter_config.typed_config(), config); - std::shared_ptr caches = context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(file_system_http_cache_singleton), [&context] { - return std::make_shared( - Common::AsyncFiles::AsyncFileManagerFactory::singleton(&context.singletonManager()), - context.api().threadFactory()); - }); + std::shared_ptr caches = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(file_system_http_cache_singleton), [&context] { + return std::make_shared( + Common::AsyncFiles::AsyncFileManagerFactory::singleton( + &context.serverFactoryContext().singletonManager()), + context.serverFactoryContext().api().threadFactory()); + }); return caches->get(caches, config, context.scope()); } }; diff --git a/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc b/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc index 25fef16ca0ee..687a0e40d896 100644 --- a/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc +++ b/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc @@ -373,16 +373,19 @@ void CacheShared::trackFileRemoved(uint64_t file_size) { // // See comment on size_bytes and size_count in stats.h for explanation of how stat // values can be out of sync with the actionable cache. - uint64_t count = size_count_; - while (count > 0 && !size_count_.compare_exchange_weak(count, count - 1)) { - } + uint64_t count, size; + do { + count = size_count_; + } while (count > 0 && !size_count_.compare_exchange_weak(count, count - 1)); + stats_.size_count_.set(size_count_); // Atomically decrease-but-clamp-at-zero the size of files in the cache, by file_size. // // See comment above for why; the same rationale applies here. - uint64_t size = size_bytes_; - while (size >= file_size && !size_bytes_.compare_exchange_weak(size, size - file_size)) { - } + do { + size = size_bytes_; + } while (size >= file_size && !size_bytes_.compare_exchange_weak(size, size - file_size)); + if (size < file_size) { size_bytes_ = 0; } diff --git a/source/extensions/http/cache/simple_http_cache/simple_http_cache.cc b/source/extensions/http/cache/simple_http_cache/simple_http_cache.cc index a66661724b11..59f15b8af3cc 100644 --- a/source/extensions/http/cache/simple_http_cache/simple_http_cache.cc +++ b/source/extensions/http/cache/simple_http_cache/simple_http_cache.cc @@ -303,7 +303,7 @@ class SimpleHttpCacheFactory : public HttpCacheFactory { std::shared_ptr getCache(const envoy::extensions::filters::http::cache::v3::CacheConfig&, Server::Configuration::FactoryContext& context) override { - return context.singletonManager().getTyped( + return context.serverFactoryContext().singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(simple_http_cache_singleton), &createCache); } diff --git a/source/extensions/http/custom_response/local_response_policy/BUILD b/source/extensions/http/custom_response/local_response_policy/BUILD index 573d6bc1d6b4..d14eb25781c2 100644 --- a/source/extensions/http/custom_response/local_response_policy/BUILD +++ b/source/extensions/http/custom_response/local_response_policy/BUILD @@ -37,6 +37,7 @@ envoy_cc_extension( "//source/extensions/filters/http/common:factory_base_lib", "//source/extensions/filters/http/custom_response:custom_response_filter", "//source/extensions/filters/http/custom_response:policy_interface", + "//source/server:generic_factory_context_lib", "@envoy_api//envoy/extensions/http/custom_response/local_response_policy/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc b/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc index 296837b96ef6..80c98d28b2a3 100644 --- a/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc +++ b/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc @@ -11,6 +11,7 @@ #include "source/common/http/utility.h" #include "source/common/router/header_parser.h" #include "source/extensions/filters/http/custom_response/custom_response_filter.h" +#include "source/server/generic_factory_context.h" namespace Envoy { namespace Extensions { @@ -19,19 +20,25 @@ namespace CustomResponse { LocalResponsePolicy::LocalResponsePolicy( const envoy::extensions::http::custom_response::local_response_policy::v3::LocalResponsePolicy& config, - Server::Configuration::CommonFactoryContext& context) + Server::Configuration::ServerFactoryContext& context) : local_body_{config.has_body() ? absl::optional(Config::DataSource::read( config.body(), true, context.api())) : absl::optional{}}, - formatter_(config.has_body_format() - ? Formatter::SubstitutionFormatStringUtils::fromProtoConfig( - config.body_format(), context) - : nullptr), status_code_{config.has_status_code() ? absl::optional( static_cast(config.status_code().value())) : absl::optional{}}, - header_parser_(Envoy::Router::HeaderParser::configure(config.response_headers_to_add())) {} + header_parser_(Envoy::Router::HeaderParser::configure(config.response_headers_to_add())) { + + // TODO(wbpcode): these is a potential bug of message validation. The validation visitor + // of server context should not be used here directly. But this is bug is not introduced + // by this PR and will be fixed in the future. + Server::GenericFactoryContextImpl generic_context(context, context.messageValidationVisitor()); + if (config.has_body_format()) { + formatter_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config.body_format(), + generic_context); + } +} // TODO(pradeepcrao): investigate if this code can be made common with // Envoy::LocalReply::BodyFormatter for consistent behavior. @@ -44,8 +51,8 @@ void LocalResponsePolicy::formatBody(const Envoy::Http::RequestHeaderMap& reques } if (formatter_) { - formatter_->formatWithContext({&request_headers, &response_headers, nullptr, body}, - stream_info); + body = formatter_->formatWithContext({&request_headers, &response_headers, nullptr, body}, + stream_info); } } diff --git a/source/extensions/http/custom_response/local_response_policy/local_response_policy.h b/source/extensions/http/custom_response/local_response_policy/local_response_policy.h index 4230cb42dafc..fa209fd3cf3a 100644 --- a/source/extensions/http/custom_response/local_response_policy/local_response_policy.h +++ b/source/extensions/http/custom_response/local_response_policy/local_response_policy.h @@ -23,7 +23,7 @@ class LocalResponsePolicy : public Extensions::HttpFilters::CustomResponse::Poli public: LocalResponsePolicy(const envoy::extensions::http::custom_response::local_response_policy::v3:: LocalResponsePolicy& config, - Server::Configuration::CommonFactoryContext& context); + Server::Configuration::ServerFactoryContext& context); Envoy::Http::FilterHeadersStatus encodeHeaders(Envoy::Http::ResponseHeaderMap&, bool, @@ -42,7 +42,7 @@ class LocalResponsePolicy : public Extensions::HttpFilters::CustomResponse::Poli const absl::optional local_body_; // body format - const Formatter::FormatterPtr formatter_; + Formatter::FormatterPtr formatter_; const absl::optional status_code_; const std::unique_ptr header_parser_; diff --git a/source/extensions/http/early_header_mutation/header_mutation/header_mutation.cc b/source/extensions/http/early_header_mutation/header_mutation/header_mutation.cc index 082dfbc51a2e..c9c22eacf823 100644 --- a/source/extensions/http/early_header_mutation/header_mutation/header_mutation.cc +++ b/source/extensions/http/early_header_mutation/header_mutation/header_mutation.cc @@ -15,8 +15,7 @@ HeaderMutation::HeaderMutation(const ProtoHeaderMutation& mutations) bool HeaderMutation::mutate(Envoy::Http::RequestHeaderMap& headers, const StreamInfo::StreamInfo& stream_info) const { - mutations_.evaluateHeaders(headers, headers, - *Envoy::Http::StaticEmptyHeaders::get().response_headers, stream_info); + mutations_.evaluateHeaders(headers, {&headers}, stream_info); return true; } diff --git a/source/extensions/http/header_validators/envoy_default/BUILD b/source/extensions/http/header_validators/envoy_default/BUILD index aa7ce06ead00..5e1a942288ce 100644 --- a/source/extensions/http/header_validators/envoy_default/BUILD +++ b/source/extensions/http/header_validators/envoy_default/BUILD @@ -157,6 +157,12 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + visibility = [ + "//source/exe:__subpackages__", + "//source/extensions/filters/network/http_connection_manager:__subpackages__", + "//source/server/admin:__subpackages__", + "//test:__subpackages__", + ], deps = [ ":header_validator_factory", "//envoy/http:header_validator_factory_interface", diff --git a/source/extensions/http/header_validators/envoy_default/header_validator.cc b/source/extensions/http/header_validators/envoy_default/header_validator.cc index e492dd1ee88a..f0104cd9e0dc 100644 --- a/source/extensions/http/header_validators/envoy_default/header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/header_validator.cc @@ -473,7 +473,7 @@ void HeaderValidator::encodeAdditionalCharactersInPath( 0b11111111111111111111111111111111, }; - absl::string_view path = header_map.path(); + absl::string_view path = header_map.getPathValue(); // Check if URL path contains any characters in the kCharactersToEncode set auto char_to_encode = path.begin(); for (; char_to_encode != path.end() && !testCharInTable(kCharactersToEncode, *char_to_encode); diff --git a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc index cf29b3e612b4..fbcf08a4624f 100644 --- a/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http1_header_validator.cc @@ -216,7 +216,7 @@ ValidationResult Http1HeaderValidator::validateRequestHeaders(const RequestHeade // parse the :path form and compare the authority component of the path against the :authority // header. auto is_connect_method = ::Envoy::Http::HeaderUtility::isConnect(header_map); - auto is_options_method = header_map.method() == header_values_.MethodValues.Options; + auto is_options_method = header_map.getMethodValue() == header_values_.MethodValues.Options; if (!is_connect_method && path.empty()) { // The :path is required for non-CONNECT requests. diff --git a/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc b/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc index e67ad1f4e0e4..b552159046ff 100644 --- a/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc +++ b/source/extensions/http/header_validators/envoy_default/http2_header_validator.cc @@ -232,7 +232,7 @@ Http2HeaderValidator::validateRequestHeaders(const ::Envoy::Http::RequestHeaderM // HTTP/2 requests. const bool is_standard_connect_request = HeaderUtility::isStandardConnectRequest(header_map); const bool is_extended_connect_request = HeaderUtility::isExtendedH2ConnectRequest(header_map); - auto is_options_request = header_map.method() == header_values_.MethodValues.Options; + auto is_options_request = header_map.getMethodValue() == header_values_.MethodValues.Options; bool path_is_empty = path.empty(); bool path_is_asterisk = path == "*"; bool path_is_absolute = !path_is_empty && path.at(0) == '/'; diff --git a/source/extensions/http/header_validators/envoy_default/path_normalizer.cc b/source/extensions/http/header_validators/envoy_default/path_normalizer.cc index 7674d3808d62..9e8713c3aa8b 100644 --- a/source/extensions/http/header_validators/envoy_default/path_normalizer.cc +++ b/source/extensions/http/header_validators/envoy_default/path_normalizer.cc @@ -177,9 +177,9 @@ PathNormalizer::normalizePathUri(RequestHeaderMap& header_map) const { // asterisk-form = "*" // // TODO(#23887) - potentially separate path normalization into multiple independent operations. - const auto original_path = header_map.path(); + const auto original_path = header_map.getPathValue(); if (original_path == "*" && - header_map.method() == ::Envoy::Http::Headers::get().MethodValues.Options) { + header_map.getMethodValue() == ::Envoy::Http::Headers::get().MethodValues.Options) { // asterisk-form, only valid for OPTIONS request return PathNormalizationResult::success(); } diff --git a/source/extensions/http/stateful_session/cookie/config.cc b/source/extensions/http/stateful_session/cookie/config.cc index e978b2d8a603..77a55cbeb247 100644 --- a/source/extensions/http/stateful_session/cookie/config.cc +++ b/source/extensions/http/stateful_session/cookie/config.cc @@ -10,12 +10,12 @@ namespace Cookie { Envoy::Http::SessionStateFactorySharedPtr CookieBasedSessionStateFactoryConfig::createSessionStateFactory( - const Protobuf::Message& config, Server::Configuration::CommonFactoryContext& context) { + const Protobuf::Message& config, Server::Configuration::GenericFactoryContext& context) { const auto& proto_config = MessageUtil::downcastAndValidate( config, context.messageValidationVisitor()); return std::make_shared( - proto_config, context.mainThreadDispatcher().timeSource()); + proto_config, context.serverFactoryContext().mainThreadDispatcher().timeSource()); } REGISTER_FACTORY(CookieBasedSessionStateFactoryConfig, Envoy::Http::SessionStateFactoryConfig); diff --git a/source/extensions/http/stateful_session/cookie/config.h b/source/extensions/http/stateful_session/cookie/config.h index 81c2d0b9abed..f4b793224c2d 100644 --- a/source/extensions/http/stateful_session/cookie/config.h +++ b/source/extensions/http/stateful_session/cookie/config.h @@ -14,7 +14,7 @@ class CookieBasedSessionStateFactoryConfig : public Envoy::Http::SessionStateFac public: Envoy::Http::SessionStateFactorySharedPtr createSessionStateFactory(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::GenericFactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique< diff --git a/source/extensions/http/stateful_session/header/config.cc b/source/extensions/http/stateful_session/header/config.cc index 7dee91aeefee..8e49d6be62bf 100644 --- a/source/extensions/http/stateful_session/header/config.cc +++ b/source/extensions/http/stateful_session/header/config.cc @@ -10,7 +10,7 @@ namespace Header { Envoy::Http::SessionStateFactorySharedPtr HeaderBasedSessionStateFactoryConfig::createSessionStateFactory( - const Protobuf::Message& config, Server::Configuration::CommonFactoryContext& context) { + const Protobuf::Message& config, Server::Configuration::GenericFactoryContext& context) { const auto& proto_config = MessageUtil::downcastAndValidate( config, context.messageValidationVisitor()); diff --git a/source/extensions/http/stateful_session/header/config.h b/source/extensions/http/stateful_session/header/config.h index be5f31d636ea..e995d685d9ec 100644 --- a/source/extensions/http/stateful_session/header/config.h +++ b/source/extensions/http/stateful_session/header/config.h @@ -14,7 +14,7 @@ class HeaderBasedSessionStateFactoryConfig : public Envoy::Http::SessionStateFac public: Envoy::Http::SessionStateFactorySharedPtr createSessionStateFactory(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) override; + Server::Configuration::GenericFactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique< diff --git a/source/extensions/io_socket/user_space/io_handle_impl.cc b/source/extensions/io_socket/user_space/io_handle_impl.cc index 8795512de3ff..3081c5b7152c 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.cc +++ b/source/extensions/io_socket/user_space/io_handle_impl.cc @@ -93,15 +93,13 @@ bool IoHandleImpl::isOpen() const { return !closed_; } Api::IoCallUint64Result IoHandleImpl::readv(uint64_t max_length, Buffer::RawSlice* slices, uint64_t num_slice) { if (!isOpen()) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_BADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::create(SOCKET_ERROR_BADF)}; } if (pending_received_data_.length() == 0) { if (receive_data_end_stream_) { - return {0, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {0, Api::IoError::none()}; } else { - return {0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEagainError()}; } } // The read bytes can not exceed the provided buffer size or pending received size. @@ -118,7 +116,7 @@ Api::IoCallUint64Result IoHandleImpl::readv(uint64_t max_length, Buffer::RawSlic const auto bytes_read = bytes_offset; ASSERT(bytes_read <= max_bytes_to_read); ENVOY_LOG(trace, "socket {} readv {} bytes", static_cast(this), bytes_read); - return {bytes_read, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {bytes_read, Api::IoError::none()}; } Api::IoCallUint64Result IoHandleImpl::read(Buffer::Instance& buffer, @@ -129,19 +127,17 @@ Api::IoCallUint64Result IoHandleImpl::read(Buffer::Instance& buffer, return Api::ioCallUint64ResultNoError(); } if (!isOpen()) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_BADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::create(SOCKET_ERROR_BADF)}; } if (pending_received_data_.length() == 0) { if (receive_data_end_stream_) { - return {0, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {0, Api::IoError::none()}; } else { - return {0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEagainError()}; } } const uint64_t bytes_to_read = moveUpTo(buffer, pending_received_data_, max_length); - return {bytes_to_read, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {bytes_to_read, Api::IoError::none()}; } Api::IoCallUint64Result IoHandleImpl::writev(const Buffer::RawSlice* slices, uint64_t num_slice) { @@ -157,24 +153,20 @@ Api::IoCallUint64Result IoHandleImpl::writev(const Buffer::RawSlice* slices, uin return Api::ioCallUint64ResultNoError(); } if (!isOpen()) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_BADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } // Closed peer. if (!peer_handle_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_INVAL), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::create(SOCKET_ERROR_INVAL)}; } // Error: write after close. if (peer_handle_->isPeerShutDownWrite()) { // TODO(lambdai): `EPIPE` or `ENOTCONN`. - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_INVAL), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::create(SOCKET_ERROR_INVAL)}; } // The peer is valid but temporarily does not accept new data. Likely due to flow control. if (!peer_handle_->isWritable()) { - return {0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEagainError()}; } auto* const dest_buffer = peer_handle_->getWriteBuffer(); @@ -188,7 +180,7 @@ Api::IoCallUint64Result IoHandleImpl::writev(const Buffer::RawSlice* slices, uin } peer_handle_->setNewDataAvailable(); ENVOY_LOG(trace, "socket {} writev {} bytes", static_cast(this), bytes_written); - return {bytes_written, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {bytes_written, Api::IoError::none()}; } Api::IoCallUint64Result IoHandleImpl::write(Buffer::Instance& buffer) { @@ -197,24 +189,20 @@ Api::IoCallUint64Result IoHandleImpl::write(Buffer::Instance& buffer) { return Api::ioCallUint64ResultNoError(); } if (!isOpen()) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_BADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } // Closed peer. if (!peer_handle_) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_INVAL), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::create(SOCKET_ERROR_INVAL)}; } // Error: write after close. if (peer_handle_->isPeerShutDownWrite()) { // TODO(lambdai): `EPIPE` or `ENOTCONN`. - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_INVAL), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::create(SOCKET_ERROR_INVAL)}; } // The peer is valid but temporarily does not accept new data. Likely due to flow control. if (!peer_handle_->isWritable()) { - return {0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEagainError()}; } const uint64_t max_bytes_to_write = buffer.length(); const uint64_t total_bytes_to_write = @@ -224,7 +212,7 @@ Api::IoCallUint64Result IoHandleImpl::write(Buffer::Instance& buffer) { peer_handle_->setNewDataAvailable(); ENVOY_LOG(trace, "socket {} write {} bytes of {}", static_cast(this), total_bytes_to_write, max_bytes_to_write); - return {total_bytes_to_write, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {total_bytes_to_write, Api::IoError::none()}; } Api::IoCallUint64Result IoHandleImpl::sendmsg(const Buffer::RawSlice*, uint64_t, int, @@ -244,16 +232,14 @@ Api::IoCallUint64Result IoHandleImpl::recvmmsg(RawSliceArrays&, uint32_t, RecvMs Api::IoCallUint64Result IoHandleImpl::recv(void* buffer, size_t length, int flags) { if (!isOpen()) { - return {0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_BADF), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEbadfError()}; } // No data and the writer closed. if (pending_received_data_.length() == 0) { if (receive_data_end_stream_) { - return {0, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {0, Api::IoError::none()}; } else { - return {0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)}; + return {0, Network::IoSocketError::getIoSocketEagainError()}; } } // Specify uint64_t since the latter length may not have the same type. @@ -262,7 +248,7 @@ Api::IoCallUint64Result IoHandleImpl::recv(void* buffer, size_t length, int flag if (!(flags & MSG_PEEK)) { pending_received_data_.drain(max_bytes_to_read); } - return {max_bytes_to_read, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}; + return {max_bytes_to_read, Api::IoError::none()}; } bool IoHandleImpl::supportsMmsg() const { return false; } diff --git a/source/extensions/key_value/file_based/config.cc b/source/extensions/key_value/file_based/config.cc index 442fd3670ab7..195f07f6c0ed 100644 --- a/source/extensions/key_value/file_based/config.cc +++ b/source/extensions/key_value/file_based/config.cc @@ -16,7 +16,9 @@ FileBasedKeyValueStore::FileBasedKeyValueStore(Event::Dispatcher& dispatcher, ENVOY_LOG(info, "File for key value store does not yet exist: {}", filename); return; } - const std::string contents = file_system_.fileReadToEnd(filename_); + auto file_or_error = file_system_.fileReadToEnd(filename_); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + const std::string contents = file_or_error.value(); if (!parseContents(contents)) { ENVOY_LOG(warn, "Failed to parse key value store file {}", filename); } diff --git a/source/extensions/listener_managers/validation_listener_manager/BUILD b/source/extensions/listener_managers/validation_listener_manager/BUILD index df3445709f7e..57dfe50ba83e 100644 --- a/source/extensions/listener_managers/validation_listener_manager/BUILD +++ b/source/extensions/listener_managers/validation_listener_manager/BUILD @@ -23,6 +23,6 @@ envoy_cc_extension( "//test:__subpackages__", ], deps = [ - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//source/common/listener_manager:listener_manager_lib", ], ) diff --git a/source/extensions/listener_managers/validation_listener_manager/validation_listener_manager.h b/source/extensions/listener_managers/validation_listener_manager/validation_listener_manager.h index c35c38c21731..c7a4d7e028fa 100644 --- a/source/extensions/listener_managers/validation_listener_manager/validation_listener_manager.h +++ b/source/extensions/listener_managers/validation_listener_manager/validation_listener_manager.h @@ -1,6 +1,6 @@ #pragma once -#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" +#include "source/common/listener_manager/listener_manager_impl.h" #include "source/server/listener_manager_factory.h" namespace Envoy { diff --git a/source/extensions/load_balancing_policies/cluster_provided/BUILD b/source/extensions/load_balancing_policies/cluster_provided/BUILD index 2cd5fcce2124..ca2f28faf353 100644 --- a/source/extensions/load_balancing_policies/cluster_provided/BUILD +++ b/source/extensions/load_balancing_policies/cluster_provided/BUILD @@ -12,6 +12,8 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + # previously considered core code and used by mobile. + visibility = ["//visibility:public"], deps = [ "//source/common/upstream:load_balancer_factory_base_lib", "@envoy_api//envoy/extensions/load_balancing_policies/cluster_provided/v3:pkg_cc_proto", diff --git a/source/extensions/load_balancing_policies/cluster_provided/config.h b/source/extensions/load_balancing_policies/cluster_provided/config.h index df59f760498f..3bad53202292 100644 --- a/source/extensions/load_balancing_policies/cluster_provided/config.h +++ b/source/extensions/load_balancing_policies/cluster_provided/config.h @@ -12,6 +12,11 @@ namespace Extensions { namespace LoadBalancingPolices { namespace ClusterProvided { +class ClusterProvidedLbConfig : public Upstream::LoadBalancerConfig { +public: + ClusterProvidedLbConfig() = default; +}; + class Factory : public Upstream::TypedLoadBalancerFactoryBase< envoy::extensions::load_balancing_policies::cluster_provided::v3::ClusterProvided> { @@ -24,8 +29,15 @@ class Factory Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) override; + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message&, + ProtobufMessage::ValidationVisitor&) override { + return std::make_unique(); + } }; +DECLARE_FACTORY(Factory); + } // namespace ClusterProvided } // namespace LoadBalancingPolices } // namespace Extensions diff --git a/source/extensions/load_balancing_policies/common/factory_base.h b/source/extensions/load_balancing_policies/common/factory_base.h index c200d600e506..5d6ed144a796 100644 --- a/source/extensions/load_balancing_policies/common/factory_base.h +++ b/source/extensions/load_balancing_policies/common/factory_base.h @@ -23,34 +23,30 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactoryBase { Envoy::Random::RandomGenerator& random, TimeSource& time_source) override { - const auto* typed_lb_config = - dynamic_cast(lb_config.ptr()); - auto typed_proto_config = typed_lb_config == nullptr - ? OptRef{} - : typed_lb_config->typedProtoConfig(); - return std::make_unique(std::make_shared( - typed_proto_config, cluster_info, priority_set, runtime, random, time_source)); + lb_config, cluster_info, priority_set, runtime, random, time_source)); } private: class LbFactory : public Upstream::LoadBalancerFactory { public: - LbFactory(OptRef proto_config, const Upstream::ClusterInfo& cluster_info, - const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime, - Envoy::Random::RandomGenerator& random, TimeSource& time_source) - : proto_config_(proto_config), cluster_info_(cluster_info), priority_set_(priority_set), + LbFactory(OptRef lb_config, + const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random, + TimeSource& time_source) + : lb_config_(lb_config), cluster_info_(cluster_info), priority_set_(priority_set), runtime_(runtime), random_(random), time_source_(time_source) {} Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override { - return Impl()(params, proto_config_, cluster_info_, priority_set_, runtime_, random_, + return Impl()(params, lb_config_, cluster_info_, priority_set_, runtime_, random_, time_source_); } bool recreateOnHostChange() const override { return false; } public: - OptRef proto_config_; + OptRef lb_config_; + const Upstream::ClusterInfo& cluster_info_; const Upstream::PrioritySet& priority_set_; Runtime::Loader& runtime_; @@ -72,6 +68,45 @@ class FactoryBase : public Upstream::TypedLoadBalancerFactoryBase { const std::string name_; }; +/** + * Helper class to hold either a legacy or production config. + */ +template class ActiveOrLegacy { +public: + template static ActiveOrLegacy get(const BaseType* base) { + auto* active_type = dynamic_cast(base); + if (active_type != nullptr) { + return {active_type}; + } + auto* legacy_type = dynamic_cast(base); + if (legacy_type != nullptr) { + return {legacy_type}; + } + + return {}; + } + + bool hasActive() const { return active_ != nullptr; } + bool hasLegacy() const { return legacy_ != nullptr; } + + const ActiveType* active() const { + ASSERT(hasActive()); + return active_; + } + const LegacyType* legacy() const { + ASSERT(hasLegacy()); + return legacy_; + } + +private: + ActiveOrLegacy() = default; + ActiveOrLegacy(const ActiveType* active) : active_(active) {} + ActiveOrLegacy(const LegacyType* legacy) : legacy_(legacy) {} + + const ActiveType* active_{}; + const LegacyType* legacy_{}; +}; + } // namespace Common } // namespace LoadBalancingPolices } // namespace Extensions diff --git a/source/extensions/load_balancing_policies/least_request/BUILD b/source/extensions/load_balancing_policies/least_request/BUILD index 5f1d06067544..e813160593b1 100644 --- a/source/extensions/load_balancing_policies/least_request/BUILD +++ b/source/extensions/load_balancing_policies/least_request/BUILD @@ -12,6 +12,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], deps = [ "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_lib", diff --git a/source/extensions/load_balancing_policies/least_request/config.cc b/source/extensions/load_balancing_policies/least_request/config.cc index 224aae7e2f6d..dc4aeb51d518 100644 --- a/source/extensions/load_balancing_policies/least_request/config.cc +++ b/source/extensions/load_balancing_policies/least_request/config.cc @@ -9,21 +9,40 @@ namespace Extensions { namespace LoadBalancingPolices { namespace LeastRequest { +LegacyLeastRequestLbConfig::LegacyLeastRequestLbConfig(const ClusterProto& cluster) { + if (cluster.has_least_request_lb_config()) { + lb_config_ = cluster.least_request_lb_config(); + } +} + +TypedLeastRequestLbConfig::TypedLeastRequestLbConfig(const LeastRequestLbProto& lb_config) + : lb_config_(lb_config) {} + Upstream::LoadBalancerPtr LeastRequestCreator::operator()( - Upstream::LoadBalancerParams params, OptRef lb_config, + Upstream::LoadBalancerParams params, OptRef lb_config, const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) { + auto active_or_legacy = + Common::ActiveOrLegacy::get( + lb_config.ptr()); + // The load balancing policy configuration will be loaded and validated in the main thread when we // load the cluster configuration. So we can assume the configuration is valid here. - ASSERT(lb_config.has_value(), + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive(), "Invalid load balancing policy configuration for least request load balancer"); - return std::make_unique( - params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, - PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), - healthy_panic_threshold, 100, 50), - lb_config.value(), time_source); + if (active_or_legacy.hasActive()) { + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), + healthy_panic_threshold, 100, 50), + active_or_legacy.active()->lb_config_, time_source); + } else { + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + cluster_info.lbConfig(), active_or_legacy.legacy()->lbConfig(), time_source); + } } /** diff --git a/source/extensions/load_balancing_policies/least_request/config.h b/source/extensions/load_balancing_policies/least_request/config.h index ffd9456d2d87..cdd2e9e93353 100644 --- a/source/extensions/load_balancing_policies/least_request/config.h +++ b/source/extensions/load_balancing_policies/least_request/config.h @@ -5,6 +5,7 @@ #include "envoy/upstream/load_balancer.h" #include "source/common/common/logger.h" +#include "source/common/upstream/load_balancer_impl.h" #include "source/extensions/load_balancing_policies/common/factory_base.h" namespace Envoy { @@ -14,19 +15,66 @@ namespace LeastRequest { using LeastRequestLbProto = envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest; +using ClusterProto = envoy::config::cluster::v3::Cluster; +using LegacyLeastRequestLbProto = ClusterProto::LeastRequestLbConfig; + +/** + * Load balancer config that used to wrap the legacy least request config. + */ +class LegacyLeastRequestLbConfig : public Upstream::LoadBalancerConfig { +public: + LegacyLeastRequestLbConfig(const ClusterProto& cluster); + + OptRef lbConfig() const { + if (lb_config_.has_value()) { + return lb_config_.value(); + } + return {}; + }; + +private: + absl::optional lb_config_; +}; + +/** + * Load balancer config that used to wrap the least request config. + */ +class TypedLeastRequestLbConfig : public Upstream::LoadBalancerConfig { +public: + TypedLeastRequestLbConfig(const LeastRequestLbProto& lb_config); + + const LeastRequestLbProto lb_config_; +}; struct LeastRequestCreator : public Logger::Loggable { - Upstream::LoadBalancerPtr - operator()(Upstream::LoadBalancerParams params, OptRef lb_config, - const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, - Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source); + Upstream::LoadBalancerPtr operator()(Upstream::LoadBalancerParams params, + OptRef lb_config, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source); }; class Factory : public Common::FactoryBase { public: Factory() : FactoryBase("envoy.load_balancing_policies.least_request") {} + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor&) override { + + auto active_or_legacy = Common::ActiveOrLegacy::get(&config); + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive()); + + return active_or_legacy.hasLegacy() + ? Upstream::LoadBalancerConfigPtr{new LegacyLeastRequestLbConfig( + *active_or_legacy.legacy())} + : Upstream::LoadBalancerConfigPtr{ + new TypedLeastRequestLbConfig(*active_or_legacy.active())}; + } }; +DECLARE_FACTORY(Factory); + } // namespace LeastRequest } // namespace LoadBalancingPolices } // namespace Extensions diff --git a/source/extensions/load_balancing_policies/maglev/BUILD b/source/extensions/load_balancing_policies/maglev/BUILD index 02852a17f644..537b9c1a7765 100644 --- a/source/extensions/load_balancing_policies/maglev/BUILD +++ b/source/extensions/load_balancing_policies/maglev/BUILD @@ -14,9 +14,28 @@ envoy_cc_library( srcs = ["maglev_lb.cc"], hdrs = ["maglev_lb.h"], deps = [ + "//envoy/upstream:load_balancer_interface", "//source/common/common:bit_array_lib", + "//source/common/runtime:runtime_features_lib", + "//source/common/upstream:thread_aware_lb_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/maglev/v3:pkg_cc_proto", + ], +) + +# This library target is for testing especially x86 coverage +# which will use the compact representation unless we force the +# original implementation. +envoy_cc_library( + name = "maglev_lb_force_original_impl_lib", + srcs = ["maglev_lb.cc"], + hdrs = ["maglev_lb.h"], + copts = ["-DMAGLEV_LB_FORCE_ORIGINAL_IMPL"], + deps = [ + "//envoy/upstream:load_balancer_interface", + "//source/common/common:bit_array_lib", + "//source/common/runtime:runtime_features_lib", "//source/common/upstream:thread_aware_lb_lib", - "//source/common/upstream:upstream_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/maglev/v3:pkg_cc_proto", ], @@ -35,6 +54,7 @@ envoy_cc_extension( "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_factory_base_lib", "//source/common/upstream:load_balancer_lib", + "//source/extensions/load_balancing_policies/common:factory_base", "@envoy_api//envoy/extensions/load_balancing_policies/maglev/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/load_balancing_policies/maglev/config.cc b/source/extensions/load_balancing_policies/maglev/config.cc index 76db3e002227..a017e1391e43 100644 --- a/source/extensions/load_balancing_policies/maglev/config.cc +++ b/source/extensions/load_balancing_policies/maglev/config.cc @@ -15,24 +15,24 @@ Factory::create(OptRef lb_config, const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource&) { - const auto* typed_lb_config = - dynamic_cast(lb_config.ptr()); - auto typed_proto_config = typed_lb_config == nullptr - ? OptRef{} - : typed_lb_config->typedProtoConfig(); + auto active_or_legacy = + Common::ActiveOrLegacy::get( + lb_config.ptr()); // Assume legacy config. - if (!typed_proto_config.has_value()) { + if (!active_or_legacy.hasActive()) { return std::make_unique( priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, - cluster_info.lbMaglevConfig(), cluster_info.lbConfig()); + !active_or_legacy.hasLegacy() ? cluster_info.lbMaglevConfig() + : active_or_legacy.legacy()->lbConfig(), + cluster_info.lbConfig()); } return std::make_unique( priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, static_cast(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( cluster_info.lbConfig(), healthy_panic_threshold, 100, 50)), - typed_proto_config.value()); + active_or_legacy.active()->lb_config_); } /** diff --git a/source/extensions/load_balancing_policies/maglev/config.h b/source/extensions/load_balancing_policies/maglev/config.h index c4eb711f2c4d..294e370c850b 100644 --- a/source/extensions/load_balancing_policies/maglev/config.h +++ b/source/extensions/load_balancing_policies/maglev/config.h @@ -7,6 +7,8 @@ #include "envoy/upstream/load_balancer.h" #include "source/common/upstream/load_balancer_factory_base.h" +#include "source/extensions/load_balancing_policies/common/factory_base.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" namespace Envoy { namespace Extensions { @@ -25,6 +27,20 @@ class Factory : public Upstream::TypedLoadBalancerFactoryBase { Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) override; + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor&) override { + auto active_or_legacy = + Common::ActiveOrLegacy::get(&config); + + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive()); + + return active_or_legacy.hasLegacy() + ? Upstream::LoadBalancerConfigPtr{new Upstream::LegacyMaglevLbConfig( + *active_or_legacy.legacy())} + : Upstream::LoadBalancerConfigPtr{ + new Upstream::TypedMaglevLbConfig(*active_or_legacy.active())}; + } }; } // namespace Maglev diff --git a/source/extensions/load_balancing_policies/maglev/maglev_lb.cc b/source/extensions/load_balancing_policies/maglev/maglev_lb.cc index 170da42e33fe..e4e98b890e5e 100644 --- a/source/extensions/load_balancing_policies/maglev/maglev_lb.cc +++ b/source/extensions/load_balancing_policies/maglev/maglev_lb.cc @@ -2,6 +2,8 @@ #include "envoy/config/cluster/v3/cluster.pb.h" +#include "source/common/runtime/runtime_features.h" + namespace Envoy { namespace Upstream { namespace { @@ -11,6 +13,10 @@ bool shouldUseCompactTable(size_t num_hosts, uint64_t table_size) { return false; } +#ifdef MAGLEV_LB_FORCE_ORIGINAL_IMPL + return false; +#endif + if (num_hosts > MaglevTable::MaxNumberOfHostsForCompactMaglev) { return false; } @@ -35,8 +41,7 @@ class MaglevFactory : private Logger::Loggable { bool use_hostname_for_hashing, MaglevLoadBalancerStats& stats) { MaglevTableSharedPtr maglev_table; - if (shouldUseCompactTable(normalized_host_weights.size(), table_size) && - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.allow_compact_maglev")) { + if (shouldUseCompactTable(normalized_host_weights.size(), table_size)) { maglev_table = std::make_shared(normalized_host_weights, max_normalized_weight, table_size, use_hostname_for_hashing, stats); @@ -56,6 +61,14 @@ class MaglevFactory : private Logger::Loggable { } // namespace +LegacyMaglevLbConfig::LegacyMaglevLbConfig(const ClusterProto& cluster) { + if (cluster.has_maglev_lb_config()) { + lb_config_ = cluster.maglev_lb_config(); + } +} + +TypedMaglevLbConfig::TypedMaglevLbConfig(const MaglevLbProto& lb_config) : lb_config_(lb_config) {} + ThreadAwareLoadBalancerBase::HashingLoadBalancerSharedPtr MaglevLoadBalancer::createLoadBalancer(const NormalizedHostWeightVector& normalized_host_weights, double /* min_normalized_weight */, diff --git a/source/extensions/load_balancing_policies/maglev/maglev_lb.h b/source/extensions/load_balancing_policies/maglev/maglev_lb.h index 01d98c421673..6bf4368dec44 100644 --- a/source/extensions/load_balancing_policies/maglev/maglev_lb.h +++ b/source/extensions/load_balancing_policies/maglev/maglev_lb.h @@ -7,14 +7,46 @@ #include "envoy/extensions/load_balancing_policies/maglev/v3/maglev.pb.validate.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" +#include "envoy/upstream/load_balancer.h" #include "source/common/common/bit_array.h" #include "source/common/upstream/thread_aware_lb_impl.h" -#include "source/common/upstream/upstream_impl.h" namespace Envoy { namespace Upstream { +using MaglevLbProto = envoy::extensions::load_balancing_policies::maglev::v3::Maglev; +using ClusterProto = envoy::config::cluster::v3::Cluster; +using LegacyMaglevLbProto = ClusterProto::MaglevLbConfig; + +/** + * Load balancer config that used to wrap legacy maglev config. + */ +class LegacyMaglevLbConfig : public Upstream::LoadBalancerConfig { +public: + LegacyMaglevLbConfig(const ClusterProto& cluster); + + OptRef lbConfig() const { + if (lb_config_.has_value()) { + return lb_config_.value(); + } + return {}; + }; + +private: + absl::optional lb_config_; +}; + +/** + * Load balancer config that used to wrap typed maglev config. + */ +class TypedMaglevLbConfig : public Upstream::LoadBalancerConfig { +public: + TypedMaglevLbConfig(const MaglevLbProto& config); + + const MaglevLbProto lb_config_; +}; + /** * All Maglev load balancer stats. @see stats_macros.h */ diff --git a/source/extensions/load_balancing_policies/random/BUILD b/source/extensions/load_balancing_policies/random/BUILD index a803dbb22f40..d73731664f21 100644 --- a/source/extensions/load_balancing_policies/random/BUILD +++ b/source/extensions/load_balancing_policies/random/BUILD @@ -12,6 +12,10 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + extra_visibility = [ + # previously considered core code. + "//test:__subpackages__", + ], deps = [ "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_lib", diff --git a/source/extensions/load_balancing_policies/random/config.cc b/source/extensions/load_balancing_policies/random/config.cc index 0bad06c2bea7..cd045886785f 100644 --- a/source/extensions/load_balancing_policies/random/config.cc +++ b/source/extensions/load_balancing_policies/random/config.cc @@ -9,21 +9,26 @@ namespace Extensions { namespace LoadBalancingPolices { namespace Random { +TypedRandomLbConfig::TypedRandomLbConfig(const RandomLbProto& lb_config) : lb_config_(lb_config) {} + Upstream::LoadBalancerPtr RandomCreator::operator()( - Upstream::LoadBalancerParams params, OptRef lb_config, + Upstream::LoadBalancerParams params, OptRef lb_config, const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&, Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random, TimeSource&) { - // The load balancing policy configuration will be loaded and validated in the main thread when we - // load the cluster configuration. So we can assume the configuration is valid here. - ASSERT(lb_config.has_value(), - "Invalid load balancing policy configuration for random load balancer"); - - return std::make_unique( - params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, - PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), - healthy_panic_threshold, 100, 50), - lb_config.value()); + const auto typed_lb_config = dynamic_cast(lb_config.ptr()); + + if (typed_lb_config != nullptr) { + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), + healthy_panic_threshold, 100, 50), + typed_lb_config->lb_config_); + } else { + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + cluster_info.lbConfig()); + } } /** diff --git a/source/extensions/load_balancing_policies/random/config.h b/source/extensions/load_balancing_policies/random/config.h index 0d637262dfc3..3af1348d0f9d 100644 --- a/source/extensions/load_balancing_policies/random/config.h +++ b/source/extensions/load_balancing_policies/random/config.h @@ -5,6 +5,7 @@ #include "envoy/upstream/load_balancer.h" #include "source/common/common/logger.h" +#include "source/common/upstream/load_balancer_impl.h" #include "source/extensions/load_balancing_policies/common/factory_base.h" namespace Envoy { @@ -14,19 +15,47 @@ namespace Random { using RandomLbProto = envoy::extensions::load_balancing_policies::random::v3::Random; +/** + * Empty load balancer config that used to represent the config for the random load balancer. + */ +class EmptyRandomLbConfig : public Upstream::LoadBalancerConfig { +public: + EmptyRandomLbConfig() = default; +}; + +/** + * Load balancer config that used to wrap the random config. + */ +class TypedRandomLbConfig : public Upstream::LoadBalancerConfig { +public: + TypedRandomLbConfig(const RandomLbProto& lb_config); + + const RandomLbProto lb_config_; +}; + struct RandomCreator : public Logger::Loggable { - Upstream::LoadBalancerPtr - operator()(Upstream::LoadBalancerParams params, OptRef lb_config, - const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, - Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random, - TimeSource& time_source); + Upstream::LoadBalancerPtr operator()( + Upstream::LoadBalancerParams params, OptRef lb_config, + const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Envoy::Random::RandomGenerator& random, TimeSource& time_source); }; class Factory : public Common::FactoryBase { public: Factory() : FactoryBase("envoy.load_balancing_policies.random") {} + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor&) override { + auto typed_config = dynamic_cast(&config); + if (typed_config == nullptr) { + return std::make_unique(); + } + return std::make_unique(*typed_config); + } }; +DECLARE_FACTORY(Factory); + } // namespace Random } // namespace LoadBalancingPolices } // namespace Extensions diff --git a/source/extensions/load_balancing_policies/ring_hash/BUILD b/source/extensions/load_balancing_policies/ring_hash/BUILD index 6e9d90339947..1c7a8cf3e392 100644 --- a/source/extensions/load_balancing_policies/ring_hash/BUILD +++ b/source/extensions/load_balancing_policies/ring_hash/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( "abseil_inlined_vector", ], deps = [ + "//envoy/upstream:load_balancer_interface", "//source/common/common:minimal_logger_lib", "//source/common/upstream:thread_aware_lb_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", @@ -37,6 +38,7 @@ envoy_cc_extension( "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_factory_base_lib", "//source/common/upstream:load_balancer_lib", + "//source/extensions/load_balancing_policies/common:factory_base", "@envoy_api//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/load_balancing_policies/ring_hash/config.cc b/source/extensions/load_balancing_policies/ring_hash/config.cc index 05dc3ab98099..80f4dc2141ee 100644 --- a/source/extensions/load_balancing_policies/ring_hash/config.cc +++ b/source/extensions/load_balancing_policies/ring_hash/config.cc @@ -13,24 +13,24 @@ Factory::create(OptRef lb_config, const Upstream::PrioritySet& priority_set, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource&) { - const auto* typed_lb_config = - dynamic_cast(lb_config.ptr()); - auto typed_proto_config = typed_lb_config == nullptr - ? OptRef{} - : typed_lb_config->typedProtoConfig(); + auto active_or_legacy = + Common::ActiveOrLegacy::get(lb_config.ptr()); // Assume legacy config. - if (!typed_proto_config.has_value()) { + if (!active_or_legacy.hasActive()) { return std::make_unique( priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, - cluster_info.lbRingHashConfig(), cluster_info.lbConfig()); + !active_or_legacy.hasLegacy() ? cluster_info.lbRingHashConfig() + : active_or_legacy.legacy()->lbConfig(), + cluster_info.lbConfig()); } return std::make_unique( priority_set, cluster_info.lbStats(), cluster_info.statsScope(), runtime, random, PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), healthy_panic_threshold, 100, 50), - typed_proto_config.value()); + active_or_legacy.active()->lb_config_); } /** diff --git a/source/extensions/load_balancing_policies/ring_hash/config.h b/source/extensions/load_balancing_policies/ring_hash/config.h index fdbac8d590cc..16af59ff05b1 100644 --- a/source/extensions/load_balancing_policies/ring_hash/config.h +++ b/source/extensions/load_balancing_policies/ring_hash/config.h @@ -7,6 +7,8 @@ #include "envoy/upstream/load_balancer.h" #include "source/common/upstream/load_balancer_factory_base.h" +#include "source/extensions/load_balancing_policies/common/factory_base.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" namespace Envoy { namespace Extensions { @@ -25,6 +27,20 @@ class Factory : public Upstream::TypedLoadBalancerFactoryBase { Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) override; + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor&) override { + auto active_or_legacy = + Common::ActiveOrLegacy::get(&config); + + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive()); + + return active_or_legacy.hasLegacy() + ? Upstream::LoadBalancerConfigPtr{new Upstream::LegacyRingHashLbConfig( + *active_or_legacy.legacy())} + : Upstream::LoadBalancerConfigPtr{ + new Upstream::TypedRingHashLbConfig(*active_or_legacy.active())}; + } }; } // namespace RingHash diff --git a/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc index 586941f7a3a1..cd1fa0543a9c 100644 --- a/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc +++ b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.cc @@ -16,6 +16,15 @@ namespace Envoy { namespace Upstream { +LegacyRingHashLbConfig::LegacyRingHashLbConfig(const ClusterProto& cluster) { + if (cluster.has_ring_hash_lb_config()) { + lb_config_ = cluster.ring_hash_lb_config(); + } +} + +TypedRingHashLbConfig::TypedRingHashLbConfig(const RingHashLbProto& lb_config) + : lb_config_(lb_config) {} + RingHashLoadBalancer::RingHashLoadBalancer( const PrioritySet& priority_set, ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime, Random::RandomGenerator& random, diff --git a/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h index 6ad5902f50fe..1626cef69741 100644 --- a/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h +++ b/source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h @@ -8,6 +8,7 @@ #include "envoy/runtime/runtime.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" +#include "envoy/upstream/load_balancer.h" #include "source/common/common/logger.h" #include "source/common/upstream/thread_aware_lb_impl.h" @@ -15,6 +16,38 @@ namespace Envoy { namespace Upstream { +using RingHashLbProto = envoy::extensions::load_balancing_policies::ring_hash::v3::RingHash; +using ClusterProto = envoy::config::cluster::v3::Cluster; +using LegacyRingHashLbProto = ClusterProto::RingHashLbConfig; + +/** + * Load balancer config that used to wrap legacy ring hash config. + */ +class LegacyRingHashLbConfig : public Upstream::LoadBalancerConfig { +public: + LegacyRingHashLbConfig(const ClusterProto& cluster); + + OptRef lbConfig() const { + if (lb_config_.has_value()) { + return lb_config_.value(); + } + return {}; + }; + +private: + absl::optional lb_config_; +}; + +/** + * Load balancer config that used to wrap typed ring hash config. + */ +class TypedRingHashLbConfig : public Upstream::LoadBalancerConfig { +public: + TypedRingHashLbConfig(const RingHashLbProto& lb_config); + + const RingHashLbProto lb_config_; +}; + /** * All ring hash load balancer stats. @see stats_macros.h */ diff --git a/source/extensions/load_balancing_policies/round_robin/BUILD b/source/extensions/load_balancing_policies/round_robin/BUILD index a04766221836..d945d54f94a1 100644 --- a/source/extensions/load_balancing_policies/round_robin/BUILD +++ b/source/extensions/load_balancing_policies/round_robin/BUILD @@ -12,6 +12,8 @@ envoy_cc_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], + # previously considered core code and used by mobile. + visibility = ["//visibility:public"], deps = [ "//source/common/common:minimal_logger_lib", "//source/common/upstream:load_balancer_lib", diff --git a/source/extensions/load_balancing_policies/round_robin/config.cc b/source/extensions/load_balancing_policies/round_robin/config.cc index fff5fb6483bc..9618e680ab60 100644 --- a/source/extensions/load_balancing_policies/round_robin/config.cc +++ b/source/extensions/load_balancing_policies/round_robin/config.cc @@ -9,21 +9,40 @@ namespace Extensions { namespace LoadBalancingPolices { namespace RoundRobin { +LegacyRoundRobinLbConfig::LegacyRoundRobinLbConfig(const ClusterProto& cluster) { + if (cluster.has_round_robin_lb_config()) { + lb_config_ = cluster.round_robin_lb_config(); + } +} + +TypedRoundRobinLbConfig::TypedRoundRobinLbConfig(const RoundRobinLbProto& lb_config) + : lb_config_(lb_config) {} + Upstream::LoadBalancerPtr RoundRobinCreator::operator()( - Upstream::LoadBalancerParams params, OptRef lb_config, + Upstream::LoadBalancerParams params, OptRef lb_config, const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet&, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) { + auto active_or_legacy = + Common::ActiveOrLegacy::get( + lb_config.ptr()); + // The load balancing policy configuration will be loaded and validated in the main thread when we // load the cluster configuration. So we can assume the configuration is valid here. - ASSERT(lb_config.has_value(), + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive(), "Invalid load balancing policy configuration for least request load balancer"); - return std::make_unique( - params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, - PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), - healthy_panic_threshold, 100, 50), - lb_config.value(), time_source); + if (active_or_legacy.hasActive()) { + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(cluster_info.lbConfig(), + healthy_panic_threshold, 100, 50), + active_or_legacy.active()->lb_config_, time_source); + } else { + return std::make_unique( + params.priority_set, params.local_priority_set, cluster_info.lbStats(), runtime, random, + cluster_info.lbConfig(), active_or_legacy.legacy()->lbConfig(), time_source); + } } /** diff --git a/source/extensions/load_balancing_policies/round_robin/config.h b/source/extensions/load_balancing_policies/round_robin/config.h index 4e06b5b48de1..bcd63339d9e1 100644 --- a/source/extensions/load_balancing_policies/round_robin/config.h +++ b/source/extensions/load_balancing_policies/round_robin/config.h @@ -5,6 +5,7 @@ #include "envoy/upstream/load_balancer.h" #include "source/common/common/logger.h" +#include "source/common/upstream/load_balancer_impl.h" #include "source/extensions/load_balancing_policies/common/factory_base.h" namespace Envoy { @@ -13,19 +14,66 @@ namespace LoadBalancingPolices { namespace RoundRobin { using RoundRobinLbProto = envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin; +using ClusterProto = envoy::config::cluster::v3::Cluster; +using LegacyRoundRobinLbProto = ClusterProto::RoundRobinLbConfig; + +/** + * Load balancer config that used to wrap the legacy proto config. + */ +class LegacyRoundRobinLbConfig : public Upstream::LoadBalancerConfig { +public: + LegacyRoundRobinLbConfig(const ClusterProto& cluster); + + OptRef lbConfig() const { + if (lb_config_.has_value()) { + return lb_config_.value(); + } + return {}; + }; + +private: + absl::optional lb_config_; +}; + +/** + * Load balancer config that used to wrap the proto config. + */ +class TypedRoundRobinLbConfig : public Upstream::LoadBalancerConfig { +public: + TypedRoundRobinLbConfig(const RoundRobinLbProto& lb_config); + + const RoundRobinLbProto lb_config_; +}; struct RoundRobinCreator : public Logger::Loggable { - Upstream::LoadBalancerPtr - operator()(Upstream::LoadBalancerParams params, OptRef lb_config, - const Upstream::ClusterInfo& cluster_info, const Upstream::PrioritySet& priority_set, - Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source); + Upstream::LoadBalancerPtr operator()(Upstream::LoadBalancerParams params, + OptRef lb_config, + const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& priority_set, + Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source); }; class Factory : public Common::FactoryBase { public: Factory() : FactoryBase("envoy.load_balancing_policies.round_robin") {} + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor&) override { + + auto active_or_legacy = Common::ActiveOrLegacy::get(&config); + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive()); + + return active_or_legacy.hasLegacy() + ? Upstream::LoadBalancerConfigPtr{new LegacyRoundRobinLbConfig( + *active_or_legacy.legacy())} + : Upstream::LoadBalancerConfigPtr{ + new TypedRoundRobinLbConfig(*active_or_legacy.active())}; + } }; +DECLARE_FACTORY(Factory); + } // namespace RoundRobin } // namespace LoadBalancingPolices } // namespace Extensions diff --git a/source/extensions/load_balancing_policies/subset/config.cc b/source/extensions/load_balancing_policies/subset/config.cc index c370e3f24e58..bd6a259155fa 100644 --- a/source/extensions/load_balancing_policies/subset/config.cc +++ b/source/extensions/load_balancing_policies/subset/config.cc @@ -1,5 +1,7 @@ #include "source/extensions/load_balancing_policies/subset/config.h" +#include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/common/factory_base.h" #include "source/extensions/load_balancing_policies/subset/subset_lb.h" namespace Envoy { @@ -7,6 +9,9 @@ namespace Extensions { namespace LoadBalancingPolices { namespace Subset { +using SubsetLbProto = envoy::extensions::load_balancing_policies::subset::v3::Subset; +using ClusterProto = envoy::config::cluster::v3::Cluster; + Upstream::LoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster, const Upstream::PrioritySet& priority_set, const Upstream::PrioritySet* local_priority_set, @@ -26,62 +31,9 @@ Upstream::LoadBalancerPtr Factory::create(const Upstream::ClusterInfo& cluster, */ REGISTER_FACTORY(Factory, Upstream::NonThreadAwareLoadBalancerFactory); -using LoadBalancerSubsetInfoImpl = - Upstream::LoadBalancerSubsetInfoImplBase; - -class SubsetLoadBalancerConfig : public Upstream::LoadBalancerConfig { -public: - SubsetLoadBalancerConfig(const Upstream::SubsetLoadbalancingPolicyProto& subset_config, - ProtobufMessage::ValidationVisitor& visitor) - : subset_info_(subset_config) { - - absl::InlinedVector missing_policies; - - for (const auto& policy : subset_config.subset_lb_policy().policies()) { - auto* factory = Config::Utility::getAndCheckFactory( - policy.typed_extension_config(), /*is_optional=*/true); - - if (factory != nullptr) { - // Load and validate the configuration. - auto sub_lb_proto_message = factory->createEmptyConfigProto(); - Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(), - visitor, *sub_lb_proto_message); - - sub_load_balancer_config_ = factory->loadConfig(std::move(sub_lb_proto_message), visitor); - sub_load_balancer_factory_ = factory; - break; - } - - missing_policies.push_back(policy.typed_extension_config().name()); - } - - if (sub_load_balancer_factory_ == nullptr) { - throw EnvoyException(fmt::format("cluster: didn't find a registered load balancer factory " - "implementation for subset lb with names from [{}]", - absl::StrJoin(missing_policies, ", "))); - } - } - - Upstream::ThreadAwareLoadBalancerPtr - createLoadBalancer(const Upstream::ClusterInfo& cluster_info, - const Upstream::PrioritySet& child_priority_set, Runtime::Loader& runtime, - Random::RandomGenerator& random, TimeSource& time_source) const { - return sub_load_balancer_factory_->create(*sub_load_balancer_config_, cluster_info, - child_priority_set, runtime, random, time_source); - } - - const Upstream::LoadBalancerSubsetInfo& subsetInfo() const { return subset_info_; } - -private: - LoadBalancerSubsetInfoImpl subset_info_; - - Upstream::LoadBalancerConfigPtr sub_load_balancer_config_; - Upstream::TypedLoadBalancerFactory* sub_load_balancer_factory_{}; -}; - class ChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCreator { public: - ChildLoadBalancerCreatorImpl(const SubsetLoadBalancerConfig& subset_config, + ChildLoadBalancerCreatorImpl(const Upstream::SubsetLoadBalancerConfig& subset_config, const Upstream::ClusterInfo& cluster_info) : subset_config_(subset_config), cluster_info_(cluster_info) {} @@ -95,13 +47,13 @@ class ChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCreator { } private: - const SubsetLoadBalancerConfig& subset_config_; + const Upstream::SubsetLoadBalancerConfig& subset_config_; const Upstream::ClusterInfo& cluster_info_; }; class LbFactory : public Upstream::LoadBalancerFactory { public: - LbFactory(const SubsetLoadBalancerConfig& subset_config, + LbFactory(const Upstream::SubsetLoadBalancerConfig& subset_config, const Upstream::ClusterInfo& cluster_info, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) : subset_config_(subset_config), cluster_info_(cluster_info), runtime_(runtime), @@ -119,7 +71,7 @@ class LbFactory : public Upstream::LoadBalancerFactory { bool recreateOnHostChange() const override { return false; } private: - const SubsetLoadBalancerConfig& subset_config_; + const Upstream::SubsetLoadBalancerConfig& subset_config_; const Upstream::ClusterInfo& cluster_info_; Runtime::Loader& runtime_; @@ -144,7 +96,8 @@ SubsetLbFactory::create(OptRef lb_config, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) { - const auto* typed_config = dynamic_cast(lb_config.ptr()); + const auto* typed_config = + dynamic_cast(lb_config.ptr()); // The load balancing policy configuration will be loaded and validated in the main thread when we // load the cluster configuration. So we can assume the configuration is valid here. ASSERT(typed_config != nullptr, @@ -161,14 +114,37 @@ SubsetLbFactory::create(OptRef lb_config, } Upstream::LoadBalancerConfigPtr -SubsetLbFactory::loadConfig(ProtobufTypes::MessagePtr config, +SubsetLbFactory::loadConfig(const Protobuf::Message& config, ProtobufMessage::ValidationVisitor& visitor) { - ASSERT(config != nullptr); - auto* proto_config = dynamic_cast(config.get()); + + auto active_or_legacy = Common::ActiveOrLegacy::get(&config); + ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive()); + + if (active_or_legacy.hasLegacy()) { + if (active_or_legacy.legacy()->lb_policy() == + envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED) { + throw EnvoyException( + fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", + envoy::config::cluster::v3::Cluster::LbPolicy_Name( + active_or_legacy.legacy()->lb_policy()))); + } + + auto sub_lb_pair = + Upstream::LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProtoWithoutSubset( + *active_or_legacy.legacy(), visitor); + + if (!sub_lb_pair.ok()) { + throw EnvoyException(std::string(sub_lb_pair.status().message())); + } + + return std::make_unique( + active_or_legacy.legacy()->lb_subset_config(), std::move(sub_lb_pair->config), + sub_lb_pair->factory); + } // Load the subset load balancer configuration. This will contains child load balancer // config and child load balancer factory. - return std::make_unique(*proto_config, visitor); + return std::make_unique(*active_or_legacy.active(), visitor); } /** diff --git a/source/extensions/load_balancing_policies/subset/config.h b/source/extensions/load_balancing_policies/subset/config.h index 8476062ab0b5..f6a261788644 100644 --- a/source/extensions/load_balancing_policies/subset/config.h +++ b/source/extensions/load_balancing_policies/subset/config.h @@ -35,7 +35,7 @@ class SubsetLbFactory Random::RandomGenerator& random, TimeSource& time_source) override; - Upstream::LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config, + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message& config, ProtobufMessage::ValidationVisitor& visitor) override; }; diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc index c1e197080878..e8ae1707703a 100644 --- a/source/extensions/load_balancing_policies/subset/subset_lb.cc +++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc @@ -126,6 +126,38 @@ LegacyChildLoadBalancerCreatorImpl::createLoadBalancer( return {nullptr, nullptr}; } +SubsetLoadBalancerConfig::SubsetLoadBalancerConfig( + const SubsetLoadbalancingPolicyProto& subset_config, + ProtobufMessage::ValidationVisitor& visitor) + : subset_info_(subset_config) { + + absl::InlinedVector missing_policies; + + for (const auto& policy : subset_config.subset_lb_policy().policies()) { + auto* factory = Config::Utility::getAndCheckFactory( + policy.typed_extension_config(), /*is_optional=*/true); + + if (factory != nullptr) { + // Load and validate the configuration. + auto sub_lb_proto_message = factory->createEmptyConfigProto(); + Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(), + visitor, *sub_lb_proto_message); + + sub_load_balancer_config_ = factory->loadConfig(*sub_lb_proto_message, visitor); + sub_load_balancer_factory_ = factory; + break; + } + + missing_policies.push_back(policy.typed_extension_config().name()); + } + + if (sub_load_balancer_factory_ == nullptr) { + throw EnvoyException(fmt::format("cluster: didn't find a registered load balancer factory " + "implementation for subset lb with names from [{}]", + absl::StrJoin(missing_policies, ", "))); + } +} + SubsetLoadBalancer::SubsetLoadBalancer(const LoadBalancerSubsetInfo& subsets, ChildLoadBalancerCreatorPtr child_lb, const PrioritySet& priority_set, diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h index 83132c031273..ddf228a2c0e7 100644 --- a/source/extensions/load_balancing_policies/subset/subset_lb.h +++ b/source/extensions/load_balancing_policies/subset/subset_lb.h @@ -29,11 +29,6 @@ namespace Envoy { namespace Upstream { -using HostHashSet = absl::flat_hash_set; - -using SubsetLoadbalancingPolicyProto = - envoy::extensions::load_balancing_policies::subset::v3::Subset; - class ChildLoadBalancerCreator { public: virtual ~ChildLoadBalancerCreator() = default; @@ -105,73 +100,36 @@ class LegacyChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCre const envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_; }; -template -class LoadBalancerSubsetInfoImplBase : public Upstream::LoadBalancerSubsetInfo { -public: - // TODO(wbpcode): use legacy enum for backward compatibility for now. - using FallbackPolicy = - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy; - using MetadataFallbackPolicy = - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy; - using SubsetFallbackPolicy = envoy::config::cluster::v3::Cluster::LbSubsetConfig:: - LbSubsetSelector::LbSubsetSelectorFallbackPolicy; - - LoadBalancerSubsetInfoImplBase(const ProtoType& subset_config) - : default_subset_(subset_config.default_subset()), - fallback_policy_(static_cast(subset_config.fallback_policy())), - metadata_fallback_policy_( - static_cast(subset_config.metadata_fallback_policy())), - enabled_(!subset_config.subset_selectors().empty()), - locality_weight_aware_(subset_config.locality_weight_aware()), - scale_locality_weight_(subset_config.scale_locality_weight()), - panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()), - allow_redundant_keys_(subset_config.allow_redundant_keys()) { - for (const auto& subset : subset_config.subset_selectors()) { - if (!subset.keys().empty()) { - subset_selectors_.emplace_back(std::make_shared( - subset.keys(), static_cast(subset.fallback_policy()), - subset.fallback_keys_subset(), subset.single_host_per_subset())); - } - } +using HostHashSet = absl::flat_hash_set; - if (allow_redundant_keys_) { - // Sort subset selectors by number of keys, descending. This will ensure that the longest - // matching subset selector will be at the beginning of the list. - std::stable_sort(subset_selectors_.begin(), subset_selectors_.end(), - [](const SubsetSelectorPtr& a, const SubsetSelectorPtr& b) -> bool { - return a->selectorKeys().size() > b->selectorKeys().size(); - }); - } +class SubsetLoadBalancerConfig : public Upstream::LoadBalancerConfig { +public: + SubsetLoadBalancerConfig(const SubsetLoadbalancingPolicyProto& subset_config, + ProtobufMessage::ValidationVisitor& visitor); + + SubsetLoadBalancerConfig(const LegacySubsetLoadbalancingPolicyProto& subset_config, + Upstream::LoadBalancerConfigPtr sub_load_balancer_config, + Upstream::TypedLoadBalancerFactory* sub_load_balancer_factory) + : subset_info_(subset_config), sub_load_balancer_config_(std::move(sub_load_balancer_config)), + sub_load_balancer_factory_(sub_load_balancer_factory) { + ASSERT(sub_load_balancer_factory_ != nullptr, "sub_load_balancer_factory_ must not be nullptr"); } - // Upstream::LoadBalancerSubsetInfo - bool isEnabled() const override { return enabled_; } - FallbackPolicy fallbackPolicy() const override { return fallback_policy_; } - MetadataFallbackPolicy metadataFallbackPolicy() const override { - return metadata_fallback_policy_; + Upstream::ThreadAwareLoadBalancerPtr + createLoadBalancer(const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& child_priority_set, Runtime::Loader& runtime, + Random::RandomGenerator& random, TimeSource& time_source) const { + return sub_load_balancer_factory_->create(*sub_load_balancer_config_, cluster_info, + child_priority_set, runtime, random, time_source); } - const ProtobufWkt::Struct& defaultSubset() const override { return default_subset_; } - const std::vector& subsetSelectors() const override { - return subset_selectors_; - } - bool localityWeightAware() const override { return locality_weight_aware_; } - bool scaleLocalityWeight() const override { return scale_locality_weight_; } - bool panicModeAny() const override { return panic_mode_any_; } - bool listAsAny() const override { return list_as_any_; } - bool allowRedundantKeys() const override { return allow_redundant_keys_; } + + const Upstream::LoadBalancerSubsetInfo& subsetInfo() const { return subset_info_; } private: - const ProtobufWkt::Struct default_subset_; - std::vector subset_selectors_; - // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. - const FallbackPolicy fallback_policy_; - const MetadataFallbackPolicy metadata_fallback_policy_; - const bool enabled_ : 1; - const bool locality_weight_aware_ : 1; - const bool scale_locality_weight_ : 1; - const bool panic_mode_any_ : 1; - const bool list_as_any_ : 1; - const bool allow_redundant_keys_{}; + LoadBalancerSubsetInfoImpl subset_info_; + + Upstream::LoadBalancerConfigPtr sub_load_balancer_config_; + Upstream::TypedLoadBalancerFactory* sub_load_balancer_factory_{}; }; class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable { diff --git a/source/extensions/matching/actions/format_string/config.cc b/source/extensions/matching/actions/format_string/config.cc index d5cb4aa17b5b..ff8cb314181e 100644 --- a/source/extensions/matching/actions/format_string/config.cc +++ b/source/extensions/matching/actions/format_string/config.cc @@ -30,8 +30,10 @@ ActionFactory::createActionFactoryCb(const Protobuf::Message& proto_config, const auto& config = MessageUtil::downcastAndValidate( proto_config, validator); + + Server::GenericFactoryContextImpl generic_context(context, validator); Formatter::FormatterConstSharedPtr formatter = - Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, context); + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, generic_context); return [formatter]() { return std::make_unique(formatter); }; } diff --git a/source/extensions/matching/http/cel_input/cel_input.h b/source/extensions/matching/http/cel_input/cel_input.h index 324d2fd264cd..d1d718a4705b 100644 --- a/source/extensions/matching/http/cel_input/cel_input.h +++ b/source/extensions/matching/http/cel_input/cel_input.h @@ -43,6 +43,7 @@ class HttpCelDataInput : public Matcher::DataInput activation = Extensions::Filters::Common::Expr::createActivation( + nullptr, // TODO: pass local_info to CEL activation. data.streamInfo(), maybe_request_headers.ptr(), maybe_response_headers.ptr(), maybe_response_trailers.ptr()); diff --git a/source/extensions/matching/network/application_protocol/BUILD b/source/extensions/matching/network/application_protocol/BUILD index b2232ec30f67..da94f6909b05 100644 --- a/source/extensions/matching/network/application_protocol/BUILD +++ b/source/extensions/matching/network/application_protocol/BUILD @@ -15,6 +15,7 @@ envoy_cc_extension( # legacy test usage extra_visibility = [ "//test/integration:__subpackages__", + "//test/common/listener_manager:__subpackages__", ], deps = [ "//envoy/http:filter_interface", diff --git a/source/extensions/matching/network/common/BUILD b/source/extensions/matching/network/common/BUILD index ffc89b27adee..016b89748179 100644 --- a/source/extensions/matching/network/common/BUILD +++ b/source/extensions/matching/network/common/BUILD @@ -13,6 +13,8 @@ envoy_cc_extension( srcs = ["inputs.cc"], hdrs = ["inputs.h"], extra_visibility = [ + "//source/common/listener_manager:__subpackages__", + "//test:__subpackages__", ], deps = [ "//envoy/http:filter_interface", diff --git a/source/extensions/path/uri_template_lib/proto/BUILD b/source/extensions/path/uri_template_lib/proto/BUILD index d8ad5c3057d5..7bc7902fb515 100644 --- a/source/extensions/path/uri_template_lib/proto/BUILD +++ b/source/extensions/path/uri_template_lib/proto/BUILD @@ -13,5 +13,5 @@ envoy_extension_package() envoy_proto_library( name = "uri_template_rewrite_segements", srcs = ["rewrite_segments.proto"], - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = ["@com_github_cncf_xds//udpa/annotations:pkg"], ) diff --git a/source/extensions/path/uri_template_lib/uri_template.cc b/source/extensions/path/uri_template_lib/uri_template.cc index 9eda8154562b..3526f90a2113 100644 --- a/source/extensions/path/uri_template_lib/uri_template.cc +++ b/source/extensions/path/uri_template_lib/uri_template.cc @@ -9,6 +9,7 @@ #include "source/extensions/path/uri_template_lib/uri_template_internal.h" #include "absl/status/statusor.h" +#include "absl/strings/escaping.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "re2/re2.h" @@ -24,6 +25,19 @@ using Internal::ParsedPathPattern; #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #endif +std::ostream& operator<<(std::ostream& os, const ParsedSegment& parsed_segment) { + os << "{kind = "; + switch (parsed_segment.kind_) { + case RewriteStringKind::Variable: + os << "Variable"; + break; + case RewriteStringKind::Literal: + os << "Literal"; + break; + } + return os << ", value = \"" << absl::CEscape(parsed_segment.value_) << "\"}"; +} + absl::StatusOr convertPathPatternSyntaxToRegex(absl::string_view path_pattern) { absl::StatusOr status = Internal::parsePathPatternSyntax(path_pattern); if (!status.ok()) { diff --git a/source/extensions/path/uri_template_lib/uri_template.h b/source/extensions/path/uri_template_lib/uri_template.h index 6e50d04bc155..d2533dc10b0f 100644 --- a/source/extensions/path/uri_template_lib/uri_template.h +++ b/source/extensions/path/uri_template_lib/uri_template.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "source/extensions/path/uri_template_lib/uri_template_internal.h" @@ -26,6 +27,8 @@ struct ParsedSegment { ParsedSegment(absl::string_view value, RewriteStringKind kind) : value_(value), kind_(kind) {} absl::string_view value_; RewriteStringKind kind_; + + friend std::ostream& operator<<(std::ostream& os, const ParsedSegment& parsed_segment); }; // Stores string literals and regex capture indexes for rewriting paths diff --git a/source/extensions/path/uri_template_lib/uri_template_internal.cc b/source/extensions/path/uri_template_lib/uri_template_internal.cc index 0b636fb2f75d..7cec7fed99c0 100644 --- a/source/extensions/path/uri_template_lib/uri_template_internal.cc +++ b/source/extensions/path/uri_template_lib/uri_template_internal.cc @@ -41,7 +41,8 @@ constexpr unsigned long kPatternMatchingMinVariableNameLen = 1; constexpr absl::string_view kLiteral = "a-zA-Z0-9-._~" // Unreserved "%" // pct-encoded "!$&'()+,;" // sub-delims excluding *= - ":@"; + ":@" + "="; // user included "=" allowed // Default operator used for the variable when none specified. constexpr Operator kDefaultVariableOperator = Operator::PathGlob; diff --git a/source/extensions/rate_limit_descriptors/expr/config.cc b/source/extensions/rate_limit_descriptors/expr/config.cc index a50dec82f0f2..e6e4b8044a59 100644 --- a/source/extensions/rate_limit_descriptors/expr/config.cc +++ b/source/extensions/rate_limit_descriptors/expr/config.cc @@ -36,7 +36,7 @@ class ExpressionDescriptor : public RateLimit::DescriptorProducer { const Http::RequestHeaderMap& headers, const StreamInfo::StreamInfo& info) const override { ProtobufWkt::Arena arena; - const auto result = Filters::Common::Expr::evaluate(*compiled_expr_.get(), arena, info, + const auto result = Filters::Common::Expr::evaluate(*compiled_expr_.get(), arena, nullptr, info, &headers, nullptr, nullptr); if (!result.has_value() || result.value().IsError()) { // If result is an error and if skip_if_error is true skip this descriptor, diff --git a/source/extensions/request_id/uuid/config.h b/source/extensions/request_id/uuid/config.h index a36a5c75a361..43ad14e493cc 100644 --- a/source/extensions/request_id/uuid/config.h +++ b/source/extensions/request_id/uuid/config.h @@ -73,12 +73,12 @@ class UUIDRequestIDExtensionFactory : public Server::Configuration::RequestIDExt } Envoy::Http::RequestIDExtensionSharedPtr createExtensionInstance(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) override { + Server::Configuration::FactoryContext& context) override { return std::make_shared( MessageUtil::downcastAndValidate< const envoy::extensions::request_id::uuid::v3::UuidRequestIdConfig&>( config, context.messageValidationVisitor()), - context.api().randomGenerator()); + context.serverFactoryContext().api().randomGenerator()); } }; diff --git a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc index a2799ac6ff15..23e7ab516d33 100644 --- a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc +++ b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc @@ -15,7 +15,7 @@ InjectedResourceMonitor::InjectedResourceMonitor( const envoy::extensions::resource_monitors::injected_resource::v3::InjectedResourceConfig& config, Server::Configuration::ResourceMonitorFactoryContext& context) - : filename_(config.filename()), file_changed_(true), + : filename_(config.filename()), watcher_(context.mainThreadDispatcher().createFilesystemWatcher()), api_(context.api()) { watcher_->addWatch(filename_, Filesystem::Watcher::Events::MovedTo, [this](uint32_t) { onFileChanged(); }); @@ -27,7 +27,9 @@ void InjectedResourceMonitor::updateResourceUsage(Server::ResourceUpdateCallback if (file_changed_) { file_changed_ = false; TRY_ASSERT_MAIN_THREAD { - const std::string contents = api_.fileSystem().fileReadToEnd(filename_); + auto file_or_error = api_.fileSystem().fileReadToEnd(filename_); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + const std::string contents = file_or_error.value(); double pressure; if (absl::SimpleAtod(contents, &pressure)) { if (pressure < 0 || pressure > 1) { diff --git a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h index 59bf32375c56..f3799969d010 100644 --- a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h +++ b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h @@ -32,7 +32,7 @@ class InjectedResourceMonitor : public Server::ResourceMonitor { private: const std::string filename_; - bool file_changed_; + bool file_changed_{true}; Filesystem::WatcherPtr watcher_; absl::optional pressure_; absl::optional error_; diff --git a/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.cc b/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.cc index 6c6e8b6481fb..96bf1346b659 100644 --- a/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.cc +++ b/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.cc @@ -11,9 +11,9 @@ bool OmitHostsRetryPredicate::shouldSelectAnotherHost(const Upstream::Host& host // Note: The additional check to verify if the labelSet is empty is performed since // metadataLabelMatch returns true in case of an empty labelSet. However, for an empty labelSet, // i.e. if there is no matching criteria defined, this method should return false. - return !labelSet_.empty() && Envoy::Config::Metadata::metadataLabelMatch( - labelSet_, host.metadata().get(), - Envoy::Config::MetadataFilters::get().ENVOY_LB, true); + return !label_set_.empty() && Envoy::Config::Metadata::metadataLabelMatch( + label_set_, host.metadata().get(), + Envoy::Config::MetadataFilters::get().ENVOY_LB, true); } } // namespace Host diff --git a/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.h b/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.h index 19ec5c659ebe..e7f3a04c2230 100644 --- a/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.h +++ b/source/extensions/retry/host/omit_host_metadata/omit_host_metadata.h @@ -18,7 +18,7 @@ class OmitHostsRetryPredicate : public Upstream::RetryHostPredicate { Envoy::Config::MetadataFilters::get().ENVOY_LB); if (filter_it != metadata_match_criteria_.filter_metadata().end()) { for (auto const& it : filter_it->second.fields()) { - labelSet_.push_back(it); + label_set_.push_back(it); } } } @@ -29,7 +29,7 @@ class OmitHostsRetryPredicate : public Upstream::RetryHostPredicate { private: const envoy::config::core::v3::Metadata metadata_match_criteria_; - std::vector> labelSet_; + std::vector> label_set_; }; } // namespace Host diff --git a/source/extensions/router/cluster_specifiers/lua/BUILD b/source/extensions/router/cluster_specifiers/lua/BUILD new file mode 100644 index 000000000000..92927266d15e --- /dev/null +++ b/source/extensions/router/cluster_specifiers/lua/BUILD @@ -0,0 +1,41 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +# Lua cluster specifier plugin. + +envoy_extension_package() + +envoy_cc_library( + name = "lua_cluster_specifier_lib", + srcs = [ + "lua_cluster_specifier.cc", + ], + hdrs = [ + "lua_cluster_specifier.h", + ], + deps = [ + "//envoy/router:cluster_specifier_plugin_interface", + "//source/common/common:utility_lib", + "//source/common/http:utility_lib", + "//source/common/router:config_lib", + "//source/extensions/filters/common/lua:lua_lib", + "//source/extensions/filters/common/lua:wrappers_lib", + "@envoy_api//envoy/extensions/router/cluster_specifiers/lua/v3:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":lua_cluster_specifier_lib", + "//envoy/registry", + ], +) diff --git a/source/extensions/router/cluster_specifiers/lua/config.cc b/source/extensions/router/cluster_specifiers/lua/config.cc new file mode 100644 index 000000000000..fb7930caf859 --- /dev/null +++ b/source/extensions/router/cluster_specifiers/lua/config.cc @@ -0,0 +1,23 @@ +#include "source/extensions/router/cluster_specifiers/lua/config.h" + +namespace Envoy { +namespace Extensions { +namespace Router { +namespace Lua { + +Envoy::Router::ClusterSpecifierPluginSharedPtr +LuaClusterSpecifierPluginFactoryConfig::createClusterSpecifierPlugin( + const Protobuf::Message& config, Server::Configuration::CommonFactoryContext& context) { + + const auto& typed_config = dynamic_cast(config); + auto cluster_config = std::make_shared(typed_config, context); + return std::make_shared(cluster_config); +} + +REGISTER_FACTORY(LuaClusterSpecifierPluginFactoryConfig, + Envoy::Router::ClusterSpecifierPluginFactoryConfig); + +} // namespace Lua +} // namespace Router +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/router/cluster_specifiers/lua/config.h b/source/extensions/router/cluster_specifiers/lua/config.h new file mode 100644 index 000000000000..2b9793b08099 --- /dev/null +++ b/source/extensions/router/cluster_specifiers/lua/config.h @@ -0,0 +1,28 @@ +#pragma once + +#include "source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.h" + +namespace Envoy { +namespace Extensions { +namespace Router { +namespace Lua { + +class LuaClusterSpecifierPluginFactoryConfig + : public Envoy::Router::ClusterSpecifierPluginFactoryConfig { +public: + LuaClusterSpecifierPluginFactoryConfig() = default; + Envoy::Router::ClusterSpecifierPluginSharedPtr + createClusterSpecifierPlugin(const Protobuf::Message& config, + Server::Configuration::CommonFactoryContext&) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "envoy.router.cluster_specifier_plugin.lua"; } +}; + +} // namespace Lua +} // namespace Router +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.cc b/source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.cc new file mode 100644 index 000000000000..c226a78c4a0a --- /dev/null +++ b/source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.cc @@ -0,0 +1,93 @@ +#include "source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.h" + +#include "source/common/router/config_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Router { +namespace Lua { + +PerLuaCodeSetup::PerLuaCodeSetup(const std::string& lua_code, ThreadLocal::SlotAllocator& tls) + : lua_state_(lua_code, tls) { + lua_state_.registerType(); + lua_state_.registerType(); + + const Filters::Common::Lua::InitializerList initializers; + + cluster_function_slot_ = lua_state_.registerGlobal("envoy_on_route", initializers); + if (lua_state_.getGlobalRef(cluster_function_slot_) == LUA_REFNIL) { + throw EnvoyException( + "envoy_on_route() function not found. Lua will not hook cluster specifier."); + } +} + +int HeaderMapWrapper::luaGet(lua_State* state) { + absl::string_view key = Filters::Common::Lua::getStringViewFromLuaString(state, 2); + const Envoy::Http::HeaderUtility::GetAllOfHeaderAsStringResult value = + Envoy::Http::HeaderUtility::getAllOfHeaderAsString(headers_, + Envoy::Http::LowerCaseString(key)); + if (value.result().has_value()) { + lua_pushlstring(state, value.result().value().data(), value.result().value().size()); + return 1; + } else { + return 0; + } +} + +int RouteHandleWrapper::luaHeaders(lua_State* state) { + if (headers_wrapper_.get() != nullptr) { + headers_wrapper_.pushStack(); + } else { + headers_wrapper_.reset(HeaderMapWrapper::create(state, headers_), true); + } + return 1; +} + +LuaClusterSpecifierConfig::LuaClusterSpecifierConfig( + const LuaClusterSpecifierConfigProto& config, + Server::Configuration::CommonFactoryContext& context) + : main_thread_dispatcher_(context.mainThreadDispatcher()), + default_cluster_(config.default_cluster()) { + const std::string code_str = Config::DataSource::read(config.source_code(), true, context.api()); + per_lua_code_setup_ptr_ = std::make_unique(code_str, context.threadLocal()); +} + +LuaClusterSpecifierPlugin::LuaClusterSpecifierPlugin(LuaClusterSpecifierConfigSharedPtr config) + : config_(config), + function_ref_(config_->perLuaCodeSetup() ? config_->perLuaCodeSetup()->clusterFunctionRef() + : LUA_REFNIL) {} + +std::string LuaClusterSpecifierPlugin::startLua(const Http::HeaderMap& headers) const { + if (function_ref_ == LUA_REFNIL) { + return config_->defaultCluster(); + } + Filters::Common::Lua::CoroutinePtr coroutine = config_->perLuaCodeSetup()->createCoroutine(); + + RouteHandleRef handle; + handle.reset(RouteHandleWrapper::create(coroutine->luaState(), headers), true); + + TRY_NEEDS_AUDIT { + coroutine->start(function_ref_, 1, []() {}); + } + END_TRY catch (const Filters::Common::Lua::LuaException& e) { + ENVOY_LOG(error, "script log: {}, use default cluster", e.what()); + return config_->defaultCluster(); + } + if (!lua_isstring(coroutine->luaState(), -1)) { + ENVOY_LOG(error, "script log: return value is not string, use default cluster"); + return config_->defaultCluster(); + } + return std::string(Filters::Common::Lua::getStringViewFromLuaString(coroutine->luaState(), -1)); +} + +Envoy::Router::RouteConstSharedPtr +LuaClusterSpecifierPlugin::route(Envoy::Router::RouteConstSharedPtr parent, + const Http::RequestHeaderMap& headers) const { + return std::make_shared( + dynamic_cast(parent.get()), parent, + startLua(headers)); +} +} // namespace Lua +} // namespace Router +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.h b/source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.h new file mode 100644 index 000000000000..5a6ddb9d6a08 --- /dev/null +++ b/source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.h @@ -0,0 +1,119 @@ +#pragma once + +#include "envoy/extensions/router/cluster_specifiers/lua/v3/lua.pb.h" +#include "envoy/router/cluster_specifier_plugin.h" + +#include "source/common/config/datasource.h" +#include "source/extensions/filters/common/lua/wrappers.h" + +namespace Envoy { +namespace Extensions { +namespace Router { +namespace Lua { + +using LuaClusterSpecifierConfigProto = + envoy::extensions::router::cluster_specifiers::lua::v3::LuaConfig; + +class PerLuaCodeSetup : Logger::Loggable { +public: + PerLuaCodeSetup(const std::string& lua_code, ThreadLocal::SlotAllocator& tls); + + Extensions::Filters::Common::Lua::CoroutinePtr createCoroutine() { + return lua_state_.createCoroutine(); + } + + int clusterFunctionRef() { return lua_state_.getGlobalRef(cluster_function_slot_); } + +private: + uint64_t cluster_function_slot_{}; + + Filters::Common::Lua::ThreadLocalState lua_state_; +}; + +using PerLuaCodeSetupPtr = std::unique_ptr; + +class HeaderMapWrapper : public Filters::Common::Lua::BaseLuaObject { +public: + HeaderMapWrapper(const Http::HeaderMap& headers) : headers_(headers) {} + + static ExportedFunctions exportedFunctions() { return {{"get", static_luaGet}}; } + +private: + /** + * Get a header value from the map. + * @param 1 (string): header name. + * @return string value if found or nil. + */ + DECLARE_LUA_FUNCTION(HeaderMapWrapper, luaGet); + + const Http::HeaderMap& headers_; +}; + +using HeaderMapRef = Filters::Common::Lua::LuaDeathRef; + +class RouteHandleWrapper : public Filters::Common::Lua::BaseLuaObject { +public: + RouteHandleWrapper(const Http::HeaderMap& headers) : headers_(headers) {} + + static ExportedFunctions exportedFunctions() { return {{"headers", static_luaHeaders}}; } + +private: + /** + * @return a handle to the headers. + */ + DECLARE_LUA_FUNCTION(RouteHandleWrapper, luaHeaders); + + const Http::HeaderMap& headers_; + HeaderMapRef headers_wrapper_; +}; + +using RouteHandleRef = Filters::Common::Lua::LuaDeathRef; + +class LuaClusterSpecifierConfig : Logger::Loggable { +public: + LuaClusterSpecifierConfig(const LuaClusterSpecifierConfigProto& config, + Server::Configuration::CommonFactoryContext& context); + + ~LuaClusterSpecifierConfig() { + // The design of the TLS system does not allow TLS state to be modified in worker threads. + // However, when the route configuration is dynamically updated via RDS, the old + // LuaClusterSpecifierConfig object may be destructed in a random worker thread. Therefore, to + // ensure thread safety, ownership of per_lua_code_setup_ptr_ must be transferred to the main + // thread and destroyed when the LuaClusterSpecifierConfig object is not destructed in the main + // thread. + if (per_lua_code_setup_ptr_ && !main_thread_dispatcher_.isThreadSafe()) { + auto shared_ptr_wrapper = + std::make_shared(std::move(per_lua_code_setup_ptr_)); + main_thread_dispatcher_.post([shared_ptr_wrapper] { shared_ptr_wrapper->reset(); }); + } + } + + PerLuaCodeSetup* perLuaCodeSetup() const { return per_lua_code_setup_ptr_.get(); } + const std::string& defaultCluster() const { return default_cluster_; } + +private: + Event::Dispatcher& main_thread_dispatcher_; + PerLuaCodeSetupPtr per_lua_code_setup_ptr_; + const std::string default_cluster_; +}; + +using LuaClusterSpecifierConfigSharedPtr = std::shared_ptr; + +class LuaClusterSpecifierPlugin : public Envoy::Router::ClusterSpecifierPlugin, + Logger::Loggable { +public: + LuaClusterSpecifierPlugin(LuaClusterSpecifierConfigSharedPtr config); + Envoy::Router::RouteConstSharedPtr route(Envoy::Router::RouteConstSharedPtr parent, + const Http::RequestHeaderMap& header) const override; + +private: + std::string startLua(const Http::HeaderMap& headers) const; + + LuaClusterSpecifierConfigSharedPtr config_; + const int function_ref_; +}; + +} // namespace Lua +} // namespace Router +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 062fdfd01fca..fa66a6205cdf 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -67,6 +67,11 @@ void UdpStatsdSink::flush(Stats::MetricSnapshot& snapshot) { } } + for (const auto& counter : snapshot.hostCounters()) { + const std::string counter_str = buildMessage(counter, counter.delta(), "|c"); + writeBuffer(buffer, writer, counter_str); + } + for (const auto& gauge : snapshot.gauges()) { if (gauge.get().used()) { const std::string gauge_str = buildMessage(gauge.get(), gauge.get().value(), "|g"); @@ -74,6 +79,11 @@ void UdpStatsdSink::flush(Stats::MetricSnapshot& snapshot) { } } + for (const auto& gauge : snapshot.hostGauges()) { + const std::string gauge_str = buildMessage(gauge, gauge.value(), "|g"); + writeBuffer(buffer, writer, gauge_str); + } + flushBuffer(buffer, writer); // TODO(efimki): Add support of text readouts stats. } @@ -126,8 +136,8 @@ void UdpStatsdSink::onHistogramComplete(const Stats::Histogram& histogram, uint6 tls_->getTyped().write(message); } -template -const std::string UdpStatsdSink::buildMessage(const Stats::Metric& metric, ValueType value, +template +const std::string UdpStatsdSink::buildMessage(const StatType& metric, ValueType value, const std::string& type) const { switch (tag_format_.tag_position) { case Statsd::TagPosition::TagAfterValue: { @@ -155,7 +165,7 @@ const std::string UdpStatsdSink::buildMessage(const Stats::Metric& metric, Value PANIC_DUE_TO_CORRUPT_ENUM; } -const std::string UdpStatsdSink::getName(const Stats::Metric& metric) const { +template const std::string UdpStatsdSink::getName(const StatType& metric) const { if (use_tag_) { return metric.tagExtractedName(); } else { @@ -201,10 +211,18 @@ void TcpStatsdSink::flush(Stats::MetricSnapshot& snapshot) { } } + for (const auto& counter : snapshot.hostCounters()) { + tls_sink.flushCounter(counter.name(), counter.delta()); + } + for (const auto& gauge : snapshot.gauges()) { if (gauge.get().used()) { tls_sink.flushGauge(gauge.get().name(), gauge.get().value()); } + + for (const auto& gauge : snapshot.hostGauges()) { + tls_sink.flushGauge(gauge.name(), gauge.value()); + } } // TODO(efimki): Add support of text readouts stats. tls_sink.endFlush(true); diff --git a/source/extensions/stat_sinks/common/statsd/statsd.h b/source/extensions/stat_sinks/common/statsd/statsd.h index e1a5cd4e46ae..4365ff3f048e 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.h +++ b/source/extensions/stat_sinks/common/statsd/statsd.h @@ -85,10 +85,10 @@ class UdpStatsdSink : public Stats::Sink { void flushBuffer(Buffer::OwnedImpl& buffer, Writer& writer) const; void writeBuffer(Buffer::OwnedImpl& buffer, Writer& writer, const std::string& data) const; - template - const std::string buildMessage(const Stats::Metric& metric, ValueType value, + template + const std::string buildMessage(const StatType& metric, ValueType value, const std::string& type) const; - const std::string getName(const Stats::Metric& metric) const; + template const std::string getName(const StatType& metric) const; const std::string buildTagStr(const std::vector& tags) const; const ThreadLocal::SlotPtr tls_; diff --git a/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.cc b/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.cc index 78c533948cbe..e50a6a85de91 100644 --- a/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.cc +++ b/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.cc @@ -53,16 +53,26 @@ MetricsExportRequestPtr OtlpMetricsFlusherImpl::flush(Stats::MetricSnapshot& sna for (const auto& gauge : snapshot.gauges()) { if (predicate_(gauge)) { - flushGauge(*scope_metrics->add_metrics(), gauge, snapshot_time_ns); + flushGauge(*scope_metrics->add_metrics(), gauge.get(), snapshot_time_ns); } } + for (const auto& gauge : snapshot.hostGauges()) { + flushGauge(*scope_metrics->add_metrics(), gauge, snapshot_time_ns); + } + for (const auto& counter : snapshot.counters()) { if (predicate_(counter.counter_)) { - flushCounter(*scope_metrics->add_metrics(), counter, snapshot_time_ns); + flushCounter(*scope_metrics->add_metrics(), counter.counter_.get(), + counter.counter_.get().value(), counter.delta_, snapshot_time_ns); } } + for (const auto& counter : snapshot.hostCounters()) { + flushCounter(*scope_metrics->add_metrics(), counter, counter.value(), counter.delta(), + snapshot_time_ns); + } + for (const auto& histogram : snapshot.histograms()) { if (predicate_(histogram)) { flushHistogram(*scope_metrics->add_metrics(), histogram, snapshot_time_ns); @@ -72,8 +82,9 @@ MetricsExportRequestPtr OtlpMetricsFlusherImpl::flush(Stats::MetricSnapshot& sna return request; } +template void OtlpMetricsFlusherImpl::flushGauge(opentelemetry::proto::metrics::v1::Metric& metric, - const Stats::Gauge& gauge_stat, + const GaugeType& gauge_stat, int64_t snapshot_time_ns) const { auto* data_point = metric.mutable_gauge()->add_data_points(); data_point->set_time_unix_nano(snapshot_time_ns); @@ -82,21 +93,21 @@ void OtlpMetricsFlusherImpl::flushGauge(opentelemetry::proto::metrics::v1::Metri data_point->set_as_int(gauge_stat.value()); } -void OtlpMetricsFlusherImpl::flushCounter( - opentelemetry::proto::metrics::v1::Metric& metric, - const Stats::MetricSnapshot::CounterSnapshot& counter_snapshot, - int64_t snapshot_time_ns) const { +template +void OtlpMetricsFlusherImpl::flushCounter(opentelemetry::proto::metrics::v1::Metric& metric, + const CounterType& counter, uint64_t value, + uint64_t delta, int64_t snapshot_time_ns) const { auto* sum = metric.mutable_sum(); sum->set_is_monotonic(true); auto* data_point = sum->add_data_points(); - setMetricCommon(metric, *data_point, snapshot_time_ns, counter_snapshot.counter_); + setMetricCommon(metric, *data_point, snapshot_time_ns, counter); if (config_->reportCountersAsDeltas()) { sum->set_aggregation_temporality(AggregationTemporality::AGGREGATION_TEMPORALITY_DELTA); - data_point->set_as_int(counter_snapshot.delta_); + data_point->set_as_int(delta); } else { sum->set_aggregation_temporality(AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE); - data_point->set_as_int(counter_snapshot.counter_.get().value()); + data_point->set_as_int(value); } } @@ -120,16 +131,23 @@ void OtlpMetricsFlusherImpl::flushHistogram(opentelemetry::proto::metrics::v1::M data_point->set_sum(histogram_stats.sampleSum()); // TODO(ohadvano): support min/max optional fields for ``HistogramDataPoint`` + std::vector bucket_counts = histogram_stats.computeDisjointBuckets(); for (size_t i = 0; i < histogram_stats.supportedBuckets().size(); i++) { data_point->add_explicit_bounds(histogram_stats.supportedBuckets()[i]); - data_point->add_bucket_counts(histogram_stats.computedBuckets()[i]); + data_point->add_bucket_counts(bucket_counts[i]); } + + // According to the spec, the number of bucket counts needs to be one element bigger + // than the size of the explicit bounds, and the last bucket should contain the count + // of values which are outside the explicit boundaries (to +infinity). + data_point->add_bucket_counts(histogram_stats.outOfBoundCount()); } +template void OtlpMetricsFlusherImpl::setMetricCommon( opentelemetry::proto::metrics::v1::Metric& metric, opentelemetry::proto::metrics::v1::NumberDataPoint& data_point, int64_t snapshot_time_ns, - const Stats::Metric& stat) const { + const StatType& stat) const { data_point.set_time_unix_nano(snapshot_time_ns); // TODO(ohadvano): support ``start_time_unix_nano`` optional field metric.set_name(absl::StrCat(config_->statPrefix(), config_->useTagExtractedName() diff --git a/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.h b/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.h index c62114c05891..92ce4fe58d2e 100644 --- a/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.h +++ b/source/extensions/stat_sinks/open_telemetry/open_telemetry_impl.h @@ -79,20 +79,22 @@ class OtlpMetricsFlusherImpl : public OtlpMetricsFlusher { MetricsExportRequestPtr flush(Stats::MetricSnapshot& snapshot) const override; private: - void flushGauge(opentelemetry::proto::metrics::v1::Metric& metric, const Stats::Gauge& gauge, + template + void flushGauge(opentelemetry::proto::metrics::v1::Metric& metric, const GaugeType& gauge, int64_t snapshot_time_ns) const; - void flushCounter(opentelemetry::proto::metrics::v1::Metric& metric, - const Stats::MetricSnapshot::CounterSnapshot& counter_snapshot, - int64_t snapshot_time_ns) const; + template + void flushCounter(opentelemetry::proto::metrics::v1::Metric& metric, const CounterType& counter, + uint64_t value, uint64_t delta, int64_t snapshot_time_ns) const; void flushHistogram(opentelemetry::proto::metrics::v1::Metric& metric, const Stats::ParentHistogram& parent_histogram, int64_t snapshot_time_ns) const; + template void setMetricCommon(opentelemetry::proto::metrics::v1::Metric& metric, opentelemetry::proto::metrics::v1::NumberDataPoint& data_point, - int64_t snapshot_time_ns, const Stats::Metric& stat) const; + int64_t snapshot_time_ns, const StatType& stat) const; void setMetricCommon(opentelemetry::proto::metrics::v1::Metric& metric, opentelemetry::proto::metrics::v1::HistogramDataPoint& data_point, diff --git a/source/extensions/tracers/common/ot/BUILD b/source/extensions/tracers/common/ot/BUILD index c07fd828a0d7..2612bf77a461 100644 --- a/source/extensions/tracers/common/ot/BUILD +++ b/source/extensions/tracers/common/ot/BUILD @@ -28,5 +28,6 @@ envoy_cc_library( "//source/common/json:json_loader_lib", "//source/common/tracing:common_values_lib", "//source/common/tracing:null_span_lib", + "//source/common/tracing:trace_context_lib", ], ) diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc index 387be89f9d2b..7d1050e76ac8 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc @@ -10,6 +10,7 @@ #include "source/common/http/header_map_impl.h" #include "source/common/tracing/common_values.h" #include "source/common/tracing/null_span_impl.h" +#include "source/common/tracing/trace_context_impl.h" namespace Envoy { namespace Extensions { @@ -22,6 +23,10 @@ Http::RegisterCustomInlineHeader Set(opentracing::string_view key, opentracing::string_view value) const override { Http::LowerCaseString lowercase_key{{key.data(), key.size()}}; - trace_context_.setByKey(lowercase_key, {value.data(), value.size()}); + trace_context_.set(lowercase_key, {value.data(), value.size()}); return {}; } @@ -57,7 +62,7 @@ class OpenTracingHeadersReader : public opentracing::HTTPHeadersReader { opentracing::expected LookupKey(opentracing::string_view key) const override { Http::LowerCaseString lowercase_key{{key.data(), key.size()}}; - const auto entry = trace_context_.getByKey(lowercase_key); + const auto entry = trace_context_.get(lowercase_key); if (entry.has_value()) { return opentracing::string_view{entry.value().data(), entry.value().length()}; } else { @@ -120,9 +125,8 @@ void OpenTracingSpan::injectContext(Tracing::TraceContext& trace_context, return; } const std::string current_span_context = oss.str(); - trace_context.setByReferenceKey( - Http::CustomHeaders::get().OtSpanContext, - Base64::encode(current_span_context.c_str(), current_span_context.length())); + otSpanContextHeader().setRefKey( + trace_context, Base64::encode(current_span_context.c_str(), current_span_context.length())); } else { // Inject the context using the tracer's standard header format. const OpenTracingHeadersWriter writer{trace_context}; @@ -155,13 +159,13 @@ Tracing::SpanPtr OpenTracingDriver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) { + Tracing::Decision tracing_decision) { const PropagationMode propagation_mode = this->propagationMode(); const opentracing::Tracer& tracer = this->tracer(); std::unique_ptr active_span; std::unique_ptr parent_span_ctx; - const auto entry = trace_context.getByKey(Http::CustomHeaders::get().OtSpanContext); + const auto entry = otSpanContextHeader().get(trace_context); if (propagation_mode == PropagationMode::SingleHeader && entry.has_value()) { opentracing::expected> parent_span_ctx_maybe; std::string parent_context = Base64::decode(std::string(entry.value())); diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.h b/source/extensions/tracers/common/ot/opentracing_driver_impl.h index 94c58b44cf4f..360b1a15e7d6 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.h +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.h @@ -68,7 +68,7 @@ class OpenTracingDriver : public Tracing::Driver, protected Logger::Loggable #include "envoy/http/header_map.h" +#include "envoy/tracing/trace_context.h" #include "absl/strings/str_join.h" @@ -61,7 +62,7 @@ TraceContextReader::TraceContextReader(const Tracing::TraceContext& context) : c datadog::tracing::Optional TraceContextReader::lookup(datadog::tracing::StringView key) const { - return context_.getByKey(key); + return context_.get(key); } void TraceContextReader::visit( diff --git a/source/extensions/tracers/datadog/span.cc b/source/extensions/tracers/datadog/span.cc index 45922a20e620..38646f5cf844 100644 --- a/source/extensions/tracers/datadog/span.cc +++ b/source/extensions/tracers/datadog/span.cc @@ -23,7 +23,7 @@ class TraceContextWriter : public datadog::tracing::DictWriter { explicit TraceContextWriter(Tracing::TraceContext& context) : context_(context) {} void set(datadog::tracing::StringView key, datadog::tracing::StringView value) override { - context_.setByKey(key, value); + context_.set(key, value); } private: @@ -41,7 +41,9 @@ void Span::setOperation(absl::string_view operation) { return; } - span_->set_name(operation); + // What Envoy calls the operation name more closely corresponds to what + // Datadog calls the resource name. + span_->set_resource_name(operation); } void Span::setTag(absl::string_view name, absl::string_view value) { @@ -49,7 +51,17 @@ void Span::setTag(absl::string_view name, absl::string_view value) { return; } - span_->set_tag(name, value); + // The special "resource.name" tag is a holdover from when the Datadog tracer + // was OpenTracing-based, and so there was no way to set the Datadog resource + // name directly. + // In Envoy, it's still the case that there's no way to set the Datadog + // resource name directly; so, here if the tag name is "resource.name", we + // actually set the resource name instead of setting a tag. + if (name == "resource.name") { + span_->set_resource_name(value); + } else { + span_->set_tag(name, value); + } } void Span::log(SystemTime, const std::string&) { @@ -78,8 +90,13 @@ Tracing::SpanPtr Span::spawnChild(const Tracing::Config&, const std::string& nam // The OpenTracing implementation ignored the `Tracing::Config` argument, // so we will as well. + // The `name` parameter to this function more closely matches Datadog's + // concept of "resource name." Datadog's "span name," or "operation name," + // instead describes the category of operation being performed, which here + // we hard-code. datadog::tracing::SpanConfig config; - config.name = name; + config.name = "envoy.proxy"; + config.resource = name; config.start = estimateTime(start_time); return std::make_unique(span_->create_child(config)); diff --git a/source/extensions/tracers/datadog/tracer.cc b/source/extensions/tracers/datadog/tracer.cc index 180a2c5a3c36..f22b09756958 100644 --- a/source/extensions/tracers/datadog/tracer.cc +++ b/source/extensions/tracers/datadog/tracer.cc @@ -20,6 +20,7 @@ #include "datadog/sampling_priority.h" #include "datadog/span_config.h" #include "datadog/trace_segment.h" +#include "datadog/tracer_config.h" namespace Envoy { namespace Extensions { @@ -77,7 +78,7 @@ Tracer::Tracer(const std::string& collector_cluster, const std::string& collecto Tracing::SpanPtr Tracer::startSpan(const Tracing::Config&, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) { + Tracing::Decision tracing_decision) { ThreadLocalTracer& thread_local_tracer = **thread_local_slot_; if (!thread_local_tracer.tracer) { return std::make_unique(); @@ -86,11 +87,35 @@ Tracing::SpanPtr Tracer::startSpan(const Tracing::Config&, Tracing::TraceContext // The OpenTracing implementation ignored the `Tracing::Config` argument, // so we will as well. datadog::tracing::SpanConfig span_config; - span_config.name = operation_name; + // The `operation_name` parameter to this function more closely matches + // Datadog's concept of "resource name." Datadog's "span name," or "operation + // name," instead describes the category of operation being performed, which + // here we hard-code. + span_config.name = "envoy.proxy"; + span_config.resource = operation_name; span_config.start = estimateTime(stream_info.startTime()); - datadog::tracing::Tracer& tracer = *thread_local_tracer.tracer; TraceContextReader reader{trace_context}; + datadog::tracing::Span span = + extract_or_create_span(*thread_local_tracer.tracer, span_config, reader); + + // If we did not extract a sampling decision, and if Envoy is telling us to + // drop the trace, then we treat that as a "user drop" (manual override). + // + // If Envoy is telling us to keep the trace, then we leave it up to the + // tracer's internal sampler (which might decide to drop the trace anyway). + if (!span.trace_segment().sampling_decision().has_value() && !tracing_decision.traced) { + span.trace_segment().override_sampling_priority( + int(datadog::tracing::SamplingPriority::USER_DROP)); + } + + return std::make_unique(std::move(span)); +} + +datadog::tracing::Span +Tracer::extract_or_create_span(datadog::tracing::Tracer& tracer, + const datadog::tracing::SpanConfig& span_config, + const datadog::tracing::DictReader& reader) { datadog::tracing::Expected maybe_span = tracer.extract_span(reader, span_config); if (datadog::tracing::Error* error = maybe_span.if_error()) { @@ -106,23 +131,10 @@ Tracing::SpanPtr Tracer::startSpan(const Tracing::Config&, Tracing::TraceContext int(error->code), error->message); } - maybe_span = tracer.create_span(span_config); - } - - ASSERT(maybe_span); - datadog::tracing::Span& span = *maybe_span; - - // If Envoy is telling us to drop the trace, then we treat that as a - // "user drop" (manual override). - // - // If Envoy is telling us to keep the trace, then we leave it up to the - // tracer's internal sampler (which might decide to drop the trace anyway). - if (!tracing_decision.traced) { - span.trace_segment().override_sampling_priority( - int(datadog::tracing::SamplingPriority::USER_DROP)); + return tracer.create_span(span_config); } - return std::make_unique(std::move(span)); + return std::move(*maybe_span); } } // namespace Datadog diff --git a/source/extensions/tracers/datadog/tracer.h b/source/extensions/tracers/datadog/tracer.h index 9e822cb9a0df..f04f7253b51e 100644 --- a/source/extensions/tracers/datadog/tracer.h +++ b/source/extensions/tracers/datadog/tracer.h @@ -10,7 +10,18 @@ #include "source/extensions/tracers/datadog/tracer_stats.h" #include "datadog/tracer.h" -#include "datadog/tracer_config.h" + +namespace datadog { +namespace tracing { + +class DictReader; +class FinalizedTracerConfig; +class Span; +struct SpanConfig; +struct TracerConfig; + +} // namespace tracing +} // namespace datadog namespace Envoy { namespace Extensions { @@ -73,8 +84,8 @@ class Tracer : public Tracing::Driver, private Logger::Loggable thread_local_slot_; }; diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 436b82666732..5c4ea0b4dfba 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -7,6 +7,7 @@ #include "source/common/common/base64.h" #include "source/common/common/empty_string.h" +#include "source/common/tracing/trace_context_impl.h" #include "absl/strings/str_cat.h" #include "google/devtools/cloudtrace/v2/tracing.grpc.pb.h" @@ -41,13 +42,13 @@ namespace { class ConstantValues { public: - const Http::LowerCaseString TRACEPARENT{"traceparent"}; - const Http::LowerCaseString GRPC_TRACE_BIN{"grpc-trace-bin"}; - const Http::LowerCaseString X_CLOUD_TRACE_CONTEXT{"x-cloud-trace-context"}; - const Http::LowerCaseString X_B3_TRACEID{"x-b3-traceid"}; - const Http::LowerCaseString X_B3_SPANID{"x-b3-spanid"}; - const Http::LowerCaseString X_B3_SAMPLED{"x-b3-sampled"}; - const Http::LowerCaseString X_B3_FLAGS{"x-b3-flags"}; + const Tracing::TraceContextHandler TRACEPARENT{"traceparent"}; + const Tracing::TraceContextHandler GRPC_TRACE_BIN{"grpc-trace-bin"}; + const Tracing::TraceContextHandler X_CLOUD_TRACE_CONTEXT{"x-cloud-trace-context"}; + const Tracing::TraceContextHandler X_B3_TRACEID{"x-b3-traceid"}; + const Tracing::TraceContextHandler X_B3_SPANID{"x-b3-spanid"}; + const Tracing::TraceContextHandler X_B3_SAMPLED{"x-b3-sampled"}; + const Tracing::TraceContextHandler X_B3_FLAGS{"x-b3-flags"}; }; using Constants = ConstSingleton; @@ -96,7 +97,7 @@ startSpanHelper(const std::string& name, bool traced, const Tracing::TraceContex bool found = false; switch (incoming) { case OpenCensusConfig::TRACE_CONTEXT: { - const auto entry = trace_context.getByKey(Constants::get().TRACEPARENT); + const auto entry = Constants::get().TRACEPARENT.get(trace_context); if (entry.has_value()) { found = true; // This is an implicitly untrusted header, so only the first value is used. @@ -105,7 +106,7 @@ startSpanHelper(const std::string& name, bool traced, const Tracing::TraceContex break; } case OpenCensusConfig::GRPC_TRACE_BIN: { - const auto entry = trace_context.getByKey(Constants::get().GRPC_TRACE_BIN); + const auto entry = Constants::get().GRPC_TRACE_BIN.get(trace_context); if (entry.has_value()) { found = true; // This is an implicitly untrusted header, so only the first value is used. @@ -115,7 +116,7 @@ startSpanHelper(const std::string& name, bool traced, const Tracing::TraceContex break; } case OpenCensusConfig::CLOUD_TRACE_CONTEXT: { - const auto entry = trace_context.getByKey(Constants::get().X_CLOUD_TRACE_CONTEXT); + const auto entry = Constants::get().X_CLOUD_TRACE_CONTEXT.get(trace_context); if (entry.has_value()) { found = true; // This is an implicitly untrusted header, so only the first value is used. @@ -129,19 +130,19 @@ startSpanHelper(const std::string& name, bool traced, const Tracing::TraceContex absl::string_view b3_span_id; absl::string_view b3_sampled; absl::string_view b3_flags; - const auto h_b3_trace_id = trace_context.getByKey(Constants::get().X_B3_TRACEID); + const auto h_b3_trace_id = Constants::get().X_B3_TRACEID.get(trace_context); if (h_b3_trace_id.has_value()) { b3_trace_id = h_b3_trace_id.value(); } - const auto h_b3_span_id = trace_context.getByKey(Constants::get().X_B3_SPANID); + const auto h_b3_span_id = Constants::get().X_B3_SPANID.get(trace_context); if (h_b3_span_id.has_value()) { b3_span_id = h_b3_span_id.value(); } - const auto h_b3_sampled = trace_context.getByKey(Constants::get().X_B3_SAMPLED); + const auto h_b3_sampled = Constants::get().X_B3_SAMPLED.get(trace_context); if (h_b3_sampled.has_value()) { b3_sampled = h_b3_sampled.value(); } - const auto h_b3_flags = trace_context.getByKey(Constants::get().X_B3_FLAGS); + const auto h_b3_flags = Constants::get().X_B3_FLAGS.get(trace_context); if (h_b3_flags.has_value()) { b3_flags = h_b3_flags.value(); } @@ -209,27 +210,26 @@ void Span::injectContext(Tracing::TraceContext& trace_context, for (const auto& outgoing : oc_config_.outgoing_trace_context()) { switch (outgoing) { case OpenCensusConfig::TRACE_CONTEXT: - trace_context.setByReferenceKey(Constants::get().TRACEPARENT, - ::opencensus::trace::propagation::ToTraceParentHeader(ctx)); + Constants::get().TRACEPARENT.setRefKey( + trace_context, ::opencensus::trace::propagation::ToTraceParentHeader(ctx)); break; case OpenCensusConfig::GRPC_TRACE_BIN: { std::string val = ::opencensus::trace::propagation::ToGrpcTraceBinHeader(ctx); val = Base64::encode(val.data(), val.size(), /*add_padding=*/false); - trace_context.setByReferenceKey(Constants::get().GRPC_TRACE_BIN, val); + Constants::get().GRPC_TRACE_BIN.setRefKey(trace_context, val); break; } case OpenCensusConfig::CLOUD_TRACE_CONTEXT: - trace_context.setByReferenceKey( - Constants::get().X_CLOUD_TRACE_CONTEXT, - ::opencensus::trace::propagation::ToCloudTraceContextHeader(ctx)); + Constants::get().X_CLOUD_TRACE_CONTEXT.setRefKey( + trace_context, ::opencensus::trace::propagation::ToCloudTraceContextHeader(ctx)); break; case OpenCensusConfig::B3: - trace_context.setByReferenceKey(Constants::get().X_B3_TRACEID, - ::opencensus::trace::propagation::ToB3TraceIdHeader(ctx)); - trace_context.setByReferenceKey(Constants::get().X_B3_SPANID, - ::opencensus::trace::propagation::ToB3SpanIdHeader(ctx)); - trace_context.setByReferenceKey(Constants::get().X_B3_SAMPLED, - ::opencensus::trace::propagation::ToB3SampledHeader(ctx)); + Constants::get().X_B3_TRACEID.setRefKey( + trace_context, ::opencensus::trace::propagation::ToB3TraceIdHeader(ctx)); + Constants::get().X_B3_SPANID.setRefKey( + trace_context, ::opencensus::trace::propagation::ToB3SpanIdHeader(ctx)); + Constants::get().X_B3_SAMPLED.setRefKey( + trace_context, ::opencensus::trace::propagation::ToB3SampledHeader(ctx)); // OpenCensus's trace context propagation doesn't produce the // "X-B3-Flags:" header. break; @@ -377,7 +377,7 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) { + Tracing::Decision tracing_decision) { return std::make_unique(config, oc_config_, trace_context, operation_name, stream_info.startTime(), tracing_decision); } diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h index 1a392ce9f59d..db3da7856a86 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h @@ -24,7 +24,7 @@ class Driver : public Tracing::Driver, Logger::Loggable { Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) override; + Tracing::Decision tracing_decision) override; private: void applyTraceConfig(const opencensus::proto::trace::v1::TraceConfig& config); diff --git a/source/extensions/tracers/opentelemetry/BUILD b/source/extensions/tracers/opentelemetry/BUILD index 3a36a3f91fd5..e84423ccf272 100644 --- a/source/extensions/tracers/opentelemetry/BUILD +++ b/source/extensions/tracers/opentelemetry/BUILD @@ -36,24 +36,40 @@ envoy_cc_library( "tracer.h", ], deps = [ - ":grpc_trace_exporter", + ":trace_exporter", "//envoy/thread_local:thread_local_interface", "//source/common/config:utility_lib", "//source/common/tracing:http_tracer_lib", "//source/extensions/tracers/common:factory_base_lib", + "//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib", + "//source/extensions/tracers/opentelemetry/samplers:sampler_lib", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", "@opentelemetry_proto//:trace_cc_proto", ], ) envoy_cc_library( - name = "grpc_trace_exporter", - srcs = ["grpc_trace_exporter.cc"], - hdrs = ["grpc_trace_exporter.h"], + name = "trace_exporter", + srcs = [ + "grpc_trace_exporter.cc", + "http_trace_exporter.cc", + ], + hdrs = [ + "grpc_trace_exporter.h", + "http_trace_exporter.h", + "trace_exporter.h", + ], deps = [ "//envoy/grpc:async_client_manager_interface", + "//envoy/upstream:cluster_manager_interface", "//source/common/grpc:typed_async_client_lib", + "//source/common/http:async_client_utility_lib", + "//source/common/http:header_map_lib", + "//source/common/http:message_lib", + "//source/common/http:utility_lib", "//source/common/protobuf", + "//source/common/tracing:trace_context_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@opentelemetry_proto//:trace_cc_proto", ], ) diff --git a/source/extensions/tracers/opentelemetry/grpc_trace_exporter.cc b/source/extensions/tracers/opentelemetry/grpc_trace_exporter.cc index ac418f33ff86..5b7e6bda0669 100644 --- a/source/extensions/tracers/opentelemetry/grpc_trace_exporter.cc +++ b/source/extensions/tracers/opentelemetry/grpc_trace_exporter.cc @@ -1,4 +1,3 @@ -#include "grpc_trace_exporter.h" #include "source/extensions/tracers/opentelemetry/grpc_trace_exporter.h" #include "source/common/common/logger.h" diff --git a/source/extensions/tracers/opentelemetry/grpc_trace_exporter.h b/source/extensions/tracers/opentelemetry/grpc_trace_exporter.h index 2d6ff1be8977..ecfc4baaa27e 100644 --- a/source/extensions/tracers/opentelemetry/grpc_trace_exporter.h +++ b/source/extensions/tracers/opentelemetry/grpc_trace_exporter.h @@ -4,6 +4,7 @@ #include "source/common/common/logger.h" #include "source/common/grpc/typed_async_client.h" +#include "source/extensions/tracers/opentelemetry/trace_exporter.h" #include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" @@ -80,18 +81,16 @@ class OpenTelemetryGrpcTraceExporterClient : Logger::Loggable { +class OpenTelemetryGrpcTraceExporter : public OpenTelemetryTraceExporter { public: OpenTelemetryGrpcTraceExporter(const Grpc::RawAsyncClientSharedPtr& client); - bool log(const ExportTraceServiceRequest& request); + bool log(const ExportTraceServiceRequest& request) override; private: OpenTelemetryGrpcTraceExporterClient client_; }; -using OpenTelemetryGrpcTraceExporterPtr = std::unique_ptr; - } // namespace OpenTelemetry } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/opentelemetry/http_trace_exporter.cc b/source/extensions/tracers/opentelemetry/http_trace_exporter.cc new file mode 100644 index 000000000000..b1fb4efdc9c1 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/http_trace_exporter.cc @@ -0,0 +1,94 @@ +#include "source/extensions/tracers/opentelemetry/http_trace_exporter.h" + +#include +#include +#include +#include + +#include "source/common/common/enum_to_int.h" +#include "source/common/common/logger.h" +#include "source/common/protobuf/protobuf.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +OpenTelemetryHttpTraceExporter::OpenTelemetryHttpTraceExporter( + Upstream::ClusterManager& cluster_manager, + const envoy::config::core::v3::HttpService& http_service) + : cluster_manager_(cluster_manager), http_service_(http_service) { + + // Prepare and store headers to be used later on each export request + for (const auto& header_value_option : http_service_.request_headers_to_add()) { + parsed_headers_to_add_.push_back({Http::LowerCaseString(header_value_option.header().key()), + header_value_option.header().value()}); + } +} + +bool OpenTelemetryHttpTraceExporter::log(const ExportTraceServiceRequest& request) { + std::string request_body; + + const auto ok = request.SerializeToString(&request_body); + if (!ok) { + ENVOY_LOG(warn, "Error while serializing the binary proto ExportTraceServiceRequest."); + return false; + } + + const auto thread_local_cluster = + cluster_manager_.getThreadLocalCluster(http_service_.http_uri().cluster()); + if (thread_local_cluster == nullptr) { + ENVOY_LOG(error, "OTLP HTTP exporter failed: [cluster = {}] is not configured", + http_service_.http_uri().cluster()); + return false; + } + + Http::RequestMessagePtr message = Http::Utility::prepareHeaders(http_service_.http_uri()); + + // The request follows the OTLP HTTP specification: + // https://github.com/open-telemetry/opentelemetry-proto/blob/v1.0.0/docs/specification.md#otlphttp. + message->headers().setReferenceMethod(Http::Headers::get().MethodValues.Post); + message->headers().setReferenceContentType(Http::Headers::get().ContentTypeValues.Protobuf); + + // Add all custom headers to the request. + for (const auto& header_pair : parsed_headers_to_add_) { + message->headers().setReference(header_pair.first, header_pair.second); + } + message->body().add(request_body); + + const auto options = Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds( + DurationUtil::durationToMilliseconds(http_service_.http_uri().timeout()))); + + Http::AsyncClient::Request* in_flight_request = + thread_local_cluster->httpAsyncClient().send(std::move(message), *this, options); + + if (in_flight_request == nullptr) { + return false; + } + + active_requests_.add(*in_flight_request); + return true; +} + +void OpenTelemetryHttpTraceExporter::onSuccess(const Http::AsyncClient::Request& request, + Http::ResponseMessagePtr&& http_response) { + active_requests_.remove(request); + const auto response_code = Http::Utility::getResponseStatus(http_response->headers()); + if (response_code != enumToInt(Http::Code::OK)) { + ENVOY_LOG(error, + "OTLP HTTP exporter received a non-success status code: {} while exporting the OTLP " + "message", + response_code); + } +} + +void OpenTelemetryHttpTraceExporter::onFailure(const Http::AsyncClient::Request& request, + Http::AsyncClient::FailureReason reason) { + active_requests_.remove(request); + ENVOY_LOG(debug, "The OTLP export request failed. Reason {}", enumToInt(reason)); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/http_trace_exporter.h b/source/extensions/tracers/opentelemetry/http_trace_exporter.h new file mode 100644 index 000000000000..ee5a5cf36564 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/http_trace_exporter.h @@ -0,0 +1,48 @@ +#pragma once + +#include "envoy/config/core/v3/http_service.pb.h" +#include "envoy/upstream/cluster_manager.h" + +#include "source/common/common/logger.h" +#include "source/common/http/async_client_impl.h" +#include "source/common/http/async_client_utility.h" +#include "source/common/http/headers.h" +#include "source/common/http/message_impl.h" +#include "source/common/http/utility.h" +#include "source/extensions/tracers/opentelemetry/trace_exporter.h" + +#include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * Exporter for OTLP traces over HTTP. + */ +class OpenTelemetryHttpTraceExporter : public OpenTelemetryTraceExporter, + public Http::AsyncClient::Callbacks { +public: + OpenTelemetryHttpTraceExporter(Upstream::ClusterManager& cluster_manager, + const envoy::config::core::v3::HttpService& http_service); + + bool log(const ExportTraceServiceRequest& request) override; + + // Http::AsyncClient::Callbacks. + void onSuccess(const Http::AsyncClient::Request&, Http::ResponseMessagePtr&&) override; + void onFailure(const Http::AsyncClient::Request&, Http::AsyncClient::FailureReason) override; + void onBeforeFinalizeUpstreamSpan(Tracing::Span&, const Http::ResponseHeaderMap*) override {} + +private: + Upstream::ClusterManager& cluster_manager_; + envoy::config::core::v3::HttpService http_service_; + // Track active HTTP requests to be able to cancel them on destruction. + Http::AsyncClientRequestTracker active_requests_; + std::vector> parsed_headers_to_add_; +}; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc index a3119c187e83..46ce3c35ae77 100644 --- a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc +++ b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.cc @@ -2,33 +2,92 @@ #include +#include "envoy/common/optref.h" #include "envoy/config/trace/v3/opentelemetry.pb.h" #include "source/common/common/empty_string.h" #include "source/common/common/logger.h" #include "source/common/config/utility.h" #include "source/common/tracing/http_tracer_impl.h" +#include "source/extensions/tracers/opentelemetry/grpc_trace_exporter.h" +#include "source/extensions/tracers/opentelemetry/http_trace_exporter.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h" +#include "source/extensions/tracers/opentelemetry/samplers/sampler.h" +#include "source/extensions/tracers/opentelemetry/span_context.h" +#include "source/extensions/tracers/opentelemetry/span_context_extractor.h" +#include "source/extensions/tracers/opentelemetry/trace_exporter.h" +#include "source/extensions/tracers/opentelemetry/tracer.h" #include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" #include "opentelemetry/proto/trace/v1/trace.pb.h" -#include "span_context.h" -#include "span_context_extractor.h" -#include "tracer.h" namespace Envoy { namespace Extensions { namespace Tracers { namespace OpenTelemetry { +namespace { + +SamplerSharedPtr +tryCreateSamper(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context) { + SamplerSharedPtr sampler; + if (opentelemetry_config.has_sampler()) { + auto& sampler_config = opentelemetry_config.sampler(); + auto* factory = Envoy::Config::Utility::getFactory(sampler_config); + if (!factory) { + throw EnvoyException(fmt::format("Sampler factory not found: '{}'", sampler_config.name())); + } + sampler = factory->createSampler(sampler_config.typed_config(), context); + } + return sampler; +} + +OTelSpanKind getSpanKind(const Tracing::Config& config) { + // If this is downstream span that be created by 'startSpan' for downstream request, then + // set the span type based on the spawnUpstreamSpan flag and traffic direction: + // * If separate tracing span will be created for upstream request, then set span type to + // SERVER because the downstream span should be server span in trace chain. + // * If separate tracing span will not be created for upstream request, that means the + // Envoy will not be treated as independent hop in trace chain and then set span type + // based on the traffic direction. + return (config.spawnUpstreamSpan() ? ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER + : config.operationName() == Tracing::OperationName::Egress + ? ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT + : ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER); +} + +} // namespace + Driver::Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, Server::Configuration::TracerFactoryContext& context) + : Driver(opentelemetry_config, context, ResourceProviderImpl{}) {} + +Driver::Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context, + const ResourceProvider& resource_provider) : tls_slot_ptr_(context.serverFactoryContext().threadLocal().allocateSlot()), tracing_stats_{OPENTELEMETRY_TRACER_STATS( POOL_COUNTER_PREFIX(context.serverFactoryContext().scope(), "tracing.opentelemetry"))} { auto& factory_context = context.serverFactoryContext(); + + Resource resource = resource_provider.getResource(opentelemetry_config, context); + ResourceConstSharedPtr resource_ptr = std::make_shared(std::move(resource)); + + if (opentelemetry_config.has_grpc_service() && opentelemetry_config.has_http_service()) { + throw EnvoyException( + "OpenTelemetry Tracer cannot have both gRPC and HTTP exporters configured. " + "OpenTelemetry tracer will be disabled."); + } + + // Create the sampler if configured + SamplerSharedPtr sampler = tryCreateSamper(opentelemetry_config, context); + // Create the tracer in Thread Local Storage. - tls_slot_ptr_->set([opentelemetry_config, &factory_context, this](Event::Dispatcher& dispatcher) { - OpenTelemetryGrpcTraceExporterPtr exporter; + tls_slot_ptr_->set([opentelemetry_config, &factory_context, this, resource_ptr, + sampler](Event::Dispatcher& dispatcher) { + OpenTelemetryTraceExporterPtr exporter; if (opentelemetry_config.has_grpc_service()) { Grpc::AsyncClientFactoryPtr&& factory = factory_context.clusterManager().grpcAsyncClientManager().factoryForGrpcService( @@ -36,11 +95,13 @@ Driver::Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetr const Grpc::RawAsyncClientSharedPtr& async_client_shared_ptr = factory->createUncachedRawAsyncClient(); exporter = std::make_unique(async_client_shared_ptr); + } else if (opentelemetry_config.has_http_service()) { + exporter = std::make_unique( + factory_context.clusterManager(), opentelemetry_config.http_service()); } TracerPtr tracer = std::make_unique( std::move(exporter), factory_context.timeSource(), factory_context.api().randomGenerator(), - factory_context.runtime(), dispatcher, tracing_stats_, opentelemetry_config.service_name()); - + factory_context.runtime(), dispatcher, tracing_stats_, resource_ptr, sampler); return std::make_shared(std::move(tracer)); }); } @@ -49,22 +110,22 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) { + Tracing::Decision tracing_decision) { // Get tracer from TLS and start span. auto& tracer = tls_slot_ptr_->getTyped().tracer(); SpanContextExtractor extractor(trace_context); + const auto span_kind = getSpanKind(config); if (!extractor.propagationHeaderPresent()) { // No propagation header, so we can create a fresh span with the given decision. - Tracing::SpanPtr new_open_telemetry_span = - tracer.startSpan(config, operation_name, stream_info.startTime(), tracing_decision); - new_open_telemetry_span->setSampled(tracing_decision.traced); + Tracing::SpanPtr new_open_telemetry_span = tracer.startSpan( + operation_name, stream_info.startTime(), tracing_decision, trace_context, span_kind); return new_open_telemetry_span; } else { // Try to extract the span context. If we can't, just return a null span. absl::StatusOr span_context = extractor.extractSpanContext(); if (span_context.ok()) { - return tracer.startSpan(config, operation_name, stream_info.startTime(), - span_context.value()); + return tracer.startSpan(operation_name, stream_info.startTime(), span_context.value(), + trace_context, span_kind); } else { ENVOY_LOG(trace, "Unable to extract span context: ", span_context.status()); return std::make_unique(); diff --git a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h index 35f734c87b82..d1d8b6c10a18 100644 --- a/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h +++ b/source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h @@ -9,6 +9,7 @@ #include "source/common/singleton/const_singleton.h" #include "source/extensions/tracers/common/factory_base.h" #include "source/extensions/tracers/opentelemetry/grpc_trace_exporter.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h" #include "source/extensions/tracers/opentelemetry/tracer.h" namespace Envoy { @@ -16,14 +17,6 @@ namespace Extensions { namespace Tracers { namespace OpenTelemetry { -class OpenTelemetryConstantValues { -public: - const Http::LowerCaseString TRACE_PARENT{"traceparent"}; - const Http::LowerCaseString TRACE_STATE{"tracestate"}; -}; - -using OpenTelemetryConstants = ConstSingleton; - /** * OpenTelemetry tracing driver. */ @@ -32,11 +25,15 @@ class Driver : Logger::Loggable, public Tracing::Driver { Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, Server::Configuration::TracerFactoryContext& context); + Driver(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context, + const ResourceProvider& resource_provider); + // Tracing::Driver Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) override; + Tracing::Decision tracing_decision) override; private: class TlsTracer : public ThreadLocal::ThreadLocalObject { diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/BUILD b/source/extensions/tracers/opentelemetry/resource_detectors/BUILD new file mode 100644 index 000000000000..c8b064de43e4 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/BUILD @@ -0,0 +1,27 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "resource_detector_lib", + srcs = [ + "resource_provider.cc", + ], + hdrs = [ + "resource_detector.h", + "resource_provider.h", + ], + deps = [ + "//envoy/config:typed_config_interface", + "//envoy/server:tracer_config_interface", + "//source/common/common:logger_lib", + "//source/common/config:utility_lib", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/BUILD b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/BUILD new file mode 100644 index 000000000000..d9eaeeacb3ce --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/BUILD @@ -0,0 +1,39 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":dynatrace_resource_detector_lib", + "//envoy/registry", + "//source/common/config:utility_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "dynatrace_resource_detector_lib", + srcs = [ + "dynatrace_metadata_file_reader.cc", + "dynatrace_resource_detector.cc", + ], + hdrs = [ + "dynatrace_metadata_file_reader.h", + "dynatrace_resource_detector.h", + ], + deps = [ + "//source/common/config:datasource_lib", + "//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.cc b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.cc new file mode 100644 index 000000000000..564c2cf291cd --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.cc @@ -0,0 +1,37 @@ +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.h" + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.pb.h" +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.pb.validate.h" + +#include "source/common/config/utility.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +ResourceDetectorPtr DynatraceResourceDetectorFactory::createResourceDetector( + const Protobuf::Message& message, Server::Configuration::TracerFactoryContext& context) { + + auto mptr = Envoy::Config::Utility::translateAnyToFactoryConfig( + dynamic_cast(message), context.messageValidationVisitor(), *this); + + const auto& proto_config = MessageUtil::downcastAndValidate< + const envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + DynatraceResourceDetectorConfig&>(*mptr, context.messageValidationVisitor()); + + DynatraceMetadataFileReaderPtr reader = std::make_unique(); + return std::make_unique(proto_config, std::move(reader)); +} + +/** + * Static registration for the Dynatrace resource detector factory. @see RegisterFactory. + */ +REGISTER_FACTORY(DynatraceResourceDetectorFactory, ResourceDetectorFactory); + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.h b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.h new file mode 100644 index 000000000000..f3063484aeb9 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.h @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.pb.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * Config registration for the Dynatrace resource detector. @see ResourceDetectorFactory. + */ +class DynatraceResourceDetectorFactory : public ResourceDetectorFactory { +public: + /** + * @brief Creates a Resource Detector that reads from the Dynatrace enrichment files. + * + * @see + * https://docs.dynatrace.com/docs/shortlink/enrichment-files#oneagent-virtual-files + * + * @param message The resource detector configuration. + * @param context The tracer factory context. + * @return ResourceDetectorPtr + */ + ResourceDetectorPtr + createResourceDetector(const Protobuf::Message& message, + Server::Configuration::TracerFactoryContext& context) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { + return "envoy.tracers.opentelemetry.resource_detectors.dynatrace"; + } +}; + +DECLARE_FACTORY(DynatraceResourceDetectorFactory); + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.cc b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.cc new file mode 100644 index 000000000000..7bac5001fc8c --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.cc @@ -0,0 +1,54 @@ +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h" + +#include +#include +#include +#include + +#include "envoy/common/exception.h" + +#include "source/common/common/logger.h" + +#include "absl/strings/str_cat.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +namespace { + +bool isIndirectionFile(const std::string& file_name) { + return absl::StartsWith(file_name, "dt_metadata_"); +} + +std::string readFile(const std::string& file_name) { + if (file_name.empty()) { + return ""; + } + + std::ifstream file(file_name); + if (file.fail()) { + throw EnvoyException(absl::StrCat("Unable to read Dynatrace enrichment file: ", file_name)); + } + + std::stringstream file_string; + file_string << file.rdbuf(); + + return file_string.str(); +} + +} // namespace + +std::string DynatraceMetadataFileReaderImpl::readEnrichmentFile(const std::string& file_name) { + if (const bool indirection_file = isIndirectionFile(file_name); indirection_file) { + return readFile(readFile(file_name)); + } else { + return readFile(file_name); + } +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h new file mode 100644 index 000000000000..6408480c284a --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/pure.h" + +#include "absl/strings/match.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * @brief A file reader that reads the content of the Dynatrace metadata enrichment files. + * When OneAgent is monitoring your application, it provides access to the enrichment files. + * These files do not physically exists in your file system, but are provided by the OneAgent on + * demand. This allows obtaining not only information about the host, but also about process. + * + * @see + * https://docs.dynatrace.com/docs/shortlink/enrichment-files#oneagent-virtual-files + * + */ +class DynatraceMetadataFileReader { +public: + virtual ~DynatraceMetadataFileReader() = default; + + /** + * @brief Reads the enrichment file and returns the enrichment metadata. + * + * @param file_name The file name. + * @return const std::string String (java-like properties) containing the enrichment metadata. + */ + virtual std::string readEnrichmentFile(const std::string& file_name) PURE; +}; + +using DynatraceMetadataFileReaderPtr = std::unique_ptr; + +class DynatraceMetadataFileReaderImpl : public DynatraceMetadataFileReader { +public: + std::string readEnrichmentFile(const std::string& file_name) override; +}; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.cc b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.cc new file mode 100644 index 000000000000..b8c98d81201b --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.cc @@ -0,0 +1,54 @@ +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.h" + +#include +#include + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { +namespace { + +void addAttributes(const std::string& content, Resource& resource) { + for (const auto& line : StringUtil::splitToken(content, "\n", false, true)) { + const auto key_value = StringUtil::splitToken(line, "="); + if (key_value.size() != 2) { + continue; + } + resource.attributes_[std::string(key_value[0])] = std::string(key_value[1]); + } +} + +} // namespace + +Resource DynatraceResourceDetector::detect() { + Resource resource; + resource.schema_url_ = ""; + int failure_count = 0; + + for (const auto& file_name : DynatraceResourceDetector::dynatraceMetadataFiles()) { + TRY_NEEDS_AUDIT { + std::string content = dynatrace_file_reader_->readEnrichmentFile(file_name); + if (content.empty()) { + failure_count++; + } else { + addAttributes(content, resource); + } + } + END_TRY catch (const EnvoyException&) { failure_count++; } + } + + if (failure_count > 0) { + ENVOY_LOG( + warn, + "Dynatrace OpenTelemetry resource detector is configured but could not detect attributes. " + "Check the Dynatrace deployment status to ensure it is correctly deployed."); + } + + return resource; +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.h b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.h new file mode 100644 index 000000000000..ba201fe884e5 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.pb.h" + +#include "source/common/common/logger.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * @brief A resource detector that reads the content of the Dynatrace metadata enrichment files. + * When OneAgent is monitoring your application, it provides access to the enrichment files. + * These files do not physically exists in your file system, but are provided by the OneAgent on + * demand. This allows obtaining not only information about the host, but also about process. + * + * Dynatrace can be deployed in multiple ways and flavors, depending on the environment. + * The available Dynatrace enrichment files vary depending on how it is deployed. E.g. In k8s via + * the Dynatrace operator. + * + * Since the resource detector is not aware how Dynatrace is deployed, the detector attempts to + * read all Dynatrace enrichment files. In such cases, reading some of these files may fail but + * this is expected and does not classify as a problem with the detector. The detector may also not + * detect any attributes, for example when a Dynatrace deployment is not successful. In such cases, + * Envoy will be started with no enrichment but Dynatrace users have the means to find out which + * Dynatrace deployment is failing. + * + * @see + * https://docs.dynatrace.com/docs/shortlink/enrichment-files + * + */ +class DynatraceResourceDetector : public ResourceDetector, Logger::Loggable { +public: + DynatraceResourceDetector(const envoy::extensions::tracers::opentelemetry::resource_detectors:: + v3::DynatraceResourceDetectorConfig& config, + DynatraceMetadataFileReaderPtr dynatrace_file_reader) + : config_(config), dynatrace_file_reader_(std::move(dynatrace_file_reader)) {} + Resource detect() override; + + static const std::vector& dynatraceMetadataFiles() { + CONSTRUCT_ON_FIRST_USE(std::vector, + { + "dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties", + "/var/lib/dynatrace/enrichment/dt_metadata.properties", + "/var/lib/dynatrace/enrichment/dt_host_metadata.properties", + }); + } + +private: + const envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + DynatraceResourceDetectorConfig config_; + DynatraceMetadataFileReaderPtr dynatrace_file_reader_; +}; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/environment/BUILD b/source/extensions/tracers/opentelemetry/resource_detectors/environment/BUILD new file mode 100644 index 000000000000..3a0026dbd0dd --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/environment/BUILD @@ -0,0 +1,33 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":environment_resource_detector_lib", + "//envoy/registry", + "//source/common/config:utility_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "environment_resource_detector_lib", + srcs = ["environment_resource_detector.cc"], + hdrs = ["environment_resource_detector.h"], + deps = [ + "//source/common/config:datasource_lib", + "//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/environment/config.cc b/source/extensions/tracers/opentelemetry/resource_detectors/environment/config.cc new file mode 100644 index 000000000000..5216a959ce1d --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/environment/config.cc @@ -0,0 +1,35 @@ +#include "source/extensions/tracers/opentelemetry/resource_detectors/environment/config.h" + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.pb.h" +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.pb.validate.h" + +#include "source/common/config/utility.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +ResourceDetectorPtr EnvironmentResourceDetectorFactory::createResourceDetector( + const Protobuf::Message& message, Server::Configuration::TracerFactoryContext& context) { + + auto mptr = Envoy::Config::Utility::translateAnyToFactoryConfig( + dynamic_cast(message), context.messageValidationVisitor(), *this); + + const auto& proto_config = MessageUtil::downcastAndValidate< + const envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + EnvironmentResourceDetectorConfig&>(*mptr, context.messageValidationVisitor()); + + return std::make_unique(proto_config, context); +} + +/** + * Static registration for the Env resource detector factory. @see RegisterFactory. + */ +REGISTER_FACTORY(EnvironmentResourceDetectorFactory, ResourceDetectorFactory); + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/environment/config.h b/source/extensions/tracers/opentelemetry/resource_detectors/environment/config.h new file mode 100644 index 000000000000..a2bf1f72025f --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/environment/config.h @@ -0,0 +1,46 @@ +#pragma once + +#include + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.pb.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * Config registration for the Environment resource detector. @see ResourceDetectorFactory. + */ +class EnvironmentResourceDetectorFactory : public ResourceDetectorFactory { +public: + /** + * @brief Create a Resource Detector that reads from the OTEL_RESOURCE_ATTRIBUTES + * environment variable. + * + * @param message The resource detector configuration. + * @param context The tracer factory context. + * @return ResourceDetectorPtr + */ + ResourceDetectorPtr + createResourceDetector(const Protobuf::Message& message, + Server::Configuration::TracerFactoryContext& context) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { + return "envoy.tracers.opentelemetry.resource_detectors.environment"; + } +}; + +DECLARE_FACTORY(EnvironmentResourceDetectorFactory); + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.cc b/source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.cc new file mode 100644 index 000000000000..c56894c25c42 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.cc @@ -0,0 +1,60 @@ +#include "environment_resource_detector.h" + +#include +#include + +#include "source/common/config/datasource.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +constexpr absl::string_view kOtelResourceAttributesEnv = "OTEL_RESOURCE_ATTRIBUTES"; + +/** + * @brief Detects a resource from the OTEL_RESOURCE_ATTRIBUTES environment variable + * Based on the OTel C++ SDK: + * https://github.com/open-telemetry/opentelemetry-cpp/blob/v1.11.0/sdk/src/resource/resource_detector.cc + * + * @return Resource A resource with the attributes from the OTEL_RESOURCE_ATTRIBUTES environment + * variable. + */ +Resource EnvironmentResourceDetector::detect() { + envoy::config::core::v3::DataSource ds; + ds.set_environment_variable(kOtelResourceAttributesEnv); + + Resource resource; + resource.schema_url_ = ""; + std::string attributes_str = ""; + + attributes_str = Config::DataSource::read(ds, true, context_.serverFactoryContext().api()); + + if (attributes_str.empty()) { + throw EnvoyException( + fmt::format("The OpenTelemetry environment resource detector is configured but the '{}'" + " environment variable is empty.", + kOtelResourceAttributesEnv)); + } + + for (const auto& pair : StringUtil::splitToken(attributes_str, ",")) { + const auto keyValue = StringUtil::splitToken(pair, "="); + if (keyValue.size() != 2) { + throw EnvoyException( + fmt::format("The OpenTelemetry environment resource detector is configured but the '{}'" + " environment variable has an invalid format.", + kOtelResourceAttributesEnv)); + } + + const std::string key = std::string(keyValue[0]); + const std::string value = std::string(keyValue[1]); + resource.attributes_[key] = value; + } + return resource; +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.h b/source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.h new file mode 100644 index 000000000000..78327b047840 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.h @@ -0,0 +1,38 @@ +#pragma once + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.pb.h" +#include "envoy/server/factory_context.h" + +#include "source/common/common/logger.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * @brief A resource detector that extracts attributes from the OTEL_RESOURCE_ATTRIBUTES environment + * variable. + * @see + * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.24.0/specification/resource/sdk.md#detecting-resource-information-from-the-environment + * + */ +class EnvironmentResourceDetector : public ResourceDetector, Logger::Loggable { +public: + EnvironmentResourceDetector(const envoy::extensions::tracers::opentelemetry::resource_detectors:: + v3::EnvironmentResourceDetectorConfig& config, + Server::Configuration::TracerFactoryContext& context) + : config_(config), context_(context) {} + Resource detect() override; + +private: + const envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + EnvironmentResourceDetectorConfig config_; + Server::Configuration::TracerFactoryContext& context_; +}; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h b/source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h new file mode 100644 index 000000000000..61079815aa10 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include + +#include "envoy/config/typed_config.h" +#include "envoy/server/tracer_config.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * @brief A string key-value map that stores the resource attributes. + */ +using ResourceAttributes = std::map; + +/** + * @brief A Resource represents the entity producing telemetry as Attributes. + * For example, a process producing telemetry that is running in a container on Kubernetes + * has a Pod name, it is in a namespace and possibly is part of a Deployment which also has a name. + * See: + * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.26.0/specification/resource/sdk.md + */ +struct Resource { + std::string schema_url_{""}; + ResourceAttributes attributes_{}; + + virtual ~Resource() = default; +}; + +using ResourceConstSharedPtr = std::shared_ptr; + +/** + * @brief The base type for all resource detectors + * + */ +class ResourceDetector { +public: + virtual ~ResourceDetector() = default; + + /** + * @brief Load attributes and returns a Resource object + * populated with them and a possible SchemaUrl. + * @return Resource + */ + virtual Resource detect() PURE; +}; + +using ResourceDetectorPtr = std::unique_ptr; + +/* + * A factory for creating resource detectors. + */ +class ResourceDetectorFactory : public Envoy::Config::TypedFactory { +public: + ~ResourceDetectorFactory() override = default; + + /** + * @brief Creates a resource detector based on the configuration type provided. + * + * @param message The resource detector configuration. + * @param context The tracer factory context. + * @return ResourceDetectorPtr A resource detector based on the configuration type provided. + */ + virtual ResourceDetectorPtr + createResourceDetector(const Protobuf::Message& message, + Server::Configuration::TracerFactoryContext& context) PURE; + + std::string category() const override { return "envoy.tracers.opentelemetry.resource_detectors"; } +}; + +using ResourceDetectorTypedFactoryPtr = std::unique_ptr; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.cc b/source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.cc new file mode 100644 index 000000000000..370d6a44d414 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.cc @@ -0,0 +1,111 @@ +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h" + +#include + +#include "source/common/common/logger.h" +#include "source/common/config/utility.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +namespace { +bool isEmptyResource(const Resource& resource) { return resource.attributes_.empty(); } + +Resource createInitialResource(const std::string& service_name) { + Resource resource{}; + + // Creates initial resource with the static service.name attribute. + resource.attributes_[std::string(kServiceNameKey.data(), kServiceNameKey.size())] = + service_name.empty() ? std::string{kDefaultServiceName} : service_name; + + return resource; +} + +/** + * @brief Resolves the new schema url when merging two resources. + * This function implements the algorithm as defined in the OpenTelemetry Resource SDK + * specification. @see + * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.24.0/specification/resource/sdk.md#merge + * + * @param old_schema_url The old resource's schema URL. + * @param updating_schema_url The updating resource's schema URL. + * @return std::string The calculated schema URL. + */ +std::string resolveSchemaUrl(const std::string& old_schema_url, + const std::string& updating_schema_url) { + if (old_schema_url.empty()) { + return updating_schema_url; + } + if (updating_schema_url.empty()) { + return old_schema_url; + } + if (old_schema_url == updating_schema_url) { + return old_schema_url; + } + // The OTel spec leaves this case (when both have value but are different) unspecified. + ENVOY_LOG_MISC(info, "Resource schemaUrl conflict. Fall-back to old schema url: {}", + old_schema_url); + return old_schema_url; +} + +/** + * @brief Updates an old resource with a new one. This function implements + * the Merge operation defined in the OpenTelemetry Resource SDK specification. + * @see + * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.24.0/specification/resource/sdk.md#merge + * + * @param old_resource The old resource. + * @param updating_resource The new resource. + */ +void mergeResource(Resource& old_resource, const Resource& updating_resource) { + // The schemaUrl is merged, regardless if the resources being merged + // have attributes or not. This behavior is compliant with the OTel spec. + // see: https://github.com/envoyproxy/envoy/pull/29547#discussion_r1344540427 + old_resource.schema_url_ = + resolveSchemaUrl(old_resource.schema_url_, updating_resource.schema_url_); + + if (isEmptyResource(updating_resource)) { + return; + } + for (auto const& attr : updating_resource.attributes_) { + old_resource.attributes_.insert_or_assign(attr.first, attr.second); + } +} +} // namespace + +Resource ResourceProviderImpl::getResource( + const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context) const { + + Resource resource = createInitialResource(opentelemetry_config.service_name()); + + const auto& detectors_configs = opentelemetry_config.resource_detectors(); + + for (const auto& detector_config : detectors_configs) { + ResourceDetectorPtr detector; + auto* factory = Envoy::Config::Utility::getFactory(detector_config); + + if (!factory) { + throw EnvoyException( + fmt::format("Resource detector factory not found: '{}'", detector_config.name())); + } + + detector = factory->createResourceDetector(detector_config.typed_config(), context); + + if (!detector) { + throw EnvoyException( + fmt::format("Resource detector could not be created: '{}'", detector_config.name())); + } + + Resource detected_resource = detector->detect(); + mergeResource(resource, detected_resource); + } + return resource; +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h b/source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h new file mode 100644 index 000000000000..9ecf6420c31d --- /dev/null +++ b/source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h @@ -0,0 +1,44 @@ +#pragma once + +#include "envoy/config/trace/v3/opentelemetry.pb.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +constexpr absl::string_view kServiceNameKey = "service.name"; +constexpr absl::string_view kDefaultServiceName = "unknown_service:envoy"; + +class ResourceProvider : public Logger::Loggable { +public: + virtual ~ResourceProvider() = default; + + /** + * @brief Iterates through all loaded resource detectors and merge all the returned + * resources into one. Resource merging is done according to the OpenTelemetry + * resource SDK specification. @see + * https://github.com/open-telemetry/opentelemetry-specification/blob/v1.24.0/specification/resource/sdk.md#merge. + * + * @param opentelemetry_config The OpenTelemetry configuration, which contains the configured + * resource detectors. + * @param context The tracer factory context. + * @return Resource const The merged resource. + */ + virtual Resource + getResource(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context) const PURE; +}; + +class ResourceProviderImpl : public ResourceProvider { +public: + Resource getResource(const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context) const override; +}; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/samplers/BUILD b/source/extensions/tracers/opentelemetry/samplers/BUILD new file mode 100644 index 000000000000..32a1005b11e8 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "sampler_lib", + srcs = [ + ], + hdrs = [ + "sampler.h", + ], + deps = [ + "//envoy/config:typed_config_interface", + "//envoy/server:tracer_config_interface", + "//source/common/common:logger_lib", + "//source/common/config:utility_lib", + "@opentelemetry_proto//:trace_cc_proto", + ], +) diff --git a/source/extensions/tracers/opentelemetry/samplers/always_on/BUILD b/source/extensions/tracers/opentelemetry/samplers/always_on/BUILD new file mode 100644 index 000000000000..744607330d57 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/always_on/BUILD @@ -0,0 +1,33 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":always_on_sampler_lib", + "//envoy/registry", + "//source/common/config:utility_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/samplers/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "always_on_sampler_lib", + srcs = ["always_on_sampler.cc"], + hdrs = ["always_on_sampler.h"], + deps = [ + "//source/common/config:datasource_lib", + "//source/extensions/tracers/opentelemetry:opentelemetry_tracer_lib", + "//source/extensions/tracers/opentelemetry/samplers:sampler_lib", + ], +) diff --git a/source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.cc b/source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.cc new file mode 100644 index 000000000000..8d06116cceb3 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.cc @@ -0,0 +1,33 @@ +#include "source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.h" + +#include +#include +#include + +#include "source/common/config/datasource.h" +#include "source/extensions/tracers/opentelemetry/span_context.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +SamplingResult AlwaysOnSampler::shouldSample(const absl::optional parent_context, + const std::string& /*trace_id*/, + const std::string& /*name*/, OTelSpanKind /*kind*/, + OptRef /*trace_context*/, + const std::vector& /*links*/) { + SamplingResult result; + result.decision = Decision::RECORD_AND_SAMPLE; + if (parent_context.has_value()) { + result.tracestate = parent_context.value().tracestate(); + } + return result; +} + +std::string AlwaysOnSampler::getDescription() const { return "AlwaysOnSampler"; } + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.h b/source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.h new file mode 100644 index 000000000000..c70864b078a0 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.h @@ -0,0 +1,38 @@ +#pragma once + +#include "envoy/server/factory_context.h" + +#include "source/common/common/logger.h" +#include "source/common/config/datasource.h" +#include "source/extensions/tracers/opentelemetry/samplers/sampler.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * @brief A sampler which samples every span. + * https://opentelemetry.io/docs/specs/otel/trace/sdk/#alwayson + * - Returns RECORD_AND_SAMPLE always. + * - Description MUST be AlwaysOnSampler. + * + */ +class AlwaysOnSampler : public Sampler, Logger::Loggable { +public: + explicit AlwaysOnSampler(const Protobuf::Message& /*config*/, + Server::Configuration::TracerFactoryContext& /*context*/) {} + SamplingResult shouldSample(const absl::optional parent_context, + const std::string& trace_id, const std::string& name, + OTelSpanKind spankind, + OptRef trace_context, + const std::vector& links) override; + std::string getDescription() const override; + +private: +}; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/samplers/always_on/config.cc b/source/extensions/tracers/opentelemetry/samplers/always_on/config.cc new file mode 100644 index 000000000000..99288c4bf469 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/always_on/config.cc @@ -0,0 +1,27 @@ +#include "source/extensions/tracers/opentelemetry/samplers/always_on/config.h" + +#include "envoy/server/tracer_config.h" + +#include "source/common/config/utility.h" +#include "source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +SamplerSharedPtr +AlwaysOnSamplerFactory::createSampler(const Protobuf::Message& config, + Server::Configuration::TracerFactoryContext& context) { + return std::make_shared(config, context); +} + +/** + * Static registration for the Env sampler factory. @see RegisterFactory. + */ +REGISTER_FACTORY(AlwaysOnSamplerFactory, SamplerFactory); + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/samplers/always_on/config.h b/source/extensions/tracers/opentelemetry/samplers/always_on/config.h new file mode 100644 index 000000000000..5f93f43c0f1f --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/always_on/config.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include "envoy/extensions/tracers/opentelemetry/samplers/v3/always_on_sampler.pb.h" +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/samplers/sampler.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * Config registration for the AlwaysOnSampler. @see SamplerFactory. + */ +class AlwaysOnSamplerFactory : public SamplerFactory { +public: + /** + * @brief Create a Sampler. @see AlwaysOnSampler + * + * @param config Protobuf config for the sampler. + * @param context A reference to the TracerFactoryContext. + * @return SamplerSharedPtr + */ + SamplerSharedPtr createSampler(const Protobuf::Message& config, + Server::Configuration::TracerFactoryContext& context) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique< + envoy::extensions::tracers::opentelemetry::samplers::v3::AlwaysOnSamplerConfig>(); + } + std::string name() const override { return "envoy.tracers.opentelemetry.samplers.always_on"; } +}; + +DECLARE_FACTORY(AlwaysOnSamplerFactory); + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/samplers/sampler.h b/source/extensions/tracers/opentelemetry/samplers/sampler.h new file mode 100644 index 000000000000..33794645344e --- /dev/null +++ b/source/extensions/tracers/opentelemetry/samplers/sampler.h @@ -0,0 +1,119 @@ +#pragma once + +#include +#include +#include +#include + +#include "envoy/common/optref.h" +#include "envoy/config/typed_config.h" +#include "envoy/server/tracer_config.h" +#include "envoy/tracing/trace_context.h" + +#include "absl/types/optional.h" +#include "opentelemetry/proto/trace/v1/trace.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +class SpanContext; + +enum class Decision { + // IsRecording will be false, the Span will not be recorded and all events and attributes will be + // dropped. + DROP, + // IsRecording will be true, but the Sampled flag MUST NOT be set. + RECORD_ONLY, + // IsRecording will be true and the Sampled flag MUST be set. + RECORD_AND_SAMPLE +}; + +/** + * @brief The type of the span. + * see + * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind + */ +using OTelSpanKind = ::opentelemetry::proto::trace::v1::Span::SpanKind; + +struct SamplingResult { + /// @see Decision + Decision decision; + // A set of span Attributes that will also be added to the Span. Can be nullptr. + std::unique_ptr> attributes; + // A Tracestate that will be associated with the Span. If the sampler + // returns an empty Tracestate here, the Tracestate will be cleared, so samplers SHOULD normally + // return the passed-in Tracestate if they do not intend to change it + std::string tracestate; + + inline bool isRecording() const { + return decision == Decision::RECORD_ONLY || decision == Decision::RECORD_AND_SAMPLE; + } + + inline bool isSampled() const { return decision == Decision::RECORD_AND_SAMPLE; } +}; + +/** + * @brief The base type for all samplers + * see https://opentelemetry.io/docs/specs/otel/trace/sdk/#sampler + * + */ +class Sampler { +public: + virtual ~Sampler() = default; + + /** + * @brief Decides if a trace should be sampled. + * + * @param parent_context Span context describing the parent span. The Span's SpanContext may be + * invalid to indicate a root span. + * @param trace_id Trace id of the Span to be created. If the parent SpanContext contains a valid + * TraceId, they MUST always match. + * @param name Name of the Span to be created. + * @param spankind Span kind of the Span to be created. + * @param trace_context TraceContext containing potential initial span attributes + * @param links Collection of links that will be associated with the Span to be created. + * @return SamplingResult @see SamplingResult + */ + virtual SamplingResult shouldSample(const absl::optional parent_context, + const std::string& trace_id, const std::string& name, + OTelSpanKind spankind, + OptRef trace_context, + const std::vector& links) PURE; + + /** + * @brief Returns a sampler description or name. + * + * @return The sampler name or short description with the configuration. + */ + virtual std::string getDescription() const PURE; +}; + +using SamplerSharedPtr = std::shared_ptr; + +/* + * A factory for creating a sampler + */ +class SamplerFactory : public Envoy::Config::TypedFactory { +public: + ~SamplerFactory() override = default; + + /** + * @brief Creates a sampler + * @param config The sampler protobuf config. + * @param context The TracerFactoryContext. + * @return SamplerSharedPtr A sampler. + */ + virtual SamplerSharedPtr createSampler(const Protobuf::Message& config, + Server::Configuration::TracerFactoryContext& context) PURE; + + std::string category() const override { return "envoy.tracers.opentelemetry.samplers"; } +}; + +using SamplerFactoryPtr = std::unique_ptr; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/span_context_extractor.cc b/source/extensions/tracers/opentelemetry/span_context_extractor.cc index 0b8b3e535b8f..6f0165267f3c 100644 --- a/source/extensions/tracers/opentelemetry/span_context_extractor.cc +++ b/source/extensions/tracers/opentelemetry/span_context_extractor.cc @@ -1,8 +1,9 @@ -#include "span_context_extractor.h" +#include "source/extensions/tracers/opentelemetry/span_context_extractor.h" #include "envoy/tracing/tracer.h" #include "source/common/http/header_map_impl.h" +#include "source/common/tracing/trace_context_impl.h" #include "absl/strings/escaping.h" #include "span_context.h" @@ -13,14 +14,6 @@ namespace Tracers { namespace OpenTelemetry { namespace { -const Http::LowerCaseString& openTelemetryPropagationHeader() { - CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, "traceparent"); -} - -const Http::LowerCaseString& openTelemetryTraceStateHeader() { - CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, "tracestate"); -} - // See https://www.w3.org/TR/trace-context/#traceparent-header constexpr int kTraceparentHeaderSize = 55; // 2 + 1 + 32 + 1 + 16 + 1 + 2 constexpr int kVersionHexSize = 2; @@ -45,12 +38,12 @@ SpanContextExtractor::SpanContextExtractor(Tracing::TraceContext& trace_context) SpanContextExtractor::~SpanContextExtractor() = default; bool SpanContextExtractor::propagationHeaderPresent() { - auto propagation_header = trace_context_.getByKey(openTelemetryPropagationHeader()); + auto propagation_header = OpenTelemetryConstants::get().TRACE_PARENT.get(trace_context_); return propagation_header.has_value(); } absl::StatusOr SpanContextExtractor::extractSpanContext() { - auto propagation_header = trace_context_.getByKey(openTelemetryPropagationHeader()); + auto propagation_header = OpenTelemetryConstants::get().TRACE_PARENT.get(trace_context_); if (!propagation_header.has_value()) { // We should have already caught this, but just in case. return absl::InvalidArgumentError("No propagation header found"); @@ -96,7 +89,7 @@ absl::StatusOr SpanContextExtractor::extractSpanContext() { // it is invalid and MUST be discarded. Because we're already checking for the // traceparent header above, we don't need to check here. // See https://www.w3.org/TR/trace-context/#processing-model-for-working-with-trace-context - absl::string_view tracestate_key = openTelemetryTraceStateHeader(); + absl::string_view tracestate_key = OpenTelemetryConstants::get().TRACE_STATE.key(); std::vector tracestate_values; // Multiple tracestate header fields MUST be handled as specified by RFC7230 Section 3.2.2 Field // Order. diff --git a/source/extensions/tracers/opentelemetry/span_context_extractor.h b/source/extensions/tracers/opentelemetry/span_context_extractor.h index 961119dc470d..dffeb6218c92 100644 --- a/source/extensions/tracers/opentelemetry/span_context_extractor.h +++ b/source/extensions/tracers/opentelemetry/span_context_extractor.h @@ -5,14 +5,22 @@ #include "source/common/common/statusor.h" #include "source/common/http/header_map_impl.h" - -#include "span_context.h" +#include "source/common/tracing/trace_context_impl.h" +#include "source/extensions/tracers/opentelemetry/span_context.h" namespace Envoy { namespace Extensions { namespace Tracers { namespace OpenTelemetry { +class OpenTelemetryConstantValues { +public: + const Tracing::TraceContextHandler TRACE_PARENT{"traceparent"}; + const Tracing::TraceContextHandler TRACE_STATE{"tracestate"}; +}; + +using OpenTelemetryConstants = ConstSingleton; + /** * This class is used to SpanContext extracted from the HTTP traceparent header * See https://www.w3.org/TR/trace-context/#traceparent-header. diff --git a/source/extensions/tracers/opentelemetry/trace_exporter.h b/source/extensions/tracers/opentelemetry/trace_exporter.h new file mode 100644 index 000000000000..df4ac67d8617 --- /dev/null +++ b/source/extensions/tracers/opentelemetry/trace_exporter.h @@ -0,0 +1,38 @@ +#pragma once + +#include "source/common/common/logger.h" + +#include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" + +using opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +/** + * @brief Base class for all OpenTelemetry Protocol (OTLP) exporters. + * @see + * https://github.com/open-telemetry/opentelemetry-proto/blob/v1.0.0/docs/specification.md#otlphttp + */ +class OpenTelemetryTraceExporter : public Logger::Loggable { +public: + virtual ~OpenTelemetryTraceExporter() = default; + + /** + * @brief Exports the trace request to the configured OTLP service. + * + * @param request The protobuf-encoded OTLP trace request. + * @return true When the request was sent. + * @return false When sending the request failed. + */ + virtual bool log(const ExportTraceServiceRequest& request) = 0; +}; + +using OpenTelemetryTraceExporterPtr = std::unique_ptr; + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/opentelemetry/tracer.cc b/source/extensions/tracers/opentelemetry/tracer.cc index a344a253ab31..8c8553015510 100644 --- a/source/extensions/tracers/opentelemetry/tracer.cc +++ b/source/extensions/tracers/opentelemetry/tracer.cc @@ -7,6 +7,7 @@ #include "source/common/common/empty_string.h" #include "source/common/common/hex.h" +#include "source/common/tracing/trace_context_impl.h" #include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" #include "opentelemetry/proto/trace/v1/trace.pb.h" @@ -16,32 +17,60 @@ namespace Extensions { namespace Tracers { namespace OpenTelemetry { -constexpr absl::string_view kTraceParent = "traceparent"; -constexpr absl::string_view kTraceState = "tracestate"; constexpr absl::string_view kDefaultVersion = "00"; -constexpr absl::string_view kServiceNameKey = "service.name"; -constexpr absl::string_view kDefaultServiceName = "unknown_service:envoy"; using opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; -Span::Span(const Tracing::Config& config, const std::string& name, SystemTime start_time, - Envoy::TimeSource& time_source, Tracer& parent_tracer) +namespace { + +const Tracing::TraceContextHandler& traceParentHeader() { + CONSTRUCT_ON_FIRST_USE(Tracing::TraceContextHandler, "traceparent"); +} + +const Tracing::TraceContextHandler& traceStateHeader() { + CONSTRUCT_ON_FIRST_USE(Tracing::TraceContextHandler, "tracestate"); +} + +void callSampler(SamplerSharedPtr sampler, const absl::optional span_context, + Span& new_span, const std::string& operation_name, + OptRef trace_context) { + if (!sampler) { + return; + } + const auto sampling_result = + sampler->shouldSample(span_context, operation_name, new_span.getTraceIdAsHex(), + new_span.spankind(), trace_context, {}); + new_span.setSampled(sampling_result.isSampled()); + + if (sampling_result.attributes) { + for (auto const& attribute : *sampling_result.attributes) { + new_span.setTag(attribute.first, attribute.second); + } + } + if (!sampling_result.tracestate.empty()) { + new_span.setTracestate(sampling_result.tracestate); + } +} + +} // namespace + +Span::Span(const std::string& name, SystemTime start_time, Envoy::TimeSource& time_source, + Tracer& parent_tracer, OTelSpanKind span_kind) : parent_tracer_(parent_tracer), time_source_(time_source) { span_ = ::opentelemetry::proto::trace::v1::Span(); - if (config.operationName() == Tracing::OperationName::Egress) { - span_.set_kind(::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT); - } else { - span_.set_kind(::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER); - } + + span_.set_kind(span_kind); + span_.set_name(name); span_.set_start_time_unix_nano(std::chrono::nanoseconds(start_time.time_since_epoch()).count()); } -Tracing::SpanPtr Span::spawnChild(const Tracing::Config& config, const std::string& name, +Tracing::SpanPtr Span::spawnChild(const Tracing::Config&, const std::string& name, SystemTime start_time) { // Build span_context from the current span, then generate the child span from that context. SpanContext span_context(kDefaultVersion, getTraceIdAsHex(), spanId(), sampled(), tracestate()); - return parent_tracer_.startSpan(config, name, start_time, span_context); + return parent_tracer_.startSpan(name, start_time, span_context, {}, + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT); } void Span::finishSpan() { @@ -62,9 +91,9 @@ void Span::injectContext(Tracing::TraceContext& trace_context, std::string traceparent_header_value = absl::StrCat(kDefaultVersion, "-", trace_id_hex, "-", span_id_hex, "-", trace_flags_hex); // Set the traceparent in the trace_context. - trace_context.setByReferenceKey(kTraceParent, traceparent_header_value); + traceParentHeader().setRefKey(trace_context, traceparent_header_value); // Also set the tracestate. - trace_context.setByReferenceKey(kTraceState, span_.trace_state()); + traceStateHeader().setRefKey(trace_context, span_.trace_state()); } void Span::setTag(absl::string_view name, absl::string_view value) { @@ -91,15 +120,12 @@ void Span::setTag(absl::string_view name, absl::string_view value) { *span_.add_attributes() = key_value; } -Tracer::Tracer(OpenTelemetryGrpcTraceExporterPtr exporter, Envoy::TimeSource& time_source, +Tracer::Tracer(OpenTelemetryTraceExporterPtr exporter, Envoy::TimeSource& time_source, Random::RandomGenerator& random, Runtime::Loader& runtime, Event::Dispatcher& dispatcher, OpenTelemetryTracerStats tracing_stats, - const std::string& service_name) + const ResourceConstSharedPtr resource, SamplerSharedPtr sampler) : exporter_(std::move(exporter)), time_source_(time_source), random_(random), runtime_(runtime), - tracing_stats_(tracing_stats), service_name_(service_name) { - if (service_name.empty()) { - service_name_ = std::string{kDefaultServiceName}; - } + tracing_stats_(tracing_stats), resource_(resource), sampler_(sampler) { flush_timer_ = dispatcher.createTimer([this]() -> void { tracing_stats_.timer_flushed_.inc(); flushSpans(); @@ -118,14 +144,20 @@ void Tracer::flushSpans() { ExportTraceServiceRequest request; // A request consists of ResourceSpans. ::opentelemetry::proto::trace::v1::ResourceSpans* resource_span = request.add_resource_spans(); - opentelemetry::proto::common::v1::KeyValue key_value = - opentelemetry::proto::common::v1::KeyValue(); - opentelemetry::proto::common::v1::AnyValue value_proto = - opentelemetry::proto::common::v1::AnyValue(); - value_proto.set_string_value(std::string{service_name_}); - key_value.set_key(std::string{kServiceNameKey}); - *key_value.mutable_value() = value_proto; - (*resource_span->mutable_resource()->add_attributes()) = key_value; + resource_span->set_schema_url(resource_->schema_url_); + + // add resource attributes + for (auto const& att : resource_->attributes_) { + opentelemetry::proto::common::v1::KeyValue key_value = + opentelemetry::proto::common::v1::KeyValue(); + opentelemetry::proto::common::v1::AnyValue value_proto = + opentelemetry::proto::common::v1::AnyValue(); + value_proto.set_string_value(std::string{att.second}); + key_value.set_key(std::string{att.first}); + *key_value.mutable_value() = value_proto; + (*resource_span->mutable_resource()->add_attributes()) = key_value; + } + ::opentelemetry::proto::trace::v1::ScopeSpans* scope_span = resource_span->add_scope_spans(); for (const auto& pending_span : span_buffer_) { (*scope_span->add_spans()) = pending_span; @@ -151,26 +183,31 @@ void Tracer::sendSpan(::opentelemetry::proto::trace::v1::Span& span) { } } -Tracing::SpanPtr Tracer::startSpan(const Tracing::Config& config, const std::string& operation_name, - SystemTime start_time, - const Tracing::Decision tracing_decision) { +Tracing::SpanPtr Tracer::startSpan(const std::string& operation_name, SystemTime start_time, + Tracing::Decision tracing_decision, + OptRef trace_context, + OTelSpanKind span_kind) { // Create an Tracers::OpenTelemetry::Span class that will contain the OTel span. - Span new_span = Span(config, operation_name, start_time, time_source_, *this); - new_span.setSampled(tracing_decision.traced); + Span new_span(operation_name, start_time, time_source_, *this, span_kind); uint64_t trace_id_high = random_.random(); uint64_t trace_id = random_.random(); new_span.setTraceId(absl::StrCat(Hex::uint64ToHex(trace_id_high), Hex::uint64ToHex(trace_id))); uint64_t span_id = random_.random(); new_span.setId(Hex::uint64ToHex(span_id)); + if (sampler_) { + callSampler(sampler_, absl::nullopt, new_span, operation_name, trace_context); + } else { + new_span.setSampled(tracing_decision.traced); + } return std::make_unique(new_span); } -Tracing::SpanPtr Tracer::startSpan(const Tracing::Config& config, const std::string& operation_name, - SystemTime start_time, - const SpanContext& previous_span_context) { +Tracing::SpanPtr Tracer::startSpan(const std::string& operation_name, SystemTime start_time, + const SpanContext& previous_span_context, + OptRef trace_context, + OTelSpanKind span_kind) { // Create a new span and populate details from the span context. - Span new_span = Span(config, operation_name, start_time, time_source_, *this); - new_span.setSampled(previous_span_context.sampled()); + Span new_span(operation_name, start_time, time_source_, *this, span_kind); new_span.setTraceId(previous_span_context.traceId()); if (!previous_span_context.parentId().empty()) { new_span.setParentId(previous_span_context.parentId()); @@ -178,10 +215,15 @@ Tracing::SpanPtr Tracer::startSpan(const Tracing::Config& config, const std::str // Generate a new identifier for the span id. uint64_t span_id = random_.random(); new_span.setId(Hex::uint64ToHex(span_id)); - // Respect the previous span's sampled flag. - new_span.setSampled(previous_span_context.sampled()); - if (!previous_span_context.tracestate().empty()) { - new_span.setTracestate(std::string{previous_span_context.tracestate()}); + if (sampler_) { + // Sampler should make a sampling decision and set tracestate + callSampler(sampler_, previous_span_context, new_span, operation_name, trace_context); + } else { + // Respect the previous span's sampled flag. + new_span.setSampled(previous_span_context.sampled()); + if (!previous_span_context.tracestate().empty()) { + new_span.setTracestate(std::string{previous_span_context.tracestate()}); + } } return std::make_unique(new_span); } diff --git a/source/extensions/tracers/opentelemetry/tracer.h b/source/extensions/tracers/opentelemetry/tracer.h index f80cb12fd3f7..f66d1abf908c 100644 --- a/source/extensions/tracers/opentelemetry/tracer.h +++ b/source/extensions/tracers/opentelemetry/tracer.h @@ -3,6 +3,7 @@ #include #include "envoy/api/api.h" +#include "envoy/common/optref.h" #include "envoy/config/trace/v3/opentelemetry.pb.h" #include "envoy/runtime/runtime.h" #include "envoy/thread_local/thread_local.h" @@ -11,9 +12,11 @@ #include "source/common/common/logger.h" #include "source/extensions/tracers/common/factory_base.h" #include "source/extensions/tracers/opentelemetry/grpc_trace_exporter.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_detector.h" +#include "source/extensions/tracers/opentelemetry/samplers/sampler.h" +#include "source/extensions/tracers/opentelemetry/span_context.h" #include "absl/strings/escaping.h" -#include "span_context.h" namespace Envoy { namespace Extensions { @@ -33,17 +36,23 @@ struct OpenTelemetryTracerStats { */ class Tracer : Logger::Loggable { public: - Tracer(OpenTelemetryGrpcTraceExporterPtr exporter, Envoy::TimeSource& time_source, + Tracer(OpenTelemetryTraceExporterPtr exporter, Envoy::TimeSource& time_source, Random::RandomGenerator& random, Runtime::Loader& runtime, Event::Dispatcher& dispatcher, - OpenTelemetryTracerStats tracing_stats, const std::string& service_name); + OpenTelemetryTracerStats tracing_stats, const ResourceConstSharedPtr resource, + SamplerSharedPtr sampler); void sendSpan(::opentelemetry::proto::trace::v1::Span& span); - Tracing::SpanPtr startSpan(const Tracing::Config& config, const std::string& operation_name, - SystemTime start_time, const Tracing::Decision tracing_decision); + Tracing::SpanPtr startSpan(const std::string& operation_name, SystemTime start_time, - Tracing::SpanPtr startSpan(const Tracing::Config& config, const std::string& operation_name, - SystemTime start_time, const SpanContext& previous_span_context); + Tracing::Decision tracing_decision, + OptRef trace_context, + OTelSpanKind span_kind); + + Tracing::SpanPtr startSpan(const std::string& operation_name, SystemTime start_time, + const SpanContext& previous_span_context, + OptRef trace_context, + OTelSpanKind span_kind); private: /** @@ -55,14 +64,15 @@ class Tracer : Logger::Loggable { */ void flushSpans(); - OpenTelemetryGrpcTraceExporterPtr exporter_; + OpenTelemetryTraceExporterPtr exporter_; Envoy::TimeSource& time_source_; Random::RandomGenerator& random_; std::vector<::opentelemetry::proto::trace::v1::Span> span_buffer_; Runtime::Loader& runtime_; Event::TimerPtr flush_timer_; OpenTelemetryTracerStats tracing_stats_; - std::string service_name_; + const ResourceConstSharedPtr resource_; + SamplerSharedPtr sampler_; }; /** @@ -71,8 +81,8 @@ class Tracer : Logger::Loggable { */ class Span : Logger::Loggable, public Tracing::Span { public: - Span(const Tracing::Config& config, const std::string& name, SystemTime start_time, - Envoy::TimeSource& time_source, Tracer& parent_tracer); + Span(const std::string& name, SystemTime start_time, Envoy::TimeSource& time_source, + Tracer& parent_tracer, OTelSpanKind span_kind); // Tracing::Span functions void setOperation(absl::string_view /*operation*/) override{}; @@ -109,6 +119,8 @@ class Span : Logger::Loggable, public Tracing::Span { std::string getTraceIdAsHex() const override { return absl::BytesToHexString(span_.trace_id()); }; + OTelSpanKind spankind() const { return span_.kind(); } + /** * Sets the span's id. */ @@ -125,7 +137,7 @@ class Span : Logger::Loggable, public Tracing::Span { span_.set_parent_span_id(absl::HexStringToBytes(parent_span_id_hex)); } - std::string tracestate() { return span_.trace_state(); } + std::string tracestate() const { return span_.trace_state(); } /** * Sets the span's tracestate. @@ -134,6 +146,11 @@ class Span : Logger::Loggable, public Tracing::Span { span_.set_trace_state(std::string{tracestate}); } + /** + * Method to access the span for testing. + */ + const ::opentelemetry::proto::trace::v1::Span& spanForTest() const { return span_; } + private: ::opentelemetry::proto::trace::v1::Span span_; Tracer& parent_tracer_; diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc index 21671c8cfe26..33c9f67051e3 100644 --- a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc +++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc @@ -46,11 +46,11 @@ Driver::Driver(const envoy::config::trace::v3::SkyWalkingConfig& proto_config, Tracing::SpanPtr Driver::startSpan(const Tracing::Config&, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo&, const std::string&, - const Tracing::Decision decision) { + Tracing::Decision decision) { auto& tracer = tls_slot_ptr_->getTyped().tracer(); TracingContextPtr tracing_context; // TODO(shikugawa): support extension span header. - auto propagation_header = trace_context.getByKey(skywalkingPropagationHeaderKey()); + auto propagation_header = skywalkingPropagationHeaderKey().get(trace_context); if (!propagation_header.has_value()) { // Although a sampling flag can be added to the propagation header, it will be ignored by most // of SkyWalking agent. The agent will enable tracing anyway if it see the propagation header. diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.h b/source/extensions/tracers/skywalking/skywalking_tracer_impl.h index 0a97052c624e..f99cb8aa2345 100644 --- a/source/extensions/tracers/skywalking/skywalking_tracer_impl.h +++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.h @@ -28,7 +28,7 @@ class Driver : public Tracing::Driver, public Logger::LoggablecreateSW8HeaderValue({remote_address.data(), remote_address.size()}); if (sw8_header.has_value()) { - trace_context.setByReferenceKey(skywalkingPropagationHeaderKey(), sw8_header.value()); + skywalkingPropagationHeaderKey().setRefKey(trace_context, sw8_header.value()); // Rewrite operation name with latest upstream request path for the EXIT span. absl::string_view upstream_request_path = trace_context.path(); diff --git a/source/extensions/tracers/skywalking/tracer.h b/source/extensions/tracers/skywalking/tracer.h index e3894d8ae564..64f623319c18 100644 --- a/source/extensions/tracers/skywalking/tracer.h +++ b/source/extensions/tracers/skywalking/tracer.h @@ -6,6 +6,7 @@ #include "source/common/tracing/common_values.h" #include "source/common/tracing/null_span_impl.h" +#include "source/common/tracing/trace_context_impl.h" #include "source/extensions/tracers/skywalking/trace_segment_reporter.h" #include "cpp2sky/tracing_context.h" @@ -19,7 +20,7 @@ namespace SkyWalking { using cpp2sky::TracingContextPtr; using cpp2sky::TracingSpanPtr; -const Http::LowerCaseString& skywalkingPropagationHeaderKey(); +const Tracing::TraceContextHandler& skywalkingPropagationHeaderKey(); class Tracer { public: diff --git a/source/extensions/tracers/xray/daemon_broker.cc b/source/extensions/tracers/xray/daemon_broker.cc index dcfa291e12d4..e344bc2d5d32 100644 --- a/source/extensions/tracers/xray/daemon_broker.cc +++ b/source/extensions/tracers/xray/daemon_broker.cc @@ -43,7 +43,7 @@ void DaemonBrokerImpl::send(const std::string& data) const { nullptr /*local_ip*/, *address_); if (rc.return_value_ != payload.length()) { - // TODO(marcomagdy): report this in stats + // TODO(suniltheta): report this in stats ENVOY_LOG_TO_LOGGER(logger, debug, "Failed to send trace payload to the X-Ray daemon."); } } diff --git a/source/extensions/tracers/xray/tracer.cc b/source/extensions/tracers/xray/tracer.cc index 2c0eb6ad3b61..01bff55a3f23 100644 --- a/source/extensions/tracers/xray/tracer.cc +++ b/source/extensions/tracers/xray/tracer.cc @@ -101,11 +101,19 @@ void Span::finishSpan() { broker_.send(json); } // namespace XRay +const Tracing::TraceContextHandler& xRayTraceHeader() { + CONSTRUCT_ON_FIRST_USE(Tracing::TraceContextHandler, "x-amzn-trace-id"); +} + +const Tracing::TraceContextHandler& xForwardedForHeader() { + CONSTRUCT_ON_FIRST_USE(Tracing::TraceContextHandler, "x-forwarded-for"); +} + void Span::injectContext(Tracing::TraceContext& trace_context, const Upstream::HostDescriptionConstSharedPtr&) { const std::string xray_header_value = fmt::format("Root={};Parent={};Sampled={}", traceId(), id(), sampled() ? "1" : "0"); - trace_context.setByReferenceKey(XRayTraceHeader, xray_header_value); + xRayTraceHeader().setRefKey(trace_context, xray_header_value); } Tracing::SpanPtr Span::spawnChild(const Tracing::Config& config, const std::string& operation_name, diff --git a/source/extensions/tracers/xray/tracer.h b/source/extensions/tracers/xray/tracer.h index d09fb34c5111..9ecb2b75b39c 100644 --- a/source/extensions/tracers/xray/tracer.h +++ b/source/extensions/tracers/xray/tracer.h @@ -13,6 +13,7 @@ #include "source/common/http/codes.h" #include "source/common/protobuf/utility.h" #include "source/common/tracing/common_values.h" +#include "source/common/tracing/trace_context_impl.h" #include "source/extensions/tracers/xray/daemon_broker.h" #include "source/extensions/tracers/xray/sampling_strategy.h" #include "source/extensions/tracers/xray/xray_configuration.h" @@ -27,10 +28,11 @@ namespace XRay { constexpr absl::string_view SpanClientIp = "client_ip"; constexpr absl::string_view SpanXForwardedFor = "x_forwarded_for"; -constexpr absl::string_view XForwardedForHeader = "x-forwarded-for"; -constexpr absl::string_view XRayTraceHeader = "x-amzn-trace-id"; constexpr absl::string_view Subsegment = "subsegment"; +const Tracing::TraceContextHandler& xRayTraceHeader(); +const Tracing::TraceContextHandler& xForwardedForHeader(); + class Span : public Tracing::Span, Logger::Loggable { public: /** diff --git a/source/extensions/tracers/xray/xray_tracer_impl.cc b/source/extensions/tracers/xray/xray_tracer_impl.cc index 7bafd64cf330..b3003c705d0e 100644 --- a/source/extensions/tracers/xray/xray_tracer_impl.cc +++ b/source/extensions/tracers/xray/xray_tracer_impl.cc @@ -65,7 +65,7 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) { + Tracing::Decision tracing_decision) { // First thing is to determine whether this request will be sampled or not. // if there's a X-Ray header and it has a sampling decision already determined (i.e. Sample=1) // then we can move on; otherwise, we ask the sampling strategy whether this request should be @@ -75,9 +75,9 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, // If we have a XRay TraceID in the headers, then we create a SpanContext to pass that trace-id // around if no TraceID (which means no x-ray header) then this is a brand new span. - // TODO(marcomagdy) - how do we factor this into the logic above + // TODO(suniltheta) - how do we factor this into the logic above UNREFERENCED_PARAMETER(tracing_decision); - const auto header = trace_context.getByKey(XRayTraceHeader); + const auto header = xRayTraceHeader().get(trace_context); absl::optional should_trace; XRayHeader xray_header; if (header.has_value()) { @@ -109,7 +109,7 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, return tracer->startSpan(config, operation_name, stream_info.startTime(), header.has_value() ? absl::optional(xray_header) : absl::nullopt, - trace_context.getByKey(XForwardedForHeader)); + xForwardedForHeader().get(trace_context)); } // Instead of returning nullptr, we return a Span that is marked as not-sampled. diff --git a/source/extensions/tracers/xray/xray_tracer_impl.h b/source/extensions/tracers/xray/xray_tracer_impl.h index 4e3b9705e5cb..df6e05f18393 100644 --- a/source/extensions/tracers/xray/xray_tracer_impl.h +++ b/source/extensions/tracers/xray/xray_tracer_impl.h @@ -21,7 +21,7 @@ class Driver : public Tracing::Driver, public Logger::Loggable SpanContextExtractor::extractSpanContext(bool is_sampled) { - if (trace_context_.getByKey(ZipkinCoreConstants::get().B3).has_value()) { + if (ZipkinCoreConstants::get().B3.get(trace_context_).has_value()) { return extractSpanContextFromB3SingleFormat(is_sampled); } uint64_t trace_id(0); @@ -82,8 +82,8 @@ std::pair SpanContextExtractor::extractSpanContext(bool is_sa uint64_t span_id(0); uint64_t parent_id(0); - auto b3_trace_id_entry = trace_context_.getByKey(ZipkinCoreConstants::get().X_B3_TRACE_ID); - auto b3_span_id_entry = trace_context_.getByKey(ZipkinCoreConstants::get().X_B3_SPAN_ID); + auto b3_trace_id_entry = ZipkinCoreConstants::get().X_B3_TRACE_ID.get(trace_context_); + auto b3_span_id_entry = ZipkinCoreConstants::get().X_B3_SPAN_ID.get(trace_context_); if (b3_span_id_entry.has_value() && b3_trace_id_entry.has_value()) { // Extract trace id - which can either be 128 or 64 bit. For 128 bit, // it needs to be divided into two 64 bit numbers (high and low). @@ -107,8 +107,7 @@ std::pair SpanContextExtractor::extractSpanContext(bool is_sa throw ExtractorException(absl::StrCat("Invalid span id ", spid.c_str())); } - auto b3_parent_id_entry = - trace_context_.getByKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID); + auto b3_parent_id_entry = ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.get(trace_context_); if (b3_parent_id_entry.has_value() && !b3_parent_id_entry.value().empty()) { // This is an implicitly untrusted header, so only the first value is used. const std::string pspid(b3_parent_id_entry.value()); @@ -125,7 +124,7 @@ std::pair SpanContextExtractor::extractSpanContext(bool is_sa std::pair SpanContextExtractor::extractSpanContextFromB3SingleFormat(bool is_sampled) { - auto b3_head_entry = trace_context_.getByKey(ZipkinCoreConstants::get().B3); + auto b3_head_entry = ZipkinCoreConstants::get().B3.get(trace_context_); ASSERT(b3_head_entry.has_value()); // This is an implicitly untrusted header, so only the first value is used. const std::string b3(b3_head_entry.value()); diff --git a/source/extensions/tracers/zipkin/util.h b/source/extensions/tracers/zipkin/util.h index 80303986034f..6f2192405285 100644 --- a/source/extensions/tracers/zipkin/util.h +++ b/source/extensions/tracers/zipkin/util.h @@ -30,7 +30,7 @@ class Util { * @return std::string byte string representation of a number. */ template static std::string toByteString(Type value) { - return std::string(reinterpret_cast(&value), sizeof(Type)); + return {reinterpret_cast(&value), sizeof(Type)}; } /** @@ -42,7 +42,7 @@ class Util { */ template static std::string toBigEndianByteString(Type value) { auto bytes = toEndianness(value); - return std::string(reinterpret_cast(&bytes), sizeof(Type)); + return {reinterpret_cast(&bytes), sizeof(Type)}; } using Replacements = std::vector>; diff --git a/source/extensions/tracers/zipkin/zipkin_core_constants.h b/source/extensions/tracers/zipkin/zipkin_core_constants.h index ee82dc36055a..3685849259f4 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_constants.h +++ b/source/extensions/tracers/zipkin/zipkin_core_constants.h @@ -5,6 +5,7 @@ #include "envoy/http/header_map.h" #include "source/common/singleton/const_singleton.h" +#include "source/common/tracing/trace_context_impl.h" namespace Envoy { namespace Extensions { @@ -44,14 +45,14 @@ constexpr bool DEFAULT_SHARED_SPAN_CONTEXT = true; class ZipkinCoreConstantValues { public: // Zipkin B3 headers - const Http::LowerCaseString X_B3_TRACE_ID{"x-b3-traceid"}; - const Http::LowerCaseString X_B3_SPAN_ID{"x-b3-spanid"}; - const Http::LowerCaseString X_B3_PARENT_SPAN_ID{"x-b3-parentspanid"}; - const Http::LowerCaseString X_B3_SAMPLED{"x-b3-sampled"}; - const Http::LowerCaseString X_B3_FLAGS{"x-b3-flags"}; + const Tracing::TraceContextHandler X_B3_TRACE_ID{"x-b3-traceid"}; + const Tracing::TraceContextHandler X_B3_SPAN_ID{"x-b3-spanid"}; + const Tracing::TraceContextHandler X_B3_PARENT_SPAN_ID{"x-b3-parentspanid"}; + const Tracing::TraceContextHandler X_B3_SAMPLED{"x-b3-sampled"}; + const Tracing::TraceContextHandler X_B3_FLAGS{"x-b3-flags"}; // Zipkin b3 single header - const Http::LowerCaseString B3{"b3"}; + const Tracing::TraceContextHandler B3{"b3"}; }; using ZipkinCoreConstants = ConstSingleton; diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc index 1eaf50a36de8..7642681d8321 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc @@ -42,19 +42,18 @@ std::string ZipkinSpan::getBaggage(absl::string_view) { return EMPTY_STRING; } void ZipkinSpan::injectContext(Tracing::TraceContext& trace_context, const Upstream::HostDescriptionConstSharedPtr&) { // Set the trace-id and span-id headers properly, based on the newly-created span structure. - trace_context.setByReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - span_.traceIdAsHexString()); - trace_context.setByReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, span_.idAsHexString()); + ZipkinCoreConstants::get().X_B3_TRACE_ID.setRefKey(trace_context, span_.traceIdAsHexString()); + ZipkinCoreConstants::get().X_B3_SPAN_ID.setRefKey(trace_context, span_.idAsHexString()); // Set the parent-span header properly, based on the newly-created span structure. if (span_.isSetParentId()) { - trace_context.setByReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, - span_.parentIdAsHexString()); + ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.setRefKey(trace_context, + span_.parentIdAsHexString()); } // Set the sampled header. - trace_context.setByReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, - span_.sampled() ? SAMPLED : NOT_SAMPLED); + ZipkinCoreConstants::get().X_B3_SAMPLED.setRefKey(trace_context, + span_.sampled() ? SAMPLED : NOT_SAMPLED); } void ZipkinSpan::setSampled(bool sampled) { span_.setSampled(sampled); } @@ -105,7 +104,7 @@ Driver::Driver(const envoy::config::trace::v3::ZipkinConfig& zipkin_config, local_info_.clusterName(), local_info_.address(), random_generator, trace_id_128bit, shared_span_context, time_source_, split_spans_for_request); tracer->setReporter( - ReporterImpl::NewInstance(std::ref(*this), std::ref(dispatcher), collector)); + ReporterImpl::newInstance(std::ref(*this), std::ref(dispatcher), collector)); return std::make_shared(std::move(tracer), *this); }); } @@ -113,7 +112,7 @@ Driver::Driver(const envoy::config::trace::v3::ZipkinConfig& zipkin_config, Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string&, - const Tracing::Decision tracing_decision) { + Tracing::Decision tracing_decision) { Tracer& tracer = *tls_->getTyped().tracer_; SpanPtr new_zipkin_span; SpanContextExtractor extractor(trace_context); @@ -155,7 +154,7 @@ ReporterImpl::ReporterImpl(Driver& driver, Event::Dispatcher& dispatcher, enableTimer(); } -ReporterPtr ReporterImpl::NewInstance(Driver& driver, Event::Dispatcher& dispatcher, +ReporterPtr ReporterImpl::newInstance(Driver& driver, Event::Dispatcher& dispatcher, const CollectorInfo& collector) { return std::make_unique(driver, dispatcher, collector); } diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index 98dc6810eaeb..9385e6d0280b 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -126,7 +126,7 @@ class Driver : public Tracing::Driver { Tracing::SpanPtr startSpan(const Tracing::Config& config, Tracing::TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) override; + Tracing::Decision tracing_decision) override; // Getters to return the ZipkinDriver's key members. Upstream::ClusterManager& clusterManager() { return cm_; } @@ -225,7 +225,7 @@ class ReporterImpl : Logger::Loggable, * * @return Pointer to the newly-created ZipkinReporter. */ - static ReporterPtr NewInstance(Driver& driver, Event::Dispatcher& dispatcher, + static ReporterPtr newInstance(Driver& driver, Event::Dispatcher& dispatcher, const CollectorInfo& collector); private: diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index 1b909e1fee8f..f1d4c0b33e01 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -1,3 +1,4 @@ +load("@com_github_rules_proto_grpc//cpp:defs.bzl", "cpp_grpc_library") load( "//bazel:envoy_build_system.bzl", "envoy_cc_extension", @@ -38,6 +39,7 @@ envoy_cc_extension( "abseil_node_hash_set", ], deps = [ + ":alts_channel_pool", ":tsi_handshaker", ":tsi_socket", "//envoy/registry", @@ -70,6 +72,7 @@ envoy_cc_library( "tsi_handshaker.h", ], deps = [ + ":alts_tsi_handshaker", ":grpc_tsi_wrapper", "//envoy/event:dispatcher_interface", "//source/common/buffer:buffer_lib", @@ -109,3 +112,51 @@ envoy_cc_library( "//envoy/network:transport_socket_interface", ], ) + +envoy_cc_library( + name = "alts_channel_pool", + srcs = ["alts_channel_pool.cc"], + hdrs = ["alts_channel_pool.h"], + external_deps = [ + "grpc", + ], + deps = [ + "@com_google_absl//absl/random", + ], +) + +cpp_grpc_library( + name = "handshaker_cc_grpc", + protos = ["@com_github_grpc_grpc//src/proto/grpc/gcp:alts_handshaker_proto"], +) + +envoy_cc_library( + name = "alts_proxy", + srcs = ["alts_proxy.cc"], + hdrs = ["alts_proxy.h"], + external_deps = [ + "abseil_memory", + "abseil_status", + "abseil_statusor", + "grpc", + ], + deps = [ + ":handshaker_cc_grpc", + ], +) + +envoy_cc_library( + name = "alts_tsi_handshaker", + srcs = ["alts_tsi_handshaker.cc"], + hdrs = ["alts_tsi_handshaker.h"], + external_deps = [ + "abseil_memory", + "abseil_status", + "grpc", + ], + deps = [ + ":alts_proxy", + ":handshaker_cc_grpc", + ":tsi_frame_protector", + ], +) diff --git a/source/extensions/transport_sockets/alts/alts_channel_pool.cc b/source/extensions/transport_sockets/alts/alts_channel_pool.cc new file mode 100644 index 000000000000..8bc4137e2f9b --- /dev/null +++ b/source/extensions/transport_sockets/alts/alts_channel_pool.cc @@ -0,0 +1,56 @@ +#include "source/extensions/transport_sockets/alts/alts_channel_pool.h" + +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" +#include "grpcpp/channel.h" +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { + +// TODO(matthewstevenson88): Extend this to be configurable through API. +constexpr std::size_t ChannelPoolSize = 10; + +std::unique_ptr +AltsChannelPool::create(absl::string_view handshaker_service_address) { + std::vector> channel_pool; + channel_pool.reserve(ChannelPoolSize); + grpc::ChannelArguments channel_args; + channel_args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + for (std::size_t i = 0; i < ChannelPoolSize; ++i) { + channel_pool.push_back(grpc::CreateCustomChannel( + std::string(handshaker_service_address), grpc::InsecureChannelCredentials(), channel_args)); + } + return absl::WrapUnique(new AltsChannelPool(std::move(channel_pool))); +} + +AltsChannelPool::AltsChannelPool(const std::vector>& channel_pool) + : channel_pool_(channel_pool) {} + +// TODO(matthewstevenson88): Add logic to limit number of outstanding channels. +std::shared_ptr AltsChannelPool::getChannel() { + std::shared_ptr channel; + { + absl::MutexLock lock(&mu_); + channel = channel_pool_[index_]; + index_ = (index_ + 1) % channel_pool_.size(); + } + return channel; +} + +std::size_t AltsChannelPool::getChannelPoolSize() const { return channel_pool_.size(); } + +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/alts/alts_channel_pool.h b/source/extensions/transport_sockets/alts/alts_channel_pool.h new file mode 100644 index 000000000000..6eeb3936249b --- /dev/null +++ b/source/extensions/transport_sockets/alts/alts_channel_pool.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" +#include "grpcpp/channel.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { + +// Manages a pool of gRPC channels to the ALTS handshaker service. +class AltsChannelPool { +public: + static std::unique_ptr create(absl::string_view handshaker_service_address); + + // Gets a channel to the ALTS handshaker service. The caller is responsible + // for checking that the channel is non-null. + std::shared_ptr getChannel(); + + std::size_t getChannelPoolSize() const; + +private: + explicit AltsChannelPool(const std::vector>& channel_pool); + + // Use round-robin to select channels from the pool. + absl::Mutex mu_; + std::size_t index_ ABSL_GUARDED_BY(mu_) = 0; + std::vector> channel_pool_; +}; + +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/alts/alts_proxy.cc b/source/extensions/transport_sockets/alts/alts_proxy.cc new file mode 100644 index 000000000000..1b96239511bb --- /dev/null +++ b/source/extensions/transport_sockets/alts/alts_proxy.cc @@ -0,0 +1,159 @@ +#include "source/extensions/transport_sockets/alts/alts_proxy.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "absl/types/span.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { + +using ::grpc::gcp::HandshakeProtocol; +using ::grpc::gcp::HandshakerReq; +using ::grpc::gcp::HandshakerResp; +using ::grpc::gcp::HandshakerService; +using ::grpc::gcp::NextHandshakeMessageReq; +using ::grpc::gcp::ServerHandshakeParameters; +using ::grpc::gcp::StartClientHandshakeReq; +using ::grpc::gcp::StartServerHandshakeReq; + +// TODO(matthewstevenson88): Make this deadline configurable. +constexpr absl::Duration AltsClientContextDeadline = absl::Seconds(30); + +void AltsProxy::setRpcProtocolVersions(grpc::gcp::RpcProtocolVersions* rpc_protocol_versions) { + rpc_protocol_versions->mutable_max_rpc_version()->set_major(MaxMajorRpcVersion); + rpc_protocol_versions->mutable_max_rpc_version()->set_minor(MaxMinorRpcVersion); + rpc_protocol_versions->mutable_min_rpc_version()->set_major(MinMajorRpcVersion); + rpc_protocol_versions->mutable_min_rpc_version()->set_minor(MinMinorRpcVersion); +} + +absl::StatusOr> +AltsProxy::create(std::shared_ptr handshaker_service_channel) { + if (handshaker_service_channel == nullptr) { + return absl::InvalidArgumentError("Handshaker service channel is null."); + } + auto client_context = std::make_unique(); + client_context->set_deadline(absl::ToChronoTime(absl::Now() + AltsClientContextDeadline)); + // TODO(matthewstevenson88): Investigate using Envoy's async gRPC client. + auto stub = HandshakerService::NewStub(handshaker_service_channel); + if (stub == nullptr) { + return absl::InvalidArgumentError("Handshaker service stub is null."); + } + auto stream = stub->DoHandshake(client_context.get()); + if (stream == nullptr) { + return absl::InvalidArgumentError("Handshaker service stream is null."); + } + return absl::WrapUnique( + new AltsProxy(std::move(client_context), std::move(stub), std::move(stream))); +} + +AltsProxy::AltsProxy( + std::unique_ptr client_context, + std::unique_ptr stub, + std::unique_ptr> stream) + : client_context_(std::move(client_context)), stub_(std::move(stub)), + stream_(std::move(stream)) {} + +AltsProxy::~AltsProxy() { + if (stream_ != nullptr) { + stream_->WritesDone(); + } +} + +absl::StatusOr AltsProxy::sendStartClientHandshakeReq() { + // Prepare the StartClientHandshakeReq message. Ignore the target name field, + // it should never be populated for Envoy's use of ALTS. + HandshakerReq request; + StartClientHandshakeReq* client_start = request.mutable_client_start(); + client_start->set_handshake_security_protocol(grpc::gcp::ALTS); + client_start->add_application_protocols(ApplicationProtocol); + client_start->add_record_protocols(RecordProtocol); + setRpcProtocolVersions(client_start->mutable_rpc_versions()); + client_start->set_max_frame_size(MaxFrameSize); + + // Send the StartClientHandshakeReq message to the handshaker service and wait + // for the response. + if (!stream_->Write(request)) { + return absl::UnavailableError( + "Failed to write client start to handshaker service. This is probably " + "because the handshaker service is unreachable or unresponsive."); + } + HandshakerResp response; + if (!stream_->Read(&response)) { + return absl::InternalError("Failed to read client start response from handshaker service."); + } + if (response.has_status() && response.status().code() != 0) { + return absl::Status(static_cast(response.status().code()), + response.status().details()); + } + return response; +} + +absl::StatusOr +AltsProxy::sendStartServerHandshakeReq(absl::Span in_bytes) { + // Prepare the StartServerHandshakeReq message. + ServerHandshakeParameters server_parameters; + server_parameters.add_record_protocols(RecordProtocol); + HandshakerReq request; + StartServerHandshakeReq* server_start = request.mutable_server_start(); + server_start->add_application_protocols(ApplicationProtocol); + (*server_start->mutable_handshake_parameters())[HandshakeProtocol::ALTS] = server_parameters; + setRpcProtocolVersions(server_start->mutable_rpc_versions()); + server_start->set_in_bytes(in_bytes.data(), in_bytes.size()); + server_start->set_max_frame_size(MaxFrameSize); + + // Send the StartServerHandshakeReq message to the handshaker service and wait + // for the response. + if (!stream_->Write(request)) { + return absl::UnavailableError( + "Failed to write server start to handshaker service. This is probably " + "because the handshaker service is unreachable or unresponsive."); + } + HandshakerResp response; + if (!stream_->Read(&response)) { + return absl::InternalError("Failed to read server start response from handshaker service."); + } + if (response.has_status() && response.status().code() != 0) { + return absl::Status(static_cast(response.status().code()), + response.status().details()); + } + return response; +} + +absl::StatusOr +AltsProxy::sendNextHandshakeReq(absl::Span in_bytes) { + // Prepare the NextHandshakeMessageReq message. + HandshakerReq request; + NextHandshakeMessageReq* next = request.mutable_next(); + next->set_in_bytes(in_bytes.data(), in_bytes.size()); + + // Send the NextHandshakeMessageReq message to the handshaker service and wait + // for the response. + if (!stream_->Write(request)) { + return absl::UnavailableError( + "Failed to write next message to handshaker service. This is probably " + "because the handshaker service is unreachable or unresponsive."); + } + HandshakerResp response; + if (!stream_->Read(&response)) { + return absl::InternalError("Failed to read next response from handshaker service."); + } + if (response.has_status() && response.status().code() != 0) { + return absl::Status(static_cast(response.status().code()), + response.status().details()); + } + return response; +} + +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/alts/alts_proxy.h b/source/extensions/transport_sockets/alts/alts_proxy.h new file mode 100644 index 000000000000..e0369e6d75a2 --- /dev/null +++ b/source/extensions/transport_sockets/alts/alts_proxy.h @@ -0,0 +1,88 @@ +#pragma once + +#include + +#include "absl/status/statusor.h" +#include "absl/types/span.h" +#include "grpcpp/channel.h" +#include "grpcpp/client_context.h" +#include "grpcpp/support/sync_stream.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { + +constexpr char ApplicationProtocol[] = "grpc"; +constexpr char RecordProtocol[] = "ALTSRP_GCM_AES128_REKEY"; +constexpr std::size_t MaxFrameSize = 1024 * 1024; +constexpr std::size_t MaxMajorRpcVersion = 2; +constexpr std::size_t MaxMinorRpcVersion = 1; +constexpr std::size_t MinMajorRpcVersion = 2; +constexpr std::size_t MinMinorRpcVersion = 1; + +// Manages a bidirectional stream to the ALTS handshaker service. An AltsProxy +// instance is tied to a single ALTS handshake and must not be reused. +// +// WARNING: Several methods block the worker thread performing the ALTS +// handshake to make a gRPC call to the ALTS handshaker service. This can slow +// down or halt the proxy if the ALTS handshaker service is unavailable or +// experiencing high latency. +class AltsProxy { +public: + static absl::StatusOr> + create(std::shared_ptr handshaker_service_channel); + + ~AltsProxy(); + + // Sends a StartClientHandshakeReq message to the ALTS handshaker service and + // returns the response. + // + // WARNING: Blocks the worker thread performing the ALTS handshake to make a + // gRPC call to the ALTS handshaker service. This can slow down or halt the + // proxy if the ALTS handshaker service is unavailable or experiencing high + // latency. + absl::StatusOr sendStartClientHandshakeReq(); + + // Sends a StartServerHandshakeReq message to the ALTS handshaker service and + // returns the response. + // + // WARNING: Blocks the worker thread performing the ALTS handshake to make a + // gRPC call to the ALTS handshaker service. This can slow down or halt the + // proxy if the ALTS handshaker service is unavailable or experiencing high + // latency. + absl::StatusOr + sendStartServerHandshakeReq(absl::Span in_bytes); + + // Sends a NextHandshakeMessageReq message to the ALTS handshaker service and + // returns the response. + // + // WARNING: Blocks the worker thread performing the ALTS handshake to make a + // gRPC call to the ALTS handshaker service. This can slow down or halt the + // proxy if the ALTS handshaker service is unavailable or experiencing high + // latency. + absl::StatusOr + sendNextHandshakeReq(absl::Span in_bytes); + +private: + static void setRpcProtocolVersions(grpc::gcp::RpcProtocolVersions* rpc_protocol_versions); + + AltsProxy( + std::unique_ptr client_context, + std::unique_ptr stub, + std::unique_ptr> + stream); + + std::unique_ptr client_context_ = nullptr; + std::unique_ptr stub_; + std::unique_ptr> + stream_ = nullptr; +}; + +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/alts/alts_tsi_handshaker.cc b/source/extensions/transport_sockets/alts/alts_tsi_handshaker.cc new file mode 100644 index 000000000000..4ad74601466c --- /dev/null +++ b/source/extensions/transport_sockets/alts/alts_tsi_handshaker.cc @@ -0,0 +1,168 @@ +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/strings/str_format.h" +#include "absl/types/span.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { + +using ::grpc::gcp::HandshakerResp; + +constexpr std::size_t AltsAes128GcmRekeyKeyLength = 44; + +std::unique_ptr +AltsTsiHandshaker::createForClient(std::shared_ptr handshaker_service_channel) { + return absl::WrapUnique(new AltsTsiHandshaker(/*is_client=*/true, handshaker_service_channel)); +} + +std::unique_ptr +AltsTsiHandshaker::createForServer(std::shared_ptr handshaker_service_channel) { + return absl::WrapUnique(new AltsTsiHandshaker(/*is_client=*/false, handshaker_service_channel)); +} + +AltsTsiHandshaker::AltsTsiHandshaker(bool is_client, + std::shared_ptr handshaker_service_channel) + : is_client_(is_client), handshaker_service_channel_(handshaker_service_channel) {} + +absl::Status AltsTsiHandshaker::next(void* handshaker, const unsigned char* received_bytes, + size_t received_bytes_size, OnNextDone on_next_done) { + // Argument and state checks. + if (handshaker == nullptr || (received_bytes == nullptr && received_bytes_size > 0) || + on_next_done == nullptr) { + return absl::InvalidArgumentError("Invalid nullptr argument to AltsTsiHandshaker::Next."); + } + if (is_handshake_complete_) { + return absl::InternalError("Handshake is already complete."); + } + + // Get a handshake message from the handshaker service. + absl::Span in_bytes = absl::MakeConstSpan(received_bytes, received_bytes_size); + HandshakerResp response; + if (!has_sent_initial_handshake_message_) { + has_sent_initial_handshake_message_ = true; + auto alts_proxy = AltsProxy::create(handshaker_service_channel_); + if (!alts_proxy.ok()) { + return alts_proxy.status(); + } + alts_proxy_ = *std::move(alts_proxy); + if (is_client_) { + auto client_start = alts_proxy_->sendStartClientHandshakeReq(); + if (!client_start.ok()) { + return client_start.status(); + } + response = *std::move(client_start); + } else { + auto server_start = alts_proxy_->sendStartServerHandshakeReq(in_bytes); + if (!server_start.ok()) { + return server_start.status(); + } + response = *std::move(server_start); + } + } else { + auto next = alts_proxy_->sendNextHandshakeReq(in_bytes); + if (!next.ok()) { + return next.status(); + } + response = *std::move(next); + } + + // Maybe prepare the handshake result. + std::unique_ptr handshake_result = nullptr; + if (response.has_result()) { + is_handshake_complete_ = true; + auto result = getHandshakeResult(response.result(), in_bytes, response.bytes_consumed()); + if (!result.ok()) { + return result.status(); + } + handshake_result = *std::move(result); + } + + // Write the out bytes. + const std::string& out_bytes = response.out_frames(); + on_next_done(absl::OkStatus(), handshaker, + reinterpret_cast(out_bytes.c_str()), out_bytes.size(), + std::move(handshake_result)); + return absl::OkStatus(); +} + +std::size_t AltsTsiHandshaker::computeMaxFrameSize(const grpc::gcp::HandshakerResult& result) { + if (result.max_frame_size() > 0) { + return std::clamp(static_cast(result.max_frame_size()), AltsMinFrameSize, + MaxFrameSize); + } + return AltsMinFrameSize; +} + +absl::StatusOr> +AltsTsiHandshaker::getHandshakeResult(const grpc::gcp::HandshakerResult& result, + absl::Span received_bytes, + std::size_t bytes_consumed) { + // Validate the HandshakerResult message. + if (!result.has_peer_identity()) { + return absl::FailedPreconditionError("Handshake result is missing peer identity."); + } + if (!result.has_local_identity()) { + return absl::FailedPreconditionError("Handshake result is missing local identity."); + } + if (!result.has_peer_rpc_versions()) { + return absl::FailedPreconditionError("Handshake result is missing peer rpc versions."); + } + if (result.application_protocol().empty()) { + return absl::FailedPreconditionError("Handshake result has empty application protocol."); + } + if (result.record_protocol() != RecordProtocol) { + return absl::FailedPreconditionError( + "Handshake result's record protocol is not ALTSRP_GCM_AES128_REKEY."); + } + if (result.key_data().size() < AltsAes128GcmRekeyKeyLength) { + return absl::FailedPreconditionError("Handshake result's key data is too short."); + } + if (bytes_consumed > received_bytes.size()) { + return absl::FailedPreconditionError( + "Handshaker service consumed more bytes than were received from the " + "peer."); + } + + // Create the frame protector. + std::size_t max_frame_size = computeMaxFrameSize(result); + tsi_zero_copy_grpc_protector* protector = nullptr; + grpc_core::ExecCtx exec_ctx; + tsi_result ok = alts_zero_copy_grpc_protector_create( + reinterpret_cast(result.key_data().data()), AltsAes128GcmRekeyKeyLength, true, + is_client_, + /*is_integrity_only=*/false, /*enable_extra_copy=*/false, &max_frame_size, &protector); + if (ok != TSI_OK) { + return absl::InternalError(absl::StrFormat("Failed to create frame protector: %zu", ok)); + } + + // Calculate the unused bytes. + std::size_t unused_bytes_size = received_bytes.size() - bytes_consumed; + const uint8_t* unused_bytes_ptr = received_bytes.data() + bytes_consumed; + std::vector unused_bytes(unused_bytes_ptr, unused_bytes_ptr + unused_bytes_size); + + // Create and return the AltsHandshakeResult. + auto handshake_result = std::make_unique(); + handshake_result->frame_protector = std::make_unique(protector); + handshake_result->peer_identity = result.peer_identity().service_account(); + handshake_result->unused_bytes = unused_bytes; + return handshake_result; +} + +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/alts/alts_tsi_handshaker.h b/source/extensions/transport_sockets/alts/alts_tsi_handshaker.h new file mode 100644 index 000000000000..dd06fa569027 --- /dev/null +++ b/source/extensions/transport_sockets/alts/alts_tsi_handshaker.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "source/extensions/transport_sockets/alts/alts_proxy.h" +#include "source/extensions/transport_sockets/alts/tsi_frame_protector.h" + +#include "absl/types/span.h" +#include "grpcpp/channel.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { + +constexpr std::size_t AltsMinFrameSize = 16 * 1024; + +struct AltsHandshakeResult { + TsiFrameProtectorPtr frame_protector; + std::string peer_identity; + std::vector unused_bytes; +}; + +// Manages a single ALTS handshake. +class AltsTsiHandshaker { +public: + using OnNextDone = std::function handshake_result)>; + + static std::unique_ptr + createForClient(std::shared_ptr handshaker_service_channel); + static std::unique_ptr + createForServer(std::shared_ptr handshaker_service_channel); + + // Get the next ALTS handshake message for the peer. + absl::Status next(void* handshaker, const unsigned char* received_bytes, + size_t received_bytes_size, OnNextDone on_next_done); + + // Exposed for testing purposes only. + absl::StatusOr> + getHandshakeResult(const grpc::gcp::HandshakerResult& result, + absl::Span received_bytes, std::size_t bytes_consumed); + static std::size_t computeMaxFrameSize(const grpc::gcp::HandshakerResult& result); + +private: + AltsTsiHandshaker(bool is_client, std::shared_ptr handshaker_service_channel); + + const bool is_client_; + const std::shared_ptr handshaker_service_channel_; + + bool has_sent_initial_handshake_message_ = false; + bool is_handshake_complete_ = false; + std::unique_ptr alts_proxy_ = nullptr; +}; + +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index 0af3cb780775..e463abb817d6 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -9,47 +9,55 @@ #include "source/common/grpc/google_grpc_context.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" +#include "source/extensions/transport_sockets/alts/alts_channel_pool.h" +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" #include "source/extensions/transport_sockets/alts/grpc_tsi.h" +#include "source/extensions/transport_sockets/alts/tsi_handshaker.h" #include "source/extensions/transport_sockets/alts/tsi_socket.h" #include "absl/container/node_hash_set.h" #include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "grpcpp/channel.h" namespace Envoy { namespace Extensions { namespace TransportSockets { namespace Alts { +namespace { -// smart pointer for grpc_alts_credentials_options that will be automatically freed. -using GrpcAltsCredentialsOptionsPtr = - CSmartPtr; +// Manage ALTS singleton state via SingletonManager +class AltsSharedState : public Singleton::Instance { +public: + explicit AltsSharedState(absl::string_view handshaker_service_address) + : channel_pool_(AltsChannelPool::create(handshaker_service_address)) {} -namespace { + ~AltsSharedState() override = default; -// TODO: gRPC v1.30.0-pre1 defines the equivalent function grpc_alts_set_rpc_protocol_versions -// that should be called directly when available. -void grpcAltsSetRpcProtocolVersions(grpc_gcp_rpc_protocol_versions* rpc_versions) { - grpc_gcp_rpc_protocol_versions_set_max(rpc_versions, GRPC_PROTOCOL_VERSION_MAX_MAJOR, - GRPC_PROTOCOL_VERSION_MAX_MINOR); - grpc_gcp_rpc_protocol_versions_set_min(rpc_versions, GRPC_PROTOCOL_VERSION_MIN_MAJOR, - GRPC_PROTOCOL_VERSION_MIN_MINOR); -} + std::shared_ptr getChannel() const { return channel_pool_->getChannel(); } + +private: + // There is blanket google-grpc initialization in MainCommonBase, but that + // doesn't cover unit tests. However, putting blanket coverage in ProcessWide + // causes background threaded memory allocation in all unit tests making it + // hard to measure memory. Thus we also initialize grpc using our idempotent + // wrapper-class in classes that need it. See + // https://github.com/envoyproxy/envoy/issues/8282 for details. +#ifdef ENVOY_GOOGLE_GRPC + Grpc::GoogleGrpcContext google_grpc_context_; +#endif + std::unique_ptr channel_pool_; +}; + +SINGLETON_MANAGER_REGISTRATION(alts_shared_state); // Returns true if the peer's service account is found in peers, otherwise // returns false and fills out err with an error message. -bool doValidate(const tsi_peer& peer, const absl::node_hash_set& peers, - TsiInfo& tsi_info, std::string& err) { - for (size_t i = 0; i < peer.property_count; ++i) { - const std::string name = std::string(peer.properties[i].name); - const std::string value = - std::string(peer.properties[i].value.data, peer.properties[i].value.length); - if (name.compare(TSI_ALTS_SERVICE_ACCOUNT_PEER_PROPERTY) == 0 && - peers.find(value) != peers.end()) { - tsi_info.name_ = value; - return true; - } +bool doValidate(const absl::node_hash_set& peers, TsiInfo& tsi_info, + std::string& err) { + if (peers.find(tsi_info.peer_identity_) != peers.end()) { + return true; } - err = "Couldn't find peer's service account in peer_service_accounts: " + absl::StrJoin(peers, ","); return false; @@ -63,81 +71,43 @@ createHandshakeValidator(const envoy::extensions::transport_sockets::alts::v3::A HandshakeValidator validator; // Skip validation if peers is empty. if (!peers.empty()) { - validator = [peers](const tsi_peer& peer, TsiInfo& tsi_info, std::string& err) { - return doValidate(peer, peers, tsi_info, err); + validator = [peers](TsiInfo& tsi_info, std::string& err) { + return doValidate(peers, tsi_info, err); }; } return validator; } -// Manage ALTS singleton state via SingletonManager -class AltsSharedState : public Singleton::Instance { -public: - AltsSharedState() { grpc_alts_shared_resource_dedicated_init(); } - - ~AltsSharedState() override { grpc_alts_shared_resource_dedicated_shutdown(); } - -private: - // There is blanket google-grpc initialization in MainCommonBase, but that - // doesn't cover unit tests. However, putting blanket coverage in ProcessWide - // causes background threaded memory allocation in all unit tests making it - // hard to measure memory. Thus we also initialize grpc using our idempotent - // wrapper-class in classes that need it. See - // https://github.com/envoyproxy/envoy/issues/8282 for details. -#ifdef ENVOY_GOOGLE_GRPC - Grpc::GoogleGrpcContext google_grpc_context_; -#endif -}; - -SINGLETON_MANAGER_REGISTRATION(alts_shared_state); - template TransportSocketFactoryPtr createTransportSocketFactoryHelper( const Protobuf::Message& message, bool is_upstream, Server::Configuration::TransportSocketFactoryContext& factory_ctxt) { - // A reference to this is held in the factory closure to keep the singleton - // instance alive. - auto alts_shared_state = - factory_ctxt.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), - [] { return std::make_shared(); }); auto config = MessageUtil::downcastAndValidate( message, factory_ctxt.messageValidationVisitor()); HandshakeValidator validator = createHandshakeValidator(config); + const std::string& handshaker_service_address = config.handshaker_service(); - const std::string& handshaker_service = config.handshaker_service(); + // A reference to this is held in the factory closure to keep the singleton + // instance alive. + auto alts_shared_state = + factory_ctxt.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(alts_shared_state), [handshaker_service_address] { + return std::make_shared(handshaker_service_address); + }); HandshakerFactory factory = - [handshaker_service, is_upstream, + [handshaker_service_address, is_upstream, alts_shared_state](Event::Dispatcher& dispatcher, const Network::Address::InstanceConstSharedPtr& local_address, const Network::Address::InstanceConstSharedPtr&) -> TsiHandshakerPtr { ASSERT(local_address != nullptr); - - GrpcAltsCredentialsOptionsPtr options; + std::unique_ptr tsi_handshaker; if (is_upstream) { - options = GrpcAltsCredentialsOptionsPtr(grpc_alts_credentials_client_options_create()); + tsi_handshaker = AltsTsiHandshaker::createForClient(alts_shared_state->getChannel()); } else { - options = GrpcAltsCredentialsOptionsPtr(grpc_alts_credentials_server_options_create()); - } - grpcAltsSetRpcProtocolVersions(&options->rpc_versions); - const char* target_name = is_upstream ? "" : nullptr; - tsi_handshaker* handshaker = nullptr; - // Specifying target name as empty since TSI won't take care of validating peer identity - // in this use case. The validation will be performed by TsiSocket with the validator. - // Set the max frame size to 16KB. - tsi_result status = alts_tsi_handshaker_create( - options.get(), target_name, handshaker_service.c_str(), is_upstream, - nullptr /* interested_parties */, &handshaker, 16384 /* default max frame size */); - CHandshakerPtr handshaker_ptr{handshaker}; - - if (status != TSI_OK) { - const std::string handshaker_name = is_upstream ? "client" : "server"; - ENVOY_LOG_MISC(warn, "Cannot create ATLS {} handshaker, status: {}", handshaker_name, status); - return nullptr; + tsi_handshaker = AltsTsiHandshaker::createForServer(alts_shared_state->getChannel()); } - - return std::make_unique(std::move(handshaker_ptr), dispatcher); + return std::make_unique(std::move(tsi_handshaker), dispatcher); }; return std::make_unique(factory, validator); diff --git a/source/extensions/transport_sockets/alts/tsi_frame_protector.cc b/source/extensions/transport_sockets/alts/tsi_frame_protector.cc index 045bbbe34a13..c03e0a1293d2 100644 --- a/source/extensions/transport_sockets/alts/tsi_frame_protector.cc +++ b/source/extensions/transport_sockets/alts/tsi_frame_protector.cc @@ -3,6 +3,7 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/common/assert.h" +#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/tsi/transport_security_grpc.h" #include "src/core/tsi/transport_security_interface.h" diff --git a/source/extensions/transport_sockets/alts/tsi_handshaker.cc b/source/extensions/transport_sockets/alts/tsi_handshaker.cc index 4440fc4011f6..90807f6bff46 100644 --- a/source/extensions/transport_sockets/alts/tsi_handshaker.cc +++ b/source/extensions/transport_sockets/alts/tsi_handshaker.cc @@ -1,61 +1,70 @@ #include "source/extensions/transport_sockets/alts/tsi_handshaker.h" +#include +#include +#include +#include + +#include "envoy/buffer/buffer.h" +#include "envoy/event/dispatcher.h" + #include "source/common/buffer/buffer_impl.h" #include "source/common/common/assert.h" +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" + +#include "absl/status/status.h" namespace Envoy { namespace Extensions { namespace TransportSockets { namespace Alts { -void TsiHandshaker::onNextDone(tsi_result status, void* user_data, +void TsiHandshaker::onNextDone(absl::Status status, void* handshaker, const unsigned char* bytes_to_send, size_t bytes_to_send_size, - tsi_handshaker_result* handshaker_result) { - TsiHandshaker* handshaker = static_cast(user_data); - + std::unique_ptr handshake_result) { + TsiHandshaker* tsi_handshaker = static_cast(handshaker); Buffer::InstancePtr to_send = std::make_unique(); if (bytes_to_send_size > 0) { to_send->add(bytes_to_send, bytes_to_send_size); } - auto next_result = - new TsiHandshakerCallbacks::NextResult{status, std::move(to_send), {handshaker_result}}; + auto next_result = new TsiHandshakerCallbacks::NextResult{status, std::move(to_send), + std::move(handshake_result)}; - handshaker->dispatcher_.post([handshaker, next_result]() { + tsi_handshaker->dispatcher_.post([tsi_handshaker, next_result]() { TsiHandshakerCallbacks::NextResultPtr next_result_ptr{next_result}; - ASSERT(handshaker->calling_); - handshaker->calling_ = false; + ASSERT(tsi_handshaker->calling_); + tsi_handshaker->calling_ = false; - if (handshaker->delete_on_done_) { - handshaker->deferredDelete(); + if (tsi_handshaker->delete_on_done_) { + tsi_handshaker->deferredDelete(); return; } - handshaker->callbacks_->onNextDone(std::move(next_result_ptr)); + tsi_handshaker->callbacks_->onNextDone(std::move(next_result_ptr)); }); } -TsiHandshaker::TsiHandshaker(CHandshakerPtr&& handshaker, Event::Dispatcher& dispatcher) +TsiHandshaker::TsiHandshaker(std::unique_ptr handshaker, + Event::Dispatcher& dispatcher) : handshaker_(std::move(handshaker)), dispatcher_(dispatcher) {} TsiHandshaker::~TsiHandshaker() { ASSERT(!calling_); } -tsi_result TsiHandshaker::next(Envoy::Buffer::Instance& received) { +absl::Status TsiHandshaker::next(Envoy::Buffer::Instance& received) { ASSERT(callbacks_); ASSERT(!calling_); calling_ = true; uint64_t received_size = received.length(); - const unsigned char* bytes_to_send = nullptr; - size_t bytes_to_send_size = 0; - tsi_handshaker_result* result = nullptr; - tsi_result status = tsi_handshaker_next( - handshaker_.get(), reinterpret_cast(received.linearize(received_size)), - received_size, &bytes_to_send, &bytes_to_send_size, &result, onNextDone, this); - received.drain(received_size); + absl::Status status = handshaker_->next( + this, reinterpret_cast(received.linearize(received_size)), + received_size, onNextDone); - if (status != TSI_ASYNC) { - onNextDone(status, this, bytes_to_send, bytes_to_send_size, result); + received.drain(received_size); + if (!status.ok()) { + onNextDone(status, this, /*bytes_to_send=*/nullptr, + /*bytes_to_send_size=*/0, /*handshake_result=*/nullptr); } return status; } diff --git a/source/extensions/transport_sockets/alts/tsi_handshaker.h b/source/extensions/transport_sockets/alts/tsi_handshaker.h index 9408a67bfa68..052469b7cc47 100644 --- a/source/extensions/transport_sockets/alts/tsi_handshaker.h +++ b/source/extensions/transport_sockets/alts/tsi_handshaker.h @@ -6,8 +6,11 @@ #include "envoy/event/dispatcher.h" #include "source/common/common/c_smart_ptr.h" +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" #include "source/extensions/transport_sockets/alts/grpc_tsi.h" +#include "absl/status/status.h" + namespace Envoy { namespace Extensions { namespace TransportSockets { @@ -22,14 +25,14 @@ class TsiHandshakerCallbacks { virtual ~TsiHandshakerCallbacks() = default; struct NextResult { - // A enum of the result. - tsi_result status_; + // A status of the result. + absl::Status status_; // The buffer to be sent to the peer. Buffer::InstancePtr to_send_; - // A pointer to tsi_handshaker_result struct. Owned by instance. - CHandshakerResultPtr result_; + // A pointer to AltsHandshakeResult. Owned by instance. + std::unique_ptr result_; }; using NextResultPtr = std::unique_ptr; @@ -49,7 +52,8 @@ class TsiHandshakerCallbacks { */ class TsiHandshaker final : public Event::DeferredDeletable { public: - explicit TsiHandshaker(CHandshakerPtr&& handshaker, Event::Dispatcher& dispatcher); + explicit TsiHandshaker(std::unique_ptr handshaker, + Event::Dispatcher& dispatcher); ~TsiHandshaker() override; /** @@ -59,7 +63,7 @@ class TsiHandshaker final : public Event::DeferredDeletable { * TsiHandshakerCallbacks::onNextDone is called. * @param received the buffer received from peer. */ - tsi_result next(Buffer::Instance& received); + absl::Status next(Buffer::Instance& received); /** * Set handshaker callbacks. This must be called before calling next. @@ -75,10 +79,11 @@ class TsiHandshaker final : public Event::DeferredDeletable { void deferredDelete(); private: - static void onNextDone(tsi_result status, void* user_data, const unsigned char* bytes_to_send, - size_t bytes_to_send_size, tsi_handshaker_result* handshaker_result); + static void onNextDone(absl::Status status, void* handshaker, const unsigned char* bytes_to_send, + size_t bytes_to_send_size, + std::unique_ptr handshake_result); - CHandshakerPtr handshaker_; + std::unique_ptr handshaker_; TsiHandshakerCallbacks* callbacks_{}; // This is set to true when there is an ongoing next call to handshaker, and set to false when diff --git a/source/extensions/transport_sockets/alts/tsi_socket.cc b/source/extensions/transport_sockets/alts/tsi_socket.cc index c882c9a96cef..331436e31975 100644 --- a/source/extensions/transport_sockets/alts/tsi_socket.cc +++ b/source/extensions/transport_sockets/alts/tsi_socket.cc @@ -1,9 +1,15 @@ #include "source/extensions/transport_sockets/alts/tsi_socket.h" +#include +#include +#include +#include + #include "source/common/common/assert.h" #include "source/common/common/cleanup.h" #include "source/common/common/empty_string.h" #include "source/common/common/enum_to_int.h" +#include "source/common/network/raw_buffer_socket.h" namespace Envoy { namespace Extensions { @@ -47,12 +53,12 @@ Network::PostIoAction TsiSocket::doHandshake() { ASSERT(!handshake_complete_); ENVOY_CONN_LOG(debug, "TSI: doHandshake", callbacks_->connection()); if (!handshaker_next_calling_ && raw_read_buffer_.length() > 0) { - doHandshakeNext(); + return doHandshakeNext(); } return Network::PostIoAction::KeepOpen; } -void TsiSocket::doHandshakeNext() { +Network::PostIoAction TsiSocket::doHandshakeNext() { ENVOY_CONN_LOG(debug, "TSI: doHandshake next: received: {}", callbacks_->connection(), raw_read_buffer_.length()); @@ -65,7 +71,7 @@ void TsiSocket::doHandshakeNext() { ENVOY_CONN_LOG(warn, "TSI: failed to create handshaker", callbacks_->connection()); callbacks_->connection().close(Network::ConnectionCloseType::NoFlush, "failed_creating_handshaker"); - return; + return Network::PostIoAction::Close; } handshaker_->setHandshakerCallbacks(*this); @@ -74,7 +80,12 @@ void TsiSocket::doHandshakeNext() { handshaker_next_calling_ = true; Buffer::OwnedImpl handshaker_buffer; handshaker_buffer.move(raw_read_buffer_); - handshaker_->next(handshaker_buffer); + absl::Status status = handshaker_->next(handshaker_buffer); + if (!status.ok()) { + ENVOY_CONN_LOG(debug, "TSI: Handshake failed: status: {}", callbacks_->connection(), status); + return Network::PostIoAction::Close; + } + return Network::PostIoAction::KeepOpen; } Network::PostIoAction TsiSocket::doHandshakeNextDone(NextResultPtr&& next_result) { @@ -83,10 +94,9 @@ Network::PostIoAction TsiSocket::doHandshakeNextDone(NextResultPtr&& next_result ENVOY_CONN_LOG(debug, "TSI: doHandshake next done: status: {} to_send: {}", callbacks_->connection(), next_result->status_, next_result->to_send_->length()); - tsi_result status = next_result->status_; - tsi_handshaker_result* handshaker_result = next_result->result_.get(); - - if (status != TSI_INCOMPLETE_DATA && status != TSI_OK) { + absl::Status status = next_result->status_; + AltsHandshakeResult* handshake_result = next_result->result_.get(); + if (!status.ok()) { ENVOY_CONN_LOG(debug, "TSI: Handshake failed: status: {}", callbacks_->connection(), status); return Network::PostIoAction::Close; } @@ -95,22 +105,12 @@ Network::PostIoAction TsiSocket::doHandshakeNextDone(NextResultPtr&& next_result raw_write_buffer_.move(*next_result->to_send_); } - if (status == TSI_OK && handshaker_result != nullptr) { - tsi_peer peer; - // returns TSI_OK assuming there is no fatal error. Asserting OK. - status = tsi_handshaker_result_extract_peer(handshaker_result, &peer); - ASSERT(status == TSI_OK); - Cleanup peer_cleanup([&peer]() { tsi_peer_destruct(&peer); }); - ENVOY_CONN_LOG(debug, "TSI: Handshake successful: peer properties: {}", - callbacks_->connection(), peer.property_count); - for (size_t i = 0; i < peer.property_count; ++i) { - ENVOY_CONN_LOG(debug, " {}: {}", callbacks_->connection(), peer.properties[i].name, - std::string(peer.properties[i].value.data, peer.properties[i].value.length)); - } + if (status.ok() && handshake_result != nullptr) { if (handshake_validator_) { std::string err; TsiInfo tsi_info; - const bool peer_validated = handshake_validator_(peer, tsi_info, err); + tsi_info.peer_identity_ = handshake_result->peer_identity; + const bool peer_validated = handshake_validator_(tsi_info, err); if (peer_validated) { ENVOY_CONN_LOG(debug, "TSI: Handshake validation succeeded.", callbacks_->connection()); } else { @@ -120,46 +120,30 @@ Network::PostIoAction TsiSocket::doHandshakeNextDone(NextResultPtr&& next_result } ProtobufWkt::Struct dynamic_metadata; ProtobufWkt::Value val; - val.set_string_value(tsi_info.name_); + val.set_string_value(tsi_info.peer_identity_); dynamic_metadata.mutable_fields()->insert({std::string("peer_identity"), val}); callbacks_->connection().streamInfo().setDynamicMetadata( "envoy.transport_sockets.peer_information", dynamic_metadata); - ENVOY_CONN_LOG(debug, "TSI hanshake with peer: {}", callbacks_->connection(), tsi_info.name_); + ENVOY_CONN_LOG(debug, "TSI handshake with peer: {}", callbacks_->connection(), + tsi_info.peer_identity_); } else { ENVOY_CONN_LOG(debug, "TSI: Handshake validation skipped.", callbacks_->connection()); } - const unsigned char* unused_bytes; - size_t unused_byte_size; - - // returns TSI_OK assuming there is no fatal error. Asserting OK. - status = - tsi_handshaker_result_get_unused_bytes(handshaker_result, &unused_bytes, &unused_byte_size); - ASSERT(status == TSI_OK); - if (unused_byte_size > 0) { + if (!handshake_result->unused_bytes.empty()) { // All handshake data is consumed. ASSERT(raw_read_buffer_.length() == 0); - raw_read_buffer_.prepend( - absl::string_view{reinterpret_cast(unused_bytes), unused_byte_size}); + absl::string_view unused_bytes( + reinterpret_cast(handshake_result->unused_bytes.data()), + handshake_result->unused_bytes.size()); + raw_read_buffer_.prepend(unused_bytes); } ENVOY_CONN_LOG(debug, "TSI: Handshake successful: unused_bytes: {}", callbacks_->connection(), - unused_byte_size); - - // returns TSI_OK assuming there is no fatal error. Asserting OK. - tsi_zero_copy_grpc_protector* frame_protector; - grpc_core::ExecCtx exec_ctx; - status = tsi_handshaker_result_create_zero_copy_grpc_protector( - handshaker_result, &default_max_frame_size_, &frame_protector); - ASSERT(status == TSI_OK); - - // TODO(yihuazhang): Check the return value once fake TSI frame protector - // used in tsi_socket_test.cc implements the interface returning the max frame size. - tsi_zero_copy_grpc_protector_max_frame_size(frame_protector, &actual_frame_size_to_use_); - + handshake_result->unused_bytes.size()); // Reset the watermarks with actual negotiated max frame size. raw_read_buffer_.setWatermarks( std::max(actual_frame_size_to_use_, callbacks_->connection().bufferLimit())); - frame_protector_ = std::make_unique(frame_protector); + frame_protector_ = std::move(handshake_result->frame_protector); handshake_complete_ = true; if (raw_write_buffer_.length() == 0) { @@ -328,15 +312,8 @@ Network::IoResult TsiSocket::repeatProtectAndWrite(Buffer::Instance& buffer, boo Network::IoResult TsiSocket::doWrite(Buffer::Instance& buffer, bool end_stream) { if (!handshake_complete_) { Network::PostIoAction action = doHandshake(); - // Envoy ALTS implements asynchronous tsi_handshaker_next() interface - // which returns immediately after scheduling a handshake request to - // the handshake service. The handshake response will be handled by a - // dedicated thread in a separate API within which handshake_complete_ - // will be set to true if the handshake completes. ASSERT(!handshake_complete_); - ASSERT(action == Network::PostIoAction::KeepOpen); - // TODO(lizan): Handle synchronous handshake when TsiHandshaker supports it. - return {Network::PostIoAction::KeepOpen, 0, false}; + return {action, 0, false}; } else { ASSERT(frame_protector_); // Check if we need to flush outstanding handshake bytes. diff --git a/source/extensions/transport_sockets/alts/tsi_socket.h b/source/extensions/transport_sockets/alts/tsi_socket.h index a9f555d2f32d..c44fb60bcb14 100644 --- a/source/extensions/transport_sockets/alts/tsi_socket.h +++ b/source/extensions/transport_sockets/alts/tsi_socket.h @@ -16,7 +16,7 @@ namespace TransportSockets { namespace Alts { struct TsiInfo { - std::string name_; + std::string peer_identity_; }; /** @@ -31,13 +31,11 @@ using HandshakerFactory = std::function; +using HandshakeValidator = std::function; /* Forward declaration */ class TsiTransportSocketCallbacks; @@ -89,7 +87,7 @@ class TsiSocket : public Network::TransportSocket, private: Network::PostIoAction doHandshake(); - void doHandshakeNext(); + Network::PostIoAction doHandshakeNext(); Network::PostIoAction doHandshakeNextDone(NextResultPtr&& next_result); // Helper function to perform repeated read and unprotect operations. diff --git a/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc b/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc index 7ffbf94f48d9..138578d1b839 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc +++ b/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc @@ -1,6 +1,8 @@ #include "source/extensions/transport_sockets/tls/cert_validator/default_validator.h" +#include #include +#include #include #include #include @@ -77,7 +79,7 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, bssl::UniquePtr list( PEM_X509_INFO_read_bio(bio.get(), nullptr, nullptr, nullptr)); if (list == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load trusted CA certificates from ", config_->caCertPath())); } @@ -101,7 +103,7 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, } } if (ca_cert_ == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load trusted CA certificates from ", config_->caCertPath())); } if (has_crl) { @@ -128,7 +130,7 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, bssl::UniquePtr list( PEM_X509_INFO_read_bio(bio.get(), nullptr, nullptr, nullptr)); if (list == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load CRL from ", config_->certificateRevocationListPath())); } @@ -155,7 +157,7 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, cert_validation_config->subjectAltNameMatchers()) { auto san_matcher = createStringSanMatcher(matcher); if (san_matcher == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to create string SAN matcher of type ", matcher.san_type())); } subject_alt_name_matchers_.push_back(std::move(san_matcher)); @@ -172,7 +174,7 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, } const auto& decoded = Hex::decode(hash); if (decoded.size() != SHA256_DIGEST_LENGTH) { - throw EnvoyException(absl::StrCat("Invalid hex-encoded SHA-256 ", hash)); + throwEnvoyExceptionOrPanic(absl::StrCat("Invalid hex-encoded SHA-256 ", hash)); } verify_certificate_hash_list_.push_back(decoded); } @@ -183,7 +185,7 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, for (const auto& hash : cert_validation_config->verifyCertificateSpkiList()) { const auto decoded = Base64::decode(hash); if (decoded.size() != SHA256_DIGEST_LENGTH) { - throw EnvoyException(absl::StrCat("Invalid base64-encoded SHA-256 ", hash)); + throwEnvoyExceptionOrPanic(absl::StrCat("Invalid base64-encoded SHA-256 ", hash)); } verify_certificate_spki_list_.emplace_back(decoded.begin(), decoded.end()); } @@ -501,8 +503,8 @@ void DefaultCertValidator::addClientValidationContext(SSL_CTX* ctx, bool require } X509_NAME* name = X509_get_subject_name(cert.get()); if (name == nullptr) { - throw EnvoyException(absl::StrCat("Failed to load trusted client CA certificates from ", - config_->caCertPath())); + throwEnvoyExceptionOrPanic(absl::StrCat("Failed to load trusted client CA certificates from ", + config_->caCertPath())); } // Check for duplicates. if (sk_X509_NAME_find(list.get(), nullptr, name)) { @@ -510,8 +512,8 @@ void DefaultCertValidator::addClientValidationContext(SSL_CTX* ctx, bool require } bssl::UniquePtr name_dup(X509_NAME_dup(name)); if (name_dup == nullptr || !sk_X509_NAME_push(list.get(), name_dup.release())) { - throw EnvoyException(absl::StrCat("Failed to load trusted client CA certificates from ", - config_->caCertPath())); + throwEnvoyExceptionOrPanic(absl::StrCat("Failed to load trusted client CA certificates from ", + config_->caCertPath())); } } @@ -520,7 +522,7 @@ void DefaultCertValidator::addClientValidationContext(SSL_CTX* ctx, bool require if (ERR_GET_LIB(err) == ERR_LIB_PEM && ERR_GET_REASON(err) == PEM_R_NO_START_LINE) { ERR_clear_error(); } else { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load trusted client CA certificates from ", config_->caCertPath())); } SSL_CTX_set_client_CA_list(ctx, list.release()); @@ -530,7 +532,15 @@ void DefaultCertValidator::addClientValidationContext(SSL_CTX* ctx, bool require } // Set the verify_depth if (config_->maxVerifyDepth().has_value()) { - SSL_CTX_set_verify_depth(ctx, config_->maxVerifyDepth().value()); + uint32_t max_verify_depth = std::min(config_->maxVerifyDepth().value(), uint32_t{INT_MAX}); +#if BORINGSSL_API_VERSION >= 29 + // Older BoringSSLs behave like OpenSSL 1.0.x and exclude the leaf from the + // depth but include the trust anchor. Newer BoringSSLs match OpenSSL 1.1.x + // and later in excluding both the leaf and trust anchor. `maxVerifyDepth` + // documents the older behavior, so adjust the value to match. + max_verify_depth = max_verify_depth > 0 ? max_verify_depth - 1 : 0; +#endif + SSL_CTX_set_verify_depth(ctx, static_cast(max_verify_depth)); } } diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index 76578b62b6fe..6322e594871f 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -50,7 +50,8 @@ std::vector getTlsCertificateConf auto secret_provider = factory_context.secretManager().findStaticTlsCertificateProvider( sds_secret_config.name()); if (!secret_provider) { - throw EnvoyException(fmt::format("Unknown static secret: {}", sds_secret_config.name())); + throwEnvoyExceptionOrPanic( + fmt::format("Unknown static secret: {}", sds_secret_config.name())); } providers.push_back(secret_provider); } @@ -74,8 +75,8 @@ Secret::CertificateValidationContextConfigProviderSharedPtr getProviderFromSds( factory_context.secretManager().findStaticCertificateValidationContextProvider( sds_secret_config.name()); if (!secret_provider) { - throw EnvoyException(fmt::format("Unknown static certificate validation context: {}", - sds_secret_config.name())); + throwEnvoyExceptionOrPanic(fmt::format("Unknown static certificate validation context: {}", + sds_secret_config.name())); } return secret_provider; } @@ -133,7 +134,7 @@ Secret::TlsSessionTicketKeysConfigProviderSharedPtr getTlsSessionTicketKeysConfi factory_context.secretManager().findStaticTlsSessionTicketKeysContextProvider( sds_secret_config.name()); if (!secret_provider) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Unknown tls session ticket keys: {}", sds_secret_config.name())); } return secret_provider; @@ -145,8 +146,8 @@ Secret::TlsSessionTicketKeysConfigProviderSharedPtr getTlsSessionTicketKeysConfi SessionTicketKeysTypeCase::SESSION_TICKET_KEYS_TYPE_NOT_SET: return nullptr; default: - throw EnvoyException(fmt::format("Unexpected case for oneof session_ticket_keys: {}", - config.session_ticket_keys_type_case())); + throwEnvoyExceptionOrPanic(fmt::format("Unexpected case for oneof session_ticket_keys: {}", + config.session_ticket_keys_type_case())); } } @@ -184,9 +185,14 @@ ContextConfigImpl::ContextConfigImpl( default_min_protocol_version)), max_protocol_version_(tlsVersionFromProto(config.tls_params().tls_maximum_protocol_version(), default_max_protocol_version)), - factory_context_(factory_context), tls_keylog_path_(config.key_log().path()), - tls_keylog_local_(config.key_log().local_address_range()), - tls_keylog_remote_(config.key_log().remote_address_range()) { + factory_context_(factory_context), tls_keylog_path_(config.key_log().path()) { + auto list_or_error = Network::Address::IpList::create(config.key_log().local_address_range()); + THROW_IF_STATUS_NOT_OK(list_or_error, throw); + tls_keylog_local_ = std::move(list_or_error.value()); + list_or_error = Network::Address::IpList::create(config.key_log().remote_address_range()); + THROW_IF_STATUS_NOT_OK(list_or_error, throw); + tls_keylog_remote_ = std::move(list_or_error.value()); + if (certificate_validation_context_provider_ != nullptr) { if (default_cvc_) { // We need to validate combined certificate validation context. @@ -211,7 +217,7 @@ ContextConfigImpl::ContextConfigImpl( auto config_or_status = Envoy::Ssl::CertificateValidationContextConfigImpl::create( *certificate_validation_context_provider_->secret(), api_); if (!config_or_status.status().ok()) { - throw EnvoyException(std::string(config_or_status.status().message())); + throwEnvoyExceptionOrPanic(std::string(config_or_status.status().message())); } validation_context_config_ = std::move(config_or_status.value()); } @@ -257,7 +263,7 @@ Ssl::CertificateValidationContextConfigPtr ContextConfigImpl::getCombinedValidat auto config_or_status = Envoy::Ssl::CertificateValidationContextConfigImpl::create(combined_cvc, api_); if (!config_or_status.status().ok()) { - throw EnvoyException(std::string(config_or_status.status().message())); + throwEnvoyExceptionOrPanic(std::string(config_or_status.status().message())); } return std::move(config_or_status.value()); } @@ -298,7 +304,7 @@ void ContextConfigImpl::setSecretUpdateCallback(std::function callback) auto config_or_status = Envoy::Ssl::CertificateValidationContextConfigImpl::create( *certificate_validation_context_provider_->secret(), api_); if (!config_or_status.status().ok()) { - throw EnvoyException(std::string(config_or_status.status().message())); + throwEnvoyExceptionOrPanic(std::string(config_or_status.status().message())); } validation_context_config_ = std::move(config_or_status.value()); callback(); @@ -362,12 +368,12 @@ ClientContextConfigImpl::ClientContextConfigImpl( // BoringSSL treats this as a C string, so embedded NULL characters will not // be handled correctly. if (server_name_indication_.find('\0') != std::string::npos) { - throw EnvoyException("SNI names containing NULL-byte are not allowed"); + throwEnvoyExceptionOrPanic("SNI names containing NULL-byte are not allowed"); } // TODO(PiotrSikora): Support multiple TLS certificates. if ((config.common_tls_context().tls_certificates().size() + config.common_tls_context().tls_certificate_sds_secret_configs().size()) > 1) { - throw EnvoyException("Multiple TLS certificates are not supported for client contexts"); + throwEnvoyExceptionOrPanic("Multiple TLS certificates are not supported for client contexts"); } } @@ -401,6 +407,7 @@ ServerContextConfigImpl::ServerContextConfigImpl( ocsp_staple_policy_(ocspStaplePolicyFromProto(config.ocsp_staple_policy())), session_ticket_keys_provider_(getTlsSessionTicketKeysConfigProvider(factory_context, config)), disable_stateless_session_resumption_(getStatelessSessionResumptionDisabled(config)), + disable_stateful_session_resumption_(config.disable_stateful_session_resumption()), full_scan_certs_on_sni_mismatch_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config, full_scan_certs_on_sni_mismatch, !Runtime::runtimeFeatureEnabled( @@ -421,10 +428,11 @@ ServerContextConfigImpl::ServerContextConfigImpl( if (!capabilities().provides_certificates) { if ((config.common_tls_context().tls_certificates().size() + config.common_tls_context().tls_certificate_sds_secret_configs().size()) == 0) { - throw EnvoyException("No TLS certificates found for server context"); + throwEnvoyExceptionOrPanic("No TLS certificates found for server context"); } else if (!config.common_tls_context().tls_certificates().empty() && !config.common_tls_context().tls_certificate_sds_secret_configs().empty()) { - throw EnvoyException("SDS and non-SDS TLS certificates may not be mixed in server contexts"); + throwEnvoyExceptionOrPanic( + "SDS and non-SDS TLS certificates may not be mixed in server contexts"); } } @@ -467,9 +475,9 @@ ServerContextConfigImpl::getSessionTicketKey(const std::string& key_data) { static_assert(sizeof(SessionTicketKey) == 80, "Input is expected to be this size"); if (key_data.size() != sizeof(SessionTicketKey)) { - throw EnvoyException(fmt::format("Incorrect TLS session ticket key length. " - "Length {}, expected length {}.", - key_data.size(), sizeof(SessionTicketKey))); + throwEnvoyExceptionOrPanic(fmt::format("Incorrect TLS session ticket key length. " + "Length {}, expected length {}.", + key_data.size(), sizeof(SessionTicketKey))); } SessionTicketKey dst_key; diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index 3cb896fbbeb5..5c43c6bab47e 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -42,8 +42,8 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { } unsigned minProtocolVersion() const override { return min_protocol_version_; }; unsigned maxProtocolVersion() const override { return max_protocol_version_; }; - const Network::Address::IpList& tlsKeyLogLocal() const override { return tls_keylog_local_; }; - const Network::Address::IpList& tlsKeyLogRemote() const override { return tls_keylog_remote_; }; + const Network::Address::IpList& tlsKeyLogLocal() const override { return *tls_keylog_local_; }; + const Network::Address::IpList& tlsKeyLogRemote() const override { return *tls_keylog_remote_; }; const std::string& tlsKeyLogPath() const override { return tls_keylog_path_; }; AccessLog::AccessLogManager& accessLogManager() const override { return factory_context_.serverFactoryContext().accessLogManager(); @@ -111,8 +111,8 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { Ssl::SslCtxCb sslctx_cb_; Server::Configuration::TransportSocketFactoryContext& factory_context_; const std::string tls_keylog_path_; - const Network::Address::IpList tls_keylog_local_; - const Network::Address::IpList tls_keylog_remote_; + std::unique_ptr tls_keylog_local_; + std::unique_ptr tls_keylog_remote_; }; class ClientContextConfigImpl : public ContextConfigImpl, public Envoy::Ssl::ClientContextConfig { @@ -165,6 +165,9 @@ class ServerContextConfigImpl : public ContextConfigImpl, public Envoy::Ssl::Ser bool disableStatelessSessionResumption() const override { return disable_stateless_session_resumption_; } + bool disableStatefulSessionResumption() const override { + return disable_stateful_session_resumption_; + } bool fullScanCertsOnSNIMismatch() const override { return full_scan_certs_on_sni_mismatch_; } @@ -190,6 +193,7 @@ class ServerContextConfigImpl : public ContextConfigImpl, public Envoy::Ssl::Ser absl::optional session_timeout_; const bool disable_stateless_session_resumption_; + const bool disable_stateful_session_resumption_; bool full_scan_certs_on_sni_mismatch_; }; diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index d68219c23ef6..cd87826018ae 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -99,7 +99,7 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c Registry::FactoryRegistry::getFactory(cert_validator_name); if (!cert_validator_factory) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to get certificate validator factory for ", cert_validator_name)); } @@ -148,21 +148,23 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c bad_ciphers.push_back(cipher_str); } } - throw EnvoyException(fmt::format("Failed to initialize cipher suites {}. The following " - "ciphers were rejected when tried individually: {}", - config.cipherSuites(), absl::StrJoin(bad_ciphers, ", "))); + throwEnvoyExceptionOrPanic(fmt::format("Failed to initialize cipher suites {}. The following " + "ciphers were rejected when tried individually: {}", + config.cipherSuites(), + absl::StrJoin(bad_ciphers, ", "))); } if (!capabilities_.provides_ciphers_and_curves && !SSL_CTX_set1_curves_list(ctx.ssl_ctx_.get(), config.ecdhCurves().c_str())) { - throw EnvoyException(absl::StrCat("Failed to initialize ECDH curves ", config.ecdhCurves())); + throwEnvoyExceptionOrPanic( + absl::StrCat("Failed to initialize ECDH curves ", config.ecdhCurves())); } // Set signature algorithms if given, otherwise fall back to BoringSSL defaults. if (!capabilities_.provides_sigalgs && !config.signatureAlgorithms().empty()) { if (!SSL_CTX_set1_sigalgs_list(ctx.ssl_ctx_.get(), config.signatureAlgorithms().c_str())) { - throw EnvoyException(absl::StrCat("Failed to initialize TLS signature algorithms ", - config.signatureAlgorithms())); + throwEnvoyExceptionOrPanic(absl::StrCat("Failed to initialize TLS signature algorithms ", + config.signatureAlgorithms())); } } } @@ -189,7 +191,7 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c #ifdef BORINGSSL_FIPS if (!capabilities_.is_fips_compliant) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Can't load a FIPS noncompliant custom handshaker while running in FIPS compliant mode."); } #endif @@ -228,9 +230,10 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c const EC_GROUP* ecdsa_group = EC_KEY_get0_group(ecdsa_public_key); if (ecdsa_group == nullptr || EC_GROUP_get_curve_name(ecdsa_group) != NID_X9_62_prime256v1) { - throw EnvoyException(fmt::format("Failed to load certificate chain from {}, only P-256 " - "ECDSA certificates are supported", - ctx.cert_chain_file_path_)); + throwEnvoyExceptionOrPanic( + fmt::format("Failed to load certificate chain from {}, only P-256 " + "ECDSA certificates are supported", + ctx.cert_chain_file_path_)); } ctx.is_ecdsa_ = true; } break; @@ -242,14 +245,14 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c const unsigned rsa_key_length = RSA_bits(rsa_public_key); #ifdef BORINGSSL_FIPS if (rsa_key_length != 2048 && rsa_key_length != 3072 && rsa_key_length != 4096) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Failed to load certificate chain from {}, only RSA certificates with " "2048-bit, 3072-bit or 4096-bit keys are supported in FIPS mode", ctx.cert_chain_file_path_)); } #else if (rsa_key_length < 2048) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Failed to load certificate chain from {}, only RSA " "certificates with 2048-bit or larger keys are supported", ctx.cert_chain_file_path_)); @@ -258,9 +261,10 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c } break; #ifdef BORINGSSL_FIPS default: - throw EnvoyException(fmt::format("Failed to load certificate chain from {}, only RSA and " - "ECDSA certificates are supported in FIPS mode", - ctx.cert_chain_file_path_)); + throwEnvoyExceptionOrPanic( + fmt::format("Failed to load certificate chain from {}, only RSA and " + "ECDSA certificates are supported in FIPS mode", + ctx.cert_chain_file_path_)); #endif } @@ -273,12 +277,12 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c Ssl::BoringSslPrivateKeyMethodSharedPtr private_key_method = private_key_method_provider->getBoringSslPrivateKeyMethod(); if (private_key_method == nullptr) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Failed to get BoringSSL private key method from provider")); } #ifdef BORINGSSL_FIPS if (!ctx.private_key_method_provider_->checkFips()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Private key method doesn't support FIPS mode with current parameters")); } #endif @@ -431,7 +435,7 @@ std::vector ContextImpl::parseAlpnProtocols(const std::string& alpn_pro } if (alpn_protocols.size() >= 65535) { - throw EnvoyException("Invalid ALPN protocol string"); + throwEnvoyExceptionOrPanic("Invalid ALPN protocol string"); } std::vector out(alpn_protocols.size() + 1); @@ -439,7 +443,7 @@ std::vector ContextImpl::parseAlpnProtocols(const std::string& alpn_pro for (size_t i = 0; i <= alpn_protocols.size(); i++) { if (i == alpn_protocols.size() || alpn_protocols[i] == ',') { if (i - start > 255) { - throw EnvoyException("Invalid ALPN protocol string"); + throwEnvoyExceptionOrPanic("Invalid ALPN protocol string"); } out[start] = i - start; @@ -781,10 +785,10 @@ ServerContextImpl::ServerContextImpl(Stats::Scope& scope, const std::vector& server_names, TimeSource& time_source) : ContextImpl(scope, config, time_source), session_ticket_keys_(config.sessionTicketKeys()), - ocsp_staple_policy_(config.ocspStaplePolicy()), has_rsa_(false), + ocsp_staple_policy_(config.ocspStaplePolicy()), full_scan_certs_on_sni_mismatch_(config.fullScanCertsOnSNIMismatch()) { if (config.tlsCertificates().empty() && !config.capabilities().provides_certificates) { - throw EnvoyException("Server TlsCertificates must have a certificate specified"); + throwEnvoyExceptionOrPanic("Server TlsCertificates must have a certificate specified"); } for (auto& ctx : tls_contexts_) { @@ -854,6 +858,10 @@ ServerContextImpl::ServerContextImpl(Stats::Scope& scope, }); } + if (config.disableStatefulSessionResumption()) { + SSL_CTX_set_session_cache_mode(ctx.ssl_ctx_.get(), SSL_SESS_CACHE_OFF); + } + if (config.sessionTimeout() && !config.capabilities().handles_session_resumption) { auto timeout = config.sessionTimeout().value().count(); SSL_CTX_set_timeout(ctx.ssl_ctx_.get(), uint32_t(timeout)); @@ -866,15 +874,15 @@ ServerContextImpl::ServerContextImpl(Stats::Scope& scope, auto& ocsp_resp_bytes = tls_certificates[i].get().ocspStaple(); if (ocsp_resp_bytes.empty()) { if (ctx.is_must_staple_) { - throw EnvoyException("OCSP response is required for must-staple certificate"); + throwEnvoyExceptionOrPanic("OCSP response is required for must-staple certificate"); } if (ocsp_staple_policy_ == Ssl::ServerContextConfig::OcspStaplePolicy::MustStaple) { - throw EnvoyException("Required OCSP response is missing from TLS context"); + throwEnvoyExceptionOrPanic("Required OCSP response is missing from TLS context"); } } else { auto response = std::make_unique(ocsp_resp_bytes, time_source_); if (!response->matchesCertificate(*ctx.cert_chain_)) { - throw EnvoyException("OCSP response does not match its TLS certificate"); + throwEnvoyExceptionOrPanic("OCSP response does not match its TLS certificate"); } ctx.ocsp_response_ = std::move(response); } @@ -967,7 +975,7 @@ ServerContextImpl::generateHashForSessionContextId(const std::vector bool { @@ -1251,10 +1259,10 @@ ServerContextImpl::selectTlsContext(const SSL_CLIENT_HELLO* ssl_client_hello) { return true; } - if (client_ecdsa_capable && !ctx.is_ecdsa_ && candicate_ctx == nullptr) { + if (client_ecdsa_capable && !ctx.is_ecdsa_ && candidate_ctx == nullptr) { // ECDSA cert is preferred if client is ECDSA capable, so RSA cert is marked as a candidate, // searching will continue until exhausting all certs or find a exact match. - candicate_ctx = &ctx; + candidate_ctx = &ctx; ocsp_staple_action = action; return false; } @@ -1277,7 +1285,7 @@ ServerContextImpl::selectTlsContext(const SSL_CLIENT_HELLO* ssl_client_hello) { auto tail_select = [&](bool go_to_next_phase) { if (selected_ctx == nullptr) { - selected_ctx = candicate_ctx; + selected_ctx = candidate_ctx; } if (selected_ctx == nullptr && !go_to_next_phase) { @@ -1307,7 +1315,7 @@ ServerContextImpl::selectTlsContext(const SSL_CLIENT_HELLO* ssl_client_hello) { // Full scan certs if client provides SNI but no cert matches to it, // it requires full_scan_certs_on_sni_mismatch is enabled. if (selected_ctx == nullptr) { - candicate_ctx = nullptr; + candidate_ctx = nullptr; // Skip loop when there is no cert compatible to key type if (client_ecdsa_capable || (!client_ecdsa_capable && has_rsa_)) { for (const auto& ctx : tls_contexts_) { @@ -1398,7 +1406,7 @@ void TlsContext::loadCertificateChain(const std::string& data, const std::string cert_chain_.reset(PEM_read_bio_X509_AUX(bio.get(), nullptr, nullptr, nullptr)); if (cert_chain_ == nullptr || !SSL_CTX_use_certificate(ssl_ctx_.get(), cert_chain_.get())) { logSslErrorChain(); - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load certificate chain from ", cert_chain_file_path_)); } // Read rest of the certificate chain. @@ -1408,7 +1416,7 @@ void TlsContext::loadCertificateChain(const std::string& data, const std::string break; } if (!SSL_CTX_add_extra_chain_cert(ssl_ctx_.get(), cert.get())) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load certificate chain from ", cert_chain_file_path_)); } // SSL_CTX_add_extra_chain_cert() takes ownership. @@ -1419,7 +1427,7 @@ void TlsContext::loadCertificateChain(const std::string& data, const std::string if (ERR_GET_LIB(err) == ERR_LIB_PEM && ERR_GET_REASON(err) == PEM_R_NO_START_LINE) { ERR_clear_error(); } else { - throw EnvoyException( + throwEnvoyExceptionOrPanic( absl::StrCat("Failed to load certificate chain from ", cert_chain_file_path_)); } } @@ -1433,8 +1441,9 @@ void TlsContext::loadPrivateKey(const std::string& data, const std::string& data !password.empty() ? const_cast(password.c_str()) : nullptr)); if (pkey == nullptr || !SSL_CTX_use_PrivateKey(ssl_ctx_.get(), pkey.get())) { - throw EnvoyException(fmt::format("Failed to load private key from {}, Cause: {}", data_path, - Utility::getLastCryptoError().value_or("unknown"))); + throwEnvoyExceptionOrPanic(fmt::format("Failed to load private key from {}, Cause: {}", + data_path, + Utility::getLastCryptoError().value_or("unknown"))); } checkPrivateKey(pkey, data_path); @@ -1454,7 +1463,7 @@ void TlsContext::loadPkcs12(const std::string& data, const std::string& data_pat !PKCS12_parse(pkcs12.get(), !password.empty() ? const_cast(password.c_str()) : nullptr, &temp_private_key, &temp_cert, &temp_ca_certs)) { logSslErrorChain(); - throw EnvoyException(absl::StrCat("Failed to load pkcs12 from ", data_path)); + throwEnvoyExceptionOrPanic(absl::StrCat("Failed to load pkcs12 from ", data_path)); } cert_chain_.reset(temp_cert); bssl::UniquePtr pkey(temp_private_key); @@ -1468,11 +1477,12 @@ void TlsContext::loadPkcs12(const std::string& data, const std::string& data_pat } if (!SSL_CTX_use_certificate(ssl_ctx_.get(), cert_chain_.get())) { logSslErrorChain(); - throw EnvoyException(absl::StrCat("Failed to load certificate from ", data_path)); + throwEnvoyExceptionOrPanic(absl::StrCat("Failed to load certificate from ", data_path)); } if (temp_private_key == nullptr || !SSL_CTX_use_PrivateKey(ssl_ctx_.get(), pkey.get())) { - throw EnvoyException(fmt::format("Failed to load private key from {}, Cause: {}", data_path, - Utility::getLastCryptoError().value_or("unknown"))); + throwEnvoyExceptionOrPanic(fmt::format("Failed to load private key from {}, Cause: {}", + data_path, + Utility::getLastCryptoError().value_or("unknown"))); } checkPrivateKey(pkey, data_path); @@ -1486,17 +1496,17 @@ void TlsContext::checkPrivateKey(const bssl::UniquePtr& pkey, case EVP_PKEY_EC: { const EC_KEY* ecdsa_private_key = EVP_PKEY_get0_EC_KEY(pkey.get()); if (!EC_KEY_check_fips(ecdsa_private_key)) { - throw EnvoyException(fmt::format("Failed to load private key from {}, ECDSA key failed " - "pairwise consistency test required in FIPS mode", - key_path)); + throwEnvoyExceptionOrPanic(fmt::format("Failed to load private key from {}, ECDSA key failed " + "pairwise consistency test required in FIPS mode", + key_path)); } } break; case EVP_PKEY_RSA: { RSA* rsa_private_key = EVP_PKEY_get0_RSA(pkey.get()); if (!RSA_check_fips(rsa_private_key)) { - throw EnvoyException(fmt::format("Failed to load private key from {}, RSA key failed " - "pairwise consistency test required in FIPS mode", - key_path)); + throwEnvoyExceptionOrPanic(fmt::format("Failed to load private key from {}, RSA key failed " + "pairwise consistency test required in FIPS mode", + key_path)); } } break; } diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index 8654d44e706e..6f034afd6b35 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -216,7 +216,7 @@ class ServerContextImpl : public ContextImpl, public Envoy::Ssl::ServerContext { const std::vector session_ticket_keys_; const Ssl::ServerContextConfig::OcspStaplePolicy ocsp_staple_policy_; ServerNamesMap server_names_map_; - bool has_rsa_; + bool has_rsa_{false}; bool full_scan_certs_on_sni_mismatch_; }; diff --git a/source/extensions/transport_sockets/tls/ocsp/ocsp.cc b/source/extensions/transport_sockets/tls/ocsp/ocsp.cc index 76063188b7f7..bfa0ee6523d6 100644 --- a/source/extensions/transport_sockets/tls/ocsp/ocsp.cc +++ b/source/extensions/transport_sockets/tls/ocsp/ocsp.cc @@ -19,13 +19,13 @@ template T unwrap(ParsingResult res) { return absl::get<0>(res); } - throw EnvoyException(std::string(absl::get<1>(res))); + throwEnvoyExceptionOrPanic(std::string(absl::get<1>(res))); } unsigned parseTag(CBS& cbs) { unsigned tag; if (!CBS_get_any_asn1_element(&cbs, nullptr, &tag, nullptr)) { - throw EnvoyException("Failed to parse ASN.1 element tag"); + throwEnvoyExceptionOrPanic("Failed to parse ASN.1 element tag"); } return tag; } @@ -36,7 +36,7 @@ std::unique_ptr readDerEncodedOcspResponse(const std::vector der_response, Time time_source_(time_source) { if (response_->status_ != OcspResponseStatus::Successful) { - throw EnvoyException("OCSP response was unsuccessful"); + throwEnvoyExceptionOrPanic("OCSP response was unsuccessful"); } if (response_->response_ == nullptr) { - throw EnvoyException("OCSP response has no body"); + throwEnvoyExceptionOrPanic("OCSP response has no body"); } // We only permit a 1:1 of certificate to response. if (response_->response_->getNumCerts() != 1) { - throw EnvoyException("OCSP Response must be for one certificate only"); + throwEnvoyExceptionOrPanic("OCSP Response must be for one certificate only"); } auto& this_update = response_->response_->getThisUpdate(); @@ -160,7 +160,7 @@ std::unique_ptr Asn1OcspUtility::parseOcspResponse(CBS& cbs) { CBS elem; if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { - throw EnvoyException("OCSP Response is not a well-formed ASN.1 SEQUENCE"); + throwEnvoyExceptionOrPanic("OCSP Response is not a well-formed ASN.1 SEQUENCE"); } OcspResponseStatus status = Asn1OcspUtility::parseResponseStatus(elem); @@ -186,7 +186,7 @@ OcspResponseStatus Asn1OcspUtility::parseResponseStatus(CBS& cbs) { // } CBS status; if (!CBS_get_asn1(&cbs, &status, CBS_ASN1_ENUMERATED)) { - throw EnvoyException("OCSP ResponseStatus is not a well-formed ASN.1 ENUMERATED"); + throwEnvoyExceptionOrPanic("OCSP ResponseStatus is not a well-formed ASN.1 ENUMERATED"); } auto status_ordinal = *CBS_data(&status); @@ -204,7 +204,8 @@ OcspResponseStatus Asn1OcspUtility::parseResponseStatus(CBS& cbs) { case 6: return OcspResponseStatus::Unauthorized; default: - throw EnvoyException(absl::StrCat("Unknown OCSP Response Status variant: ", status_ordinal)); + throwEnvoyExceptionOrPanic( + absl::StrCat("Unknown OCSP Response Status variant: ", status_ordinal)); } } @@ -217,18 +218,18 @@ ResponsePtr Asn1OcspUtility::parseResponseBytes(CBS& cbs) { // } CBS elem, response; if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { - throw EnvoyException("OCSP ResponseBytes is not a well-formed SEQUENCE"); + throwEnvoyExceptionOrPanic("OCSP ResponseBytes is not a well-formed SEQUENCE"); } auto oid_str = unwrap(Asn1Utility::parseOid(elem)); if (!CBS_get_asn1(&elem, &response, CBS_ASN1_OCTETSTRING)) { - throw EnvoyException("Expected ASN.1 OCTETSTRING for response"); + throwEnvoyExceptionOrPanic("Expected ASN.1 OCTETSTRING for response"); } if (oid_str == BasicOcspResponse::OID) { return Asn1OcspUtility::parseBasicOcspResponse(response); } - throw EnvoyException(absl::StrCat("Unknown OCSP Response type with OID: ", oid_str)); + throwEnvoyExceptionOrPanic(absl::StrCat("Unknown OCSP Response type with OID: ", oid_str)); } std::unique_ptr Asn1OcspUtility::parseBasicOcspResponse(CBS& cbs) { @@ -242,7 +243,7 @@ std::unique_ptr Asn1OcspUtility::parseBasicOcspResponse(CBS& // } CBS elem; if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { - throw EnvoyException("OCSP BasicOCSPResponse is not a wellf-formed ASN.1 SEQUENCE"); + throwEnvoyExceptionOrPanic("OCSP BasicOCSPResponse is not a wellf-formed ASN.1 SEQUENCE"); } auto response_data = Asn1OcspUtility::parseResponseData(elem); // The `signatureAlgorithm` and `signature` are ignored because OCSP @@ -262,7 +263,7 @@ ResponseData Asn1OcspUtility::parseResponseData(CBS& cbs) { // } CBS elem; if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { - throw EnvoyException("OCSP ResponseData is not a well-formed ASN.1 SEQUENCE"); + throwEnvoyExceptionOrPanic("OCSP ResponseData is not a well-formed ASN.1 SEQUENCE"); } // only support v1, the value of v1 is 0x00 @@ -271,7 +272,8 @@ ResponseData Asn1OcspUtility::parseResponseData(CBS& cbs) { if (version_cbs.has_value()) { auto version = unwrap(Asn1Utility::parseInteger(*version_cbs)); if (version != "00") { - throw EnvoyException(fmt::format("OCSP ResponseData version 0x{} is not supported", version)); + throwEnvoyExceptionOrPanic( + fmt::format("OCSP ResponseData version 0x{} is not supported", version)); } } @@ -294,7 +296,7 @@ SingleResponse Asn1OcspUtility::parseSingleResponse(CBS& cbs) { // } CBS elem; if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { - throw EnvoyException("OCSP SingleResponse is not a well-formed ASN.1 SEQUENCE"); + throwEnvoyExceptionOrPanic("OCSP SingleResponse is not a well-formed ASN.1 SEQUENCE"); } auto cert_id = Asn1OcspUtility::parseCertId(elem); @@ -317,7 +319,7 @@ CertId Asn1OcspUtility::parseCertId(CBS& cbs) { // } CBS elem; if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { - throw EnvoyException("OCSP CertID is not a well-formed ASN.1 SEQUENCE"); + throwEnvoyExceptionOrPanic("OCSP CertID is not a well-formed ASN.1 SEQUENCE"); } unwrap(Asn1Utility::skip(elem, CBS_ASN1_SEQUENCE)); diff --git a/source/extensions/transport_sockets/tls/ssl_handshaker.cc b/source/extensions/transport_sockets/tls/ssl_handshaker.cc index f94c152c2be2..b0bcca7dbbf5 100644 --- a/source/extensions/transport_sockets/tls/ssl_handshaker.cc +++ b/source/extensions/transport_sockets/tls/ssl_handshaker.cc @@ -67,7 +67,7 @@ Ssl::ValidateResultCallbackPtr SslExtendedSocketInfoImpl::createValidateResultCa SslHandshakerImpl::SslHandshakerImpl(bssl::UniquePtr ssl, int ssl_extended_socket_info_index, Ssl::HandshakeCallbacks* handshake_callbacks) : ssl_(std::move(ssl)), handshake_callbacks_(handshake_callbacks), - state_(Ssl::SocketState::PreHandshake), extended_socket_info_(*this) { + extended_socket_info_(*this) { SSL_set_ex_data(ssl_.get(), ssl_extended_socket_info_index, &(this->extended_socket_info_)); } diff --git a/source/extensions/transport_sockets/tls/ssl_handshaker.h b/source/extensions/transport_sockets/tls/ssl_handshaker.h index 2499dd186880..4203ffac9462 100644 --- a/source/extensions/transport_sockets/tls/ssl_handshaker.h +++ b/source/extensions/transport_sockets/tls/ssl_handshaker.h @@ -106,7 +106,7 @@ class SslHandshakerImpl : public ConnectionInfoImplBase, private: Ssl::HandshakeCallbacks* handshake_callbacks_; - Ssl::SocketState state_; + Ssl::SocketState state_{Ssl::SocketState::PreHandshake}; mutable SslExtendedSocketInfoImpl extended_socket_info_; }; diff --git a/source/extensions/transport_sockets/tls/ssl_socket.cc b/source/extensions/transport_sockets/tls/ssl_socket.cc index c8ac52eee455..bc56bca6b417 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.cc +++ b/source/extensions/transport_sockets/tls/ssl_socket.cc @@ -204,6 +204,8 @@ PostIoAction SslSocket::doHandshake() { return info_->doHandshake(); } void SslSocket::drainErrorQueue() { bool saw_error = false; bool saw_counted_error = false; + bool new_ssl_failure_format = Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.ssl_transport_failure_reason_format"); while (uint64_t err = ERR_get_error()) { if (ERR_GET_LIB(err) == ERR_LIB_SSL) { if (ERR_GET_REASON(err) == SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE) { @@ -221,15 +223,19 @@ void SslSocket::drainErrorQueue() { saw_error = true; if (failure_reason_.empty()) { - failure_reason_ = "TLS error:"; + failure_reason_ = new_ssl_failure_format ? "TLS_error:" : "TLS error:"; } - absl::StrAppend(&failure_reason_, " ", err, ":", + + absl::StrAppend(&failure_reason_, new_ssl_failure_format ? "|" : " ", err, ":", absl::NullSafeStringView(ERR_lib_error_string(err)), ":", absl::NullSafeStringView(ERR_func_error_string(err)), ":", absl::NullSafeStringView(ERR_reason_error_string(err))); } if (!failure_reason_.empty()) { + if (new_ssl_failure_format) { + absl::StrAppend(&failure_reason_, ":TLS_error_end"); + } ENVOY_CONN_LOG(debug, "remote address:{},{}", callbacks_->connection(), callbacks_->connection().connectionInfoProvider().remoteAddress()->asString(), failure_reason_); diff --git a/source/extensions/udp_packet_writer/default/BUILD b/source/extensions/udp_packet_writer/default/BUILD index 032e0de2dea2..ccdecc3bdc08 100644 --- a/source/extensions/udp_packet_writer/default/BUILD +++ b/source/extensions/udp_packet_writer/default/BUILD @@ -18,6 +18,7 @@ envoy_cc_extension( ], extra_visibility = [ "//source/server:__subpackages__", + "//source/common/listener_manager:__subpackages__", ], deps = [ "//envoy/config:typed_config_interface", diff --git a/source/extensions/udp_packet_writer/gso/BUILD b/source/extensions/udp_packet_writer/gso/BUILD index d16cc337767d..59cbce650e39 100644 --- a/source/extensions/udp_packet_writer/gso/BUILD +++ b/source/extensions/udp_packet_writer/gso/BUILD @@ -19,6 +19,7 @@ envoy_cc_extension( ], extra_visibility = [ "//source/server:__subpackages__", + "//source/common/listener_manager:__subpackages__", ], tags = ["nofips"], deps = [ diff --git a/source/extensions/upstreams/http/config.cc b/source/extensions/upstreams/http/config.cc index 2d8a704e0a3e..a60a135e0e22 100644 --- a/source/extensions/upstreams/http/config.cc +++ b/source/extensions/upstreams/http/config.cc @@ -88,12 +88,13 @@ getAlternateProtocolsCacheOptions( Server::Configuration::ServerFactoryContext& server_context) { if (options.has_auto_config() && options.auto_config().has_http3_protocol_options()) { if (!options.auto_config().has_alternate_protocols_cache_options()) { - throw EnvoyException(fmt::format("alternate protocols cache must be configured when HTTP/3 " - "is enabled with auto_config")); + throwEnvoyExceptionOrPanic( + fmt::format("alternate protocols cache must be configured when HTTP/3 " + "is enabled with auto_config")); } auto cache_options = options.auto_config().alternate_protocols_cache_options(); if (cache_options.has_key_value_store_config() && server_context.options().concurrency() != 1) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("options has key value store but Envoy has concurrency = {} : {}", server_context.options().concurrency(), cache_options.DebugString())); } @@ -133,26 +134,26 @@ Envoy::Http::HeaderValidatorFactoryPtr createHeaderValidatorFactory( auto* factory = Envoy::Config::Utility::getFactory( header_validator_config); if (!factory) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("Header validator extension not found: '{}'", header_validator_config.name())); } header_validator_factory = factory->createFromProto(header_validator_config.typed_config(), server_context); if (!header_validator_factory) { - throw EnvoyException(fmt::format("Header validator extension could not be created: '{}'", - header_validator_config.name())); + throwEnvoyExceptionOrPanic(fmt::format("Header validator extension could not be created: '{}'", + header_validator_config.name())); } #else if (options.has_header_validation_config()) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("This Envoy binary does not support header validator extensions: '{}'", options.header_validation_config().name())); } if (Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.enable_universal_header_validator")) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( "Header validator can not be enabled since this Envoy binary does not support it."); } #endif diff --git a/source/extensions/upstreams/http/generic/config.cc b/source/extensions/upstreams/http/generic/config.cc index 6b87ffeb1855..890ca5abceed 100644 --- a/source/extensions/upstreams/http/generic/config.cc +++ b/source/extensions/upstreams/http/generic/config.cc @@ -14,18 +14,17 @@ using UpstreamProtocol = Envoy::Router::GenericConnPoolFactory::UpstreamProtocol Router::GenericConnPoolPtr GenericGenericConnPoolFactory::createGenericConnPool( Upstream::ThreadLocalCluster& thread_local_cluster, UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, - absl::optional downstream_protocol, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const { Router::GenericConnPoolPtr conn_pool; switch (upstream_protocol) { case UpstreamProtocol::HTTP: conn_pool = std::make_unique( - thread_local_cluster, route_entry, downstream_protocol, ctx); + thread_local_cluster, priority, downstream_protocol, ctx); return (conn_pool->valid() ? std::move(conn_pool) : nullptr); case UpstreamProtocol::TCP: conn_pool = - std::make_unique(thread_local_cluster, route_entry, ctx); + std::make_unique(thread_local_cluster, priority, ctx); return (conn_pool->valid() ? std::move(conn_pool) : nullptr); case UpstreamProtocol::UDP: conn_pool = std::make_unique(thread_local_cluster, ctx); diff --git a/source/extensions/upstreams/http/generic/config.h b/source/extensions/upstreams/http/generic/config.h index 524f3f22444d..a7d5425d052d 100644 --- a/source/extensions/upstreams/http/generic/config.h +++ b/source/extensions/upstreams/http/generic/config.h @@ -20,7 +20,7 @@ class GenericGenericConnPoolFactory : public Router::GenericConnPoolFactory { Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const override; diff --git a/source/extensions/upstreams/http/http/config.cc b/source/extensions/upstreams/http/http/config.cc index 7004c49caa9c..3bd2099030ca 100644 --- a/source/extensions/upstreams/http/http/config.cc +++ b/source/extensions/upstreams/http/http/config.cc @@ -12,11 +12,10 @@ using UpstreamProtocol = Envoy::Router::GenericConnPoolFactory::UpstreamProtocol Router::GenericConnPoolPtr HttpGenericConnPoolFactory::createGenericConnPool( Upstream::ThreadLocalCluster& thread_local_cluster, UpstreamProtocol, - const Router::RouteEntry& route_entry, - absl::optional downstream_protocol, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const { auto ret = - std::make_unique(thread_local_cluster, route_entry, downstream_protocol, ctx); + std::make_unique(thread_local_cluster, priority, downstream_protocol, ctx); return (ret->valid() ? std::move(ret) : nullptr); } diff --git a/source/extensions/upstreams/http/http/config.h b/source/extensions/upstreams/http/http/config.h index 717d028e4d5e..3c4ea1ffa0d0 100644 --- a/source/extensions/upstreams/http/http/config.h +++ b/source/extensions/upstreams/http/http/config.h @@ -20,7 +20,7 @@ class HttpGenericConnPoolFactory : public Router::GenericConnPoolFactory { Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const override; diff --git a/source/extensions/upstreams/http/http/upstream_request.h b/source/extensions/upstreams/http/http/upstream_request.h index 79932f506a04..d7b29d7c8d53 100644 --- a/source/extensions/upstreams/http/http/upstream_request.h +++ b/source/extensions/upstreams/http/http/upstream_request.h @@ -21,11 +21,10 @@ namespace Http { class HttpConnPool : public Router::GenericConnPool, public Envoy::Http::ConnectionPool::Callbacks { public: HttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) { - pool_data_ = - thread_local_cluster.httpConnPool(route_entry.priority(), downstream_protocol, ctx); + pool_data_ = thread_local_cluster.httpConnPool(priority, downstream_protocol, ctx); } ~HttpConnPool() override { ASSERT(conn_pool_stream_handle_ == nullptr, "conn_pool_stream_handle not null"); diff --git a/source/extensions/upstreams/http/tcp/config.cc b/source/extensions/upstreams/http/tcp/config.cc index 8cd71bbb21ba..aa517582945d 100644 --- a/source/extensions/upstreams/http/tcp/config.cc +++ b/source/extensions/upstreams/http/tcp/config.cc @@ -10,9 +10,9 @@ namespace Tcp { Router::GenericConnPoolPtr TcpGenericConnPoolFactory::createGenericConnPool( Upstream::ThreadLocalCluster& thread_local_cluster, - Router::GenericConnPoolFactory::UpstreamProtocol, const Router::RouteEntry& route_entry, + Router::GenericConnPoolFactory::UpstreamProtocol, Upstream::ResourcePriority priority, absl::optional, Upstream::LoadBalancerContext* ctx) const { - auto ret = std::make_unique(thread_local_cluster, route_entry, ctx); + auto ret = std::make_unique(thread_local_cluster, priority, ctx); return (ret->valid() ? std::move(ret) : nullptr); } diff --git a/source/extensions/upstreams/http/tcp/config.h b/source/extensions/upstreams/http/tcp/config.h index bf92975f9180..c31b8034cd9e 100644 --- a/source/extensions/upstreams/http/tcp/config.h +++ b/source/extensions/upstreams/http/tcp/config.h @@ -20,7 +20,7 @@ class TcpGenericConnPoolFactory : public Router::GenericConnPoolFactory { Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/source/extensions/upstreams/http/tcp/upstream_request.h b/source/extensions/upstreams/http/tcp/upstream_request.h index b87ad570cedb..87e9431fbd30 100644 --- a/source/extensions/upstreams/http/tcp/upstream_request.h +++ b/source/extensions/upstreams/http/tcp/upstream_request.h @@ -23,8 +23,8 @@ namespace Tcp { class TcpConnPool : public Router::GenericConnPool, public Envoy::Tcp::ConnectionPool::Callbacks { public: TcpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, - const Router::RouteEntry& route_entry, Upstream::LoadBalancerContext* ctx) { - conn_pool_data_ = thread_local_cluster.tcpConnPool(route_entry.priority(), ctx); + Upstream::ResourcePriority priority, Upstream::LoadBalancerContext* ctx) { + conn_pool_data_ = thread_local_cluster.tcpConnPool(priority, ctx); } // Router::GenericConnPool void newStream(Router::GenericConnectionPoolCallbacks* callbacks) override { diff --git a/source/extensions/upstreams/http/udp/BUILD b/source/extensions/upstreams/http/udp/BUILD index 1967e17642e2..6cd59424ef05 100644 --- a/source/extensions/upstreams/http/udp/BUILD +++ b/source/extensions/upstreams/http/udp/BUILD @@ -49,8 +49,7 @@ envoy_cc_library( "//source/common/router:router_lib", "//source/common/upstream:load_balancer_lib", "//source/extensions/common/proxy_protocol:proxy_protocol_header_lib", - "@com_github_google_quiche//:quic_core_http_spdy_session_lib", - "@com_github_google_quiche//:quic_core_types_lib", + "@com_github_google_quiche//:quiche_common_capsule_lib", "@com_github_google_quiche//:quiche_common_connect_udp_datagram_payload_lib", ], ) diff --git a/source/extensions/upstreams/http/udp/config.cc b/source/extensions/upstreams/http/udp/config.cc index 54ea9e1e4c19..bf3d72c0db72 100644 --- a/source/extensions/upstreams/http/udp/config.cc +++ b/source/extensions/upstreams/http/udp/config.cc @@ -10,7 +10,7 @@ namespace Udp { Router::GenericConnPoolPtr UdpGenericConnPoolFactory::createGenericConnPool( Upstream::ThreadLocalCluster& thread_local_cluster, - Router::GenericConnPoolFactory::UpstreamProtocol, const Router::RouteEntry&, + Router::GenericConnPoolFactory::UpstreamProtocol, Upstream::ResourcePriority, absl::optional, Upstream::LoadBalancerContext* ctx) const { auto ret = std::make_unique(thread_local_cluster, ctx); return (ret->valid() ? std::move(ret) : nullptr); diff --git a/source/extensions/upstreams/http/udp/config.h b/source/extensions/upstreams/http/udp/config.h index 7bc42d649dd8..09fef13a1fb2 100644 --- a/source/extensions/upstreams/http/udp/config.h +++ b/source/extensions/upstreams/http/udp/config.h @@ -20,7 +20,7 @@ class UdpGenericConnPoolFactory : public Router::GenericConnPoolFactory { Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/source/extensions/upstreams/http/udp/upstream_request.cc b/source/extensions/upstreams/http/udp/upstream_request.cc index 5a11cc0f5ff7..1fd5de949251 100644 --- a/source/extensions/upstreams/http/udp/upstream_request.cc +++ b/source/extensions/upstreams/http/udp/upstream_request.cc @@ -18,7 +18,6 @@ #include "quiche/common/masque/connect_udp_datagram_payload.h" #include "quiche/common/simple_buffer_allocator.h" -#include "quiche/quic/core/http/quic_spdy_stream.h" namespace Envoy { namespace Extensions { @@ -27,15 +26,34 @@ namespace Http { namespace Udp { void UdpConnPool::newStream(Router::GenericConnectionPoolCallbacks* callbacks) { - Router::UpstreamToDownstream& upstream_to_downstream = callbacks->upstreamToDownstream(); - Network::SocketPtr socket = createSocket(host_); + Envoy::Network::SocketPtr socket = createSocket(host_); + auto source_address_selector = host_->cluster().getUpstreamLocalAddressSelector(); + auto upstream_local_address = source_address_selector->getUpstreamLocalAddress( + host_->address(), /*socket_options=*/nullptr); + if (!Envoy::Network::Socket::applyOptions(upstream_local_address.socket_options_, *socket, + envoy::config::core::v3::SocketOption::STATE_PREBIND)) { + callbacks->onPoolFailure(ConnectionPool::PoolFailureReason::LocalConnectionFailure, + "Failed to apply socket option for UDP upstream", host_); + return; + } + if (upstream_local_address.address_) { + Envoy::Api::SysCallIntResult bind_result = socket->bind(upstream_local_address.address_); + if (bind_result.return_value_ < 0) { + callbacks->onPoolFailure(ConnectionPool::PoolFailureReason::LocalConnectionFailure, + "Failed to bind for UDP upstream", host_); + return; + } + } + const Network::ConnectionInfoProvider& connection_info_provider = socket->connectionInfoProvider(); + Router::UpstreamToDownstream& upstream_to_downstream = callbacks->upstreamToDownstream(); ASSERT(upstream_to_downstream.connection().has_value()); Event::Dispatcher& dispatcher = upstream_to_downstream.connection()->dispatcher(); auto upstream = std::make_unique(&upstream_to_downstream, std::move(socket), host_, dispatcher); StreamInfo::StreamInfoImpl stream_info(dispatcher.timeSource(), nullptr); + callbacks->onPoolReady(std::move(upstream), host_, connection_info_provider, stream_info, {}); } diff --git a/source/extensions/upstreams/http/udp/upstream_request.h b/source/extensions/upstreams/http/udp/upstream_request.h index 73bbe0db1451..eed57dbb6eec 100644 --- a/source/extensions/upstreams/http/udp/upstream_request.h +++ b/source/extensions/upstreams/http/udp/upstream_request.h @@ -15,8 +15,8 @@ #include "source/common/router/upstream_request.h" #include "source/common/stream_info/stream_info_impl.h" +#include "quiche/common/capsule.h" #include "quiche/common/simple_buffer_allocator.h" -#include "quiche/quic/core/http/quic_spdy_stream.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/upstreams/tcp/generic/BUILD b/source/extensions/upstreams/tcp/generic/BUILD index 551a4ca75359..a29fa7133934 100644 --- a/source/extensions/upstreams/tcp/generic/BUILD +++ b/source/extensions/upstreams/tcp/generic/BUILD @@ -19,7 +19,9 @@ envoy_cc_extension( visibility = ["//visibility:public"], deps = [ "//envoy/stream_info:bool_accessor_interface", + "//envoy/stream_info:filter_state_interface", "//source/common/http:codec_client_lib", + "//source/common/stream_info:bool_accessor_lib", "//source/common/tcp_proxy:upstream_lib", "@envoy_api//envoy/extensions/upstreams/tcp/generic/v3:pkg_cc_proto", ], diff --git a/source/extensions/upstreams/tcp/generic/config.cc b/source/extensions/upstreams/tcp/generic/config.cc index 709323a2a04b..e688ab84a510 100644 --- a/source/extensions/upstreams/tcp/generic/config.cc +++ b/source/extensions/upstreams/tcp/generic/config.cc @@ -1,9 +1,11 @@ #include "source/extensions/upstreams/tcp/generic/config.h" #include "envoy/stream_info/bool_accessor.h" +#include "envoy/stream_info/filter_state.h" #include "envoy/upstream/cluster_manager.h" #include "source/common/http/codec_client.h" +#include "source/common/stream_info/bool_accessor_impl.h" #include "source/common/tcp_proxy/upstream.h" namespace Envoy { @@ -47,6 +49,19 @@ bool GenericConnPoolFactory::disableTunnelingByFilterState( REGISTER_FACTORY(GenericConnPoolFactory, TcpProxy::GenericConnPoolFactory); +class DisableTunnelingObjectFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { + return std::string(TcpProxy::DisableTunnelingFilterStateKey); + } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data == "true"); + } +}; + +REGISTER_FACTORY(DisableTunnelingObjectFactory, StreamInfo::FilterState::ObjectFactory); + } // namespace Generic } // namespace Tcp } // namespace Upstreams diff --git a/source/server/BUILD b/source/server/BUILD index 582a56c00ee1..f0d63ba47174 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -99,6 +99,7 @@ envoy_cc_library( "//envoy/network:listen_socket_interface", "//envoy/network:listener_interface", "//envoy/server:listener_manager_interface", + "//source/common/network:listener_lib", "//source/common/network:utility_lib", "//source/server:active_listener_base", ], @@ -232,8 +233,8 @@ envoy_cc_library( ) envoy_cc_library( - name = "options_lib", - srcs = ["options_impl.cc"] + select({ + name = "options_base", + srcs = ["options_impl_base.cc"] + select({ "//bazel:linux_x86_64": ["options_impl_platform_linux.cc"], "//bazel:linux_aarch64": ["options_impl_platform_linux.cc"], "//bazel:linux_ppc": ["options_impl_platform_linux.cc"], @@ -241,7 +242,7 @@ envoy_cc_library( "//conditions:default": ["options_impl_platform_default.cc"], }), hdrs = [ - "options_impl.h", + "options_impl_base.h", "options_impl_platform.h", ] + select({ "//bazel:linux_x86_64": ["options_impl_platform_linux.h"], @@ -250,10 +251,30 @@ envoy_cc_library( "//bazel:linux_mips64": ["options_impl_platform_linux.h"], "//conditions:default": [], }), + deps = [ + "//envoy/network:address_interface", + "//envoy/registry", + "//envoy/server:options_interface", + "//envoy/stats:stats_interface", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:logger_lib", + "//source/common/common:macros", + "//source/common/protobuf:utility_lib", + "//source/common/stats:stats_lib", + "//source/common/stats:tag_utility_lib", + "//source/common/version:version_lib", + ], +) + +envoy_cc_library( + name = "options_lib", # TCLAP command line parser needs this to support int64_t/uint64_t in several build environments. + srcs = ["options_impl.cc"], + hdrs = ["options_impl.h"], copts = ["-DHAVE_LONG_LONG"], external_deps = ["tclap"], deps = [ + ":options_base", "//envoy/network:address_interface", "//envoy/registry", "//envoy/server:options_interface", @@ -265,6 +286,17 @@ envoy_cc_library( "//source/common/stats:stats_lib", "//source/common/stats:tag_utility_lib", "//source/common/version:version_lib", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "null_overload_manager_lib", + hdrs = ["null_overload_manager.h"], + deps = [ + "//envoy/server/overload:overload_manager_interface", + "//source/common/event:scaled_range_timer_manager_lib", ], ) @@ -309,6 +341,7 @@ envoy_cc_library( "//source/common/config:utility_lib", "//source/common/init:manager_lib", "//source/common/init:target_lib", + "//source/common/listener_manager:listener_info_lib", "//source/common/network:utility_lib", "//source/common/stream_info:stream_info_lib", "//source/extensions/filters/network/http_connection_manager:config", @@ -327,6 +360,8 @@ envoy_cc_library( deps = [ "//envoy/server:factory_context_interface", "//envoy/server:instance_interface", + "//source/common/config:metadata_lib", + "//source/common/listener_manager:listener_info_lib", ], ) @@ -355,14 +390,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "factory_context_base_impl_lib", - hdrs = ["factory_context_base_impl.h"], - deps = [ - "//envoy/server:factory_context_interface", - ], -) - envoy_cc_library( name = "listener_manager_factory_lib", hdrs = ["listener_manager_factory.h"], @@ -375,7 +402,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "server_lib", + name = "server_base_lib", srcs = ["server.cc"], hdrs = ["server.h"], external_deps = [ @@ -385,7 +412,6 @@ envoy_cc_library( deps = [ ":api_listener_lib", ":configuration_lib", - ":guarddog_lib", ":listener_hooks_lib", ":listener_manager_factory_lib", ":regex_engine_lib", @@ -423,11 +449,11 @@ envoy_cc_library( "//source/common/http:headers_lib", "//source/common/init:manager_lib", "//source/common/local_info:local_info_lib", - "//source/common/memory:heap_shrinker_lib", "//source/common/memory:stats_lib", "//source/common/protobuf:utility_lib", "//source/common/quic:quic_stat_names_lib", "//source/common/router:rds_lib", + "//source/common/runtime:runtime_keys_lib", "//source/common/runtime:runtime_lib", "//source/common/secret:secret_manager_impl_lib", "//source/common/signal:fatal_error_handler_lib", @@ -436,13 +462,24 @@ envoy_cc_library( "//source/common/upstream:cluster_manager_lib", "//source/common/upstream:health_discovery_service_lib", "//source/common/version:version_lib", - "//source/server:overload_manager_lib", "//source/server/admin:admin_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], ) +envoy_cc_library( + name = "server_lib", + srcs = ["instance_impl.cc"], + hdrs = ["instance_impl.h"], + deps = [ + ":guarddog_lib", + ":server_base_lib", + "//source/common/memory:heap_shrinker_lib", + "//source/server:overload_manager_lib", + ], +) + envoy_cc_library( name = "ssl_context_manager_lib", srcs = ["ssl_context_manager.cc"], @@ -520,3 +557,12 @@ envoy_cc_library( "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], ) + +envoy_cc_library( + name = "generic_factory_context_lib", + srcs = ["generic_factory_context.cc"], + hdrs = ["generic_factory_context.h"], + deps = [ + "//envoy/server:factory_context_interface", + ], +) diff --git a/source/server/active_udp_listener.cc b/source/server/active_udp_listener.cc index e4bb6f1f3886..b932a801d357 100644 --- a/source/server/active_udp_listener.cc +++ b/source/server/active_udp_listener.cc @@ -4,6 +4,7 @@ #include "envoy/server/listener_manager.h" #include "envoy/stats/scope.h" +#include "source/common/network/udp_listener_impl.h" #include "source/common/network/utility.h" #include "spdlog/spdlog.h" @@ -74,8 +75,8 @@ ActiveRawUdpListener::ActiveRawUdpListener(uint32_t worker_index, uint32_t concu Event::Dispatcher& dispatcher, Network::ListenerConfig& config) : ActiveRawUdpListener(worker_index, concurrency, parent, listen_socket, - dispatcher.createUdpListener( - listen_socket_ptr, *this, + std::make_unique( + dispatcher, listen_socket_ptr, *this, dispatcher.timeSource(), config.udpListenerConfig()->config().downstream_socket_config()), config) {} diff --git a/source/server/admin/BUILD b/source/server/admin/BUILD index 571911234a88..3913191068d4 100644 --- a/source/server/admin/BUILD +++ b/source/server/admin/BUILD @@ -19,6 +19,18 @@ envoy_cc_library( deps = [] + envoy_select_admin_functionality(["//source/server/admin:admin_lib_internal"]), ) +envoy_cc_library( + name = "admin_factory_context", + hdrs = ["admin_factory_context.h"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/server:factory_context_interface", + "//envoy/server:instance_interface", + "//source/common/listener_manager:listener_info_lib", + "//source/server:factory_context_lib", + ], +) + envoy_cc_library( name = "admin_lib_internal", srcs = ["admin.cc"] + envoy_select_admin_html([ @@ -28,6 +40,7 @@ envoy_cc_library( ]), hdrs = ["admin.h"], deps = [ + ":admin_factory_context", ":admin_filter_lib", ":clusters_handler_lib", ":config_dump_handler_lib", @@ -66,6 +79,8 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", + "//source/common/listener_manager:listener_info_lib", + "//source/common/listener_manager:listener_manager_lib", "//source/common/memory:utils_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:listen_socket_lib", @@ -75,8 +90,9 @@ envoy_cc_library( "//source/common/router:scoped_config_lib", "//source/common/stats:isolated_store_lib", "//source/extensions/access_loggers/common:file_access_log_lib", - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//source/extensions/http/header_validators/envoy_default:config", "//source/extensions/request_id/uuid:config", + "//source/server:null_overload_manager_lib", ] + envoy_select_admin_html([ ":admin_html_util", ]), @@ -151,6 +167,7 @@ envoy_cc_library( "//source/common/http:codes_lib", "//source/common/http:header_map_lib", "//source/common/stats:histogram_lib", + "//source/common/upstream:host_utility_lib", "@com_google_absl//absl/container:btree", ], ) @@ -207,6 +224,7 @@ envoy_cc_library( "//envoy/stats:custom_stat_namespaces_interface", "//source/common/buffer:buffer_lib", "//source/common/stats:histogram_lib", + "//source/common/upstream:host_utility_lib", ], ) diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc index fd17f6fbe9d0..78755f587a23 100644 --- a/source/server/admin/admin.cc +++ b/source/server/admin/admin.cc @@ -7,6 +7,8 @@ #include #include +#include "envoy/extensions/http/header_validators/envoy_default/v3/header_validator.pb.h" +#include "envoy/http/header_validator_factory.h" #include "envoy/server/hot_restart.h" #include "envoy/server/instance.h" #include "envoy/server/options.h" @@ -26,12 +28,12 @@ #include "source/common/http/conn_manager_utility.h" #include "source/common/http/header_map_impl.h" #include "source/common/http/headers.h" +#include "source/common/listener_manager/listener_impl.h" #include "source/common/memory/utils.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" #include "source/common/router/config_impl.h" -#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include "source/extensions/request_id/uuid/config.h" #include "source/server/admin/utils.h" @@ -48,27 +50,26 @@ ConfigTracker& AdminImpl::getConfigTracker() { return config_tracker_; } AdminImpl::NullRouteConfigProvider::NullRouteConfigProvider(TimeSource& time_source) : config_(new Router::NullConfigImpl()), time_source_(time_source) {} -void AdminImpl::startHttpListener(const std::list& access_logs, - const std::string& address_out_path, +void AdminImpl::startHttpListener(std::list access_logs, Network::Address::InstanceConstSharedPtr address, - const Network::Socket::OptionsSharedPtr& socket_options, - Stats::ScopeSharedPtr&& listener_scope) { - for (const auto& access_log : access_logs) { - access_logs_.emplace_back(access_log); - } + Network::Socket::OptionsSharedPtr socket_options) { + access_logs_ = std::move(access_logs); + null_overload_manager_.start(); socket_ = std::make_shared(address, socket_options, true); RELEASE_ASSERT(0 == socket_->ioHandle().listen(ENVOY_TCP_BACKLOG_SIZE).return_value_, "listen() failed on admin listener"); socket_factories_.emplace_back(std::make_unique(socket_)); - listener_ = std::make_unique(*this, std::move(listener_scope)); + listener_ = std::make_unique(*this, factory_context_.listenerScope()); + ENVOY_LOG(info, "admin address: {}", socket().connectionInfoProvider().localAddress()->asString()); - if (!address_out_path.empty()) { - std::ofstream address_out_file(address_out_path); + + if (!server_.options().adminAddressPath().empty()) { + std::ofstream address_out_file(server_.options().adminAddressPath()); if (!address_out_file) { ENVOY_LOG(critical, "cannot open admin address output file {} for writing.", - address_out_path); + server_.options().adminAddressPath()); } else { address_out_file << socket_->connectionInfoProvider().localAddress()->asString(); } @@ -82,16 +83,38 @@ std::vector prepend(const absl::string_view first, strings.insert(strings.begin(), first); return strings; } + +Http::HeaderValidatorFactoryPtr createHeaderValidatorFactory( + [[maybe_unused]] Server::Configuration::ServerFactoryContext& context) { + Http::HeaderValidatorFactoryPtr header_validator_factory; +#ifdef ENVOY_ENABLE_UHV + // Default UHV config matches the admin HTTP validation and normalization config + ::envoy::extensions::http::header_validators::envoy_default::v3::HeaderValidatorConfig uhv_config; + + ::envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("default_universal_header_validator_for_admin"); + config.mutable_typed_config()->PackFrom(uhv_config); + + auto* factory = Envoy::Config::Utility::getFactory(config); + ENVOY_BUG(factory != nullptr, "Default UHV is not linked into binary."); + + header_validator_factory = factory->createFromProto(config.typed_config(), context); + ENVOY_BUG(header_validator_factory != nullptr, "Unable to create default UHV."); +#endif + return header_validator_factory; +} + } // namespace AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, bool ignore_global_conn_limit) - : server_(server), + : server_(server), listener_info_(std::make_shared()), + factory_context_(server, listener_info_), request_id_extension_(Extensions::RequestId::UUIDRequestIDExtension::defaultInstance( server_.api().randomGenerator())), profile_path_(profile_path), stats_(Http::ConnectionManagerImpl::generateStats( "http.admin.", *server_.stats().rootScope())), - null_overload_manager_(server_.threadLocal()), + null_overload_manager_(server_.threadLocal(), false), tracing_stats_(Http::ConnectionManagerImpl::generateTracingStats("http.admin.", *no_op_store_.rootScope())), route_config_provider_(server.timeSource()), @@ -178,6 +201,9 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, "When draining listeners, enter a graceful drain period prior to closing " "listeners. This behaviour and duration is configurable via server options " "or CLI"}, + {ParamDescriptor::Type::Boolean, "skip_exit", + "When draining listeners, do not exit after the drain period. " + "This must be used with graceful"}, {ParamDescriptor::Type::Boolean, "inboundonly", "Drains all inbound listeners. traffic_direction field in " "envoy_v3_api_msg_config.listener.v3.Listener is used to determine whether a " @@ -229,7 +255,8 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, date_provider_(server.dispatcher().timeSource()), admin_filter_chain_(std::make_shared()), local_reply_(LocalReply::Factory::createDefault()), - ignore_global_conn_limit_(ignore_global_conn_limit) { + ignore_global_conn_limit_(ignore_global_conn_limit), + header_validator_factory_(createHeaderValidatorFactory(server.serverFactoryContext())) { #ifndef NDEBUG // Verify that no duplicate handlers exist. absl::flat_hash_set handlers; @@ -273,7 +300,6 @@ bool AdminImpl::createFilterChain(Http::FilterChainManager& manager, bool, } namespace { - // Implements a chunked request for static text. class StaticTextRequest : public Admin::Request { public: @@ -490,9 +516,28 @@ void AdminImpl::closeSocket() { void AdminImpl::addListenerToHandler(Network::ConnectionHandler* handler) { if (listener_) { - handler->addListener(absl::nullopt, *listener_, server_.runtime()); + handler->addListener(absl::nullopt, *listener_, server_.runtime(), + server_.api().randomGenerator()); } } +#ifdef ENVOY_ENABLE_UHV +::Envoy::Http::HeaderValidatorStats& +AdminImpl::getHeaderValidatorStats([[maybe_unused]] Http::Protocol protocol) { + switch (protocol) { + case Http::Protocol::Http10: + case Http::Protocol::Http11: + return Http::Http1::CodecStats::atomicGet(http1_codec_stats_, *server_.stats().rootScope()); + case Http::Protocol::Http3: + IS_ENVOY_BUG("HTTP/3 is not supported for admin UI"); + // Return H/2 stats object, since we do not have H/3 stats. + ABSL_FALLTHROUGH_INTENDED; + case Http::Protocol::Http2: + return Http::Http2::CodecStats::atomicGet(http2_codec_stats_, *server_.stats().rootScope()); + } + PANIC_DUE_TO_CORRUPT_ENUM; +} +#endif + } // namespace Server } // namespace Envoy diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 221cac26466b..6b3a0da53e54 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -14,7 +14,6 @@ #include "envoy/server/admin.h" #include "envoy/server/instance.h" #include "envoy/server/listener_manager.h" -#include "envoy/server/overload/overload_manager.h" #include "envoy/upstream/outlier_detection.h" #include "envoy/upstream/resource_manager.h" @@ -36,6 +35,7 @@ #include "source/common/router/scoped_config_impl.h" #include "source/common/stats/isolated_store_impl.h" #include "source/extensions/filters/http/common/pass_through_filter.h" +#include "source/server/admin/admin_factory_context.h" #include "source/server/admin/admin_filter.h" #include "source/server/admin/clusters_handler.h" #include "source/server/admin/config_dump_handler.h" @@ -48,6 +48,7 @@ #include "source/server/admin/server_cmd_handler.h" #include "source/server/admin/server_info_handler.h" #include "source/server/admin/stats_handler.h" +#include "source/server/null_overload_manager.h" #include "absl/strings/string_view.h" @@ -76,6 +77,8 @@ class AdminImpl : public Admin, const Network::Socket& socket() override { return *socket_; } Network::Socket& mutableSocket() { return *socket_; } + Configuration::FactoryContext& factoryContext() { return factory_context_; } + // Server::Admin // TODO(jsedgwick) These can be managed with a generic version of ConfigTracker. // Wins would be no manual removeHandler() and code reuse. @@ -90,11 +93,9 @@ class AdminImpl : public Admin, bool removeHandler(const std::string& prefix) override; ConfigTracker& getConfigTracker() override; - void startHttpListener(const std::list& access_logs, - const std::string& address_out_path, + void startHttpListener(std::list access_logs, Network::Address::InstanceConstSharedPtr address, - const Network::Socket::OptionsSharedPtr& socket_options, - Stats::ScopeSharedPtr&& listener_scope) override; + Network::Socket::OptionsSharedPtr socket_options) override; uint32_t concurrency() const override { return server_.options().concurrency(); } // Network::FilterChainManager @@ -222,9 +223,17 @@ class AdminImpl : public Admin, const HttpConnectionManagerProto::ProxyStatusConfig* proxyStatusConfig() const override { return proxy_status_config_.get(); } - Http::ServerHeaderValidatorPtr makeHeaderValidator(Http::Protocol) override { - // TODO(yanavlasov): admin interface should use the default validator + Http::ServerHeaderValidatorPtr + makeHeaderValidator([[maybe_unused]] Http::Protocol protocol) override { +#ifdef ENVOY_ENABLE_UHV + ENVOY_BUG(header_validator_factory_ != nullptr, + "Admin HCM config can not have null UHV factory."); + return header_validator_factory_ ? header_validator_factory_->createServerHeaderValidator( + protocol, getHeaderValidatorStats(protocol)) + : nullptr; +#else return nullptr; +#endif } bool appendXForwardedPort() const override { return false; } bool addProxyProtocolConnectionState() const override { return true; } @@ -232,6 +241,10 @@ class AdminImpl : public Admin, private: friend class AdminTestingPeer; +#ifdef ENVOY_ENABLE_UHV + ::Envoy::Http::HeaderValidatorStats& getHeaderValidatorStats(Http::Protocol protocol); +#endif + /** * Creates a Request from the request in the admin stream. */ @@ -318,46 +331,6 @@ class AdminImpl : public Admin, Router::ScopeKeyPtr computeScopeKey(const Http::HeaderMap&) const override { return nullptr; }; }; - /** - * Implementation of OverloadManager that is never overloaded. Using this instead of the real - * OverloadManager keeps the admin interface accessible even when the proxy is overloaded. - */ - struct NullOverloadManager : public OverloadManager { - struct OverloadState : public ThreadLocalOverloadState { - OverloadState(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} - const OverloadActionState& getState(const std::string&) override { return inactive_; } - bool tryAllocateResource(OverloadProactiveResourceName, int64_t) override { return false; } - bool tryDeallocateResource(OverloadProactiveResourceName, int64_t) override { return false; } - bool isResourceMonitorEnabled(OverloadProactiveResourceName) override { return false; } - Event::Dispatcher& dispatcher_; - const OverloadActionState inactive_ = OverloadActionState::inactive(); - }; - - NullOverloadManager(ThreadLocal::SlotAllocator& slot_allocator) - : tls_(slot_allocator.allocateSlot()) {} - - void start() override { - tls_->set([](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(dispatcher); - }); - } - - ThreadLocalOverloadState& getThreadLocalOverloadState() override { - return tls_->getTyped(); - } - LoadShedPoint* getLoadShedPoint(absl::string_view) override { return nullptr; } - - Event::ScaledRangeTimerManagerFactory scaledTimerFactory() override { return nullptr; } - - bool registerForAction(const std::string&, Event::Dispatcher&, OverloadActionCb) override { - // This method shouldn't be called by the admin listener - IS_ENVOY_BUG("Unexpected function call"); - return false; - } - - ThreadLocal::SlotPtr tls_; - }; - std::vector sortedHandlers() const; envoy::admin::v3::ServerInfo::State serverState(); @@ -397,9 +370,9 @@ class AdminImpl : public Admin, class AdminListener : public Network::ListenerConfig { public: - AdminListener(AdminImpl& parent, Stats::ScopeSharedPtr&& listener_scope) - : parent_(parent), name_("admin"), scope_(std::move(listener_scope)), - stats_(Http::ConnectionManagerImpl::generateListenerStats("http.admin.", *scope_)), + AdminListener(AdminImpl& parent, Stats::Scope& listener_scope) + : parent_(parent), name_("admin"), scope_(listener_scope), + stats_(Http::ConnectionManagerImpl::generateListenerStats("http.admin.", scope_)), init_manager_(nullptr), ignore_global_conn_limit_(parent.ignore_global_conn_limit_) {} // Network::ListenerConfig @@ -413,14 +386,14 @@ class AdminImpl : public Admin, uint32_t perConnectionBufferLimitBytes() const override { return 0; } std::chrono::milliseconds listenerFiltersTimeout() const override { return {}; } bool continueOnListenerFiltersTimeout() const override { return false; } - Stats::Scope& listenerScope() override { return *scope_; } + Stats::Scope& listenerScope() override { return scope_; } uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } + const Network::ListenerInfoConstSharedPtr& listenerInfo() const override { + return parent_.listener_info_; + } Network::UdpListenerConfigOptRef udpListenerConfig() override { return {}; } Network::InternalListenerConfigOptRef internalListenerConfig() override { return {}; } - envoy::config::core::v3::TrafficDirection direction() const override { - return envoy::config::core::v3::UNSPECIFIED; - } Network::ConnectionBalancer& connectionBalancer(const Network::Address::Instance&) override { return connection_balancer_; } @@ -437,7 +410,7 @@ class AdminImpl : public Admin, AdminImpl& parent_; const std::string name_; - Stats::ScopeSharedPtr scope_; + Stats::Scope& scope_; Http::ConnectionManagerListenerStats stats_; Network::NopConnectionBalancerImpl connection_balancer_; BasicResourceLimitImpl open_connections_; @@ -476,6 +449,8 @@ class AdminImpl : public Admin, }; Server::Instance& server_; + const Network::ListenerInfoConstSharedPtr listener_info_; + AdminFactoryContext factory_context_; Http::RequestIDExtensionSharedPtr request_id_extension_; std::list access_logs_; const bool flush_access_log_on_new_request_ = false; @@ -525,6 +500,7 @@ class AdminImpl : public Admin, const absl::optional scheme_{}; const bool ignore_global_conn_limit_; std::unique_ptr proxy_status_config_; + const Http::HeaderValidatorFactoryPtr header_validator_factory_; }; } // namespace Server diff --git a/source/server/admin/admin_factory_context.h b/source/server/admin/admin_factory_context.h new file mode 100644 index 000000000000..ab38ff179748 --- /dev/null +++ b/source/server/admin/admin_factory_context.h @@ -0,0 +1,31 @@ +#pragma once + +#include "envoy/server/factory_context.h" +#include "envoy/server/instance.h" + +#include "source/server/factory_context_impl.h" + +namespace Envoy { +namespace Server { + +class AdminFactoryContext final : public FactoryContextImplBase { +public: + AdminFactoryContext(Envoy::Server::Instance& server, + const Network::ListenerInfoConstSharedPtr& listener_info) + : FactoryContextImplBase(server, server.messageValidationContext().staticValidationVisitor(), + server.stats().createScope(""), + server.stats().createScope("listener.admin."), listener_info) {} + + Init::Manager& initManager() override { + // Reuse the server init manager to avoid creating a new one for this special listener. + return server_.initManager(); + } + Network::DrainDecision& drainDecision() override { + // Reuse the server drain manager to avoid creating a new one for this special listener. + return server_.drainManager(); + } +}; +using AdminFactoryContextPtr = std::unique_ptr; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/admin/admin_filter.cc b/source/server/admin/admin_filter.cc index 054c886e06bb..7a11c251dc88 100644 --- a/source/server/admin/admin_filter.cc +++ b/source/server/admin/admin_filter.cc @@ -61,10 +61,11 @@ const Http::RequestHeaderMap& AdminFilter::getRequestHeaders() const { return *request_headers_; } -Http::Utility::QueryParams AdminFilter::queryParams() const { +Http::Utility::QueryParamsMulti AdminFilter::queryParams() const { absl::string_view path = request_headers_->getPathValue(); - Http::Utility::QueryParams query = Http::Utility::parseAndDecodeQueryString(path); - if (!query.empty()) { + Http::Utility::QueryParamsMulti query = + Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(path); + if (!query.data().empty()) { return query; } @@ -73,7 +74,7 @@ Http::Utility::QueryParams AdminFilter::queryParams() const { Http::Headers::get().ContentTypeValues.FormUrlEncoded) { const Buffer::Instance* body = getRequestBody(); if (body != nullptr) { - query = Http::Utility::parseFromBody(body->toString()); + query = Http::Utility::QueryParamsMulti::parseParameters(body->toString(), 0, true); } } diff --git a/source/server/admin/admin_filter.h b/source/server/admin/admin_filter.h index 2bc528125ec4..e163029b2283 100644 --- a/source/server/admin/admin_filter.h +++ b/source/server/admin/admin_filter.h @@ -51,7 +51,7 @@ class AdminFilter : public Http::PassThroughFilter, Http::Http1StreamEncoderOptionsOptRef http1StreamEncoderOptions() override { return encoder_callbacks_->http1StreamEncoderOptions(); } - Http::Utility::QueryParams queryParams() const override; + Http::Utility::QueryParamsMulti queryParams() const override; private: /** diff --git a/source/server/admin/admin_html.cc b/source/server/admin/admin_html.cc index b84a3329592f..c6ab88fdfca9 100644 --- a/source/server/admin/admin_html.cc +++ b/source/server/admin/admin_html.cc @@ -15,7 +15,7 @@ Http::Code AdminImpl::handlerAdminHome(Http::ResponseHeaderMap& response_headers AdminHtmlUtil::renderTableBegin(response, "Command"); // Prefix order is used during searching, but for printing do them in alpha order. - OptRef no_query_params; + OptRef no_query_params; uint32_t index = 0; for (const UrlHandler* handler : sortedHandlers()) { AdminHtmlUtil::renderEndpointTableRow(response, *handler, no_query_params, ++index, false, diff --git a/source/server/admin/admin_html_util.cc b/source/server/admin/admin_html_util.cc index 008be7ab45f3..143b58d5da73 100644 --- a/source/server/admin/admin_html_util.cc +++ b/source/server/admin/admin_html_util.cc @@ -124,14 +124,14 @@ void AdminHtmlUtil::finalize(Buffer::Instance& response) { response.add(" void AdminHtmlUtil::renderHandlerParam(Buffer::Instance& response, absl::string_view id, absl::string_view name, absl::string_view path, Admin::ParamDescriptor::Type type, - OptRef query, + OptRef query, const std::vector& enum_choices, bool submit_on_change) { std::string value; if (query.has_value()) { - auto iter = query->find(std::string(name)); - if (iter != query->end()) { - value = iter->second; + auto result = query->getFirstValue(name); + if (result.has_value()) { + value = result.value(); } } @@ -171,7 +171,7 @@ void AdminHtmlUtil::renderHandlerParam(Buffer::Instance& response, absl::string_ void AdminHtmlUtil::renderEndpointTableRow(Buffer::Instance& response, const Admin::UrlHandler& handler, - OptRef query, + OptRef query, int index, bool submit_on_change, bool active) { absl::string_view path = handler.prefix_; diff --git a/source/server/admin/admin_html_util.h b/source/server/admin/admin_html_util.h index b106665794c8..9467658e3d40 100644 --- a/source/server/admin/admin_html_util.h +++ b/source/server/admin/admin_html_util.h @@ -88,7 +88,7 @@ class AdminHtmlUtil { * @param active indicates */ static void renderEndpointTableRow(Buffer::Instance& response, const Admin::UrlHandler& handler, - OptRef query, int index, + OptRef query, int index, bool submit_on_change, bool active); /** @@ -110,7 +110,7 @@ class AdminHtmlUtil { static void renderHandlerParam(Buffer::Instance& response, absl::string_view id, absl::string_view name, absl::string_view path, Admin::ParamDescriptor::Type type, - OptRef query, + OptRef query, const std::vector& enum_choices, bool submit_on_change); }; diff --git a/source/server/admin/config_dump_handler.cc b/source/server/admin/config_dump_handler.cc index 4c65b891efdd..6fefdc692ea9 100644 --- a/source/server/admin/config_dump_handler.cc +++ b/source/server/admin/config_dump_handler.cc @@ -119,24 +119,14 @@ bool trimResourceMessage(const Protobuf::FieldMask& field_mask, Protobuf::Messag return checkFieldMaskAndTrimMessage(outer_field_mask, message); } -// Helper method to get the resource parameter. -absl::optional resourceParam(const Http::Utility::QueryParams& params) { - return Utility::queryParam(params, "resource"); -} - -// Helper method to get the mask parameter. -absl::optional maskParam(const Http::Utility::QueryParams& params) { - return Utility::queryParam(params, "mask"); -} - // Helper method to get the eds parameter. -bool shouldIncludeEdsInDump(const Http::Utility::QueryParams& params) { - return params.find("include_eds") != params.end(); +bool shouldIncludeEdsInDump(const Http::Utility::QueryParamsMulti& params) { + return params.getFirstValue("include_eds").has_value(); } absl::StatusOr -buildNameMatcher(const Http::Utility::QueryParams& params) { - const auto name_regex = Utility::queryParam(params, "name_regex"); +buildNameMatcher(const Http::Utility::QueryParamsMulti& params) { + const auto name_regex = params.getFirstValue("name_regex"); if (!name_regex.has_value() || name_regex->empty()) { return std::make_unique(); } @@ -160,9 +150,9 @@ ConfigDumpHandler::ConfigDumpHandler(ConfigTracker& config_tracker, Server::Inst Http::Code ConfigDumpHandler::handlerConfigDump(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream& admin_stream) const { - Http::Utility::QueryParams query_params = admin_stream.queryParams(); - const auto resource = resourceParam(query_params); - const auto mask = maskParam(query_params); + Http::Utility::QueryParamsMulti query_params = admin_stream.queryParams(); + const auto resource = query_params.getFirstValue("resource"); + const auto mask = query_params.getFirstValue("mask"); const bool include_eds = shouldIncludeEdsInDump(query_params); const absl::StatusOr name_matcher = buildNameMatcher(query_params); if (!name_matcher.ok()) { @@ -311,6 +301,16 @@ ConfigDumpHandler::dumpEndpointConfigs(const Matchers::StringMatcher& name_match } auto& policy = *cluster_load_assignment.mutable_policy(); + // Using MILLION as denominator in config dump. + float value = cluster.dropOverload().value() * 1000000; + if (value > 0) { + auto* drop_overload = policy.add_drop_overloads(); + drop_overload->set_category("drop_overload"); + auto* percent = drop_overload->mutable_drop_percentage(); + percent->set_denominator(envoy::type::v3::FractionalPercent::MILLION); + percent->set_numerator(uint32_t(value)); + } + for (auto& host_set : cluster.prioritySet().hostSetsPerPriority()) { policy.mutable_overprovisioning_factor()->set_value(host_set->overprovisioningFactor()); diff --git a/source/server/admin/init_dump_handler.cc b/source/server/admin/init_dump_handler.cc index f8d57b7b155d..d2323b638f23 100644 --- a/source/server/admin/init_dump_handler.cc +++ b/source/server/admin/init_dump_handler.cc @@ -8,20 +8,12 @@ namespace Envoy { namespace Server { -namespace { -// Helper method to get the mask parameter. -absl::optional maskParam(const Http::Utility::QueryParams& params) { - return Utility::queryParam(params, "mask"); -} - -} // namespace - InitDumpHandler::InitDumpHandler(Server::Instance& server) : HandlerContextBase(server) {} Http::Code InitDumpHandler::handlerInitDump(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream& admin_stream) const { - const auto mask = maskParam(admin_stream.queryParams()); + const auto mask = admin_stream.queryParams().getFirstValue("mask"); envoy::admin::v3::UnreadyTargetsDumps dump = *dumpUnreadyTargets(mask); MessageUtil::redact(dump); diff --git a/source/server/admin/listeners_handler.cc b/source/server/admin/listeners_handler.cc index 127de3e3c6f5..0e4dfa3bd7c4 100644 --- a/source/server/admin/listeners_handler.cc +++ b/source/server/admin/listeners_handler.cc @@ -15,19 +15,27 @@ ListenersHandler::ListenersHandler(Server::Instance& server) : HandlerContextBas Http::Code ListenersHandler::handlerDrainListeners(Http::ResponseHeaderMap&, Buffer::Instance& response, AdminStream& admin_query) { - const Http::Utility::QueryParams params = admin_query.queryParams(); + const Http::Utility::QueryParamsMulti params = admin_query.queryParams(); ListenerManager::StopListenersType stop_listeners_type = - params.find("inboundonly") != params.end() ? ListenerManager::StopListenersType::InboundOnly - : ListenerManager::StopListenersType::All; + params.getFirstValue("inboundonly").has_value() + ? ListenerManager::StopListenersType::InboundOnly + : ListenerManager::StopListenersType::All; - const bool graceful = params.find("graceful") != params.end(); + const bool graceful = params.getFirstValue("graceful").has_value(); + const bool skip_exit = params.getFirstValue("skip_exit").has_value(); + if (skip_exit && !graceful) { + response.add("skip_exit requires graceful\n"); + return Http::Code::BadRequest; + } if (graceful) { // Ignore calls to /drain_listeners?graceful if the drain sequence has // already started. if (!server_.drainManager().draining()) { - server_.drainManager().startDrainSequence([this, stop_listeners_type]() { - server_.listenerManager().stopListeners(stop_listeners_type, {}); + server_.drainManager().startDrainSequence([this, stop_listeners_type, skip_exit]() { + if (!skip_exit) { + server_.listenerManager().stopListeners(stop_listeners_type, {}); + } }); } } else { @@ -41,7 +49,7 @@ Http::Code ListenersHandler::handlerDrainListeners(Http::ResponseHeaderMap&, Http::Code ListenersHandler::handlerListenerInfo(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream& admin_query) { - const Http::Utility::QueryParams query_params = admin_query.queryParams(); + const Http::Utility::QueryParamsMulti query_params = admin_query.queryParams(); const auto format_value = Utility::formatParam(query_params); if (format_value.has_value() && format_value.value() == "json") { diff --git a/source/server/admin/logs_handler.cc b/source/server/admin/logs_handler.cc index 6ae59c25f5bd..9c95b544f727 100644 --- a/source/server/admin/logs_handler.cc +++ b/source/server/admin/logs_handler.cc @@ -39,7 +39,7 @@ std::vector LogsHandler::levelStrings() { Http::Code LogsHandler::handlerLogging(Http::ResponseHeaderMap&, Buffer::Instance& response, AdminStream& admin_stream) { - Http::Utility::QueryParams query_params = admin_stream.queryParams(); + Http::Utility::QueryParamsMulti query_params = admin_stream.queryParams(); Http::Code rc = Http::Code::OK; const absl::Status status = changeLogLevel(query_params); @@ -82,31 +82,31 @@ Http::Code LogsHandler::handlerReopenLogs(Http::ResponseHeaderMap&, Buffer::Inst return Http::Code::OK; } -absl::Status LogsHandler::changeLogLevel(Http::Utility::QueryParams& params) { +absl::Status LogsHandler::changeLogLevel(Http::Utility::QueryParamsMulti& params) { // "level" and "paths" will be set to the empty string when this is invoked // from HTML without setting them, so clean out empty values. - auto level = params.find("level"); - if (level != params.end() && level->second.empty()) { - params.erase(level); - level = params.end(); + auto level = params.getFirstValue("level"); + if (level.has_value() && level.value().empty()) { + params.remove("level"); + level = std::nullopt; } - auto paths = params.find("paths"); - if (paths != params.end() && paths->second.empty()) { - params.erase(paths); - paths = params.end(); + auto paths = params.getFirstValue("paths"); + if (paths.has_value() && paths.value().empty()) { + params.remove("paths"); + paths = std::nullopt; } - if (params.empty()) { + if (params.data().empty()) { return absl::OkStatus(); } - if (params.size() != 1) { + if (params.data().size() != 1) { return absl::InvalidArgumentError("invalid number of parameters"); } - if (level != params.end()) { + if (level.has_value()) { // Change all log levels. - const absl::StatusOr level_to_use = parseLogLevel(level->second); + const absl::StatusOr level_to_use = parseLogLevel(level.value()); if (!level_to_use.ok()) { return level_to_use.status(); } @@ -118,11 +118,13 @@ absl::Status LogsHandler::changeLogLevel(Http::Utility::QueryParams& params) { // Build a map of name:level pairs, a few allocations is ok here since it's // not common to call this function at a high rate. absl::flat_hash_map name_levels; + std::vector> glob_levels; + const bool use_fine_grain_logger = Logger::Context::useFineGrainLogger(); - if (paths != params.end()) { + if (paths.has_value()) { // Bulk change log level by name:level pairs, separated by comma. std::vector pairs = - absl::StrSplit(paths->second, ',', absl::SkipWhitespace()); + absl::StrSplit(paths.value(), ',', absl::SkipWhitespace()); for (const auto& name_level : pairs) { const std::pair name_level_pair = absl::StrSplit(name_level, absl::MaxSplits(':', 1), absl::SkipWhitespace()); @@ -136,78 +138,74 @@ absl::Status LogsHandler::changeLogLevel(Http::Utility::QueryParams& params) { return level_to_use.status(); } - name_levels[name] = *level_to_use; + if (use_fine_grain_logger) { + ENVOY_LOG(info, "adding fine-grain log update, glob='{}' level='{}'", name, + spdlog::level::level_string_views[*level_to_use]); + glob_levels.emplace_back(name, *level_to_use); + } else { + name_levels[name] = *level_to_use; + } } } else { // The HTML admin interface will always populate "level" and "paths" though // they may be empty. There's a legacy non-HTML-accessible mechanism to // set a single logger to a level, which we'll handle now. In this scenario, // "level" and "paths" will not be populated. - if (params.size() != 1) { + if (params.data().size() != 1) { return absl::InvalidArgumentError("invalid number of parameters"); } // Change particular log level by name. - const auto it = params.begin(); + const auto it = params.data().begin(); const std::string& key = it->first; - const std::string& value = it->second; + const std::string& value = it->second[0]; const absl::StatusOr level_to_use = parseLogLevel(value); if (!level_to_use.ok()) { return level_to_use.status(); } - name_levels[key] = *level_to_use; + if (use_fine_grain_logger) { + ENVOY_LOG(info, "adding fine-grain log update, glob='{}' level='{}'", key, + spdlog::level::level_string_views[*level_to_use]); + glob_levels.emplace_back(key, *level_to_use); + } else { + name_levels[key] = *level_to_use; + } } - return changeLogLevels(name_levels); + if (!use_fine_grain_logger) { + return changeLogLevelsForComponentLoggers(name_levels); + } + getFineGrainLogContext().updateVerbositySetting(glob_levels); + + return absl::OkStatus(); } -absl::Status LogsHandler::changeLogLevels( +absl::Status LogsHandler::changeLogLevelsForComponentLoggers( const absl::flat_hash_map& changes) { - if (!Logger::Context::useFineGrainLogger()) { - std::vector> loggers_to_change; - for (Logger::Logger& logger : Logger::Registry::loggers()) { - auto name_level_itr = changes.find(logger.name()); - if (name_level_itr == changes.end()) { - continue; - } - - loggers_to_change.emplace_back(std::make_pair(&logger, name_level_itr->second)); - } - - // Check if we have any invalid logger in changes. - if (loggers_to_change.size() != changes.size()) { - return absl::InvalidArgumentError("unknown logger name"); + std::vector> loggers_to_change; + for (Logger::Logger& logger : Logger::Registry::loggers()) { + auto name_level_itr = changes.find(logger.name()); + if (name_level_itr == changes.end()) { + continue; } - for (auto& it : loggers_to_change) { - Logger::Logger* logger = it.first; - spdlog::level::level_enum level = it.second; - - ENVOY_LOG(info, "change log level: name='{}' level='{}'", logger->name(), - spdlog::level::level_string_views[level]); - logger->setLevel(level); - } - } else { - std::vector> loggers_to_change; - for (auto& it : changes) { - SpdLoggerSharedPtr logger = getFineGrainLogContext().getFineGrainLogEntry(it.first); - if (!logger) { - return absl::InvalidArgumentError("unknown logger name"); - } + loggers_to_change.emplace_back(std::make_pair(&logger, name_level_itr->second)); + } - loggers_to_change.emplace_back(std::make_pair(logger, it.second)); - } + // Check if we have any invalid logger in changes. + if (loggers_to_change.size() != changes.size()) { + return absl::InvalidArgumentError("unknown logger name"); + } - for (auto& it : loggers_to_change) { - SpdLoggerSharedPtr logger = it.first; - spdlog::level::level_enum level = it.second; + for (auto& it : loggers_to_change) { + Logger::Logger* logger = it.first; + spdlog::level::level_enum level = it.second; - FINE_GRAIN_LOG(info, "change log level: name='{}' level='{}'", logger->name(), - spdlog::level::level_string_views[level]); - logger->set_level(level); - } + ENVOY_LOG(info, "change log level: name='{}' level='{}'", logger->name(), + spdlog::level::level_string_views[level]); + logger->setLevel(level); } return absl::OkStatus(); diff --git a/source/server/admin/logs_handler.h b/source/server/admin/logs_handler.h index 317a854b5230..ffb8700d775f 100644 --- a/source/server/admin/logs_handler.h +++ b/source/server/admin/logs_handler.h @@ -42,9 +42,9 @@ class LogsHandler : public HandlerContextBase, Logger::Loggable& changes); + absl::Status changeLogLevel(Http::Utility::QueryParamsMulti& params); + absl::Status changeLogLevelsForComponentLoggers( + const absl::flat_hash_map& changes); inline absl::StatusOr parseLogLevel(absl::string_view level_string) { auto level_it = log_levels_.find(level_string); diff --git a/source/server/admin/profiling_handler.cc b/source/server/admin/profiling_handler.cc index dc4bf97df035..0aedcfc4776f 100644 --- a/source/server/admin/profiling_handler.cc +++ b/source/server/admin/profiling_handler.cc @@ -11,14 +11,15 @@ ProfilingHandler::ProfilingHandler(const std::string& profile_path) : profile_pa Http::Code ProfilingHandler::handlerCpuProfiler(Http::ResponseHeaderMap&, Buffer::Instance& response, AdminStream& admin_stream) { - Http::Utility::QueryParams query_params = admin_stream.queryParams(); - if (query_params.size() != 1 || query_params.begin()->first != "enable" || - (query_params.begin()->second != "y" && query_params.begin()->second != "n")) { + Http::Utility::QueryParamsMulti query_params = admin_stream.queryParams(); + const auto enableVal = query_params.getFirstValue("enable"); + if (query_params.data().size() != 1 || !enableVal.has_value() || + (enableVal.value() != "y" && enableVal.value() != "n")) { response.add("?enable=\n"); return Http::Code::BadRequest; } - bool enable = query_params.begin()->second == "y"; + bool enable = enableVal.value() == "y"; if (enable && !Profiler::Cpu::profilerEnabled()) { if (!Profiler::Cpu::startProfiler(profile_path_)) { response.add("failure to start the profiler"); @@ -41,15 +42,16 @@ Http::Code ProfilingHandler::handlerHeapProfiler(Http::ResponseHeaderMap&, return Http::Code::NotImplemented; } - Http::Utility::QueryParams query_params = admin_stream.queryParams(); - if (query_params.size() != 1 || query_params.begin()->first != "enable" || - (query_params.begin()->second != "y" && query_params.begin()->second != "n")) { + Http::Utility::QueryParamsMulti query_params = admin_stream.queryParams(); + const auto enableVal = query_params.getFirstValue("enable"); + if (query_params.data().size() != 1 || !enableVal.has_value() || + (enableVal.value() != "y" && enableVal.value() != "n")) { response.add("?enable=\n"); return Http::Code::BadRequest; } Http::Code res = Http::Code::OK; - bool enable = query_params.begin()->second == "y"; + bool enable = enableVal.value() == "y"; if (enable) { if (Profiler::Heap::isProfilerStarted()) { response.add("Fail to start heap profiler: already started"); diff --git a/source/server/admin/prometheus_stats.cc b/source/server/admin/prometheus_stats.cc index 195f047a4c6b..6c7115295d27 100644 --- a/source/server/admin/prometheus_stats.cc +++ b/source/server/admin/prometheus_stats.cc @@ -4,6 +4,7 @@ #include "source/common/common/macros.h" #include "source/common/common/regex.h" #include "source/common/stats/histogram_impl.h" +#include "source/common/upstream/host_utility.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" @@ -43,34 +44,6 @@ std::string sanitizeValue(const absl::string_view value) { }); } -/** - * Determines whether a metric should be shown based on the specified query-parameters. This covers: - * ``usedonly``, hidden, and filter. - * - * @param metric the metric to test - * @param params captures query parameters indicating which metrics should be included. - */ -template -static bool shouldShowMetric(const StatType& metric, const StatsParams& params) { - // This duplicates logic in StatsRequest::populateStatsFromScopes, but differs - // in one subtle way: in Prometheus we only use metric.name() for filtering, - // not rendering, so we only construct the name if there's a filter. - if (params.used_only_ && !metric.used()) { - return false; - } - if (params.hidden_ == HiddenFlag::ShowOnly && !metric.hidden()) { - return false; - } - if (params.hidden_ == HiddenFlag::Exclude && metric.hidden()) { - return false; - } - if (params.re2_filter_ != nullptr && - !re2::RE2::PartialMatch(metric.name(), *params.re2_filter_)) { - return false; - } - return true; -} - /* * Comparator for Stats::Metric that does not require a string representation * to make the comparison, for memory efficiency. @@ -82,6 +55,79 @@ struct MetricLessThan { } }; +struct PrimitiveMetricSnapshotLessThan { + bool operator()(const Stats::PrimitiveMetricMetadata* a, + const Stats::PrimitiveMetricMetadata* b) { + return a->name() < b->name(); + } +}; + +std::string generateNumericOutput(uint64_t value, const Stats::TagVector& tags, + const std::string& prefixed_tag_extracted_name) { + const std::string formatted_tags = PrometheusStatsFormatter::formattedTags(tags); + return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, formatted_tags, value); +} + +/* + * Return the prometheus output for a numeric Stat (Counter or Gauge). + */ +template +std::string generateStatNumericOutput(const StatType& metric, + const std::string& prefixed_tag_extracted_name) { + return generateNumericOutput(metric.value(), metric.tags(), prefixed_tag_extracted_name); +} + +/* + * Returns the prometheus output for a TextReadout in gauge format. + * It is a workaround of a limitation of prometheus which stores only numeric metrics. + * The output is a gauge named the same as a given text-readout. The value of returned gauge is + * always equal to 0. Returned gauge contains all tags of a given text-readout and one additional + * tag {"text_value":"textReadout.value"}. + */ +std::string generateTextReadoutOutput(const Stats::TextReadout& text_readout, + const std::string& prefixed_tag_extracted_name) { + auto tags = text_readout.tags(); + tags.push_back(Stats::Tag{"text_value", text_readout.value()}); + const std::string formattedTags = PrometheusStatsFormatter::formattedTags(tags); + return fmt::format("{0}{{{1}}} 0\n", prefixed_tag_extracted_name, formattedTags); +} + +/* + * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded + * newlines) that contains all the individual bucket counts and sum/count for a single histogram + * (metric_name plus all tags). + */ +std::string generateHistogramOutput(const Stats::ParentHistogram& histogram, + const std::string& prefixed_tag_extracted_name) { + const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags()); + const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ","); + + const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics(); + Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets(); + const std::vector& computed_buckets = stats.computedBuckets(); + std::string output; + for (size_t i = 0; i < supported_buckets.size(); ++i) { + double bucket = supported_buckets[i]; + uint64_t value = computed_buckets[i]; + // We want to print the bucket in a fixed point (non-scientific) format. The fmt library + // doesn't have a specific modifier to format as a fixed-point value only so we use the + // 'g' operator which prints the number in general fixed point format or scientific format + // with precision 50 to round the number up to 32 significant digits in fixed point format + // which should cover pretty much all cases + output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name, + hist_tags, bucket, value)); + } + + output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name, + hist_tags, stats.sampleCount())); + output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags, + stats.sampleSum())); + output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, + stats.sampleCount())); + + return output; +}; + /** * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting * them by tag-extracted metric name, and then outputting them in the correct sorted order into @@ -138,7 +184,7 @@ uint64_t outputStatType( for (const auto& metric : metrics) { ASSERT(&global_symbol_table == &metric->constSymbolTable()); - if (!shouldShowMetric(*metric, params)) { + if (!params.shouldShowMetric(*metric)) { continue; } groups[metric->tagExtractedStatName()].push_back(metric.get()); @@ -167,66 +213,66 @@ uint64_t outputStatType( return result; } -/* - * Return the prometheus output for a numeric Stat (Counter or Gauge). - */ template -std::string generateNumericOutput(const StatType& metric, - const std::string& prefixed_tag_extracted_name) { - const std::string tags = PrometheusStatsFormatter::formattedTags(metric.tags()); - return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value()); -} +uint64_t outputPrimitiveStatType(Buffer::Instance& response, const StatsParams& params, + const std::vector& metrics, absl::string_view type, + const Stats::CustomStatNamespaces& custom_namespaces) { -/* - * Returns the prometheus output for a TextReadout in gauge format. - * It is a workaround of a limitation of prometheus which stores only numeric metrics. - * The output is a gauge named the same as a given text-readout. The value of returned gauge is - * always equal to 0. Returned gauge contains all tags of a given text-readout and one additional - * tag {"text_value":"textReadout.value"}. - */ -std::string generateTextReadoutOutput(const Stats::TextReadout& text_readout, - const std::string& prefixed_tag_extracted_name) { - auto tags = text_readout.tags(); - tags.push_back(Stats::Tag{"text_value", text_readout.value()}); - const std::string formattedTags = PrometheusStatsFormatter::formattedTags(tags); - return fmt::format("{0}{{{1}}} 0\n", prefixed_tag_extracted_name, formattedTags); -} + /* + * From + * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting: + * + * All lines for a given metric must be provided as one single group, with the optional HELP and + * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated + * expositions is preferred but not required, i.e. do not sort if the computational cost is + * prohibitive. + */ -/* - * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded - * newlines) that contains all the individual bucket counts and sum/count for a single histogram - * (metric_name plus all tags). - */ -std::string generateHistogramOutput(const Stats::ParentHistogram& histogram, - const std::string& prefixed_tag_extracted_name) { - const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags()); - const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ","); + // This is an unsorted collection of dumb-pointers (no need to increment then decrement every + // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will + // be sorted before producing the final output to satisfy the "preferred" ordering from the + // prometheus spec: metrics will be sorted by their tags' textual representation, which will be + // consistent across calls. + using StatTypeUnsortedCollection = std::vector; - const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics(); - Stats::ConstSupportedBuckets& supported_buckets = stats.supportedBuckets(); - const std::vector& computed_buckets = stats.computedBuckets(); - std::string output; - for (size_t i = 0; i < supported_buckets.size(); ++i) { - double bucket = supported_buckets[i]; - uint64_t value = computed_buckets[i]; - // We want to print the bucket in a fixed point (non-scientific) format. The fmt library - // doesn't have a specific modifier to format as a fixed-point value only so we use the - // 'g' operator which prints the number in general fixed point format or scientific format - // with precision 50 to round the number up to 32 significant digits in fixed point format - // which should cover pretty much all cases - output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name, - hist_tags, bucket, value)); + // Return early to avoid crashing when getting the symbol table from the first metric. + if (metrics.empty()) { + return 0; } - output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name, - hist_tags, stats.sampleCount())); - output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags, - stats.sampleSum())); - output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, - stats.sampleCount())); + // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements + // of the exposition format. + std::map groups; - return output; -}; + for (const auto& metric : metrics) { + if (!params.shouldShowMetric(metric)) { + continue; + } + groups[metric.tagExtractedName()].push_back(&metric); + } + + auto result = groups.size(); + for (auto& group : groups) { + const absl::optional prefixed_tag_extracted_name = + PrometheusStatsFormatter::metricName(group.first, custom_namespaces); + if (!prefixed_tag_extracted_name.has_value()) { + --result; + continue; + } + response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name.value(), type)); + + // Sort before producing the final output to satisfy the "preferred" ordering from the + // prometheus spec: metrics will be sorted by their tags' textual representation, which will + // be consistent across calls. + std::sort(group.second.begin(), group.second.end(), PrimitiveMetricSnapshotLessThan()); + + for (const auto& metric : group.second) { + response.add(generateNumericOutput(metric->value(), metric->tags(), + prefixed_tag_extracted_name.value())); + } + } + return result; +} } // namespace @@ -269,16 +315,18 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( const std::vector& counters, const std::vector& gauges, const std::vector& histograms, - const std::vector& text_readouts, Buffer::Instance& response, + const std::vector& text_readouts, + const Upstream::ClusterManager& cluster_manager, Buffer::Instance& response, const StatsParams& params, const Stats::CustomStatNamespaces& custom_namespaces) { uint64_t metric_name_count = 0; metric_name_count += outputStatType(response, params, counters, - generateNumericOutput, + generateStatNumericOutput, "counter", custom_namespaces); - metric_name_count += outputStatType( - response, params, gauges, generateNumericOutput, "gauge", custom_namespaces); + metric_name_count += outputStatType(response, params, gauges, + generateStatNumericOutput, + "gauge", custom_namespaces); // TextReadout stats are returned in gauge format, so "gauge" type is set intentionally. metric_name_count += outputStatType( @@ -287,6 +335,24 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( metric_name_count += outputStatType( response, params, histograms, generateHistogramOutput, "histogram", custom_namespaces); + // Note: This assumes that there is no overlap in stat name between per-endpoint stats and all + // other stats. If this is not true, then the counters/gauges for per-endpoint need to be combined + // with the above counter/gauge calls so that stats can be properly grouped. + std::vector host_counters; + std::vector host_gauges; + Upstream::HostUtility::forEachHostMetric( + cluster_manager, + [&](Stats::PrimitiveCounterSnapshot&& metric) { + host_counters.emplace_back(std::move(metric)); + }, + [&](Stats::PrimitiveGaugeSnapshot&& metric) { host_gauges.emplace_back(std::move(metric)); }); + + metric_name_count += + outputPrimitiveStatType(response, params, host_counters, "counter", custom_namespaces); + + metric_name_count += + outputPrimitiveStatType(response, params, host_gauges, "gauge", custom_namespaces); + return metric_name_count; } diff --git a/source/server/admin/prometheus_stats.h b/source/server/admin/prometheus_stats.h index 9677e86685c1..a463da68d0f1 100644 --- a/source/server/admin/prometheus_stats.h +++ b/source/server/admin/prometheus_stats.h @@ -28,6 +28,7 @@ class PrometheusStatsFormatter { const std::vector& gauges, const std::vector& histograms, const std::vector& text_readouts, + const Upstream::ClusterManager& cluster_manager, Buffer::Instance& response, const StatsParams& params, const Stats::CustomStatNamespaces& custom_namespaces); /** diff --git a/source/server/admin/runtime_handler.cc b/source/server/admin/runtime_handler.cc index abe3586a9355..7cb9c2531e5a 100644 --- a/source/server/admin/runtime_handler.cc +++ b/source/server/admin/runtime_handler.cc @@ -17,7 +17,7 @@ RuntimeHandler::RuntimeHandler(Server::Instance& server) : HandlerContextBase(se Http::Code RuntimeHandler::handlerRuntime(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream& admin_stream) { - const Http::Utility::QueryParams params = admin_stream.queryParams(); + const Http::Utility::QueryParamsMulti params = admin_stream.queryParams(); response_headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json); // TODO(jsedgwick): Use proto to structure this output instead of arbitrary JSON. @@ -79,15 +79,17 @@ Http::Code RuntimeHandler::handlerRuntime(Http::ResponseHeaderMap& response_head Http::Code RuntimeHandler::handlerRuntimeModify(Http::ResponseHeaderMap&, Buffer::Instance& response, AdminStream& admin_stream) { - Http::Utility::QueryParams params = admin_stream.queryParams(); - if (params.empty()) { + Http::Utility::QueryParamsMulti params = admin_stream.queryParams(); + if (params.data().empty()) { response.add("usage: /runtime_modify?key1=value1&key2=value2&keyN=valueN\n"); response.add(" or send the parameters as form values\n"); response.add("use an empty value to remove a previously added override"); return Http::Code::BadRequest; } absl::node_hash_map overrides; - overrides.insert(params.begin(), params.end()); + for (const auto& it : params.data()) { + overrides.insert({it.first, it.second[0]}); + } TRY_ASSERT_MAIN_THREAD { server_.runtime().mergeValues(overrides); } END_TRY catch (const EnvoyException& e) { diff --git a/source/server/admin/stats_handler.cc b/source/server/admin/stats_handler.cc index 0c0a8b4dc9f4..92c941fb6ac0 100644 --- a/source/server/admin/stats_handler.cc +++ b/source/server/admin/stats_handler.cc @@ -97,20 +97,21 @@ Admin::RequestPtr StatsHandler::makeRequest(AdminStream& admin_stream, bool acti server_.flushStats(); } + bool active_mode; #ifdef ENVOY_ADMIN_HTML - return makeRequest(server_.stats(), params, - [this, active_html = params.active_html_]() -> Admin::UrlHandler { - return statsHandler("/stats", active_html); - }); + active_mode = params.format_ == StatsFormat::ActiveHtml; #else - return makeRequest(server_.stats(), params, - [this]() -> Admin::UrlHandler { return statsHandler("/stats", false); }); + active_mode = false; #endif + return makeRequest( + server_.stats(), params, server_.clusterManager(), + [this, active_mode]() -> Admin::UrlHandler { return statsHandler(active_mode); }); } Admin::RequestPtr StatsHandler::makeRequest(Stats::Store& stats, const StatsParams& params, + const Upstream::ClusterManager& cluster_manager, StatsRequest::UrlHandlerFn url_handler_fn) { - return std::make_unique(stats, params, url_handler_fn); + return std::make_unique(stats, params, cluster_manager, url_handler_fn); } Http::Code StatsHandler::handlerPrometheusStats(Http::ResponseHeaderMap&, @@ -139,17 +140,19 @@ void StatsHandler::prometheusFlushAndRender(const StatsParams& params, Buffer::I if (server_.statsConfig().flushOnAdmin()) { server_.flushStats(); } - prometheusRender(server_.stats(), server_.api().customStatNamespaces(), params, response); + prometheusRender(server_.stats(), server_.api().customStatNamespaces(), server_.clusterManager(), + params, response); } void StatsHandler::prometheusRender(Stats::Store& stats, const Stats::CustomStatNamespaces& custom_namespaces, + const Upstream::ClusterManager& cluster_manager, const StatsParams& params, Buffer::Instance& response) { const std::vector& text_readouts_vec = params.prometheus_text_readouts_ ? stats.textReadouts() : std::vector(); PrometheusStatsFormatter::statsAsPrometheus(stats.counters(), stats.gauges(), stats.histograms(), - text_readouts_vec, response, params, + text_readouts_vec, cluster_manager, response, params, custom_namespaces); } diff --git a/source/server/admin/stats_handler.h b/source/server/admin/stats_handler.h index 79d3fb54de31..5954407784bb 100644 --- a/source/server/admin/stats_handler.h +++ b/source/server/admin/stats_handler.h @@ -67,6 +67,7 @@ class StatsHandler : public HandlerContextBase { */ static void prometheusRender(Stats::Store& stats, const Stats::CustomStatNamespaces& custom_namespaces, + const Upstream::ClusterManager& cluster_manager, const StatsParams& params, Buffer::Instance& response); Http::Code handlerContention(Http::ResponseHeaderMap& response_headers, @@ -95,6 +96,7 @@ class StatsHandler : public HandlerContextBase { Admin::RequestPtr makeRequest(AdminStream&, bool active_html); static Admin::RequestPtr makeRequest(Stats::Store& stats, const StatsParams& params, + const Upstream::ClusterManager& cm, StatsRequest::UrlHandlerFn url_handler_fn = nullptr); private: diff --git a/source/server/admin/stats_params.cc b/source/server/admin/stats_params.cc index 3c5c95eb6e54..6d7f6940cd04 100644 --- a/source/server/admin/stats_params.cc +++ b/source/server/admin/stats_params.cc @@ -4,14 +4,14 @@ namespace Envoy { namespace Server { Http::Code StatsParams::parse(absl::string_view url, Buffer::Instance& response) { - query_ = Http::Utility::parseAndDecodeQueryString(url); - used_only_ = query_.find("usedonly") != query_.end(); - pretty_ = query_.find("pretty") != query_.end(); - prometheus_text_readouts_ = query_.find("text_readouts") != query_.end(); + query_ = Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(url); + used_only_ = query_.getFirstValue("usedonly").has_value(); + pretty_ = query_.getFirstValue("pretty").has_value(); + prometheus_text_readouts_ = query_.getFirstValue("text_readouts").has_value(); - auto filter_iter = query_.find("filter"); - if (filter_iter != query_.end()) { - filter_string_ = filter_iter->second; + auto filter_val = query_.getFirstValue("filter"); + if (filter_val.has_value() && !filter_val.value().empty()) { + filter_string_ = filter_val.value(); re2::RE2::Options options; options.set_log_errors(false); re2_filter_ = std::make_shared(filter_string_, options); @@ -44,14 +44,14 @@ Http::Code StatsParams::parse(absl::string_view url, Buffer::Instance& response) return true; }; - auto type_iter = query_.find("type"); - if (type_iter != query_.end() && !parse_type(type_iter->second, type_)) { + auto type_val = query_.getFirstValue("type"); + if (type_val.has_value() && !type_val.value().empty() && !parse_type(type_val.value(), type_)) { response.add("invalid &type= param"); return Http::Code::BadRequest; } - const absl::optional hidden_value = Utility::queryParam(query_, "hidden"); - if (hidden_value.has_value()) { + const absl::optional hidden_value = query_.getFirstValue("hidden"); + if (hidden_value.has_value() && !hidden_value.value().empty()) { if (hidden_value.value() == "include") { hidden_ = HiddenFlag::Include; } else if (hidden_value.value() == "only") { @@ -65,7 +65,7 @@ Http::Code StatsParams::parse(absl::string_view url, Buffer::Instance& response) } const absl::optional format_value = Utility::formatParam(query_); - if (format_value.has_value()) { + if (format_value.has_value() && !format_value.value().empty()) { if (format_value.value() == "prometheus") { format_ = StatsFormat::Prometheus; } else if (format_value.value() == "json") { diff --git a/source/server/admin/stats_params.h b/source/server/admin/stats_params.h index 09b82a96ce04..14192cd2690a 100644 --- a/source/server/admin/stats_params.h +++ b/source/server/admin/stats_params.h @@ -69,7 +69,43 @@ struct StatsParams { std::string filter_string_; std::shared_ptr re2_filter_; Utility::HistogramBucketsMode histogram_buckets_mode_{Utility::HistogramBucketsMode::NoBuckets}; - Http::Utility::QueryParams query_; + Http::Utility::QueryParamsMulti query_; + + /** + * Determines whether a metric should be shown based on the specified query-parameters. This + * covers: + * ``usedonly``, hidden, and filter. + * + * @param metric the metric to test + * @param name_out if non-null, and the return value is true, + * will contain the metric name. This improves performance because computing the name is + * somewhat expensive, and in some cases it isn't needed. + */ + template bool shouldShowMetric(const StatType& metric) const { + if (!shouldShowMetricWithoutFilter(metric)) { + return false; + } + + if (re2_filter_ != nullptr && !re2::RE2::PartialMatch(metric.name(), *re2_filter_)) { + return false; + } + + return true; + } + + template bool shouldShowMetricWithoutFilter(const StatType& metric) const { + if (used_only_ && !metric.used()) { + return false; + } + if (hidden_ == HiddenFlag::ShowOnly && !metric.hidden()) { + return false; + } + if (hidden_ == HiddenFlag::Exclude && metric.hidden()) { + return false; + } + + return true; + } }; } // namespace Server diff --git a/source/server/admin/stats_request.cc b/source/server/admin/stats_request.cc index cd2d0c263372..c1489c0ee0cf 100644 --- a/source/server/admin/stats_request.cc +++ b/source/server/admin/stats_request.cc @@ -1,5 +1,7 @@ #include "source/server/admin/stats_request.h" +#include "source/common/upstream/host_utility.h" + #ifdef ENVOY_ADMIN_HTML #include "source/server/admin/stats_html_render.h" #endif @@ -8,8 +10,10 @@ namespace Envoy { namespace Server { StatsRequest::StatsRequest(Stats::Store& stats, const StatsParams& params, + const Upstream::ClusterManager& cluster_manager, UrlHandlerFn url_handler_fn) - : params_(params), stats_(stats), url_handler_fn_(url_handler_fn) { + : params_(params), stats_(stats), cluster_manager_(cluster_manager), + url_handler_fn_(url_handler_fn) { switch (params_.type_) { case StatsType::TextReadouts: case StatsType::All: @@ -78,15 +82,27 @@ bool StatsRequest::nextChunk(Buffer::Instance& response) { const uint64_t starting_response_length = response.length(); while (response.length() - starting_response_length < chunk_size_) { while (stat_map_.empty()) { - if (phase_stat_count_ == 0) { - render_->noStats(response, phase_string_); - } else { - phase_stat_count_ = 0; - } if (params_.type_ != StatsType::All) { + if (phase_ == Phase::CountersAndGauges) { + // In the case of filtering by type, we need to call this before checking for + // no stats in the phase, and then after that this function returns without the normal + // advancing to the next phase. + renderPerHostMetrics(response); + } + + if (phase_stat_count_ == 0) { + render_->noStats(response, phase_string_); + } + render_->finalize(response); return false; } + + if (phase_stat_count_ == 0) { + render_->noStats(response, phase_string_); + } + + phase_stat_count_ = 0; switch (phase_) { case Phase::TextReadouts: phase_ = Phase::CountersAndGauges; @@ -94,6 +110,8 @@ bool StatsRequest::nextChunk(Buffer::Instance& response) { startPhase(); break; case Phase::CountersAndGauges: + renderPerHostMetrics(response); + phase_ = Phase::Histograms; phase_string_ = "Histograms"; startPhase(); @@ -181,15 +199,7 @@ void StatsRequest::populateStatsForCurrentPhase(const ScopeVec& scope_vec) { template void StatsRequest::populateStatsFromScopes(const ScopeVec& scope_vec) { Stats::IterateFn check_stat = [this](const Stats::RefcountPtr& stat) -> bool { - if (params_.used_only_ && !stat->used()) { - return true; - } - - if (params_.hidden_ == HiddenFlag::Exclude && stat->hidden()) { - return true; - } - - if (params_.hidden_ == HiddenFlag::ShowOnly && !stat->hidden()) { + if (!params_.shouldShowMetricWithoutFilter(*stat)) { return true; } @@ -198,7 +208,7 @@ template void StatsRequest::populateStatsFromScopes(const Scope // stat->name() takes a symbol table lock and builds a string, so we only // want to call it once. // - // This duplicates logic in shouldShowMetric in prometheus_stats.cc, but + // This duplicates logic in shouldShowMetric in `StatsParams`, but // differs in that Prometheus only uses stat->name() for filtering, not // rendering, so it only grab the name if there's a filter. std::string name = stat->name(); @@ -213,6 +223,29 @@ template void StatsRequest::populateStatsFromScopes(const Scope } } +void StatsRequest::renderPerHostMetrics(Buffer::Instance& response) { + // This code does not adhere to the streaming contract, but there isn't a good way to stream + // these. There isn't a shared pointer to hold, so there's no way to safely pause iteration here + // without copying all of the data somewhere. But copying all of the data would be more expensive + // than generating it all in one batch here. + Upstream::HostUtility::forEachHostMetric( + cluster_manager_, + [&](Stats::PrimitiveCounterSnapshot&& metric) { + if ((params_.type_ == StatsType::All || params_.type_ == StatsType::Counters) && + params_.shouldShowMetric(metric)) { + ++phase_stat_count_; + render_->generate(response, metric.name(), metric.value()); + } + }, + [&](Stats::PrimitiveGaugeSnapshot&& metric) { + if ((params_.type_ == StatsType::All || params_.type_ == StatsType::Gauges) && + params_.shouldShowMetric(metric)) { + ++phase_stat_count_; + render_->generate(response, metric.name(), metric.value()); + } + }); +} + template void StatsRequest::renderStat(const std::string& name, Buffer::Instance& response, StatOrScopes& variant) { diff --git a/source/server/admin/stats_request.h b/source/server/admin/stats_request.h index 41d43632a83f..4692a77d88a0 100644 --- a/source/server/admin/stats_request.h +++ b/source/server/admin/stats_request.h @@ -40,6 +40,7 @@ class StatsRequest : public Admin::Request { static constexpr uint64_t DefaultChunkSize = 2 * 1000 * 1000; StatsRequest(Stats::Store& stats, const StatsParams& params, + const Upstream::ClusterManager& cluster_manager, UrlHandlerFn url_handler_fn = nullptr); // Admin::Request @@ -92,6 +93,8 @@ class StatsRequest : public Admin::Request { // duplication. template void populateStatsFromScopes(const ScopeVec& scope_vec); + void renderPerHostMetrics(Buffer::Instance& response); + // Renders the templatized type, exploiting the fact that Render::generate is // generic to avoid code duplication. template @@ -110,6 +113,7 @@ class StatsRequest : public Admin::Request { uint64_t phase_stat_count_{0}; absl::string_view phase_string_{"text readouts"}; Buffer::OwnedImpl response_; + const Upstream::ClusterManager& cluster_manager_; UrlHandlerFn url_handler_fn_; uint64_t chunk_size_{DefaultChunkSize}; }; diff --git a/source/server/admin/utils.cc b/source/server/admin/utils.cc index b348146413c9..99f35842d053 100644 --- a/source/server/admin/utils.cc +++ b/source/server/admin/utils.cc @@ -26,10 +26,10 @@ void populateFallbackResponseHeaders(Http::Code code, Http::ResponseHeaderMap& h // Helper method to get the histogram_buckets parameter. Returns false if histogram_buckets query // param is found and value is not "cumulative" or "disjoint", true otherwise. -absl::Status histogramBucketsParam(const Http::Utility::QueryParams& params, +absl::Status histogramBucketsParam(const Http::Utility::QueryParamsMulti& params, HistogramBucketsMode& histogram_buckets_mode) { absl::optional histogram_buckets_query_param = - queryParam(params, "histogram_buckets"); + params.getFirstValue("histogram_buckets"); histogram_buckets_mode = HistogramBucketsMode::NoBuckets; if (histogram_buckets_query_param.has_value()) { if (histogram_buckets_query_param.value() == "cumulative") { @@ -47,21 +47,8 @@ absl::Status histogramBucketsParam(const Http::Utility::QueryParams& params, } // Helper method to get the format parameter. -absl::optional formatParam(const Http::Utility::QueryParams& params) { - return queryParam(params, "format"); -} - -// Helper method to get a query parameter. -absl::optional queryParam(const Http::Utility::QueryParams& params, - const std::string& key) { - const auto iter = params.find(key); - if (iter != params.end()) { - const std::string& value = iter->second; - if (!value.empty()) { - return value; - } - } - return absl::nullopt; +absl::optional formatParam(const Http::Utility::QueryParamsMulti& params) { + return params.getFirstValue("format"); } } // namespace Utility diff --git a/source/server/admin/utils.h b/source/server/admin/utils.h index 5deaf651d8a7..4566c7167cc6 100644 --- a/source/server/admin/utils.h +++ b/source/server/admin/utils.h @@ -17,16 +17,13 @@ enum class HistogramBucketsMode { NoBuckets, Cumulative, Disjoint, Detailed }; void populateFallbackResponseHeaders(Http::Code code, Http::ResponseHeaderMap& header_map); -bool filterParam(Http::Utility::QueryParams params, Buffer::Instance& response, +bool filterParam(Http::Utility::QueryParamsMulti params, Buffer::Instance& response, std::shared_ptr& regex); -absl::Status histogramBucketsParam(const Http::Utility::QueryParams& params, +absl::Status histogramBucketsParam(const Http::Utility::QueryParamsMulti& params, HistogramBucketsMode& histogram_buckets_mode); -absl::optional formatParam(const Http::Utility::QueryParams& params); - -absl::optional queryParam(const Http::Utility::QueryParams& params, - const std::string& key); +absl::optional formatParam(const Http::Utility::QueryParamsMulti& params); } // namespace Utility } // namespace Server diff --git a/source/server/api_listener_impl.cc b/source/server/api_listener_impl.cc index b22c9d786585..d947a0549c7a 100644 --- a/source/server/api_listener_impl.cc +++ b/source/server/api_listener_impl.cc @@ -5,6 +5,7 @@ #include "envoy/stats/scope.h" #include "source/common/http/conn_manager_impl.h" +#include "source/common/listener_manager/listener_info_impl.h" #include "source/common/network/resolver_impl.h" #include "source/common/protobuf/utility.h" #include "source/extensions/filters/network/http_connection_manager/config.h" @@ -12,17 +13,13 @@ namespace Envoy { namespace Server { -bool isQuic(const envoy::config::listener::v3::Listener& config) { - return config.has_udp_listener_config() && config.udp_listener_config().has_quic_options(); -} - ApiListenerImplBase::ApiListenerImplBase(const envoy::config::listener::v3::Listener& config, Server::Instance& server, const std::string& name) : config_(config), name_(name), address_(Network::Address::resolveProtoAddress(config.address())), - global_scope_(server.stats().createScope("")), - listener_scope_(server.stats().createScope(fmt::format("listener.api.{}.", name_))), - factory_context_(server, config_, *this, *global_scope_, *listener_scope_, isQuic(config)) {} + factory_context_(server, *this, server.stats().createScope(""), + server.stats().createScope(fmt::format("listener.api.{}.", name_)), + std::make_shared(config)) {} void ApiListenerImplBase::SyntheticReadCallbacks::SyntheticConnection::raiseConnectionEvent( Network::ConnectionEvent event) { diff --git a/source/server/api_listener_impl.h b/source/server/api_listener_impl.h index ee2038d454b3..ccf1b2cbdcd3 100644 --- a/source/server/api_listener_impl.h +++ b/source/server/api_listener_impl.h @@ -85,7 +85,8 @@ class ApiListenerImplBase : public ApiListener, : parent_(parent), dispatcher_(dispatcher), connection_info_provider_(std::make_shared( parent.parent_.address_, parent.parent_.address_)), - stream_info_(parent_.parent_.factory_context_.timeSource(), connection_info_provider_), + stream_info_(parent_.parent_.factory_context_.serverFactoryContext().timeSource(), + connection_info_provider_), options_(std::make_shared>()) {} void raiseConnectionEvent(Network::ConnectionEvent event); @@ -190,8 +191,6 @@ class ApiListenerImplBase : public ApiListener, const envoy::config::listener::v3::Listener& config_; const std::string name_; Network::Address::InstanceConstSharedPtr address_; - Stats::ScopeSharedPtr global_scope_; - Stats::ScopeSharedPtr listener_scope_; FactoryContextImpl factory_context_; }; diff --git a/source/server/config_validation/BUILD b/source/server/config_validation/BUILD index f5e8bd30b1d9..eba39b9bc6d2 100644 --- a/source/server/config_validation/BUILD +++ b/source/server/config_validation/BUILD @@ -90,6 +90,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/config:utility_lib", "//source/common/grpc:common_lib", + "//source/common/listener_manager:listener_info_lib", "//source/common/local_info:local_info_lib", "//source/common/protobuf:utility_lib", "//source/common/quic:quic_stat_names_lib", @@ -100,9 +101,11 @@ envoy_cc_library( "//source/common/thread_local:thread_local_lib", "//source/common/version:version_lib", "//source/server:configuration_lib", + "//source/server:hot_restart_nop_lib", + "//source/server:overload_manager_lib", "//source/server:server_lib", "//source/server:utils_lib", - "//source/server/admin:admin_lib", + "//source/server/admin:admin_factory_context", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", diff --git a/source/server/config_validation/admin.cc b/source/server/config_validation/admin.cc index 678942a2934b..4aa108346b43 100644 --- a/source/server/config_validation/admin.cc +++ b/source/server/config_validation/admin.cc @@ -20,11 +20,9 @@ const Network::Socket& ValidationAdmin::socket() { return *socket_; } ConfigTracker& ValidationAdmin::getConfigTracker() { return config_tracker_; } -void ValidationAdmin::startHttpListener(const std::list&, - const std::string&, +void ValidationAdmin::startHttpListener(std::list, Network::Address::InstanceConstSharedPtr, - const Network::Socket::OptionsSharedPtr&, - Stats::ScopeSharedPtr&&) {} + Network::Socket::OptionsSharedPtr) {} Http::Code ValidationAdmin::request(absl::string_view, absl::string_view, Http::ResponseHeaderMap&, std::string&) { diff --git a/source/server/config_validation/admin.h b/source/server/config_validation/admin.h index 150a8dbfac07..173afb29d355 100644 --- a/source/server/config_validation/admin.h +++ b/source/server/config_validation/admin.h @@ -2,6 +2,7 @@ #include "envoy/network/listen_socket.h" #include "envoy/server/admin.h" +#include "envoy/server/instance.h" #include "source/common/common/assert.h" #include "source/common/network/listen_socket_impl.h" @@ -30,11 +31,9 @@ class ValidationAdmin : public Admin { bool removeHandler(const std::string&) override; const Network::Socket& socket() override; ConfigTracker& getConfigTracker() override; - void startHttpListener(const std::list& access_logs, - const std::string& address_out_path, + void startHttpListener(std::list access_logs, Network::Address::InstanceConstSharedPtr address, - const Network::Socket::OptionsSharedPtr&, - Stats::ScopeSharedPtr&& listener_scope) override; + Network::Socket::OptionsSharedPtr) override; Http::Code request(absl::string_view path_and_query, absl::string_view method, Http::ResponseHeaderMap& response_headers, std::string& body) override; void addListenerToHandler(Network::ConnectionHandler* handler) override; diff --git a/source/server/config_validation/api.cc b/source/server/config_validation/api.cc index 1d27b2f32e83..cfc1c091d69c 100644 --- a/source/server/config_validation/api.cc +++ b/source/server/config_validation/api.cc @@ -9,8 +9,10 @@ namespace Api { ValidationImpl::ValidationImpl(Thread::ThreadFactory& thread_factory, Stats::Store& stats_store, Event::TimeSystem& time_system, Filesystem::Instance& file_system, Random::RandomGenerator& random_generator, - const envoy::config::bootstrap::v3::Bootstrap& bootstrap) - : Impl(thread_factory, stats_store, time_system, file_system, random_generator, bootstrap), + const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + const ProcessContextOptRef& process_context) + : Impl(thread_factory, stats_store, time_system, file_system, random_generator, bootstrap, + process_context), time_system_(time_system) {} Event::DispatcherPtr ValidationImpl::allocateDispatcher(const std::string& name) { diff --git a/source/server/config_validation/api.h b/source/server/config_validation/api.h index c1a64a65df54..11722fb6de4f 100644 --- a/source/server/config_validation/api.h +++ b/source/server/config_validation/api.h @@ -19,7 +19,8 @@ class ValidationImpl : public Impl { ValidationImpl(Thread::ThreadFactory& thread_factory, Stats::Store& stats_store, Event::TimeSystem& time_system, Filesystem::Instance& file_system, Random::RandomGenerator& random_generator, - const envoy::config::bootstrap::v3::Bootstrap& bootstrap); + const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + const ProcessContextOptRef& process_context = absl::nullopt); Event::DispatcherPtr allocateDispatcher(const std::string& name) override; Event::DispatcherPtr allocateDispatcher(const std::string& name, diff --git a/source/server/config_validation/cluster_manager.cc b/source/server/config_validation/cluster_manager.cc index afb0dd525726..719919f25874 100644 --- a/source/server/config_validation/cluster_manager.cc +++ b/source/server/config_validation/cluster_manager.cc @@ -10,11 +10,13 @@ namespace Upstream { ClusterManagerPtr ValidationClusterManagerFactory::clusterManagerFromProto( const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - return std::make_unique( + auto cluster_manager = std::unique_ptr{new ValidationClusterManager( bootstrap, *this, stats_, tls_, context_.runtime(), context_.localInfo(), context_.accessLogManager(), context_.mainThreadDispatcher(), context_.admin(), context_.messageValidationContext(), context_.api(), http_context_, context_.grpcContext(), - context_.routerContext(), server_); + context_.routerContext(), server_)}; + THROW_IF_NOT_OK(cluster_manager->init(bootstrap)); + return cluster_manager; } CdsApiPtr ValidationClusterManagerFactory::createCds( diff --git a/source/server/config_validation/cluster_manager.h b/source/server/config_validation/cluster_manager.h index 474a29a5151e..c94ea9381c19 100644 --- a/source/server/config_validation/cluster_manager.h +++ b/source/server/config_validation/cluster_manager.h @@ -52,6 +52,9 @@ class ValidationClusterManager : public ClusterManagerImpl { // any calling code creating real outbound networking during validation. return nullptr; } + + // Gives access to the protected constructor. + friend ValidationClusterManagerFactory; }; } // namespace Upstream diff --git a/source/server/config_validation/dispatcher.h b/source/server/config_validation/dispatcher.h index 743ef2b34445..13808db9c9db 100644 --- a/source/server/config_validation/dispatcher.h +++ b/source/server/config_validation/dispatcher.h @@ -23,10 +23,6 @@ class ValidationDispatcher : public DispatcherImpl { Network::Address::InstanceConstSharedPtr, Network::Address::InstanceConstSharedPtr, Network::TransportSocketPtr&&, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_options) override; - Network::ListenerPtr createListener(Network::SocketSharedPtr&&, Network::TcpListenerCallbacks&, - Runtime::Loader&, const Network::ListenerConfig&) override { - return nullptr; - } }; } // namespace Event diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 53cc73ee0752..f8d6e31813d6 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -8,11 +8,14 @@ #include "source/common/config/utility.h" #include "source/common/config/well_known_names.h" #include "source/common/event/real_time_system.h" +#include "source/common/listener_manager/listener_info_impl.h" #include "source/common/local_info/local_info_impl.h" #include "source/common/protobuf/utility.h" #include "source/common/singleton/manager_impl.h" #include "source/common/version/version.h" +#include "source/server/admin/admin_factory_context.h" #include "source/server/listener_manager_factory.h" +#include "source/server/overload_manager_impl.h" #include "source/server/regex_engine.h" #include "source/server/ssl_context_manager.h" #include "source/server/utils.h" @@ -23,14 +26,15 @@ namespace Server { bool validateConfig(const Options& options, const Network::Address::InstanceConstSharedPtr& local_address, ComponentFactory& component_factory, Thread::ThreadFactory& thread_factory, - Filesystem::Instance& file_system) { + Filesystem::Instance& file_system, + const ProcessContextOptRef& process_context) { Thread::MutexBasicLockable access_log_lock; Stats::IsolatedStoreImpl stats_store; TRY_ASSERT_MAIN_THREAD { Event::RealTimeSystem time_system; ValidationInstance server(options, time_system, local_address, stats_store, access_log_lock, - component_factory, thread_factory, file_system); + component_factory, thread_factory, file_system, process_context); std::cout << "configuration '" << options.configPath() << "' OK" << std::endl; server.shutdown(); return true; @@ -45,21 +49,21 @@ ValidationInstance::ValidationInstance( const Options& options, Event::TimeSystem& time_system, const Network::Address::InstanceConstSharedPtr& local_address, Stats::IsolatedStoreImpl& store, Thread::BasicLockable& access_log_lock, ComponentFactory& component_factory, - Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system) + Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system, + const ProcessContextOptRef& process_context) : options_(options), validation_context_(options_.allowUnknownStaticFields(), !options.rejectUnknownDynamicFields(), !options.ignoreUnknownDynamicFields()), stats_store_(store), api_(new Api::ValidationImpl(thread_factory, store, time_system, file_system, - random_generator_, bootstrap_)), + random_generator_, bootstrap_, process_context)), dispatcher_(api_->allocateDispatcher("main_thread")), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), access_log_manager_(options.fileFlushIntervalMsec(), *api_, *dispatcher_, access_log_lock, store), - mutex_tracer_(nullptr), grpc_context_(stats_store_.symbolTable()), - http_context_(stats_store_.symbolTable()), router_context_(stats_store_.symbolTable()), - time_system_(time_system), server_contexts_(*this), - quic_stat_names_(stats_store_.symbolTable()) { + grpc_context_(stats_store_.symbolTable()), http_context_(stats_store_.symbolTable()), + router_context_(stats_store_.symbolTable()), time_system_(time_system), + server_contexts_(*this), quic_stat_names_(stats_store_.symbolTable()) { TRY_ASSERT_MAIN_THREAD { initialize(options, local_address, component_factory); } END_TRY catch (const EnvoyException& e) { @@ -109,7 +113,8 @@ void ValidationInstance::initialize(const Options& options, dispatcher(), *stats().rootScope(), threadLocal(), bootstrap_.overload_manager(), messageValidationContext().staticValidationVisitor(), *api_, options_); Configuration::InitialImpl initial_config(bootstrap_); - initial_config.initAdminAccessLog(bootstrap_, *this); + AdminFactoryContext factory_context(*this, std::make_shared()); + initial_config.initAdminAccessLog(bootstrap_, factory_context); admin_ = std::make_unique(initial_config.admin().address()); listener_manager_ = Config::Utility::getAndCheckFactoryByName( Config::ServerExtensionValues::get().VALIDATION_LISTENER) @@ -117,6 +122,11 @@ void ValidationInstance::initialize(const Options& options, thread_local_.registerThread(*dispatcher_, true); runtime_ = component_factory.createRuntime(*this, initial_config); + ENVOY_BUG(runtime_ != nullptr, + "Component factory should not return nullptr from createRuntime()"); + drain_manager_ = component_factory.createDrainManager(*this); + ENVOY_BUG(drain_manager_ != nullptr, + "Component factory should not return nullptr from createDrainManager()"); secret_manager_ = std::make_unique(admin()->getConfigTracker()); ssl_context_manager_ = createContextManager("ssl_context_manager", api_->timeSource()); diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index 181f6adcbf1c..fc301751e57d 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -28,6 +28,7 @@ #include "source/server/config_validation/api.h" #include "source/server/config_validation/cluster_manager.h" #include "source/server/config_validation/dns.h" +#include "source/server/hot_restart_nop_impl.h" #include "source/server/server.h" #include "absl/types/optional.h" @@ -42,7 +43,8 @@ namespace Server { bool validateConfig(const Options& options, const Network::Address::InstanceConstSharedPtr& local_address, ComponentFactory& component_factory, Thread::ThreadFactory& thread_factory, - Filesystem::Instance& file_system); + Filesystem::Instance& file_system, + const ProcessContextOptRef& process_context = absl::nullopt); /** * ValidationInstance does the bulk of the work for config-validation runs of Envoy. It implements @@ -65,9 +67,11 @@ class ValidationInstance final : Logger::Loggable, const Network::Address::InstanceConstSharedPtr& local_address, Stats::IsolatedStoreImpl& store, Thread::BasicLockable& access_log_lock, ComponentFactory& component_factory, Thread::ThreadFactory& thread_factory, - Filesystem::Instance& file_system); + Filesystem::Instance& file_system, + const ProcessContextOptRef& process_context = absl::nullopt); // Server::Instance + void run() override { PANIC("not implemented"); } OptRef admin() override { return makeOptRefFromPtr(static_cast(admin_.get())); } @@ -85,10 +89,10 @@ class ValidationInstance final : Logger::Loggable, return dns_resolver_factory.createDnsResolver(dispatcher(), api(), typed_dns_resolver_config); } void drainListeners(OptRef) override {} - DrainManager& drainManager() override { PANIC("not implemented"); } + DrainManager& drainManager() override { return *drain_manager_; } AccessLog::AccessLogManager& accessLogManager() override { return access_log_manager_; } void failHealthcheck(bool) override {} - HotRestart& hotRestart() override { PANIC("not implemented"); } + HotRestart& hotRestart() override { return nop_hot_restart_; } Init::Manager& initManager() override { return init_manager_; } ServerLifecycleNotifier& lifecycleNotifier() override { return *this; } ListenerManager& listenerManager() override { return *listener_manager_; } @@ -107,7 +111,7 @@ class ValidationInstance final : Logger::Loggable, Grpc::Context& grpcContext() override { return grpc_context_; } Http::Context& httpContext() override { return http_context_; } Router::Context& routerContext() override { return router_context_; } - ProcessContextOptRef processContext() override { return absl::nullopt; } + ProcessContextOptRef processContext() override { return api_->processContext(); } ThreadLocal::Instance& threadLocal() override { return thread_local_; } LocalInfo::LocalInfo& localInfo() const override { return *local_info_; } TimeSource& timeSource() override { return api_->timeSource(); } @@ -166,19 +170,21 @@ class ValidationInstance final : Logger::Loggable, ThreadLocal::InstanceImpl thread_local_; envoy::config::bootstrap::v3::Bootstrap bootstrap_; Api::ApiPtr api_; + // ssl_context_manager_ must come before dispatcher_, since ClusterInfo + // references SslSocketFactory and is deleted on the main thread via the dispatcher. + std::unique_ptr ssl_context_manager_; Event::DispatcherPtr dispatcher_; std::unique_ptr admin_; Singleton::ManagerPtr singleton_manager_; std::unique_ptr runtime_; Random::RandomGeneratorImpl random_generator_; - std::unique_ptr ssl_context_manager_; Configuration::MainImpl config_; LocalInfo::LocalInfoPtr local_info_; AccessLog::AccessLogManagerImpl access_log_manager_; std::unique_ptr cluster_manager_factory_; std::unique_ptr listener_manager_; std::unique_ptr overload_manager_; - MutexTracer* mutex_tracer_; + MutexTracer* mutex_tracer_{nullptr}; Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; Router::ContextImpl router_context_; @@ -186,6 +192,8 @@ class ValidationInstance final : Logger::Loggable, ServerFactoryContextImpl server_contexts_; Quic::QuicStatNames quic_stat_names_; Filter::TcpListenerFilterConfigProviderManagerImpl tcp_listener_config_provider_manager_; + Server::DrainManagerPtr drain_manager_; + HotRestartNopImpl nop_hot_restart_; }; } // namespace Server diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index dddbe09c77df..80818215be7b 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -86,7 +86,8 @@ StatsConfigImpl::StatsConfigImpl(const envoy::config::bootstrap::v3::Bootstrap& if (bootstrap.has_stats_flush_interval() && bootstrap.stats_flush_case() != envoy::config::bootstrap::v3::Bootstrap::STATS_FLUSH_NOT_SET) { - throw EnvoyException("Only one of stats_flush_interval or stats_flush_on_admin should be set!"); + throwEnvoyExceptionOrPanic( + "Only one of stats_flush_interval or stats_flush_on_admin should be set!"); } flush_interval_ = @@ -118,7 +119,7 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr ENVOY_LOG(info, "loading {} static secret(s)", secrets.size()); for (ssize_t i = 0; i < secrets.size(); i++) { ENVOY_LOG(debug, "static secret #{}: {}", i, secrets[i].name()); - server.secretManager().addStaticSecret(secrets[i]); + THROW_IF_NOT_OK(server.secretManager().addStaticSecret(secrets[i])); } ENVOY_LOG(info, "loading {} cluster(s)", bootstrap.static_resources().clusters().size()); @@ -131,7 +132,7 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr absl::StatusOr update_or_error = server.listenerManager().addOrUpdateListener(listeners[i], "", false); if (!update_or_error.status().ok()) { - throw EnvoyException(std::string(update_or_error.status().message())); + throwEnvoyExceptionOrPanic(std::string(update_or_error.status().message())); } } initializeWatchdogs(bootstrap, server); @@ -183,7 +184,7 @@ void MainImpl::initializeTracers(const envoy::config::trace::v3::Tracing& config void MainImpl::initializeWatchdogs(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, Instance& server) { if (bootstrap.has_watchdog() && bootstrap.has_watchdogs()) { - throw EnvoyException("Only one of watchdog or watchdogs should be set!"); + throwEnvoyExceptionOrPanic("Only one of watchdog or watchdogs should be set!"); } if (bootstrap.has_watchdog()) { @@ -253,12 +254,12 @@ InitialImpl::InitialImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstra } void InitialImpl::initAdminAccessLog(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, - Instance& server) { + FactoryContext& factory_context) { const auto& admin = bootstrap.admin(); for (const auto& access_log : admin.access_log()) { AccessLog::InstanceSharedPtr current_access_log = - AccessLog::AccessLogFactory::fromProto(access_log, server.serverFactoryContext()); + AccessLog::AccessLogFactory::fromProto(access_log, factory_context); admin_.access_logs_.emplace_back(current_access_log); } @@ -267,7 +268,7 @@ void InitialImpl::initAdminAccessLog(const envoy::config::bootstrap::v3::Bootstr admin.access_log_path()}; admin_.access_logs_.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), - server.accessLogManager())); + factory_context.serverFactoryContext().accessLogManager())); } } diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index 63b2bafd33e9..6b039328a491 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -198,7 +198,7 @@ class InitialImpl : public Initial { * Initialize admin access log. */ void initAdminAccessLog(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, - Instance& server); + FactoryContext& factory_context); private: struct AdminImpl : public Admin { diff --git a/source/server/drain_manager_impl.cc b/source/server/drain_manager_impl.cc index 27eed8d92d41..23ac96cdb88f 100644 --- a/source/server/drain_manager_impl.cc +++ b/source/server/drain_manager_impl.cc @@ -70,13 +70,23 @@ bool DrainManagerImpl::drainClose() const { const auto remaining_time = std::chrono::duration_cast(drain_deadline_ - current_time); + const auto drain_time = server_.options().drainTime(); ASSERT(server_.options().drainTime() >= remaining_time); - const auto elapsed_time = server_.options().drainTime() - remaining_time; + const auto drain_time_count = drain_time.count(); + + // If the user hasn't specified a drain timeout it will be zero, so we'll + // confirm the drainClose immediately. Otherwise we'll use the drain timeout + // as a modulus to a random number to salt the drain timing. + if (drain_time_count == 0) { + return true; + } + const auto elapsed_time = drain_time - remaining_time; return static_cast(elapsed_time.count()) > - (server_.api().randomGenerator().random() % server_.options().drainTime().count()); + (server_.api().randomGenerator().random() % drain_time_count); } Common::CallbackHandlePtr DrainManagerImpl::addOnDrainCloseCb(DrainCloseCb cb) const { + ASSERT_IS_MAIN_OR_TEST_THREAD(); ASSERT(dispatcher_.isThreadSafe()); if (draining_) { @@ -84,14 +94,21 @@ Common::CallbackHandlePtr DrainManagerImpl::addOnDrainCloseCb(DrainCloseCb cb) c // Calculate the delay. If using an immediate drain-strategy or past our deadline, use // a zero millisecond delay. Otherwise, pick a random value within the remaining time-span. - std::chrono::milliseconds drain_delay = - (server_.options().drainStrategy() != Server::DrainStrategy::Immediate && - current_time < drain_deadline_) - ? std::chrono::milliseconds(server_.api().randomGenerator().random() % - std::chrono::duration_cast( - drain_deadline_ - current_time) - .count()) - : std::chrono::milliseconds{0}; + std::chrono::milliseconds drain_delay{0}; + if (server_.options().drainStrategy() != Server::DrainStrategy::Immediate) { + if (current_time < drain_deadline_) { + const auto delta = drain_deadline_ - current_time; + const auto ms = std::chrono::duration_cast(delta).count(); + + // Note; current_time may be less than drain_deadline_ by only a + // microsecond (delta will be 1000 nanoseconds), in which case when we + // convert to milliseconds that will be 0, which will throw a SIGFPE + // if used as a modulus unguarded. + if (ms > 0) { + drain_delay = std::chrono::milliseconds(server_.api().randomGenerator().random() % ms); + } + } + } cb(drain_delay); return nullptr; } @@ -100,6 +117,7 @@ Common::CallbackHandlePtr DrainManagerImpl::addOnDrainCloseCb(DrainCloseCb cb) c } void DrainManagerImpl::addDrainCompleteCallback(std::function cb) { + ASSERT_IS_MAIN_OR_TEST_THREAD(); ASSERT(draining_); // If the drain-tick-timer is active, add the callback to the queue. If not defined @@ -112,6 +130,7 @@ void DrainManagerImpl::addDrainCompleteCallback(std::function cb) { } void DrainManagerImpl::startDrainSequence(std::function drain_complete_cb) { + ASSERT_IS_MAIN_OR_TEST_THREAD(); ASSERT(drain_complete_cb); // If we've already started draining (either through direct invocation or through @@ -122,7 +141,24 @@ void DrainManagerImpl::startDrainSequence(std::function drain_complete_c } ASSERT(!drain_tick_timer_); - draining_ = true; + const std::chrono::seconds drain_delay(server_.options().drainTime()); + + // Note https://github.com/envoyproxy/envoy/issues/31457, previous to which, + // drain_deadline_ was set *after* draining_ resulting in a read/write race between + // the main thread running this function from admin, and the worker thread calling + // drainClose. Note that drain_deadline_ is default-constructed which guarantees + // to set the time-since epoch to a count of 0 + // (https://en.cppreference.com/w/cpp/chrono/time_point/time_point). + ASSERT(drain_deadline_.time_since_epoch().count() == 0, "drain_deadline_ cannot be set twice."); + + // Since draining_ is atomic, it is safe to set drain_deadline_ without a mutex + // as drain_close() only reads from drain_deadline_ if draining_ is true, and + // C++ will not re-order an assign to an atomic. See + // https://stackoverflow.com/questions/40320254/reordering-atomic-operations-in-c . + drain_deadline_ = dispatcher_.timeSource().monotonicTime() + drain_delay; + + // Atomic assign must come after the assign to drain_deadline_. + draining_.store(true, std::memory_order_seq_cst); // Signal to child drain-managers to start their drain sequence children_->runCallbacks(); @@ -136,9 +172,7 @@ void DrainManagerImpl::startDrainSequence(std::function drain_complete_c drain_tick_timer_.reset(); }); addDrainCompleteCallback(drain_complete_cb); - const std::chrono::seconds drain_delay(server_.options().drainTime()); drain_tick_timer_->enableTimer(drain_delay); - drain_deadline_ = dispatcher_.timeSource().monotonicTime() + drain_delay; // Call registered on-drain callbacks - with gradual delays // Note: This will distribute drain events in the first 1/4th of the drain window diff --git a/source/server/factory_context_base_impl.h b/source/server/factory_context_base_impl.h deleted file mode 100644 index 80fc55e0a313..000000000000 --- a/source/server/factory_context_base_impl.h +++ /dev/null @@ -1,57 +0,0 @@ -#pragma once - -#include "envoy/server/factory_context.h" - -namespace Envoy { -namespace Server { - -class FactoryContextBaseImpl : public Configuration::FactoryContextBase { -public: - FactoryContextBaseImpl(const Server::Options& options, Event::Dispatcher& main_thread_dispatcher, - Api::Api& api, const LocalInfo::LocalInfo& local_info, - OptRef admin, Runtime::Loader& runtime, - Singleton::Manager& singleton_manager, - ProtobufMessage::ValidationVisitor& validation_visitor, - Stats::Store& store, ThreadLocal::SlotAllocator& local) - : options_(options), main_thread_dispatcher_(main_thread_dispatcher), api_(api), - local_info_(local_info), admin_(admin), runtime_(runtime), - singleton_manager_(singleton_manager), validation_visitor_(validation_visitor), - scope_(*store.rootScope()), thread_local_(local) {} - - FactoryContextBaseImpl(Configuration::FactoryContextBase& config) - : options_(config.options()), main_thread_dispatcher_(config.mainThreadDispatcher()), - api_(config.api()), local_info_(config.localInfo()), admin_(config.admin()), - runtime_(config.runtime()), singleton_manager_(config.singletonManager()), - validation_visitor_(config.messageValidationVisitor()), scope_(config.scope()), - thread_local_(config.threadLocal()) {} - - // FactoryContextBase - const Options& options() override { return options_; }; - Event::Dispatcher& mainThreadDispatcher() override { return main_thread_dispatcher_; }; - Api::Api& api() override { return api_; }; - const LocalInfo::LocalInfo& localInfo() const override { return local_info_; }; - OptRef admin() override { return admin_; }; - Envoy::Runtime::Loader& runtime() override { return runtime_; }; - Singleton::Manager& singletonManager() override { return singleton_manager_; }; - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return validation_visitor_; - }; - Stats::Scope& scope() override { return scope_; }; - Stats::Scope& serverScope() override { return scope_; } - ThreadLocal::SlotAllocator& threadLocal() override { return thread_local_; }; - -private: - const Server::Options& options_; - Event::Dispatcher& main_thread_dispatcher_; - Api::Api& api_; - const LocalInfo::LocalInfo& local_info_; - OptRef admin_; - Runtime::Loader& runtime_; - Singleton::Manager& singleton_manager_; - ProtobufMessage::ValidationVisitor& validation_visitor_; - Stats::Scope& scope_; - ThreadLocal::SlotAllocator& thread_local_; -}; - -} // namespace Server -} // namespace Envoy diff --git a/source/server/factory_context_impl.cc b/source/server/factory_context_impl.cc index 3d525653814c..63232e6c5f38 100644 --- a/source/server/factory_context_impl.cc +++ b/source/server/factory_context_impl.cc @@ -3,64 +3,48 @@ namespace Envoy { namespace Server { -FactoryContextImpl::FactoryContextImpl(Server::Instance& server, - const envoy::config::listener::v3::Listener& config, - Network::DrainDecision& drain_decision, - Stats::Scope& global_scope, Stats::Scope& listener_scope, - bool is_quic) - : server_(server), config_(config), drain_decision_(drain_decision), - global_scope_(global_scope), listener_scope_(listener_scope), is_quic_(is_quic) {} - -AccessLog::AccessLogManager& FactoryContextImpl::accessLogManager() { - return server_.accessLogManager(); -} -Upstream::ClusterManager& FactoryContextImpl::clusterManager() { return server_.clusterManager(); } -Event::Dispatcher& FactoryContextImpl::mainThreadDispatcher() { return server_.dispatcher(); } -const Server::Options& FactoryContextImpl::options() { return server_.options(); } -Grpc::Context& FactoryContextImpl::grpcContext() { return server_.grpcContext(); } -Router::Context& FactoryContextImpl::routerContext() { return server_.routerContext(); } -bool FactoryContextImpl::healthCheckFailed() { return server_.healthCheckFailed(); } -Http::Context& FactoryContextImpl::httpContext() { return server_.httpContext(); } -Init::Manager& FactoryContextImpl::initManager() { return server_.initManager(); } -const LocalInfo::LocalInfo& FactoryContextImpl::localInfo() const { return server_.localInfo(); } -Envoy::Runtime::Loader& FactoryContextImpl::runtime() { return server_.runtime(); } -Stats::Scope& FactoryContextImpl::scope() { return global_scope_; } -Singleton::Manager& FactoryContextImpl::singletonManager() { return server_.singletonManager(); } -OverloadManager& FactoryContextImpl::overloadManager() { return server_.overloadManager(); } -ThreadLocal::SlotAllocator& FactoryContextImpl::threadLocal() { return server_.threadLocal(); } -OptRef FactoryContextImpl::admin() { return server_.admin(); } -TimeSource& FactoryContextImpl::timeSource() { return server_.timeSource(); } -ProtobufMessage::ValidationContext& FactoryContextImpl::messageValidationContext() { - return server_.messageValidationContext(); -} -ProtobufMessage::ValidationVisitor& FactoryContextImpl::messageValidationVisitor() { - return server_.messageValidationContext().staticValidationVisitor(); +FactoryContextImplBase::FactoryContextImplBase( + Server::Instance& server, ProtobufMessage::ValidationVisitor& validation_visitor, + Stats::ScopeSharedPtr scope, Stats::ScopeSharedPtr listener_scope, + const Network::ListenerInfoConstSharedPtr& listener_info) + : server_(server), validation_visitor_(validation_visitor), scope_(std::move(scope)), + listener_scope_(std::move(listener_scope)), listener_info_(listener_info) { + ASSERT(listener_info_ != nullptr); } -Api::Api& FactoryContextImpl::api() { return server_.api(); } -ServerLifecycleNotifier& FactoryContextImpl::lifecycleNotifier() { - return server_.lifecycleNotifier(); -} -ProcessContextOptRef FactoryContextImpl::processContext() { return server_.processContext(); } -Configuration::ServerFactoryContext& FactoryContextImpl::getServerFactoryContext() const { + +Configuration::ServerFactoryContext& FactoryContextImplBase::serverFactoryContext() const { return server_.serverFactoryContext(); } + +ProtobufMessage::ValidationVisitor& FactoryContextImplBase::messageValidationVisitor() const { + return validation_visitor_; +} + +Stats::Scope& FactoryContextImplBase::scope() { return *scope_; } + Configuration::TransportSocketFactoryContext& -FactoryContextImpl::getTransportSocketFactoryContext() const { +FactoryContextImplBase::getTransportSocketFactoryContext() const { return server_.transportSocketFactoryContext(); } -const envoy::config::core::v3::Metadata& FactoryContextImpl::listenerMetadata() const { - return config_.metadata(); -} -const Envoy::Config::TypedMetadata& FactoryContextImpl::listenerTypedMetadata() const { - // TODO(nareddyt): Needs an implementation for this context. Currently not used. - PANIC("not implemented"); -} -envoy::config::core::v3::TrafficDirection FactoryContextImpl::direction() const { - return config_.traffic_direction(); + +Stats::Scope& FactoryContextImplBase::listenerScope() { return *listener_scope_; } + +const Network::ListenerInfo& FactoryContextImplBase::listenerInfo() const { + return *listener_info_; } + +FactoryContextImpl::FactoryContextImpl(Server::Instance& server, + Network::DrainDecision& drain_decision, + Stats::ScopeSharedPtr scope, + Stats::ScopeSharedPtr listener_scope, + const Network::ListenerInfoConstSharedPtr& listener_info) + : FactoryContextImplBase(server, server.messageValidationContext().staticValidationVisitor(), + std::move(scope), std::move(listener_scope), listener_info), + drain_decision_(drain_decision) {} + +Init::Manager& FactoryContextImpl::initManager() { return server_.initManager(); } + Network::DrainDecision& FactoryContextImpl::drainDecision() { return drain_decision_; } -Stats::Scope& FactoryContextImpl::listenerScope() { return listener_scope_; } -bool FactoryContextImpl::isQuicListener() const { return is_quic_; } } // namespace Server } // namespace Envoy diff --git a/source/server/factory_context_impl.h b/source/server/factory_context_impl.h index 5fdfd022e5b0..fd476a148d04 100644 --- a/source/server/factory_context_impl.h +++ b/source/server/factory_context_impl.h @@ -3,58 +3,52 @@ #include "envoy/server/factory_context.h" #include "envoy/server/instance.h" +#include "source/common/config/metadata.h" + namespace Envoy { namespace Server { +class FactoryContextImplBase : virtual public Configuration::FactoryContext { +public: + FactoryContextImplBase(Server::Instance& server, + ProtobufMessage::ValidationVisitor& validation_visitor, + Stats::ScopeSharedPtr scope, Stats::ScopeSharedPtr listener_scope, + const Network::ListenerInfoConstSharedPtr& listener_info); + + // Configuration::FactoryContext + Configuration::ServerFactoryContext& serverFactoryContext() const override; + Stats::Scope& scope() override; + ProtobufMessage::ValidationVisitor& messageValidationVisitor() const override; + Configuration::TransportSocketFactoryContext& getTransportSocketFactoryContext() const override; + const Network::ListenerInfo& listenerInfo() const override; + + Stats::Scope& listenerScope() override; + +protected: + Server::Instance& server_; + ProtobufMessage::ValidationVisitor& validation_visitor_; + // Listener scope without the listener prefix. + Stats::ScopeSharedPtr scope_; + // Listener scope with the listener prefix. + Stats::ScopeSharedPtr listener_scope_; + const Network::ListenerInfoConstSharedPtr listener_info_; +}; + /** * Implementation of FactoryContext wrapping a Server::Instance and some listener components. */ -class FactoryContextImpl : public Configuration::FactoryContext { +class FactoryContextImpl : public FactoryContextImplBase { public: - FactoryContextImpl(Server::Instance& server, const envoy::config::listener::v3::Listener& config, - Network::DrainDecision& drain_decision, Stats::Scope& global_scope, - Stats::Scope& listener_scope, bool is_quic); + FactoryContextImpl(Server::Instance& server, Network::DrainDecision& drain_decision, + Stats::ScopeSharedPtr scope, Stats::ScopeSharedPtr listener_scope, + const Network::ListenerInfoConstSharedPtr& listener_info); // Configuration::FactoryContext - AccessLog::AccessLogManager& accessLogManager() override; - Upstream::ClusterManager& clusterManager() override; - Event::Dispatcher& mainThreadDispatcher() override; - const Server::Options& options() override; - Grpc::Context& grpcContext() override; - Router::Context& routerContext() override; - bool healthCheckFailed() override; - Http::Context& httpContext() override; Init::Manager& initManager() override; - const LocalInfo::LocalInfo& localInfo() const override; - Envoy::Runtime::Loader& runtime() override; - Stats::Scope& scope() override; - Stats::Scope& serverScope() override { return *server_.stats().rootScope(); } - Singleton::Manager& singletonManager() override; - OverloadManager& overloadManager() override; - ThreadLocal::SlotAllocator& threadLocal() override; - OptRef admin() override; - TimeSource& timeSource() override; - ProtobufMessage::ValidationContext& messageValidationContext() override; - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override; - Api::Api& api() override; - ServerLifecycleNotifier& lifecycleNotifier() override; - ProcessContextOptRef processContext() override; - Configuration::ServerFactoryContext& getServerFactoryContext() const override; - Configuration::TransportSocketFactoryContext& getTransportSocketFactoryContext() const override; - const envoy::config::core::v3::Metadata& listenerMetadata() const override; - const Envoy::Config::TypedMetadata& listenerTypedMetadata() const override; - envoy::config::core::v3::TrafficDirection direction() const override; Network::DrainDecision& drainDecision() override; - Stats::Scope& listenerScope() override; - bool isQuicListener() const override; private: - Server::Instance& server_; - const envoy::config::listener::v3::Listener& config_; Network::DrainDecision& drain_decision_; - Stats::Scope& global_scope_; - Stats::Scope& listener_scope_; - bool is_quic_; }; } // namespace Server diff --git a/source/server/generic_factory_context.cc b/source/server/generic_factory_context.cc new file mode 100644 index 000000000000..f86a2e89090a --- /dev/null +++ b/source/server/generic_factory_context.cc @@ -0,0 +1,31 @@ +#include "source/server/generic_factory_context.h" + +namespace Envoy { +namespace Server { + +GenericFactoryContextImpl::GenericFactoryContextImpl( + Server::Configuration::ServerFactoryContext& server_context, + ProtobufMessage::ValidationVisitor& validation_visitor) + : server_context_(server_context), validation_visitor_(validation_visitor), + scope_(server_context.serverScope()), init_manager_(server_context.initManager()) {} + +GenericFactoryContextImpl::GenericFactoryContextImpl( + Server::Configuration::GenericFactoryContext& generic_context) + : server_context_(generic_context.serverFactoryContext()), + validation_visitor_(generic_context.messageValidationVisitor()), + scope_(generic_context.scope()), init_manager_(generic_context.initManager()) {} + +// Server::Configuration::GenericFactoryContext +Server::Configuration::ServerFactoryContext& +GenericFactoryContextImpl::serverFactoryContext() const { + return server_context_; +} +ProtobufMessage::ValidationVisitor& GenericFactoryContextImpl::messageValidationVisitor() const { + return validation_visitor_; +} + +Stats::Scope& GenericFactoryContextImpl::scope() { return scope_; } +Init::Manager& GenericFactoryContextImpl::initManager() { return init_manager_; } + +} // namespace Server +} // namespace Envoy diff --git a/source/server/generic_factory_context.h b/source/server/generic_factory_context.h new file mode 100644 index 000000000000..b39cd8e78c0e --- /dev/null +++ b/source/server/generic_factory_context.h @@ -0,0 +1,28 @@ +#pragma once + +#include "envoy/server/factory_context.h" + +namespace Envoy { +namespace Server { + +class GenericFactoryContextImpl : public Server::Configuration::GenericFactoryContext { +public: + GenericFactoryContextImpl(Server::Configuration::ServerFactoryContext& server_context, + ProtobufMessage::ValidationVisitor& validation_visitor); + GenericFactoryContextImpl(Server::Configuration::GenericFactoryContext& generic_context); + + // Server::Configuration::GenericFactoryContext + Server::Configuration::ServerFactoryContext& serverFactoryContext() const override; + ProtobufMessage::ValidationVisitor& messageValidationVisitor() const override; + Stats::Scope& scope() override; + Init::Manager& initManager() override; + +private: + Server::Configuration::ServerFactoryContext& server_context_; + ProtobufMessage::ValidationVisitor& validation_visitor_; + Stats::Scope& scope_; + Init::Manager& init_manager_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/hot_restart.proto b/source/server/hot_restart.proto index 2439eacdf463..fff940ed9f6f 100644 --- a/source/server/hot_restart.proto +++ b/source/server/hot_restart.proto @@ -17,12 +17,22 @@ message HotRestartMessage { } message Terminate { } + // A separate socket is established for forwarding undeliverable quic udp packets + // from the parent instance to the child instance. + message ForwardedUdpPacket { + string local_addr = 1; + string peer_addr = 2; + uint64 receive_time_epoch_microseconds = 3; + bytes payload = 4; + uint32 worker_index = 5; + } oneof request { PassListenSocket pass_listen_socket = 1; ShutdownAdmin shutdown_admin = 2; Stats stats = 3; DrainListeners drain_listeners = 4; Terminate terminate = 5; + ForwardedUdpPacket forwarded_udp_packet = 6; } } diff --git a/source/server/hot_restart_impl.cc b/source/server/hot_restart_impl.cc index d08c6a15b778..417bb4a5f2f3 100644 --- a/source/server/hot_restart_impl.cc +++ b/source/server/hot_restart_impl.cc @@ -126,6 +126,7 @@ void HotRestartImpl::registerUdpForwardingListener( void HotRestartImpl::initialize(Event::Dispatcher& dispatcher, Server::Instance& server) { as_parent_.initialize(dispatcher, server); + as_child_.initialize(dispatcher); } absl::optional HotRestartImpl::sendParentAdminShutdownRequest() { @@ -147,7 +148,10 @@ HotRestartImpl::mergeParentStatsIfAny(Stats::StoreRoot& stats_store) { return response; } -void HotRestartImpl::shutdown() { as_parent_.shutdown(); } +void HotRestartImpl::shutdown() { + as_parent_.shutdown(); + as_child_.shutdown(); +} uint32_t HotRestartImpl::baseId() { return base_id_; } std::string HotRestartImpl::version() { return hotRestartVersion(); } diff --git a/source/server/hot_restarting_base.cc b/source/server/hot_restarting_base.cc index 69084b50e187..01116fcde81b 100644 --- a/source/server/hot_restarting_base.cc +++ b/source/server/hot_restarting_base.cc @@ -157,6 +157,7 @@ bool RpcStream::replyIsExpectedType(const HotRestartMessage* proto, // should only get control data in a PassListenSocketReply, it should only be the fd passing type, // and there should only be one at a time. Crash on any other control data. void RpcStream::getPassedFdIfPresent(HotRestartMessage* out, msghdr* message) { + // NOLINTNEXTLINE(clang-analyzer-core.UndefinedBinaryOperatorResult) cmsghdr* cmsg = CMSG_FIRSTHDR(message); if (cmsg != nullptr) { RELEASE_ASSERT(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS && @@ -208,6 +209,7 @@ std::unique_ptr RpcStream::receiveHotRestartMessage(Blocking msghdr message; uint8_t control_buffer[CMSG_SPACE(sizeof(int))]; std::unique_ptr ret = nullptr; + Api::OsSysCalls& os_sys_calls = Api::OsSysCallsSingleton::get(); while (!ret) { iov[0].iov_base = recv_buf_.data() + cur_msg_recvd_bytes_; iov[0].iov_len = MaxSendmsgSize; @@ -220,25 +222,27 @@ std::unique_ptr RpcStream::receiveHotRestartMessage(Blocking message.msg_control = control_buffer; message.msg_controllen = CMSG_SPACE(sizeof(int)); - const int recvmsg_rc = recvmsg(domain_socket_, &message, 0); - if (block == Blocking::No && recvmsg_rc == -1 && errno == SOCKET_ERROR_AGAIN) { + const Api::SysCallSizeResult recv_result = os_sys_calls.recvmsg(domain_socket_, &message, 0); + if (block == Blocking::No && recv_result.return_value_ == -1 && + recv_result.errno_ == SOCKET_ERROR_AGAIN) { return nullptr; } - RELEASE_ASSERT(recvmsg_rc != -1, fmt::format("recvmsg() returned -1, errno = {}", errno)); + RELEASE_ASSERT(recv_result.return_value_ != -1, + fmt::format("recvmsg() returned -1, errno = {}", recv_result.errno_)); RELEASE_ASSERT(message.msg_flags == 0, fmt::format("recvmsg() left msg_flags = {}", message.msg_flags)); - cur_msg_recvd_bytes_ += recvmsg_rc; + cur_msg_recvd_bytes_ += recv_result.return_value_; // If we don't already know 'length', we're at the start of a new length+protobuf message! if (!expected_proto_length_.has_value()) { // We are not ok with messages so fragmented that the length doesn't even come in one piece. - RELEASE_ASSERT(recvmsg_rc >= 8, "received a brokenly tiny message fragment."); + RELEASE_ASSERT(recv_result.return_value_ >= 8, "received a brokenly tiny message fragment."); expected_proto_length_ = be64toh(*reinterpret_cast(recv_buf_.data())); // Expand the buffer from its default 4096 if this message is going to be longer. if (expected_proto_length_.value() > MaxSendmsgSize - sizeof(uint64_t)) { recv_buf_.resize(expected_proto_length_.value() + sizeof(uint64_t)); - cur_msg_recvd_bytes_ = recvmsg_rc; + cur_msg_recvd_bytes_ = recv_result.return_value_; } } // If we have received beyond the end of the current in-flight proto, then next is misaligned. diff --git a/source/server/hot_restarting_base.h b/source/server/hot_restarting_base.h index afb460aa898d..878d9e0ed659 100644 --- a/source/server/hot_restarting_base.h +++ b/source/server/hot_restarting_base.h @@ -91,13 +91,28 @@ class RpcStream : public Logger::Loggable { */ class HotRestartingBase : public Logger::Loggable { protected: - HotRestartingBase(uint64_t base_id) : main_rpc_stream_(base_id) {} + HotRestartingBase(uint64_t base_id) + : main_rpc_stream_(base_id), udp_forwarding_rpc_stream_(base_id) {} // Returns a Gauge that tracks hot-restart generation, where every successive // child increments this number. static Stats::Gauge& hotRestartGeneration(Stats::Scope& scope); + // A stream over a unix socket between the parent and child instances, used + // for the child instance to request socket information and control draining + // and shutdown of the parent. RpcStream main_rpc_stream_; + + // A separate channel is used for udp forwarding because udp forwarding can + // begin while communication on the main channel is still occurring. The hot + // restarter is single-threaded, so we don't have to worry about packets coming + // in a jumbled order, but there are two instances of the hot restarter, the + // parent and the child; it is possible for the child to send a udp packet + // while the parent is sending a request on the main channel, for which it will + // expect to receive a response (and not an unrelated udp packet). Therefore, a + // separate channel is used to deliver udp packets, ensuring no interference + // between the two data sources. + RpcStream udp_forwarding_rpc_stream_; }; } // namespace Server diff --git a/source/server/hot_restarting_child.cc b/source/server/hot_restarting_child.cc index ab480760ccc0..0d842a2755eb 100644 --- a/source/server/hot_restarting_child.cc +++ b/source/server/hot_restarting_child.cc @@ -1,6 +1,8 @@ #include "source/server/hot_restarting_child.h" +#include "source/common/buffer/buffer_impl.h" #include "source/common/common/utility.h" +#include "source/common/network/utility.h" namespace Envoy { namespace Server { @@ -48,11 +50,54 @@ HotRestartingChild::HotRestartingChild(int base_id, int restart_epoch, const std::string& socket_path, mode_t socket_mode) : HotRestartingBase(base_id), restart_epoch_(restart_epoch) { main_rpc_stream_.initDomainSocketAddress(&parent_address_); + std::string socket_path_udp = socket_path + "_udp"; + udp_forwarding_rpc_stream_.initDomainSocketAddress(&parent_address_udp_forwarding_); if (restart_epoch_ != 0) { parent_address_ = main_rpc_stream_.createDomainSocketAddress(restart_epoch_ + -1, "parent", socket_path, socket_mode); + parent_address_udp_forwarding_ = udp_forwarding_rpc_stream_.createDomainSocketAddress( + restart_epoch_ + -1, "parent", socket_path_udp, socket_mode); } main_rpc_stream_.bindDomainSocket(restart_epoch_, "child", socket_path, socket_mode); + udp_forwarding_rpc_stream_.bindDomainSocket(restart_epoch_, "child", socket_path_udp, + socket_mode); +} + +void HotRestartingChild::initialize(Event::Dispatcher& dispatcher) { + socket_event_udp_forwarding_ = dispatcher.createFileEvent( + udp_forwarding_rpc_stream_.domain_socket_, + [this](uint32_t events) -> void { + ASSERT(events == Event::FileReadyType::Read); + onSocketEventUdpForwarding(); + }, + Event::FileTriggerType::Edge, Event::FileReadyType::Read); +} + +void HotRestartingChild::shutdown() { socket_event_udp_forwarding_.reset(); } + +void HotRestartingChild::onForwardedUdpPacket(uint32_t worker_index, Network::UdpRecvData&& data) { + auto addr_and_listener = + udp_forwarding_context_.getListenerForDestination(*data.addresses_.local_); + if (addr_and_listener.has_value()) { + auto [addr, listener_config] = *addr_and_listener; + // We send to the worker index from the parent instance. + // In the case that the number of workers changes between instances, + // or the quic connection id generator changes how it selects worker + // ids, the hot restart packet transfer will fail. + // + // One option would be to dispatch an onData call to have the receiving + // worker forward the packet if the calculated destination differs from + // the parent instance worker index; however, this would require + // temporarily disabling kernel_worker_routing_ in each instance of + // ActiveQuicListener, and a much more convoluted pipeline to collect + // the set of destinations (listenerWorkerRouter doesn't currently + // expose the actual listeners.) + // + // Since the vast majority of hot restarts will change neither of these + // things, this implementation is "pretty good", and much better than no + // hot restart capability at all. + listener_config->listenerWorkerRouter(*addr).deliver(worker_index, std::move(data)); + } } int HotRestartingChild::duplicateParentListenSocket(const std::string& address, @@ -172,5 +217,40 @@ void HotRestartingChild::mergeParentStats(Stats::Store& stats_store, stat_merger_->mergeStats(stats_proto.counter_deltas(), stats_proto.gauges(), dynamics); } +void HotRestartingChild::onSocketEventUdpForwarding() { + std::unique_ptr wrapped_request; + while ((wrapped_request = + udp_forwarding_rpc_stream_.receiveHotRestartMessage(RpcStream::Blocking::No))) { + if (wrapped_request->requestreply_case() == HotRestartMessage::kReply) { + ENVOY_LOG_PERIODIC( + error, std::chrono::seconds(5), + "HotRestartMessage reply received on UdpForwarding (we want only requests); ignoring."); + continue; + } + switch (wrapped_request->request().request_case()) { + case HotRestartMessage::Request::kForwardedUdpPacket: { + const auto& req = wrapped_request->request().forwarded_udp_packet(); + Network::UdpRecvData packet; + packet.addresses_.local_ = Network::Utility::resolveUrl(req.local_addr()); + packet.addresses_.peer_ = Network::Utility::resolveUrl(req.peer_addr()); + if (!packet.addresses_.local_ || !packet.addresses_.peer_) { + break; + } + packet.receive_time_ = + MonotonicTime(std::chrono::microseconds{req.receive_time_epoch_microseconds()}); + packet.buffer_ = std::make_unique(req.payload()); + onForwardedUdpPacket(req.worker_index(), std::move(packet)); + break; + } + default: { + ENVOY_LOG( + error, + "child sent a request other than ForwardedUdpPacket on udp forwarding socket; ignoring."); + break; + } + } + } +} + } // namespace Server } // namespace Envoy diff --git a/source/server/hot_restarting_child.h b/source/server/hot_restarting_child.h index b12b508bc91b..7f61dfc59f55 100644 --- a/source/server/hot_restarting_child.h +++ b/source/server/hot_restarting_child.h @@ -1,5 +1,7 @@ #pragma once +#include "envoy/server/instance.h" + #include "source/common/stats/stat_merger.h" #include "source/server/hot_restarting_base.h" @@ -9,11 +11,8 @@ namespace Server { /** * The child half of hot restarting. Issues requests and commands to the parent. */ -class HotRestartingChild : HotRestartingBase { +class HotRestartingChild : public HotRestartingBase { public: - HotRestartingChild(int base_id, int restart_epoch, const std::string& socket_path, - mode_t socket_mode); - // A structure to record the set of registered UDP listeners keyed on their addresses, // to support QUIC packet forwarding. class UdpForwardingContext { @@ -41,6 +40,13 @@ class HotRestartingChild : HotRestartingBase { absl::flat_hash_map listener_map_; }; + HotRestartingChild(int base_id, int restart_epoch, const std::string& socket_path, + mode_t socket_mode); + ~HotRestartingChild() = default; + + void initialize(Event::Dispatcher& dispatcher); + void shutdown(); + int duplicateParentListenSocket(const std::string& address, uint32_t worker_index); void registerUdpForwardingListener(Network::Address::InstanceConstSharedPtr address, std::shared_ptr listener_config); @@ -51,14 +57,20 @@ class HotRestartingChild : HotRestartingBase { void mergeParentStats(Stats::Store& stats_store, const envoy::HotRestartMessage::Reply::Stats& stats_proto); +protected: + void onSocketEventUdpForwarding(); + void onForwardedUdpPacket(uint32_t worker_index, Network::UdpRecvData&& data); + private: + friend class HotRestartUdpForwardingTestHelper; const int restart_epoch_; bool parent_terminated_{}; sockaddr_un parent_address_; + sockaddr_un parent_address_udp_forwarding_; std::unique_ptr stat_merger_{}; Stats::StatName hot_restart_generation_stat_name_; + Event::FileEventPtr socket_event_udp_forwarding_; UdpForwardingContext udp_forwarding_context_; - friend class HotRestartUdpForwardingTestHelper; }; } // namespace Server diff --git a/source/server/hot_restarting_parent.cc b/source/server/hot_restarting_parent.cc index c044cbb14c10..f9d74b60b31b 100644 --- a/source/server/hot_restarting_parent.cc +++ b/source/server/hot_restarting_parent.cc @@ -16,21 +16,38 @@ using HotRestartMessage = envoy::HotRestartMessage; HotRestartingParent::HotRestartingParent(int base_id, int restart_epoch, const std::string& socket_path, mode_t socket_mode) : HotRestartingBase(base_id), restart_epoch_(restart_epoch) { + std::string socket_path_udp = socket_path + "_udp"; child_address_ = main_rpc_stream_.createDomainSocketAddress(restart_epoch_ + 1, "child", socket_path, socket_mode); + child_address_udp_forwarding_ = udp_forwarding_rpc_stream_.createDomainSocketAddress( + restart_epoch_ + 1, "child", socket_path_udp, socket_mode); main_rpc_stream_.bindDomainSocket(restart_epoch_, "parent", socket_path, socket_mode); + udp_forwarding_rpc_stream_.bindDomainSocket(restart_epoch_, "parent", socket_path_udp, + socket_mode); } -// Network::NonDispatchedUdpPacketHandler -void HotRestartingParent::Internal::handle(uint32_t /*worker_index*/, - const Network::UdpRecvData& /*packet*/) { - dispatcher_.post([]() { - // TODO(ravenblack): encapsulate the packet, dispatch it to the hot restarter thread, and - // forward the message over a domain socket to HotRestartingChild. - // (Pending PR #29328) +void HotRestartingParent::sendHotRestartMessage(envoy::HotRestartMessage&& msg) { + ASSERT(dispatcher_.has_value()); + dispatcher_->post([this, msg = std::move(msg)]() { + udp_forwarding_rpc_stream_.sendHotRestartMessage(child_address_udp_forwarding_, std::move(msg)); }); } +// Network::NonDispatchedUdpPacketHandler +void HotRestartingParent::Internal::handle(uint32_t worker_index, + const Network::UdpRecvData& packet) { + envoy::HotRestartMessage msg; + auto* packet_msg = msg.mutable_request()->mutable_forwarded_udp_packet(); + packet_msg->set_local_addr(Network::Utility::urlFromDatagramAddress(*packet.addresses_.local_)); + packet_msg->set_peer_addr(Network::Utility::urlFromDatagramAddress(*packet.addresses_.peer_)); + packet_msg->set_receive_time_epoch_microseconds( + std::chrono::duration_cast(packet.receive_time_.time_since_epoch()) + .count()); + *packet_msg->mutable_payload() = packet.buffer_->toString(); + packet_msg->set_worker_index(worker_index); + udp_sender_.sendHotRestartMessage(std::move(msg)); +} + void HotRestartingParent::initialize(Event::Dispatcher& dispatcher, Server::Instance& server) { socket_event_ = dispatcher.createFileEvent( main_rpc_stream_.domain_socket_, @@ -39,7 +56,8 @@ void HotRestartingParent::initialize(Event::Dispatcher& dispatcher, Server::Inst onSocketEvent(); }, Event::FileTriggerType::Edge, Event::FileReadyType::Read); - internal_ = std::make_unique(&server, dispatcher); + dispatcher_ = dispatcher; + internal_ = std::make_unique(&server, *this); } void HotRestartingParent::onSocketEvent() { @@ -95,8 +113,9 @@ void HotRestartingParent::onSocketEvent() { void HotRestartingParent::shutdown() { socket_event_.reset(); } -HotRestartingParent::Internal::Internal(Server::Instance* server, Event::Dispatcher& dispatcher) - : server_(server), dispatcher_(dispatcher) { +HotRestartingParent::Internal::Internal(Server::Instance* server, + HotRestartMessageSender& udp_sender) + : server_(server), udp_sender_(udp_sender) { Stats::Gauge& hot_restart_generation = hotRestartGeneration(*server->stats().rootScope()); hot_restart_generation.inc(); } diff --git a/source/server/hot_restarting_parent.h b/source/server/hot_restarting_parent.h index f648f9bccd81..246eab225fb8 100644 --- a/source/server/hot_restarting_parent.h +++ b/source/server/hot_restarting_parent.h @@ -6,23 +6,30 @@ namespace Envoy { namespace Server { +class HotRestartMessageSender { +public: + virtual void sendHotRestartMessage(envoy::HotRestartMessage&& msg) PURE; + virtual ~HotRestartMessageSender() = default; +}; + /** * The parent half of hot restarting. Listens for requests and commands from the child. * This outer class only handles evented socket I/O. The actual hot restart logic lives in * HotRestartingParent::Internal. */ -class HotRestartingParent : HotRestartingBase { +class HotRestartingParent : public HotRestartingBase, public HotRestartMessageSender { public: HotRestartingParent(int base_id, int restart_epoch, const std::string& socket_path, mode_t socket_mode); void initialize(Event::Dispatcher& dispatcher, Server::Instance& server); void shutdown(); + void sendHotRestartMessage(envoy::HotRestartMessage&& msg) override; // The hot restarting parent's hot restart logic. Each function is meant to be called to fulfill a // request from the child for that action. class Internal : public Network::NonDispatchedUdpPacketHandler { public: - explicit Internal(Server::Instance* server, Event::Dispatcher& dispatcher); + explicit Internal(Server::Instance* server, HotRestartMessageSender& udp_sender); // Return value is the response to return to the child. envoy::HotRestartMessage shutdownAdmin(); // Return value is the response to return to the child. @@ -39,7 +46,7 @@ class HotRestartingParent : HotRestartingBase { private: Server::Instance* const server_{}; - Event::Dispatcher& dispatcher_; + HotRestartMessageSender& udp_sender_; }; private: @@ -47,7 +54,9 @@ class HotRestartingParent : HotRestartingBase { const int restart_epoch_; sockaddr_un child_address_; + sockaddr_un child_address_udp_forwarding_; Event::FileEventPtr socket_event_; + OptRef dispatcher_; std::unique_ptr internal_; }; diff --git a/source/server/instance_impl.cc b/source/server/instance_impl.cc new file mode 100644 index 000000000000..82e1817f6f6c --- /dev/null +++ b/source/server/instance_impl.cc @@ -0,0 +1,25 @@ +#include "source/server/instance_impl.h" + +#include "source/server/guarddog_impl.h" +#include "source/server/overload_manager_impl.h" + +namespace Envoy { +namespace Server { +void InstanceImpl::maybeCreateHeapShrinker() { + heap_shrinker_ = + std::make_unique(dispatcher(), overloadManager(), *stats().rootScope()); +} + +std::unique_ptr InstanceImpl::createOverloadManager() { + return std::make_unique( + dispatcher(), *stats().rootScope(), threadLocal(), bootstrap().overload_manager(), + messageValidationContext().staticValidationVisitor(), api(), options()); +} + +std::unique_ptr InstanceImpl::maybeCreateGuardDog(absl::string_view name) { + return std::make_unique(*stats().rootScope(), + config().mainThreadWatchdogConfig(), api(), name); +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/instance_impl.h b/source/server/instance_impl.h new file mode 100644 index 000000000000..c21d8b17ef2b --- /dev/null +++ b/source/server/instance_impl.h @@ -0,0 +1,24 @@ +#pragma once + +#include "source/common/memory/heap_shrinker.h" +#include "source/server/server.h" + +namespace Envoy { +namespace Server { + +// The production server instance, which creates all of the required components. +class InstanceImpl : public InstanceBase { +public: + using InstanceBase::InstanceBase; + +protected: + void maybeCreateHeapShrinker() override; + std::unique_ptr createOverloadManager() override; + std::unique_ptr maybeCreateGuardDog(absl::string_view name) override; + +private: + std::unique_ptr heap_shrinker_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/null_overload_manager.h b/source/server/null_overload_manager.h new file mode 100644 index 000000000000..aead55df581b --- /dev/null +++ b/source/server/null_overload_manager.h @@ -0,0 +1,72 @@ +#pragma once + +#include "envoy/server/overload/overload_manager.h" + +#include "source/common/event/scaled_range_timer_manager_impl.h" + +namespace Envoy { +namespace Server { + +/** Implementation of OverloadManager that is never overloaded. Using this instead of the real + * OverloadManager keeps the admin interface accessible even when the proxy is overloaded. + */ +class NullOverloadManager : public OverloadManager { +public: + struct OverloadState : public ThreadLocalOverloadState { + OverloadState(Event::Dispatcher& dispatcher, bool permissive) + : dispatcher_(dispatcher), permissive_(permissive) {} + const OverloadActionState& getState(const std::string&) override { return inactive_; } + bool tryAllocateResource(OverloadProactiveResourceName, int64_t) override { + return permissive_; + } + bool tryDeallocateResource(OverloadProactiveResourceName, int64_t) override { + return permissive_; + } + bool isResourceMonitorEnabled(OverloadProactiveResourceName) override { return false; } + ProactiveResourceMonitorOptRef + getProactiveResourceMonitorForTest(OverloadProactiveResourceName) override { + return makeOptRefFromPtr(nullptr); + } + Event::Dispatcher& dispatcher_; + const bool permissive_; + const OverloadActionState inactive_ = OverloadActionState::inactive(); + }; + + NullOverloadManager(ThreadLocal::SlotAllocator& slot_allocator, bool permissive) + : tls_(slot_allocator.allocateSlot()), permissive_(permissive) {} + + void start() override { + tls_->set([this](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(dispatcher, permissive_); + }); + } + + ThreadLocalOverloadState& getThreadLocalOverloadState() override { + return tls_->getTyped(); + } + + LoadShedPoint* getLoadShedPoint(absl::string_view) override { return nullptr; } + + Event::ScaledRangeTimerManagerFactory scaledTimerFactory() override { + if (!permissive_) { + return nullptr; + } + return [](Event::Dispatcher& dispatcher) { + return std::make_unique(dispatcher, nullptr); + }; + } + + bool registerForAction(const std::string&, Event::Dispatcher&, OverloadActionCb) override { + return true; + } + void stop() override {} + + ThreadLocal::SlotPtr tls_; + // The admin code runs in non-permissive mode, rejecting connections and + // ensuring timer code is not called. Envoy mobile uses permissive mode and + // does the opposite. + const bool permissive_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index dc4b223535de..770245549b65 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -32,6 +32,7 @@ std::vector toArgsVector(int argc, const char* const* argv) { } return args; } + } // namespace OptionsImpl::OptionsImpl(int argc, const char* const* argv, @@ -54,6 +55,7 @@ OptionsImpl::OptionsImpl(std::vector args, "\nDefault is \"{}\"", Logger::Logger::DEFAULT_LOG_FORMAT); + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) TCLAP::CmdLine cmd("envoy", ' ', VersionInfo::version()); TCLAP::ValueArg base_id( "", "base-id", "Base ID so that multiple envoys can run on the same host if needed", false, 0, @@ -178,7 +180,8 @@ OptionsImpl::OptionsImpl(std::vector args, } END_TRY MULTI_CATCH( - TCLAP::ArgException & e, { failure_function(e); }, { throw NoServingException(); }); + TCLAP::ArgException & e, { failure_function(e); }, + { throw NoServingException("NoServingException"); }); hot_restart_disabled_ = disable_hot_restart.getValue(); mutex_tracing_enabled_ = enable_mutex_tracing.getValue(); @@ -187,7 +190,11 @@ OptionsImpl::OptionsImpl(std::vector args, cpuset_threads_ = cpuset_threads.getValue(); if (log_level.isSet()) { - log_level_ = parseAndValidateLogLevel(log_level.getValue()); + auto status_or_error = parseAndValidateLogLevel(log_level.getValue()); + if (!status_or_error.status().ok()) { + logError(std::string(status_or_error.status().message())); + } + log_level_ = status_or_error.value(); } else { log_level_ = default_log_level; } @@ -285,7 +292,7 @@ OptionsImpl::OptionsImpl(std::vector args, if (hot_restart_version_option.getValue()) { std::cerr << hot_restart_version_cb(!hot_restart_disabled_); - throw NoServingException(); + throw NoServingException("NoServingException"); } if (!disable_extensions.getValue().empty()) { @@ -321,26 +328,6 @@ OptionsImpl::OptionsImpl(std::vector args, } } -spdlog::level::level_enum OptionsImpl::parseAndValidateLogLevel(absl::string_view log_level) { - if (log_level == "warn") { - return spdlog::level::level_enum::warn; - } - - size_t level_to_use = std::numeric_limits::max(); - for (size_t i = 0; i < ARRAY_SIZE(spdlog::level::level_string_views); i++) { - spdlog::string_view_t spd_log_level = spdlog::level::level_string_views[i]; - if (log_level == absl::string_view(spd_log_level.data(), spd_log_level.size())) { - level_to_use = i; - break; - } - } - - if (level_to_use == std::numeric_limits::max()) { - logError(fmt::format("error: invalid log level specified '{}'", log_level)); - } - return static_cast(level_to_use); -} - std::string OptionsImpl::allowedLogLevels() { std::string allowed_log_levels; for (auto level_string_view : spdlog::level::level_string_views) { @@ -365,7 +352,11 @@ void OptionsImpl::parseComponentLogLevels(const std::string& component_log_level logError(fmt::format("error: component log level not correctly specified '{}'", level)); } std::string log_name = log_name_level[0]; - spdlog::level::level_enum log_level = parseAndValidateLogLevel(log_name_level[1]); + auto status_or_error = parseAndValidateLogLevel(log_name_level[1]); + if (!status_or_error.status().ok()) { + logError(std::string(status_or_error.status().message())); + } + spdlog::level::level_enum log_level = status_or_error.value(); Logger::Logger* logger_to_change = Logger::Registry::logger(log_name); if (!logger_to_change) { logError(fmt::format("error: invalid component specified '{}'", log_name)); @@ -374,8 +365,6 @@ void OptionsImpl::parseComponentLogLevels(const std::string& component_log_level } } -uint32_t OptionsImpl::count() const { return count_; } - void OptionsImpl::logError(const std::string& error) { throw MalformedArgvException(error); } Server::CommandLineOptionsPtr OptionsImpl::toCommandLineOptions() const { @@ -440,25 +429,11 @@ Server::CommandLineOptionsPtr OptionsImpl::toCommandLineOptions() const { } OptionsImpl::OptionsImpl(const std::string& service_cluster, const std::string& service_node, - const std::string& service_zone, spdlog::level::level_enum log_level) - : log_level_(log_level), service_cluster_(service_cluster), service_node_(service_node), - service_zone_(service_zone) {} - -void OptionsImpl::disableExtensions(const std::vector& names) { - for (const auto& name : names) { - const std::vector parts = absl::StrSplit(name, absl::MaxSplits('/', 1)); - - if (parts.size() != 2) { - ENVOY_LOG_MISC(warn, "failed to disable invalid extension name '{}'", name); - continue; - } - - if (Registry::FactoryCategoryRegistry::disableFactory(parts[0], parts[1])) { - ENVOY_LOG_MISC(info, "disabled extension '{}'", name); - } else { - ENVOY_LOG_MISC(warn, "failed to disable unknown extension '{}'", name); - } - } + const std::string& service_zone, spdlog::level::level_enum log_level) { + setLogLevel(log_level); + setServiceClusterName(service_cluster); + setServiceNodeName(service_node); + setServiceZone(service_zone); } } // namespace Envoy diff --git a/source/server/options_impl.h b/source/server/options_impl.h index d02f37dce5bc..7f59fc9f8363 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -11,14 +11,16 @@ #include "source/common/common/logger.h" #include "source/common/config/well_known_names.h" +#include "source/server/options_impl_base.h" #include "spdlog/spdlog.h" namespace Envoy { + /** - * Implementation of Server::Options. + * Implementation of Server::Options which can parse from the command line. */ -class OptionsImpl : public Server::Options, protected Logger::Loggable { +class OptionsImpl : public OptionsImplBase { public: /** * Parameters are hot_restart_enabled @@ -50,187 +52,10 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable&& config_proto) { - config_proto_ = std::move(config_proto); - } - void setConfigYaml(const std::string& config_yaml) { config_yaml_ = config_yaml; } - void setAdminAddressPath(const std::string& admin_address_path) { - admin_address_path_ = admin_address_path; - } - void setLocalAddressIpVersion(Network::Address::IpVersion local_address_ip_version) { - local_address_ip_version_ = local_address_ip_version; - } - void setDrainTime(std::chrono::seconds drain_time) { drain_time_ = drain_time; } - void setParentShutdownTime(std::chrono::seconds parent_shutdown_time) { - parent_shutdown_time_ = parent_shutdown_time; - } - void setDrainStrategy(Server::DrainStrategy drain_strategy) { drain_strategy_ = drain_strategy; } - void setLogLevel(spdlog::level::level_enum log_level) { log_level_ = log_level; } - void setLogFormat(const std::string& log_format) { - log_format_ = log_format; - log_format_set_ = true; - } - void setLogPath(const std::string& log_path) { log_path_ = log_path; } - void setRestartEpoch(uint64_t restart_epoch) { restart_epoch_ = restart_epoch; } - void setMode(Server::Mode mode) { mode_ = mode; } - void setFileFlushIntervalMsec(std::chrono::milliseconds file_flush_interval_msec) { - file_flush_interval_msec_ = file_flush_interval_msec; - } - void setServiceClusterName(const std::string& service_cluster) { - service_cluster_ = service_cluster; - } - void setServiceNodeName(const std::string& service_node) { service_node_ = service_node; } - void setServiceZone(const std::string& service_zone) { service_zone_ = service_zone; } - void setHotRestartDisabled(bool hot_restart_disabled) { - hot_restart_disabled_ = hot_restart_disabled; - } - void setSignalHandling(bool signal_handling_enabled) { - signal_handling_enabled_ = signal_handling_enabled; - } - void setCpusetThreads(bool cpuset_threads_enabled) { cpuset_threads_ = cpuset_threads_enabled; } - void setAllowUnknownFields(bool allow_unknown_static_fields) { - allow_unknown_static_fields_ = allow_unknown_static_fields; - } - void setRejectUnknownFieldsDynamic(bool reject_unknown_dynamic_fields) { - reject_unknown_dynamic_fields_ = reject_unknown_dynamic_fields; - } - void setIgnoreUnknownFieldsDynamic(bool ignore_unknown_dynamic_fields) { - ignore_unknown_dynamic_fields_ = ignore_unknown_dynamic_fields; - } - - void setSocketPath(const std::string& socket_path) { socket_path_ = socket_path; } - - void setSocketMode(mode_t socket_mode) { socket_mode_ = socket_mode; } - - void setStatsTags(const Stats::TagVector& stats_tags) { stats_tags_ = stats_tags; } - - // Server::Options - uint64_t baseId() const override { return base_id_; } - bool useDynamicBaseId() const override { return use_dynamic_base_id_; } - const std::string& baseIdPath() const override { return base_id_path_; } - uint32_t concurrency() const override { return concurrency_; } - const std::string& configPath() const override { return config_path_; } - const envoy::config::bootstrap::v3::Bootstrap& configProto() const override { - return *config_proto_; - } - const std::string& configYaml() const override { return config_yaml_; } - bool allowUnknownStaticFields() const override { return allow_unknown_static_fields_; } - bool rejectUnknownDynamicFields() const override { return reject_unknown_dynamic_fields_; } - bool ignoreUnknownDynamicFields() const override { return ignore_unknown_dynamic_fields_; } - const std::string& adminAddressPath() const override { return admin_address_path_; } - Network::Address::IpVersion localAddressIpVersion() const override { - return local_address_ip_version_; - } - std::chrono::seconds drainTime() const override { return drain_time_; } - std::chrono::seconds parentShutdownTime() const override { return parent_shutdown_time_; } - Server::DrainStrategy drainStrategy() const override { return drain_strategy_; } - - spdlog::level::level_enum logLevel() const override { return log_level_; } - const std::vector>& - componentLogLevels() const override { - return component_log_levels_; - } - const std::string& logFormat() const override { return log_format_; } - bool logFormatSet() const override { return log_format_set_; } - bool logFormatEscaped() const override { return log_format_escaped_; } - bool enableFineGrainLogging() const override { return enable_fine_grain_logging_; } - const std::string& logPath() const override { return log_path_; } - uint64_t restartEpoch() const override { return restart_epoch_; } - Server::Mode mode() const override { return mode_; } - std::chrono::milliseconds fileFlushIntervalMsec() const override { - return file_flush_interval_msec_; - } - const std::string& serviceClusterName() const override { return service_cluster_; } - const std::string& serviceNodeName() const override { return service_node_; } - const std::string& serviceZone() const override { return service_zone_; } - bool hotRestartDisabled() const override { return hot_restart_disabled_; } - bool signalHandlingEnabled() const override { return signal_handling_enabled_; } - bool mutexTracingEnabled() const override { return mutex_tracing_enabled_; } - bool coreDumpEnabled() const override { return core_dump_enabled_; } - const Stats::TagVector& statsTags() const override { return stats_tags_; } Server::CommandLineOptionsPtr toCommandLineOptions() const override; void parseComponentLogLevels(const std::string& component_log_levels); - bool cpusetThreadsEnabled() const override { return cpuset_threads_; } - const std::vector& disabledExtensions() const override { - return disabled_extensions_; - } - uint32_t count() const; - const std::string& socketPath() const override { return socket_path_; } - mode_t socketMode() const override { return socket_mode_; } - - /** - * disableExtensions parses the given set of extension names of - * the form $CATEGORY/$NAME, and disables the corresponding extension - * factories. - */ - static void disableExtensions(const std::vector&); - static std::string allowedLogLevels(); - - /** - * Parses and validates the provided log_level, returning the corresponding - * spdlog::level::level_enum. - * @throws MalformedArgvException if the provided string is not a valid spdlog string. - */ - static spdlog::level::level_enum parseAndValidateLogLevel(absl::string_view log_level); - -private: static void logError(const std::string& error); - - uint64_t base_id_{0}; - bool use_dynamic_base_id_{false}; - std::string base_id_path_; - uint32_t concurrency_{1}; - std::string config_path_; - std::unique_ptr config_proto_{ - new envoy::config::bootstrap::v3::Bootstrap()}; - std::string config_yaml_; - bool allow_unknown_static_fields_{false}; - bool reject_unknown_dynamic_fields_{false}; - bool ignore_unknown_dynamic_fields_{false}; - std::string admin_address_path_; - Network::Address::IpVersion local_address_ip_version_{Network::Address::IpVersion::v4}; - spdlog::level::level_enum log_level_{spdlog::level::info}; - std::vector> component_log_levels_; - std::string component_log_level_str_; - std::string log_format_{Logger::Logger::DEFAULT_LOG_FORMAT}; - bool log_format_set_{false}; - bool log_format_escaped_{false}; - std::string log_path_; - uint64_t restart_epoch_{0}; - std::string service_cluster_; - std::string service_node_; - std::string service_zone_; - std::chrono::milliseconds file_flush_interval_msec_{10000}; - std::chrono::seconds drain_time_{600}; - std::chrono::seconds parent_shutdown_time_{900}; - Server::DrainStrategy drain_strategy_{Server::DrainStrategy::Gradual}; - Server::Mode mode_{Server::Mode::Serve}; - bool hot_restart_disabled_{false}; - bool signal_handling_enabled_{true}; - bool mutex_tracing_enabled_{false}; - bool core_dump_enabled_{false}; - bool cpuset_threads_{false}; - std::vector disabled_extensions_; - Stats::TagVector stats_tags_; - uint32_t count_{0}; - - // Initialization added here to avoid integration_admin_test failure caused by uninitialized - // enable_fine_grain_logging_. - bool enable_fine_grain_logging_ = false; - std::string socket_path_{"@envoy_domain_socket"}; - mode_t socket_mode_{0}; + static std::string allowedLogLevels(); }; /** @@ -240,7 +65,7 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable +#include +#include +#include + +#include "envoy/admin/v3/server_info.pb.h" + +#include "source/common/common/fmt.h" +#include "source/common/common/logger.h" +#include "source/common/common/macros.h" +#include "source/common/protobuf/utility.h" +#include "source/common/stats/tag_utility.h" +#include "source/common/version/version.h" + +#include "absl/strings/str_replace.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "spdlog/spdlog.h" + +namespace Envoy { + +absl::StatusOr +OptionsImplBase::parseAndValidateLogLevel(absl::string_view log_level) { + if (log_level == "warn") { + return spdlog::level::level_enum::warn; + } + + size_t level_to_use = std::numeric_limits::max(); + for (size_t i = 0; i < ARRAY_SIZE(spdlog::level::level_string_views); i++) { + spdlog::string_view_t spd_log_level = spdlog::level::level_string_views[i]; + if (log_level == absl::string_view(spd_log_level.data(), spd_log_level.size())) { + level_to_use = i; + break; + } + } + + if (level_to_use == std::numeric_limits::max()) { + return absl::InvalidArgumentError( + fmt::format("error: invalid log level specified '{}'", log_level)); + } + return static_cast(level_to_use); +} + +absl::Status OptionsImplBase::setLogLevel(absl::string_view log_level) { + auto status_or_level = parseAndValidateLogLevel(log_level); + if (!status_or_level.status().ok()) { + return status_or_level.status(); + } + setLogLevel(status_or_level.value()); + return absl::OkStatus(); +} + +uint32_t OptionsImplBase::count() const { return count_; } + +void OptionsImplBase::disableExtensions(const std::vector& names) { + for (const auto& name : names) { + const std::vector parts = absl::StrSplit(name, absl::MaxSplits('/', 1)); + + if (parts.size() != 2) { + ENVOY_LOG_MISC(warn, "failed to disable invalid extension name '{}'", name); + continue; + } + + if (Registry::FactoryCategoryRegistry::disableFactory(parts[0], parts[1])) { + ENVOY_LOG_MISC(info, "disabled extension '{}'", name); + } else { + ENVOY_LOG_MISC(warn, "failed to disable unknown extension '{}'", name); + } + } +} + +} // namespace Envoy diff --git a/source/server/options_impl_base.h b/source/server/options_impl_base.h new file mode 100644 index 000000000000..c38f066e7155 --- /dev/null +++ b/source/server/options_impl_base.h @@ -0,0 +1,213 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/exception.h" +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/registry/registry.h" +#include "envoy/server/options.h" + +#include "source/common/common/logger.h" +#include "source/common/config/well_known_names.h" + +#include "spdlog/spdlog.h" + +namespace Envoy { + +/** + * Implementation of Server::Options without command line parsing. + */ +class OptionsImplBase : public Server::Options, protected Logger::Loggable { +public: + /** + * Parameters are hot_restart_enabled + */ + using HotRestartVersionCb = std::function; + + // Constructor for mobile + OptionsImplBase() = default; + + // Setters for option fields. These are not part of the Options interface. + void setBaseId(uint64_t base_id) { base_id_ = base_id; }; + void setUseDynamicBaseId(bool use_dynamic_base_id) { use_dynamic_base_id_ = use_dynamic_base_id; } + void setBaseIdPath(const std::string& base_id_path) { base_id_path_ = base_id_path; } + void setConcurrency(uint32_t concurrency) { concurrency_ = concurrency; } + void setConfigPath(const std::string& config_path) { config_path_ = config_path; } + void setConfigProto(const envoy::config::bootstrap::v3::Bootstrap& config_proto) { + *config_proto_ = config_proto; + } + void setConfigProto(std::unique_ptr&& config_proto) { + config_proto_ = std::move(config_proto); + } + void setConfigYaml(const std::string& config_yaml) { config_yaml_ = config_yaml; } + void setAdminAddressPath(const std::string& admin_address_path) { + admin_address_path_ = admin_address_path; + } + void setLocalAddressIpVersion(Network::Address::IpVersion local_address_ip_version) { + local_address_ip_version_ = local_address_ip_version; + } + void setDrainTime(std::chrono::seconds drain_time) { drain_time_ = drain_time; } + void setParentShutdownTime(std::chrono::seconds parent_shutdown_time) { + parent_shutdown_time_ = parent_shutdown_time; + } + void setDrainStrategy(Server::DrainStrategy drain_strategy) { drain_strategy_ = drain_strategy; } + void setLogLevel(spdlog::level::level_enum log_level) { log_level_ = log_level; } + absl::Status setLogLevel(absl::string_view log_level); + void setLogFormat(const std::string& log_format) { + log_format_ = log_format; + log_format_set_ = true; + } + void setLogPath(const std::string& log_path) { log_path_ = log_path; } + void setRestartEpoch(uint64_t restart_epoch) { restart_epoch_ = restart_epoch; } + void setMode(Server::Mode mode) { mode_ = mode; } + void setFileFlushIntervalMsec(std::chrono::milliseconds file_flush_interval_msec) { + file_flush_interval_msec_ = file_flush_interval_msec; + } + void setServiceClusterName(const std::string& service_cluster) { + service_cluster_ = service_cluster; + } + void setServiceNodeName(const std::string& service_node) { service_node_ = service_node; } + void setServiceZone(const std::string& service_zone) { service_zone_ = service_zone; } + void setHotRestartDisabled(bool hot_restart_disabled) { + hot_restart_disabled_ = hot_restart_disabled; + } + void setSignalHandling(bool signal_handling_enabled) { + signal_handling_enabled_ = signal_handling_enabled; + } + void setCpusetThreads(bool cpuset_threads_enabled) { cpuset_threads_ = cpuset_threads_enabled; } + void setAllowUnknownFields(bool allow_unknown_static_fields) { + allow_unknown_static_fields_ = allow_unknown_static_fields; + } + void setRejectUnknownFieldsDynamic(bool reject_unknown_dynamic_fields) { + reject_unknown_dynamic_fields_ = reject_unknown_dynamic_fields; + } + void setIgnoreUnknownFieldsDynamic(bool ignore_unknown_dynamic_fields) { + ignore_unknown_dynamic_fields_ = ignore_unknown_dynamic_fields; + } + + void setSocketPath(const std::string& socket_path) { socket_path_ = socket_path; } + + void setSocketMode(mode_t socket_mode) { socket_mode_ = socket_mode; } + + void setStatsTags(const Stats::TagVector& stats_tags) { stats_tags_ = stats_tags; } + + // Server::Options + uint64_t baseId() const override { return base_id_; } + bool useDynamicBaseId() const override { return use_dynamic_base_id_; } + const std::string& baseIdPath() const override { return base_id_path_; } + uint32_t concurrency() const override { return concurrency_; } + const std::string& configPath() const override { return config_path_; } + const envoy::config::bootstrap::v3::Bootstrap& configProto() const override { + return *config_proto_; + } + const std::string& configYaml() const override { return config_yaml_; } + bool allowUnknownStaticFields() const override { return allow_unknown_static_fields_; } + bool rejectUnknownDynamicFields() const override { return reject_unknown_dynamic_fields_; } + bool ignoreUnknownDynamicFields() const override { return ignore_unknown_dynamic_fields_; } + const std::string& adminAddressPath() const override { return admin_address_path_; } + Network::Address::IpVersion localAddressIpVersion() const override { + return local_address_ip_version_; + } + std::chrono::seconds drainTime() const override { return drain_time_; } + std::chrono::seconds parentShutdownTime() const override { return parent_shutdown_time_; } + Server::DrainStrategy drainStrategy() const override { return drain_strategy_; } + + spdlog::level::level_enum logLevel() const override { return log_level_; } + const std::vector>& + componentLogLevels() const override { + return component_log_levels_; + } + const std::string& logFormat() const override { return log_format_; } + bool logFormatSet() const override { return log_format_set_; } + bool logFormatEscaped() const override { return log_format_escaped_; } + bool enableFineGrainLogging() const override { return enable_fine_grain_logging_; } + const std::string& logPath() const override { return log_path_; } + uint64_t restartEpoch() const override { return restart_epoch_; } + Server::Mode mode() const override { return mode_; } + std::chrono::milliseconds fileFlushIntervalMsec() const override { + return file_flush_interval_msec_; + } + const std::string& serviceClusterName() const override { return service_cluster_; } + const std::string& serviceNodeName() const override { return service_node_; } + const std::string& serviceZone() const override { return service_zone_; } + bool hotRestartDisabled() const override { return hot_restart_disabled_; } + bool signalHandlingEnabled() const override { return signal_handling_enabled_; } + bool mutexTracingEnabled() const override { return mutex_tracing_enabled_; } + bool coreDumpEnabled() const override { return core_dump_enabled_; } + const Stats::TagVector& statsTags() const override { return stats_tags_; } + bool cpusetThreadsEnabled() const override { return cpuset_threads_; } + const std::vector& disabledExtensions() const override { + return disabled_extensions_; + } + uint32_t count() const; + const std::string& socketPath() const override { return socket_path_; } + mode_t socketMode() const override { return socket_mode_; } + // implemented by OptionsImpl + Server::CommandLineOptionsPtr toCommandLineOptions() const override { return nullptr; } + + /** + * disableExtensions parses the given set of extension names of + * the form $CATEGORY/$NAME, and disables the corresponding extension + * factories. + */ + static void disableExtensions(const std::vector&); + + /** + * Parses and validates the provided log_level, returning the corresponding + * spdlog::level::level_enum or an error status if the provided string is not a valid spdlog + * string. + */ + static absl::StatusOr + parseAndValidateLogLevel(absl::string_view log_level); + +private: + friend class OptionsImpl; + + uint64_t base_id_{0}; + bool use_dynamic_base_id_{false}; + std::string base_id_path_; + uint32_t concurrency_{1}; + std::string config_path_; + std::unique_ptr config_proto_{ + new envoy::config::bootstrap::v3::Bootstrap()}; + std::string config_yaml_; + bool allow_unknown_static_fields_{false}; + bool reject_unknown_dynamic_fields_{false}; + bool ignore_unknown_dynamic_fields_{false}; + std::string admin_address_path_; + Network::Address::IpVersion local_address_ip_version_{Network::Address::IpVersion::v4}; + spdlog::level::level_enum log_level_{spdlog::level::info}; + std::vector> component_log_levels_; + std::string component_log_level_str_; + std::string log_format_{Logger::Logger::DEFAULT_LOG_FORMAT}; + bool log_format_set_{false}; + bool log_format_escaped_{false}; + std::string log_path_; + uint64_t restart_epoch_{0}; + std::string service_cluster_; + std::string service_node_; + std::string service_zone_; + std::chrono::milliseconds file_flush_interval_msec_{10000}; + std::chrono::seconds drain_time_{600}; + std::chrono::seconds parent_shutdown_time_{900}; + Server::DrainStrategy drain_strategy_{Server::DrainStrategy::Gradual}; + Server::Mode mode_{Server::Mode::Serve}; + bool hot_restart_disabled_{false}; + bool signal_handling_enabled_{true}; + bool mutex_tracing_enabled_{false}; + bool core_dump_enabled_{false}; + bool cpuset_threads_{false}; + std::vector disabled_extensions_; + Stats::TagVector stats_tags_; + uint32_t count_{0}; + + // Initialization added here to avoid integration_admin_test failure caused by uninitialized + // enable_fine_grain_logging_. + bool enable_fine_grain_logging_ = false; + std::string socket_path_{"@envoy_domain_socket"}; + mode_t socket_mode_{0}; +}; + +} // namespace Envoy diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index 0642f9977613..84d00032a2f6 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -194,8 +194,7 @@ class ThreadLocalOverloadStateImpl : public ThreadLocalOverloadState { int64_t increment) override { const auto proactive_resource = proactive_resources_->find(resource_name); if (proactive_resource == proactive_resources_->end()) { - ENVOY_LOG_MISC(warn, " {Failed to allocate unknown proactive resource }"); - // Resource monitor is not configured. + ENVOY_LOG_MISC(warn, "Failed to allocate resource usage, resource monitor is not configured"); return false; } @@ -206,7 +205,8 @@ class ThreadLocalOverloadStateImpl : public ThreadLocalOverloadState { int64_t decrement) override { const auto proactive_resource = proactive_resources_->find(resource_name); if (proactive_resource == proactive_resources_->end()) { - ENVOY_LOG_MISC(warn, " {Failed to deallocate unknown proactive resource }"); + ENVOY_LOG_MISC(warn, + "Failed to deallocate resource usage, resource monitor is not configured"); return false; } @@ -218,6 +218,16 @@ class ThreadLocalOverloadStateImpl : public ThreadLocalOverloadState { return proactive_resource != proactive_resources_->end(); } + ProactiveResourceMonitorOptRef + getProactiveResourceMonitorForTest(OverloadProactiveResourceName resource_name) override { + const auto proactive_resource = proactive_resources_->find(resource_name); + if (proactive_resource == proactive_resources_->end()) { + ENVOY_LOG_MISC(warn, "Failed to get resource usage, resource monitor is not configured"); + return makeOptRefFromPtr(nullptr); + } + return proactive_resource->second.getProactiveResourceMonitorForTest(); + } + private: static const OverloadActionState always_inactive_; const NamedOverloadActionSymbolTable& action_symbol_table_; @@ -355,9 +365,9 @@ OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::S const envoy::config::overload::v3::OverloadManager& config, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api, const Server::Options& options) - : started_(false), dispatcher_(dispatcher), time_source_(api.timeSource()), - tls_(slot_allocator), refresh_interval_(std::chrono::milliseconds( - PROTOBUF_GET_MS_OR_DEFAULT(config, refresh_interval, 1000))), + : dispatcher_(dispatcher), time_source_(api.timeSource()), tls_(slot_allocator), + refresh_interval_( + std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, refresh_interval, 1000))), refresh_interval_delays_(makeHistogram(stats_scope, "refresh_interval_delay", Stats::Histogram::Unit::Milliseconds)), proactive_resources_( @@ -644,7 +654,7 @@ void OverloadManagerImpl::flushResourceUpdates() { OverloadManagerImpl::Resource::Resource(const std::string& name, ResourceMonitorPtr monitor, OverloadManagerImpl& manager, Stats::Scope& stats_scope) - : name_(name), monitor_(std::move(monitor)), manager_(manager), pending_update_(false), + : name_(name), monitor_(std::move(monitor)), manager_(manager), pressure_gauge_( makeGauge(stats_scope, name, "pressure", Stats::Gauge::ImportMode::NeverImport)), failed_updates_counter_(makeCounter(stats_scope, name, "failed_updates")), diff --git a/source/server/overload_manager_impl.h b/source/server/overload_manager_impl.h index d6295fd0f6b7..fb6e43b5d1d5 100644 --- a/source/server/overload_manager_impl.h +++ b/source/server/overload_manager_impl.h @@ -157,11 +157,7 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM ThreadLocalOverloadState& getThreadLocalOverloadState() override; LoadShedPoint* getLoadShedPoint(absl::string_view point_name) override; Event::ScaledRangeTimerManagerFactory scaledTimerFactory() override; - - // Stop the overload manager timer and wait for any pending resource updates to complete. - // After this returns, overload manager clients should not receive any more callbacks - // about overload state changes. - void stop(); + void stop() override; protected: // Factory for timer managers. This allows test-only subclasses to inject a mock implementation. @@ -186,7 +182,7 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM const std::string name_; ResourceMonitorPtr monitor_; OverloadManagerImpl& manager_; - bool pending_update_; + bool pending_update_{false}; FlushEpochId flush_epoch_; Stats::Gauge& pressure_gauge_; Stats::Counter& failed_updates_counter_; @@ -205,7 +201,7 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM // Flushes any enqueued action state updates to all worker threads. void flushResourceUpdates(); - bool started_; + bool started_{false}; Event::Dispatcher& dispatcher_; TimeSource& time_source_; ThreadLocal::TypedSlot tls_; diff --git a/source/server/server.cc b/source/server/server.cc index 64d716c4b973..dd2028bb26bd 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -40,10 +40,10 @@ #include "source/common/network/dns_resolver/dns_factory_util.h" #include "source/common/network/socket_interface.h" #include "source/common/network/socket_interface_impl.h" -#include "source/common/network/tcp_listener_impl.h" #include "source/common/protobuf/utility.h" #include "source/common/router/rds_impl.h" #include "source/common/runtime/runtime_impl.h" +#include "source/common/runtime/runtime_keys.h" #include "source/common/signal/fatal_error_handler.h" #include "source/common/singleton/manager_impl.h" #include "source/common/stats/thread_local_store.h" @@ -51,7 +51,6 @@ #include "source/common/upstream/cluster_manager_impl.h" #include "source/common/version/version.h" #include "source/server/configuration_impl.h" -#include "source/server/guarddog_impl.h" #include "source/server/listener_hooks.h" #include "source/server/listener_manager_factory.h" #include "source/server/regex_engine.h" @@ -60,6 +59,7 @@ namespace Envoy { namespace Server { + namespace { std::unique_ptr getHandler(Event::Dispatcher& dispatcher) { @@ -74,18 +74,19 @@ std::unique_ptr getHandler(Event::Dispatcher& dispatcher) { } // namespace -InstanceImpl::InstanceImpl( - Init::Manager& init_manager, const Options& options, Event::TimeSystem& time_system, - Network::Address::InstanceConstSharedPtr local_address, ListenerHooks& hooks, - HotRestart& restarter, Stats::StoreRoot& store, Thread::BasicLockable& access_log_lock, - ComponentFactory& component_factory, Random::RandomGeneratorPtr&& random_generator, - ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, - Filesystem::Instance& file_system, std::unique_ptr process_context, - Buffer::WatermarkFactorySharedPtr watermark_factory) - : init_manager_(init_manager), workers_started_(false), live_(false), shutdown_(false), - options_(options), validation_context_(options_.allowUnknownStaticFields(), - !options.rejectUnknownDynamicFields(), - options.ignoreUnknownDynamicFields()), +InstanceBase::InstanceBase(Init::Manager& init_manager, const Options& options, + Event::TimeSystem& time_system, ListenerHooks& hooks, + HotRestart& restarter, Stats::StoreRoot& store, + Thread::BasicLockable& access_log_lock, + Random::RandomGeneratorPtr&& random_generator, + ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, + Filesystem::Instance& file_system, + std::unique_ptr process_context, + Buffer::WatermarkFactorySharedPtr watermark_factory) + : init_manager_(init_manager), live_(false), options_(options), + validation_context_(options_.allowUnknownStaticFields(), + !options.rejectUnknownDynamicFields(), + options.ignoreUnknownDynamicFields()), time_source_(time_system), restarter_(restarter), start_time_(time(nullptr)), original_start_time_(start_time_), stats_store_(store), thread_local_(tls), random_generator_(std::move(random_generator)), @@ -98,58 +99,21 @@ InstanceImpl::InstanceImpl( store), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), handler_(getHandler(*dispatcher_)), worker_factory_(thread_local_, *api_, hooks), - terminated_(false), mutex_tracer_(options.mutexTracingEnabled() ? &Envoy::MutexTracerImpl::getOrCreateTracer() : nullptr), grpc_context_(store.symbolTable()), http_context_(store.symbolTable()), router_context_(store.symbolTable()), process_context_(std::move(process_context)), hooks_(hooks), quic_stat_names_(store.symbolTable()), server_contexts_(*this), - enable_reuse_port_default_(true), stats_flush_in_progress_(false) { - std::function set_up_logger = [&] { - TRY_ASSERT_MAIN_THREAD { - file_logger_ = std::make_unique( - options.logPath(), access_log_manager_, Logger::Registry::getSink()); - } - END_TRY - CATCH(const EnvoyException& e, { - throw EnvoyException( - fmt::format("Failed to open log-file '{}'. e.what(): {}", options.logPath(), e.what())); - }); - }; + enable_reuse_port_default_(true), stats_flush_in_progress_(false) {} - TRY_ASSERT_MAIN_THREAD { - if (!options.logPath().empty()) { - set_up_logger(); - } - restarter_.initialize(*dispatcher_, *this); - drain_manager_ = component_factory.createDrainManager(*this); - initialize(std::move(local_address), component_factory); - } - END_TRY - MULTI_CATCH( - const EnvoyException& e, - { - ENVOY_LOG(critical, "error initializing config '{} {} {}': {}", - options.configProto().DebugString(), options.configYaml(), options.configPath(), - e.what()); - terminate(); - throw; - }, - { - ENVOY_LOG(critical, "error initializing due to unknown exception"); - terminate(); - throw; - }); -} - -InstanceImpl::~InstanceImpl() { +InstanceBase::~InstanceBase() { terminate(); // Stop logging to file before all the AccessLogManager and its dependencies are // destructed to avoid crashing at shutdown. file_logger_.reset(); - // Destruct the ListenerManager explicitly, before InstanceImpl's local init_manager_ is + // Destruct the ListenerManager explicitly, before InstanceBase's local init_manager_ is // destructed. // // The ListenerManager's DestinationPortsMap contains FilterChainSharedPtrs. There is a rare race @@ -174,17 +138,17 @@ InstanceImpl::~InstanceImpl() { #endif } -Upstream::ClusterManager& InstanceImpl::clusterManager() { +Upstream::ClusterManager& InstanceBase::clusterManager() { ASSERT(config_.clusterManager() != nullptr); return *config_.clusterManager(); } -const Upstream::ClusterManager& InstanceImpl::clusterManager() const { +const Upstream::ClusterManager& InstanceBase::clusterManager() const { ASSERT(config_.clusterManager() != nullptr); return *config_.clusterManager(); } -void InstanceImpl::drainListeners(OptRef options) { +void InstanceBase::drainListeners(OptRef options) { ENVOY_LOG(info, "closing and draining listeners"); listener_manager_->stopListeners(ListenerManager::StopListenersType::All, options.has_value() ? *options @@ -192,12 +156,14 @@ void InstanceImpl::drainListeners(OptRefstartDrainSequence([] {}); } -void InstanceImpl::failHealthcheck(bool fail) { +void InstanceBase::failHealthcheck(bool fail) { live_.store(!fail); server_stats_->live_.set(live_.load()); } -MetricSnapshotImpl::MetricSnapshotImpl(Stats::Store& store, TimeSource& time_source) { +MetricSnapshotImpl::MetricSnapshotImpl(Stats::Store& store, + Upstream::ClusterManager& cluster_manager, + TimeSource& time_source) { store.forEachSinkedCounter( [this](std::size_t size) { snapped_counters_.reserve(size); @@ -238,22 +204,31 @@ MetricSnapshotImpl::MetricSnapshotImpl(Stats::Store& store, TimeSource& time_sou text_readouts_.push_back(text_readout); }); + Upstream::HostUtility::forEachHostMetric( + cluster_manager, + [this](Stats::PrimitiveCounterSnapshot&& metric) { + host_counters_.emplace_back(std::move(metric)); + }, + [this](Stats::PrimitiveGaugeSnapshot&& metric) { + host_gauges_.emplace_back(std::move(metric)); + }); + snapshot_time_ = time_source.systemTime(); } void InstanceUtil::flushMetricsToSinks(const std::list& sinks, Stats::Store& store, - TimeSource& time_source) { + Upstream::ClusterManager& cm, TimeSource& time_source) { // Create a snapshot and flush to all sinks. // NOTE: Even if there are no sinks, creating the snapshot has the important property that it // latches all counters on a periodic basis. The hot restart code assumes this is being // done so this should not be removed. - MetricSnapshotImpl snapshot(store, time_source); + MetricSnapshotImpl snapshot(store, cm, time_source); for (const auto& sink : sinks) { sink->flush(snapshot); } } -void InstanceImpl::flushStats() { +void InstanceBase::flushStats() { if (stats_flush_in_progress_) { ENVOY_LOG(debug, "skipping stats flush as flush is already in progress"); server_stats_->dropped_stat_flushes_.inc(); @@ -276,7 +251,7 @@ void InstanceImpl::flushStats() { } } -void InstanceImpl::updateServerStats() { +void InstanceBase::updateServerStats() { // mergeParentStatsIfAny() does nothing and returns a struct of 0s if there is no parent. HotRestart::ServerStatsFromParent parent_stats = restarter_.mergeParentStatsIfAny(stats_store_); @@ -303,10 +278,11 @@ void InstanceImpl::updateServerStats() { stats_store_.symbolTable().getRecentLookups([](absl::string_view, uint64_t) {})); } -void InstanceImpl::flushStatsInternal() { +void InstanceBase::flushStatsInternal() { updateServerStats(); auto& stats_config = config_.statsConfig(); - InstanceUtil::flushMetricsToSinks(stats_config.sinks(), stats_store_, timeSource()); + InstanceUtil::flushMetricsToSinks(stats_config.sinks(), stats_store_, clusterManager(), + timeSource()); // TODO(ramaraochavali): consider adding different flush interval for histograms. if (stat_flush_timer_ != nullptr) { stat_flush_timer_->enableTimer(stats_config.flushInterval()); @@ -315,9 +291,9 @@ void InstanceImpl::flushStatsInternal() { stats_flush_in_progress_ = false; } -bool InstanceImpl::healthCheckFailed() { return !live_.load(); } +bool InstanceBase::healthCheckFailed() { return !live_.load(); } -ProcessContextOptRef InstanceImpl::processContext() { +ProcessContextOptRef InstanceBase::processContext() { if (process_context_ == nullptr) { return absl::nullopt; } @@ -340,8 +316,8 @@ void registerCustomInlineHeadersFromBootstrap( for (const auto& inline_header : bootstrap.inline_headers()) { const Http::LowerCaseString lower_case_name(inline_header.inline_header_name()); if (!canBeRegisteredAsInlineHeader(lower_case_name)) { - throw EnvoyException(fmt::format("Header {} cannot be registered as an inline header.", - inline_header.inline_header_name())); + throwEnvoyExceptionOrPanic(fmt::format("Header {} cannot be registered as an inline header.", + inline_header.inline_header_name())); } switch (inline_header.inline_header_type()) { case envoy::config::bootstrap::v3::CustomInlineHeader::REQUEST_HEADER: @@ -378,8 +354,9 @@ void InstanceUtil::loadBootstrapConfig(envoy::config::bootstrap::v3::Bootstrap& // One of config_path and config_yaml or bootstrap should be specified. if (config_path.empty() && config_yaml.empty() && config_proto.ByteSizeLong() == 0) { - throw EnvoyException("At least one of --config-path or --config-yaml or Options::configProto() " - "should be non-empty"); + throwEnvoyExceptionOrPanic( + "At least one of --config-path or --config-yaml or Options::configProto() " + "should be non-empty"); } if (!config_path.empty()) { @@ -387,7 +364,7 @@ void InstanceUtil::loadBootstrapConfig(envoy::config::bootstrap::v3::Bootstrap& MessageUtil::loadFromFile(config_path, bootstrap, validation_visitor, api); #else if (!config_path.empty()) { - throw EnvoyException("Cannot load from file with YAML disabled\n"); + throwEnvoyExceptionOrPanic("Cannot load from file with YAML disabled\n"); } UNREFERENCED_PARAMETER(api); #endif @@ -399,7 +376,7 @@ void InstanceUtil::loadBootstrapConfig(envoy::config::bootstrap::v3::Bootstrap& // TODO(snowp): The fact that we do a merge here doesn't seem to be covered under test. bootstrap.MergeFrom(bootstrap_override); #else - throw EnvoyException("Cannot load from YAML with YAML disabled\n"); + throwEnvoyExceptionOrPanic("Cannot load from YAML with YAML disabled\n"); #endif } if (config_proto.ByteSizeLong() != 0) { @@ -408,8 +385,47 @@ void InstanceUtil::loadBootstrapConfig(envoy::config::bootstrap::v3::Bootstrap& MessageUtil::validate(bootstrap, validation_visitor); } -void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_address, +void InstanceBase::initialize(Network::Address::InstanceConstSharedPtr local_address, ComponentFactory& component_factory) { + std::function set_up_logger = [&] { + TRY_ASSERT_MAIN_THREAD { + file_logger_ = std::make_unique( + options_.logPath(), access_log_manager_, Logger::Registry::getSink()); + } + END_TRY + CATCH(const EnvoyException& e, { + throw EnvoyException( + fmt::format("Failed to open log-file '{}'. e.what(): {}", options_.logPath(), e.what())); + }); + }; + + TRY_ASSERT_MAIN_THREAD { + if (!options_.logPath().empty()) { + set_up_logger(); + } + restarter_.initialize(*dispatcher_, *this); + drain_manager_ = component_factory.createDrainManager(*this); + initializeOrThrow(std::move(local_address), component_factory); + } + END_TRY + MULTI_CATCH( + const EnvoyException& e, + { + ENVOY_LOG(critical, "error initializing config '{} {} {}': {}", + options_.configProto().DebugString(), options_.configYaml(), + options_.configPath(), e.what()); + terminate(); + throw; + }, + { + ENVOY_LOG(critical, "error initializing due to unknown exception"); + terminate(); + throw; + }); +} + +void InstanceBase::initializeOrThrow(Network::Address::InstanceConstSharedPtr local_address, + ComponentFactory& component_factory) { ENVOY_LOG(info, "initializing epoch {} (base id={}, hot restart version={})", options_.restartEpoch(), restarter_.baseId(), restarter_.version()); @@ -450,7 +466,7 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add tracing_session_ = perfetto::Tracing::NewTrace(); tracing_fd_ = open(pftrace_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0600); if (tracing_fd_ == -1) { - throw EnvoyException( + throwEnvoyExceptionOrPanic( fmt::format("unable to open tracing file {}: {}", pftrace_path, errorDetails(errno))); } // Configure the tracing session. @@ -508,7 +524,7 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add server_stats_->initialization_time_ms_, timeSource()); server_stats_->concurrency_.set(options_.concurrency()); server_stats_->hot_restart_epoch_.set(options_.restartEpoch()); - InstanceImpl::failHealthcheck(false); + InstanceBase::failHealthcheck(false); // Check if bootstrap has server version override set, if yes, we should use that as // 'server.version' stat. @@ -517,7 +533,7 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add version_int = bootstrap_.stats_server_version_override().value(); } else { if (!StringUtil::atoull(VersionInfo::revision().substr(0, 6).c_str(), version_int, 16)) { - throw EnvoyException("compiled GIT SHA is invalid. Invalid build."); + throwEnvoyExceptionOrPanic("compiled GIT SHA is invalid. Invalid build."); } } server_stats_->version_.set(version_int); @@ -590,12 +606,9 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add loadServerFlags(initial_config.flagsPath()); // Initialize the overload manager early so other modules can register for actions. - overload_manager_ = std::make_unique( - *dispatcher_, *stats_store_.rootScope(), thread_local_, bootstrap_.overload_manager(), - messageValidationContext().staticValidationVisitor(), *api_, options_); + overload_manager_ = createOverloadManager(); - heap_shrinker_ = std::make_unique(*dispatcher_, *overload_manager_, - *stats_store_.rootScope()); + maybeCreateHeapShrinker(); for (const auto& bootstrap_extension : bootstrap_.bootstrap_extensions()) { auto& factory = Config::Utility::getAndCheckFactory( @@ -679,8 +692,6 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add // Runtime gets initialized before the main configuration since during main configuration // load things may grab a reference to the loader for later use. runtime_ = component_factory.createRuntime(*this, initial_config); - - initial_config.initAdminAccessLog(bootstrap_, *this); validation_context_.setRuntime(runtime()); if (!runtime().snapshot().getBoolean("envoy.disallow_global_stats", false)) { @@ -691,15 +702,19 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add } if (initial_config.admin().address()) { - if (!admin_) { - throw EnvoyException("Admin address configured but admin support compiled out"); - } - admin_->startHttpListener(initial_config.admin().accessLogs(), options_.adminAddressPath(), - initial_config.admin().address(), - initial_config.admin().socketOptions(), - stats_store_.createScope("listener.admin.")); +#ifdef ENVOY_ADMIN_FUNCTIONALITY + // Admin instance always be created if admin support is not compiled out. + RELEASE_ASSERT(admin_ != nullptr, "Admin instance should be created but actually not."); + auto typed_admin = dynamic_cast(admin_.get()); + RELEASE_ASSERT(typed_admin != nullptr, "Admin implementation is not an AdminImpl."); + initial_config.initAdminAccessLog(bootstrap_, typed_admin->factoryContext()); + admin_->startHttpListener(initial_config.admin().accessLogs(), initial_config.admin().address(), + initial_config.admin().socketOptions()); +#else + throwEnvoyExceptionOrPanic("Admin address configured but admin support compiled out"); +#endif } else { - ENVOY_LOG(warn, "No admin address given, so no admin HTTP server started."); + ENVOY_LOG(info, "No admin address given, so no admin HTTP server started."); } if (admin_) { config_tracker_entry_ = admin_->getConfigTracker().add( @@ -719,7 +734,7 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add // Now the configuration gets parsed. The configuration may start setting // thread local data per above. See MainImpl::initialize() for why ConfigImpl - // is constructed as part of the InstanceImpl and then populated once + // is constructed as part of the InstanceBase and then populated once // cluster_manager_factory_ is available. config_.initialize(bootstrap_, *this, *cluster_manager_factory_); @@ -762,21 +777,21 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add // GuardDog (deadlock detection) object and thread setup before workers are // started and before our own run() loop runs. - main_thread_guard_dog_ = std::make_unique( - *stats_store_.rootScope(), config_.mainThreadWatchdogConfig(), *api_, "main_thread"); - worker_guard_dog_ = std::make_unique( - *stats_store_.rootScope(), config_.workerWatchdogConfig(), *api_, "workers"); + main_thread_guard_dog_ = maybeCreateGuardDog("main_thread"); + worker_guard_dog_ = maybeCreateGuardDog("workers"); } -void InstanceImpl::onClusterManagerPrimaryInitializationComplete() { +void InstanceBase::onClusterManagerPrimaryInitializationComplete() { // If RTDS was not configured the `onRuntimeReady` callback is immediately invoked. runtime().startRtdsSubscriptions([this]() { onRuntimeReady(); }); } -void InstanceImpl::onRuntimeReady() { +void InstanceBase::onRuntimeReady() { // Begin initializing secondary clusters after RTDS configuration has been applied. // Initializing can throw exceptions, so catch these. - TRY_ASSERT_MAIN_THREAD { clusterManager().initializeSecondaryClusters(bootstrap_); } + TRY_ASSERT_MAIN_THREAD { + THROW_IF_NOT_OK(clusterManager().initializeSecondaryClusters(bootstrap_)); + } END_TRY CATCH(const EnvoyException& e, { ENVOY_LOG(warn, "Skipping initialization of secondary cluster: {}", e.what()); @@ -786,7 +801,8 @@ void InstanceImpl::onRuntimeReady() { if (bootstrap_.has_hds_config()) { const auto& hds_config = bootstrap_.hds_config(); async_client_manager_ = std::make_unique( - *config_.clusterManager(), thread_local_, time_source_, *api_, grpc_context_.statNames()); + *config_.clusterManager(), thread_local_, time_source_, *api_, grpc_context_.statNames(), + bootstrap_.grpc_async_client_manager_config()); TRY_ASSERT_MAIN_THREAD { THROW_IF_NOT_OK(Config::Utility::checkTransportVersion(hds_config)); hds_delegate_ = std::make_unique( @@ -802,20 +818,11 @@ void InstanceImpl::onRuntimeReady() { shutdown(); }); } - - // If there is no global limit to the number of active connections, warn on startup. - // TODO (tonya11en): Move this functionality into the overload manager. - if (!runtime().snapshot().get(Network::TcpListenerImpl::GlobalMaxCxRuntimeKey)) { - ENVOY_LOG(warn, - "there is no configured limit to the number of allowed active connections. Set a " - "limit via the runtime key {}", - Network::TcpListenerImpl::GlobalMaxCxRuntimeKey); - } } -void InstanceImpl::startWorkers() { +void InstanceBase::startWorkers() { // The callback will be called after workers are started. - listener_manager_->startWorkers(*worker_guard_dog_, [this]() { + listener_manager_->startWorkers(makeOptRefFromPtr(worker_guard_dog_.get()), [this]() { if (isShutdown()) { return; } @@ -843,7 +850,7 @@ Runtime::LoaderPtr InstanceUtil::createRuntime(Instance& server, server.messageValidationContext().dynamicValidationVisitor(), server.api()); } -void InstanceImpl::loadServerFlags(const absl::optional& flags_path) { +void InstanceBase::loadServerFlags(const absl::optional& flags_path) { if (!flags_path) { return; } @@ -851,7 +858,7 @@ void InstanceImpl::loadServerFlags(const absl::optional& flags_path ENVOY_LOG(info, "server flags path: {}", flags_path.value()); if (api_->fileSystem().fileExists(flags_path.value() + "/drain")) { ENVOY_LOG(info, "starting server in drain mode"); - InstanceImpl::failHealthcheck(true); + InstanceBase::failHealthcheck(true); } } @@ -894,6 +901,15 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Start overload manager before workers. overload_manager.start(); + // If there is no global limit to the number of active connections, warn on startup. + if (!overload_manager.getThreadLocalOverloadState().isResourceMonitorEnabled( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections) && + !instance.runtime().snapshot().get(Runtime::Keys::GlobalMaxCxRuntimeKey)) { + ENVOY_LOG(warn, "There is no configured limit to the number of allowed active downstream " + "connections. Configure a " + "limit in `envoy.resource_monitors.downstream_connections` resource monitor."); + } + // Register for cluster manager init notification. We don't start serving worker traffic until // upstream clusters are initialized which may involve running the event loop. Note however that // this can fire immediately if all clusters have already initialized. Also note that we need @@ -922,7 +938,7 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch }); } -void InstanceImpl::run() { +void InstanceBase::run() { // RunHelper exists primarily to facilitate testing of how we respond to early shutdown during // startup (see RunHelperTest in server_test.cc). const auto run_helper = RunHelper(*this, options_, *dispatcher_, clusterManager(), @@ -933,18 +949,23 @@ void InstanceImpl::run() { // Run the main dispatch loop waiting to exit. ENVOY_LOG(info, "starting main dispatch loop"); - auto watchdog = main_thread_guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), - "main_thread", *dispatcher_); + WatchDogSharedPtr watchdog; + if (main_thread_guard_dog_) { + watchdog = main_thread_guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), + "main_thread", *dispatcher_); + } dispatcher_->post([this] { notifyCallbacksForStage(Stage::Startup); }); dispatcher_->run(Event::Dispatcher::RunType::Block); ENVOY_LOG(info, "main dispatch loop exited"); - main_thread_guard_dog_->stopWatching(watchdog); + if (main_thread_guard_dog_) { + main_thread_guard_dog_->stopWatching(watchdog); + } watchdog.reset(); terminate(); } -void InstanceImpl::terminate() { +void InstanceBase::terminate() { if (terminated_) { return; } @@ -992,16 +1013,16 @@ void InstanceImpl::terminate() { FatalErrorHandler::clearFatalActionsOnTerminate(); } -Runtime::Loader& InstanceImpl::runtime() { return *runtime_; } +Runtime::Loader& InstanceBase::runtime() { return *runtime_; } -void InstanceImpl::shutdown() { +void InstanceBase::shutdown() { ENVOY_LOG(info, "shutting down server instance"); shutdown_ = true; restarter_.sendParentTerminateRequest(); notifyCallbacksForStage(Stage::ShutdownExit, [this] { dispatcher_->exit(); }); } -void InstanceImpl::shutdownAdmin() { +void InstanceBase::shutdownAdmin() { ENVOY_LOG(warn, "shutting down admin due to child startup"); stat_flush_timer_.reset(); handler_->stopListeners(); @@ -1014,21 +1035,21 @@ void InstanceImpl::shutdownAdmin() { restarter_.sendParentTerminateRequest(); } -ServerLifecycleNotifier::HandlePtr InstanceImpl::registerCallback(Stage stage, +ServerLifecycleNotifier::HandlePtr InstanceBase::registerCallback(Stage stage, StageCallback callback) { auto& callbacks = stage_callbacks_[stage]; return std::make_unique>(callbacks, callback); } ServerLifecycleNotifier::HandlePtr -InstanceImpl::registerCallback(Stage stage, StageCallbackWithCompletion callback) { +InstanceBase::registerCallback(Stage stage, StageCallbackWithCompletion callback) { ASSERT(stage == Stage::ShutdownExit); auto& callbacks = stage_completable_callbacks_[stage]; return std::make_unique>(callbacks, callback); } -void InstanceImpl::notifyCallbacksForStage(Stage stage, std::function completion_cb) { +void InstanceBase::notifyCallbacksForStage(Stage stage, std::function completion_cb) { ASSERT_IS_MAIN_OR_TEST_THREAD(); const auto it = stage_callbacks_.find(stage); if (it != stage_callbacks_.end()) { @@ -1057,7 +1078,7 @@ void InstanceImpl::notifyCallbacksForStage(Stage stage, std::function co } } -ProtobufTypes::MessagePtr InstanceImpl::dumpBootstrapConfig() { +ProtobufTypes::MessagePtr InstanceBase::dumpBootstrapConfig() { auto config_dump = std::make_unique(); config_dump->mutable_bootstrap()->MergeFrom(bootstrap_); TimestampUtil::systemClockToTimestamp(bootstrap_config_update_time_, @@ -1065,7 +1086,7 @@ ProtobufTypes::MessagePtr InstanceImpl::dumpBootstrapConfig() { return config_dump; } -Network::DnsResolverSharedPtr InstanceImpl::getOrCreateDnsResolver() { +Network::DnsResolverSharedPtr InstanceBase::getOrCreateDnsResolver() { if (!dns_resolver_) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; Network::DnsResolverFactory& dns_resolver_factory = @@ -1076,7 +1097,7 @@ Network::DnsResolverSharedPtr InstanceImpl::getOrCreateDnsResolver() { return dns_resolver_; } -bool InstanceImpl::enableReusePortDefault() { return enable_reuse_port_default_; } +bool InstanceBase::enableReusePortDefault() { return enable_reuse_port_default_; } } // namespace Server } // namespace Envoy diff --git a/source/server/server.h b/source/server/server.h index 923e68fae141..dcfd5be7e5a9 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -32,7 +32,6 @@ #include "source/common/grpc/context_impl.h" #include "source/common/http/context_impl.h" #include "source/common/init/manager_impl.h" -#include "source/common/memory/heap_shrinker.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/quic/quic_stat_names.h" #include "source/common/router/context_impl.h" @@ -45,7 +44,6 @@ #endif #include "source/server/configuration_impl.h" #include "source/server/listener_hooks.h" -#include "source/server/overload_manager_impl.h" #include "source/server/worker_impl.h" #include "absl/container/node_hash_map.h" @@ -133,7 +131,7 @@ class InstanceUtil : Logger::Loggable { * @param store provides the store being flushed. */ static void flushMetricsToSinks(const std::list& sinks, Stats::Store& store, - TimeSource& time_source); + Upstream::ClusterManager& cm, TimeSource& time_source); /** * Load a bootstrap config and perform validation. @@ -149,7 +147,7 @@ class InstanceUtil : Logger::Loggable { }; /** - * This is a helper used by InstanceImpl::run() on the stack. It's broken out to make testing + * This is a helper used by InstanceBase::run() on the stack. It's broken out to make testing * easier. */ class RunHelper : Logger::Loggable { @@ -193,12 +191,16 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, TimeSource& timeSource() override { return api().timeSource(); } AccessLog::AccessLogManager& accessLogManager() override { return server_.accessLogManager(); } Api::Api& api() override { return server_.api(); } + Http::Context& httpContext() override { return server_.httpContext(); } Grpc::Context& grpcContext() override { return server_.grpcContext(); } Router::Context& routerContext() override { return server_.routerContext(); } + ProcessContextOptRef processContext() override { return server_.processContext(); } Envoy::Server::DrainManager& drainManager() override { return server_.drainManager(); } ServerLifecycleNotifier& lifecycleNotifier() override { return server_.lifecycleNotifier(); } Configuration::StatsConfig& statsConfig() override { return server_.statsConfig(); } envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return server_.bootstrap(); } + OverloadManager& overloadManager() override { return server_.overloadManager(); } + bool healthCheckFailed() const override { return server_.healthCheckFailed(); } // Configuration::TransportSocketFactoryContext ServerFactoryContext& serverFactoryContext() override { return *this; } @@ -223,27 +225,34 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, }; /** - * This is the actual full standalone server which stitches together various common components. + * This is the base class for the standalone server which stitches together various common + * components. Some components are optional (so PURE) and can be created or not by subclasses. */ -class InstanceImpl final : Logger::Loggable, - public Instance, - public ServerLifecycleNotifier { +class InstanceBase : Logger::Loggable, + public Instance, + public ServerLifecycleNotifier { public: /** * @throw EnvoyException if initialization fails. */ - InstanceImpl(Init::Manager& init_manager, const Options& options, Event::TimeSystem& time_system, - Network::Address::InstanceConstSharedPtr local_address, ListenerHooks& hooks, - HotRestart& restarter, Stats::StoreRoot& store, - Thread::BasicLockable& access_log_lock, ComponentFactory& component_factory, + InstanceBase(Init::Manager& init_manager, const Options& options, Event::TimeSystem& time_system, + ListenerHooks& hooks, HotRestart& restarter, Stats::StoreRoot& store, + Thread::BasicLockable& access_log_lock, Random::RandomGeneratorPtr&& random_generator, ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system, std::unique_ptr process_context, Buffer::WatermarkFactorySharedPtr watermark_factory = nullptr); - ~InstanceImpl() override; + // initialize the server. This must be called before run(). + void initialize(Network::Address::InstanceConstSharedPtr local_address, + ComponentFactory& component_factory); + ~InstanceBase() override; - void run(); + virtual void maybeCreateHeapShrinker() PURE; + virtual std::unique_ptr createOverloadManager() PURE; + virtual std::unique_ptr maybeCreateGuardDog(absl::string_view name) PURE; + + void run() override; // Server::Instance OptRef admin() override { return makeOptRefFromPtr(admin_.get()); } @@ -307,14 +316,19 @@ class InstanceImpl final : Logger::Loggable, ServerLifecycleNotifier::HandlePtr registerCallback(Stage stage, StageCallbackWithCompletion callback) override; +protected: + const Configuration::MainImpl& config() { return config_; } + private: Network::DnsResolverSharedPtr getOrCreateDnsResolver(); ProtobufTypes::MessagePtr dumpBootstrapConfig(); void flushStatsInternal(); void updateServerStats(); - void initialize(Network::Address::InstanceConstSharedPtr local_address, - ComponentFactory& component_factory); + // This does most of the work of initialization, but can throw errors caught + // by initialize(). + void initializeOrThrow(Network::Address::InstanceConstSharedPtr local_address, + ComponentFactory& component_factory); void loadServerFlags(const absl::optional& flags_path); void startWorkers(); void terminate(); @@ -336,9 +350,9 @@ class InstanceImpl final : Logger::Loggable, // - There may be active clusters referencing it in config_.cluster_manager_. // - There may be active connections referencing it. std::unique_ptr secret_manager_; - bool workers_started_; + bool workers_started_{false}; std::atomic live_; - bool shutdown_; + bool shutdown_{false}; const Options& options_; ProtobufMessage::ProdValidationContextImpl validation_context_; TimeSource& time_source_; @@ -378,28 +392,27 @@ class InstanceImpl final : Logger::Loggable, std::unique_ptr cluster_manager_factory_; std::unique_ptr main_thread_guard_dog_; std::unique_ptr worker_guard_dog_; - bool terminated_; + bool terminated_{false}; std::unique_ptr file_logger_; ConfigTracker::EntryOwnerPtr config_tracker_entry_; SystemTime bootstrap_config_update_time_; Grpc::AsyncClientManagerPtr async_client_manager_; Upstream::ProdClusterInfoFactory info_factory_; Upstream::HdsDelegatePtr hds_delegate_; - std::unique_ptr overload_manager_; + std::unique_ptr overload_manager_; std::vector bootstrap_extensions_; Envoy::MutexTracer* mutex_tracer_; Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; Router::ContextImpl router_context_; std::unique_ptr process_context_; - std::unique_ptr heap_shrinker_; // initialization_time is a histogram for tracking the initialization time across hot restarts // whenever we have support for histogram merge across hot restarts. Stats::TimespanPtr initialization_timer_; ListenerHooks& hooks_; Quic::QuicStatNames quic_stat_names_; ServerFactoryContextImpl server_contexts_; - bool enable_reuse_port_default_; + bool enable_reuse_port_default_{false}; Regex::EnginePtr regex_engine_; bool stats_flush_in_progress_ : 1; @@ -426,7 +439,8 @@ class InstanceImpl final : Logger::Loggable, // copying and probably be a cleaner API in general. class MetricSnapshotImpl : public Stats::MetricSnapshot { public: - explicit MetricSnapshotImpl(Stats::Store& store, TimeSource& time_source); + explicit MetricSnapshotImpl(Stats::Store& store, Upstream::ClusterManager& cluster_manager, + TimeSource& time_source); // Stats::MetricSnapshot const std::vector& counters() override { return counters_; } @@ -439,6 +453,10 @@ class MetricSnapshotImpl : public Stats::MetricSnapshot { const std::vector>& textReadouts() override { return text_readouts_; } + const std::vector& hostCounters() override { + return host_counters_; + } + const std::vector& hostGauges() override { return host_gauges_; } SystemTime snapshotTime() const override { return snapshot_time_; } private: @@ -450,6 +468,8 @@ class MetricSnapshotImpl : public Stats::MetricSnapshot { std::vector> histograms_; std::vector snapped_text_readouts_; std::vector> text_readouts_; + std::vector host_counters_; + std::vector host_gauges_; SystemTime snapshot_time_; }; diff --git a/source/server/ssl_context_manager.cc b/source/server/ssl_context_manager.cc index f2f086c7a7f7..d3b8df9d17f5 100644 --- a/source/server/ssl_context_manager.cc +++ b/source/server/ssl_context_manager.cc @@ -40,13 +40,13 @@ class SslContextManagerNoTlsStub final : public Envoy::Ssl::ContextManager { void removeContext(const Envoy::Ssl::ContextSharedPtr& old_context) override { if (old_context) { - throw EnvoyException("SSL is not supported in this configuration"); + throwEnvoyExceptionOrPanic("SSL is not supported in this configuration"); } } private: [[noreturn]] void throwException() { - throw EnvoyException("SSL is not supported in this configuration"); + throwEnvoyExceptionOrPanic("SSL is not supported in this configuration"); } }; diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index be1921b8bdc9..dcdd526b27e0 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -60,12 +60,13 @@ WorkerImpl::WorkerImpl(ThreadLocal::Instance& tls, ListenerHooks& hooks, void WorkerImpl::addListener(absl::optional overridden_listener, Network::ListenerConfig& listener, AddListenerCompletion completion, - Runtime::Loader& runtime) { - dispatcher_->post([this, overridden_listener, &listener, &runtime, completion]() -> void { - handler_->addListener(overridden_listener, listener, runtime); - hooks_.onWorkerListenerAdded(); - completion(); - }); + Runtime::Loader& runtime, Random::RandomGenerator& random) { + dispatcher_->post( + [this, overridden_listener, &listener, &runtime, &random, completion]() -> void { + handler_->addListener(overridden_listener, listener, runtime, random); + hooks_.onWorkerListenerAdded(); + completion(); + }); } uint64_t WorkerImpl::numConnections() const { @@ -97,7 +98,7 @@ void WorkerImpl::removeFilterChains(uint64_t listener_tag, }); } -void WorkerImpl::start(GuardDog& guard_dog, const std::function& cb) { +void WorkerImpl::start(OptRef guard_dog, const std::function& cb) { ASSERT(!thread_); // In posix, thread names are limited to 15 characters, so contrive to make @@ -112,7 +113,7 @@ void WorkerImpl::start(GuardDog& guard_dog, const std::function& cb) { // architecture is centralized, resulting in clearer names. Thread::Options options{absl::StrCat("wrk:", dispatcher_->name())}; thread_ = api_.threadFactory().createThread( - [this, &guard_dog, cb]() -> void { threadRoutine(guard_dog, cb); }, options); + [this, guard_dog, cb]() -> void { threadRoutine(guard_dog, cb); }, options); } void WorkerImpl::initializeStats(Stats::Scope& scope) { dispatcher_->initializeStats(scope); } @@ -138,18 +139,22 @@ void WorkerImpl::stopListener(Network::ListenerConfig& listener, }); } -void WorkerImpl::threadRoutine(GuardDog& guard_dog, const std::function& cb) { +void WorkerImpl::threadRoutine(OptRef guard_dog, const std::function& cb) { ENVOY_LOG(debug, "worker entering dispatch loop"); // The watch dog must be created after the dispatcher starts running and has post events flushed, // as this is when TLS stat scopes start working. dispatcher_->post([this, &guard_dog, cb]() { cb(); - watch_dog_ = guard_dog.createWatchDog(api_.threadFactory().currentThreadId(), - dispatcher_->name(), *dispatcher_); + if (guard_dog.has_value()) { + watch_dog_ = guard_dog->createWatchDog(api_.threadFactory().currentThreadId(), + dispatcher_->name(), *dispatcher_); + } }); dispatcher_->run(Event::Dispatcher::RunType::Block); ENVOY_LOG(debug, "worker exited dispatch loop"); - guard_dog.stopWatching(watch_dog_); + if (guard_dog.has_value()) { + guard_dog->stopWatching(watch_dog_); + } dispatcher_->shutdown(); // We must close all active connections before we actually exit the thread. This prevents any diff --git a/source/server/worker_impl.h b/source/server/worker_impl.h index c4c6c43a9d5a..a1baa6ec5ff4 100644 --- a/source/server/worker_impl.h +++ b/source/server/worker_impl.h @@ -53,14 +53,15 @@ class WorkerImpl : public Worker, Logger::Loggable { // Server::Worker void addListener(absl::optional overridden_listener, Network::ListenerConfig& listener, - AddListenerCompletion completion, Runtime::Loader& loader) override; + AddListenerCompletion completion, Runtime::Loader& loader, + Random::RandomGenerator& random) override; uint64_t numConnections() const override; void removeListener(Network::ListenerConfig& listener, std::function completion) override; void removeFilterChains(uint64_t listener_tag, const std::list& filter_chains, std::function completion) override; - void start(GuardDog& guard_dog, const std::function& cb) override; + void start(OptRef guard_dog, const std::function& cb) override; void initializeStats(Stats::Scope& scope) override; void stop() override; void stopListener(Network::ListenerConfig& listener, @@ -68,7 +69,7 @@ class WorkerImpl : public Worker, Logger::Loggable { std::function completion) override; private: - void threadRoutine(GuardDog& guard_dog, const std::function& cb); + void threadRoutine(OptRef guard_dog, const std::function& cb); void stopAcceptingConnectionsCb(OverloadActionState state); void rejectIncomingConnectionsCb(OverloadActionState state); void resetStreamsUsingExcessiveMemory(OverloadActionState state); diff --git a/support/README.md b/support/README.md index c1dce995fd60..1a0d2a19e12e 100644 --- a/support/README.md +++ b/support/README.md @@ -69,5 +69,5 @@ To run clang-tidy under Docker, run the following (this creates a full compilation db and takes a long time): ```console -./ci/run_envoy_docker.sh ci/do_ci.sh bazel.clang_tidy +./ci/run_envoy_docker.sh ci/do_ci.sh clang_tidy ``` diff --git a/test/BUILD b/test/BUILD index afe0ad6a9143..d8ad1e080cd3 100644 --- a/test/BUILD +++ b/test/BUILD @@ -21,9 +21,11 @@ envoy_cc_test_library( "main.cc", "test_listener.cc", "test_runner.cc", + ], + hdrs = [ + "test_listener.h", "test_runner.h", ], - hdrs = ["test_listener.h"], deps = [ "//source/common/common:logger_lib", "//source/common/common:thread_lib", diff --git a/test/benchmark/main.cc b/test/benchmark/main.cc index 90bdbb10e670..8c0f0c764382 100644 --- a/test/benchmark/main.cc +++ b/test/benchmark/main.cc @@ -26,14 +26,14 @@ int main(int argc, char** argv) { bool contains_help_flag = false; // Checking if any of the command-line arguments contains `--help` - for (int i = 1; i < argc; ++i) { + for (int i = 1; i < argc; ++i) { // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall) if (strcmp(argv[i], "--help") == 0) { contains_help_flag = true; break; } } - if (contains_help_flag) { + if (contains_help_flag) { // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall) // if the `--help` flag isn't considered separately, it runs "benchmark --help" // (Google Benchmark Help) and the help output doesn't contains details about // custom defined flags like `--skip_expensive_benchmarks`, `--runtime_feature`, etc diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 5426ca86097b..d003228f1a29 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -51,8 +51,9 @@ class AccessLogImplTest : public Event::TestUsingSimulatedTime, public testing:: AccessLogImplTest() : stream_info_(time_source_), file_(new MockAccessLogFile()), engine_(std::make_unique()) { - ON_CALL(context_, runtime()).WillByDefault(ReturnRef(runtime_)); - ON_CALL(context_, accessLogManager()).WillByDefault(ReturnRef(log_manager_)); + ON_CALL(context_.server_factory_context_, runtime()).WillByDefault(ReturnRef(runtime_)); + ON_CALL(context_.server_factory_context_, accessLogManager()) + .WillByDefault(ReturnRef(log_manager_)); ON_CALL(log_manager_, createAccessLog(_)).WillByDefault(Return(file_)); ON_CALL(*file_, write(_)).WillByDefault(SaveArg<0>(&output_)); stream_info_.addBytesReceived(1); @@ -62,6 +63,9 @@ class AccessLogImplTest : public Event::TestUsingSimulatedTime, public testing:: stream_info_.stream_id_provider_ = nullptr; } +protected: + void routeNameTest(std::string yaml, bool omit_empty); + NiceMock time_source_; Http::TestRequestHeaderMapImpl request_headers_{{":method", "GET"}, {":path", "/"}}; Http::TestResponseHeaderMapImpl response_headers_; @@ -93,8 +97,7 @@ name: accesslog request_headers_.addCopy(Http::Headers::get().Host, "host"); request_headers_.addCopy(Http::Headers::get().ForwardedFor, "x.x.x.x"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 UF 1 2 3 - \"x.x.x.x\" " "\"user-agent-set\" \"id\" \"host\" \"-\"\n", output_); @@ -117,24 +120,13 @@ name: accesslog Upstream::makeTestHostDescription(cluster, "tcp://10.0.0.5:1234", simTime())); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::DownstreamConnectionTermination); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 DC 1 2 3 - \"-\" \"-\" \"-\" \"-\" " "\"10.0.0.5:1234\"\n", output_); } -TEST_F(AccessLogImplTest, RouteName) { - const std::string yaml = R"EOF( -name: accesslog -typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: /dev/null - log_format: - text_format_source: - inline_string: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %ROUTE_NAME% %BYTES_RECEIVED% %BYTES_SENT% %UPSTREAM_WIRE_BYTES_RECEIVED% %UPSTREAM_WIRE_BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n" - )EOF"; - +void AccessLogImplTest::routeNameTest(std::string yaml, bool omit_empty) { InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); @@ -149,13 +141,48 @@ name: accesslog request_headers_.addCopy(Http::Headers::get().Host, "host"); request_headers_.addCopy(Http::Headers::get().ForwardedFor, "x.x.x.x"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); + + if (omit_empty) { + EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 UF route-test-name 1 2 0 0 3 " + "\"x.x.x.x\" " + "\"user-agent-set\" \"id\" \"host\"\n", + output_); + } else { + EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 UF route-test-name 1 2 0 0 3 - " + "\"x.x.x.x\" " + "\"user-agent-set\" \"id\" \"host\"\n", + output_); + } +} - EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 UF route-test-name 1 2 0 0 3 - " - "\"x.x.x.x\" " - "\"user-agent-set\" \"id\" \"host\"\n", - output_); +TEST_F(AccessLogImplTest, RouteName) { + const std::string yaml = R"EOF( +name: accesslog +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + log_format: + text_format_source: + inline_string: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %ROUTE_NAME% %BYTES_RECEIVED% %BYTES_SENT% %UPSTREAM_WIRE_BYTES_RECEIVED% %UPSTREAM_WIRE_BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n" + )EOF"; + + routeNameTest(yaml, false /* omit_empty */); +} + +TEST_F(AccessLogImplTest, RouteNameWithOmitEmptyString) { + const std::string yaml = R"EOF( +name: accesslog +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + log_format: + text_format_source: + inline_string: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %ROUTE_NAME% %BYTES_RECEIVED% %BYTES_SENT% %UPSTREAM_WIRE_BYTES_RECEIVED% %UPSTREAM_WIRE_BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n" + omit_empty_values: true + )EOF"; + + routeNameTest(yaml, true /* omit_empty */); } TEST_F(AccessLogImplTest, HeadersBytes) { @@ -187,8 +214,7 @@ name: accesslog // response trailers: // response_trailer_key: response_trailer_val - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ(output_, "52 38 40"); } @@ -206,8 +232,7 @@ name: accesslog EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().EnvoyUpstreamServiceTime, "999"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 - 1 2 3 999 \"-\" \"-\" \"-\" \"-\" " "\"-\"\n", output_); @@ -224,8 +249,7 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ( "[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 - 1 2 3 - \"-\" \"-\" \"-\" \"-\" \"-\"\n", output_); @@ -246,8 +270,7 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET / HTTP/1.1\" 0 - 1 2 3 - \"-\" \"-\" \"-\" \"-\" " "\"10.0.0.5:1234\"\n", output_); @@ -279,12 +302,10 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseCode(200); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, WithFilterHit) { @@ -319,18 +340,15 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(3); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseCode(500); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseCode(200); stream_info_.end_time_ = stream_info_.startTimeMonotonic() + std::chrono::microseconds(1001000000000000); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, RuntimeFilter) { @@ -348,33 +366,29 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); // Value is taken from random generator. - EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(42)); + EXPECT_CALL(context_.server_factory_context_.api_.random_, random()).WillOnce(Return(42)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 42, 100)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); - EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(43)); + EXPECT_CALL(context_.server_factory_context_.api_.random_, random()).WillOnce(Return(43)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 43, 100)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.stream_id_provider_ = std::make_shared("000000ff-0000-0000-0000-000000000000"); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 55, 100)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 0, 55, 100)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, RuntimeFilterV2) { @@ -395,33 +409,29 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); // Value is taken from random generator. - EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(42)); + EXPECT_CALL(context_.server_factory_context_.api_.random_, random()).WillOnce(Return(42)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 42, 10000)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); - EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(43)); + EXPECT_CALL(context_.server_factory_context_.api_.random_, random()).WillOnce(Return(43)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 43, 10000)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.stream_id_provider_ = std::make_shared("000000ff-0000-0000-0000-000000000000"); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 255, 10000)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 255, 10000)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, RuntimeFilterV2IndependentRandomness) { @@ -443,19 +453,17 @@ name: accesslog stream_info_.stream_id_provider_ = std::make_shared("000000ff-0000-0000-0000-000000000000"); - EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(42)); + EXPECT_CALL(context_.server_factory_context_.api_.random_, random()).WillOnce(Return(42)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 42, 1000000)) .WillOnce(Return(true)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); - EXPECT_CALL(context_.api_.random_, random()).WillOnce(Return(43)); + EXPECT_CALL(context_.server_factory_context_.api_.random_, random()).WillOnce(Return(43)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("access_log.test_key", 5, 43, 1000000)) .WillOnce(Return(false)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, PathRewrite) { @@ -471,8 +479,7 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("[1999-01-01T00:00:00.000Z] \"GET /bar HTTP/1.1\" 0 - 1 2 3 - \"-\" \"-\" \"-\" \"-\" " "\"-\"\n", output_); @@ -494,8 +501,7 @@ name: accesslog stream_info_.health_check_request_ = true; EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, HealthCheckFalse) { @@ -513,8 +519,7 @@ name: accesslog Http::TestRequestHeaderMapImpl header_map{}; EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, RequestTracing) { @@ -534,22 +539,19 @@ name: accesslog { stream_info_.setTraceReason(Tracing::Reason::ServiceForced); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } { stream_info_.setTraceReason(Tracing::Reason::NotTraceable); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } { stream_info_.setTraceReason(Tracing::Reason::Sampling); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } } @@ -610,16 +612,14 @@ name: accesslog EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{{"user-agent", "NOT/Envoy/HC"}}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } { EXPECT_CALL(*file_, write(_)).Times(0); Http::TestRequestHeaderMapImpl header_map{}; stream_info_.health_check_request_ = true; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } } @@ -648,15 +648,13 @@ name: accesslog EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{{"user-agent", "NOT/Envoy/HC"}}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } { EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{{"user-agent", "Envoy/HC"}}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } } @@ -693,8 +691,7 @@ name: accesslog EXPECT_CALL(*file_, write(_)); Http::TestRequestHeaderMapImpl header_map{}; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } { @@ -702,8 +699,7 @@ name: accesslog Http::TestRequestHeaderMapImpl header_map{}; stream_info_.health_check_request_ = true; - log->log(&header_map, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&header_map, &response_headers_, &response_trailers_}, stream_info_); } } @@ -723,35 +719,30 @@ TEST(AccessLogFilterTest, DurationWithRuntimeKey) { TestUtility::loadFromYaml(filter_yaml, config); DurationFilter filter(config.duration_filter(), runtime); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; NiceMock time_source; TestStreamInfo stream_info(time_source); + const Formatter::HttpFormatterContext log_context{&request_headers}; + stream_info.end_time_ = stream_info.startTimeMonotonic() + std::chrono::microseconds(100000); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(1)); - EXPECT_TRUE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)); + EXPECT_TRUE(filter.evaluate(log_context, stream_info)); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(1000)); - EXPECT_FALSE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)); + EXPECT_FALSE(filter.evaluate(log_context, stream_info)); stream_info.end_time_ = stream_info.startTimeMonotonic() + std::chrono::microseconds(100000001000); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(100000000)); - EXPECT_TRUE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)); + EXPECT_TRUE(filter.evaluate(log_context, stream_info)); stream_info.end_time_ = stream_info.startTimeMonotonic() + std::chrono::microseconds(10000); EXPECT_CALL(runtime.snapshot_, getInteger("key", 1000000)).WillOnce(Return(100000000)); - EXPECT_FALSE(filter.evaluate(stream_info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)); + EXPECT_FALSE(filter.evaluate(log_context, stream_info)); StreamInfo::MockStreamInfo mock_stream_info; EXPECT_CALL(mock_stream_info, currentDuration()).WillOnce(testing::Return(std::nullopt)); - EXPECT_FALSE(filter.evaluate(mock_stream_info, request_headers, response_headers, - response_trailers, AccessLog::AccessLogType::NotSet)); + EXPECT_FALSE(filter.evaluate(log_context, mock_stream_info)); } TEST(AccessLogFilterTest, MidStreamDuration) { @@ -769,19 +760,16 @@ TEST(AccessLogFilterTest, MidStreamDuration) { TestUtility::loadFromYaml(filter_yaml, config); DurationFilter filter(config.duration_filter(), runtime); Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; + + const Formatter::HttpFormatterContext log_context{&request_headers}; StreamInfo::MockStreamInfo mock_stream_info; EXPECT_CALL(mock_stream_info, currentDuration()) .WillOnce(testing::Return(std::chrono::microseconds(1000))) // 1ms .WillOnce(testing::Return(std::chrono::microseconds(1000000))); // 1000ms - EXPECT_FALSE(filter.evaluate(mock_stream_info, request_headers, response_headers, - response_trailers, AccessLog::AccessLogType::NotSet)); - - EXPECT_TRUE(filter.evaluate(mock_stream_info, request_headers, response_headers, - response_trailers, AccessLog::AccessLogType::NotSet)); + EXPECT_FALSE(filter.evaluate(log_context, mock_stream_info)); + EXPECT_TRUE(filter.evaluate(log_context, mock_stream_info)); } TEST(AccessLogFilterTest, StatusCodeWithRuntimeKey) { @@ -802,18 +790,16 @@ TEST(AccessLogFilterTest, StatusCodeWithRuntimeKey) { NiceMock time_source; Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; TestStreamInfo info(time_source); + const Formatter::HttpFormatterContext log_context{&request_headers}; + info.setResponseCode(400); EXPECT_CALL(runtime.snapshot_, getInteger("key", 300)).WillOnce(Return(350)); - EXPECT_TRUE(filter.evaluate(info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)); + EXPECT_TRUE(filter.evaluate(log_context, info)); EXPECT_CALL(runtime.snapshot_, getInteger("key", 300)).WillOnce(Return(500)); - EXPECT_FALSE(filter.evaluate(info, request_headers, response_headers, response_trailers, - AccessLog::AccessLogType::NotSet)); + EXPECT_FALSE(filter.evaluate(log_context, info)); } TEST_F(AccessLogImplTest, StatusCodeLessThan) { @@ -836,14 +822,12 @@ name: accesslog stream_info_.setResponseCode(499); EXPECT_CALL(runtime_.snapshot_, getInteger("hello", 499)).WillOnce(Return(499)); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseCode(500); EXPECT_CALL(runtime_.snapshot_, getInteger("hello", 499)).WillOnce(Return(499)); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, HeaderPresence) { @@ -861,13 +845,11 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.addCopy("test-header", "present"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, HeaderExactMatch) { @@ -888,19 +870,16 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.addCopy("test-header", "exact-match-value"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "not-exact-match-value"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, HeaderRegexMatch) { @@ -921,25 +900,21 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.addCopy("test-header", "123"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "1234"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "123.456"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, HeaderRangeMatch) { @@ -960,37 +935,31 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.addCopy("test-header", "-1"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "0"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "somestring"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "10.9"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("test-header"); request_headers_.addCopy("test-header", "-1somestring"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, ResponseFlagFilterAnyFlag) { @@ -1006,13 +975,11 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, ResponseFlagFilterSpecificFlag) { @@ -1030,18 +997,15 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, ResponseFlagFilterSeveralFlags) { @@ -1060,18 +1024,15 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, ResponseFlagFilterAllFlagsInPGV) { @@ -1106,6 +1067,8 @@ name: accesslog - UPE - NC - OM + - DF + - DO typed_config: "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog path: /dev/null @@ -1120,8 +1083,7 @@ name: accesslog TestStreamInfo stream_info(time_source_); stream_info.setResponseFlag(response_flag); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info); } } @@ -1164,8 +1126,9 @@ name: accesslog "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog )EOF"; - ON_CALL(context_, runtime()).WillByDefault(ReturnRef(runtime_)); - ON_CALL(context_, accessLogManager()).WillByDefault(ReturnRef(log_manager_)); + ON_CALL(context_.server_factory_context_, runtime()).WillByDefault(ReturnRef(runtime_)); + ON_CALL(context_.server_factory_context_, accessLogManager()) + .WillByDefault(ReturnRef(log_manager_)); EXPECT_CALL(log_manager_, createAccessLog(_)) .WillOnce(Invoke( [this](const Envoy::Filesystem::FilePathAndType& file_info) -> AccessLogFileSharedPtr { @@ -1184,8 +1147,9 @@ name: accesslog "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StderrAccessLog )EOF"; - ON_CALL(context_, runtime()).WillByDefault(ReturnRef(runtime_)); - ON_CALL(context_, accessLogManager()).WillByDefault(ReturnRef(log_manager_)); + ON_CALL(context_.server_factory_context_, runtime()).WillByDefault(ReturnRef(runtime_)); + ON_CALL(context_.server_factory_context_, accessLogManager()) + .WillByDefault(ReturnRef(log_manager_)); EXPECT_CALL(log_manager_, createAccessLog(_)) .WillOnce(Invoke( [this](const Envoy::Filesystem::FilePathAndType& file_info) -> AccessLogFileSharedPtr { @@ -1226,8 +1190,7 @@ name: accesslog { EXPECT_CALL(*file_, write(_)); response_trailers_.addCopy(Http::Headers::get().GrpcStatus, "0"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("OK 0 OK OK OK 0\n", output_); response_trailers_.remove(Http::Headers::get().GrpcStatus); } @@ -1235,8 +1198,7 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().GrpcStatus, "1"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("Canceled 1 Canceled Canceled CANCELLED 1\n", output_); response_headers_.remove(Http::Headers::get().GrpcStatus); } @@ -1244,8 +1206,7 @@ name: accesslog InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().GrpcStatus, "-1"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); EXPECT_EQ("-1 -1 -1 -1 -1 -1\n", output_); response_headers_.remove(Http::Headers::get().GrpcStatus); } @@ -1277,8 +1238,7 @@ name: accesslog EXPECT_CALL(*file_, write(_)); response_trailers_.addCopy(Http::Headers::get().GrpcStatus, std::to_string(i)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); response_trailers_.remove(Http::Headers::get().GrpcStatus); } } @@ -1317,8 +1277,7 @@ name: accesslog response_trailers_.addCopy(Http::Headers::get().GrpcStatus, "1"); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, GrpcStatusFilterHttpCodes) { @@ -1349,8 +1308,7 @@ name: accesslog parseAccessLogFromV3Yaml(fmt::format(yaml_template, response_string)), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } } @@ -1370,8 +1328,7 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, GrpcStatusFilterExclude) { @@ -1394,8 +1351,7 @@ name: accesslog EXPECT_CALL(*file_, write(_)).Times(i == 0 ? 0 : 1); response_trailers_.addCopy(Http::Headers::get().GrpcStatus, std::to_string(i)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); response_trailers_.remove(Http::Headers::get().GrpcStatus); } } @@ -1419,8 +1375,7 @@ name: accesslog response_trailers_.addCopy(Http::Headers::get().GrpcStatus, "0"); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, GrpcStatusFilterHeader) { @@ -1441,8 +1396,7 @@ name: accesslog EXPECT_CALL(*file_, write(_)); response_headers_.addCopy(Http::Headers::get().GrpcStatus, "0"); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, LogTypeFilterUnsupportedValue) { @@ -1478,8 +1432,12 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::TcpConnectionEnd); // Blocked + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::TcpConnectionEnd}, + stream_info_); // Blocked } TEST_F(AccessLogImplTest, LogTypeFilterAllow) { @@ -1499,12 +1457,24 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(2); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::TcpUpstreamConnected); // Allowed - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::DownstreamEnd); // Allowed - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::UpstreamPoolReady); // Blocked + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::TcpUpstreamConnected}, + stream_info_); // Allowed + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::DownstreamEnd}, + stream_info_); // Allowed + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::UpstreamPoolReady}, + stream_info_); // Blocked } TEST_F(AccessLogImplTest, LogTypeFilterExclude) { @@ -1525,12 +1495,24 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::TcpUpstreamConnected); // Blocked - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::DownstreamEnd); // Blocked - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::UpstreamPoolReady); // Allowed + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::TcpUpstreamConnected}, + stream_info_); // Blocked + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::DownstreamEnd}, + stream_info_); // Blocked + log->log({&request_headers_, + &response_headers_, + &response_trailers_, + {}, + AccessLog::AccessLogType::UpstreamPoolReady}, + stream_info_); // Allowed } TEST_F(AccessLogImplTest, MetadataFilter) { @@ -1568,8 +1550,7 @@ name: accesslog EXPECT_CALL(*file_, write(_)); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info); fields_c["c"].set_bool_value(false); EXPECT_CALL(*file_, write(_)).Times(0); @@ -1598,8 +1579,7 @@ name: accesslog // If no matcher is set, then expect no logs. EXPECT_CALL(*file_, write(_)).Times(0); - log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, - AccessLog::AccessLogType::NotSet); + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info); } TEST_F(AccessLogImplTest, MetadataFilterNoKey) { @@ -1650,15 +1630,13 @@ name: accesslog AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(default_false_yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - default_false_log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, - AccessLog::AccessLogType::NotSet); + default_false_log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info); const InstanceSharedPtr default_true_log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(default_true_yaml), context_); EXPECT_CALL(*file_, write(_)); - default_true_log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, - AccessLog::AccessLogType::NotSet); + default_true_log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info); } class TestHeaderFilterFactory : public ExtensionFilterFactory { @@ -1666,7 +1644,7 @@ class TestHeaderFilterFactory : public ExtensionFilterFactory { ~TestHeaderFilterFactory() override = default; FilterPtr createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, - Server::Configuration::CommonFactoryContext& context) override { + Server::Configuration::FactoryContext& context) override { auto factory_config = Config::Utility::translateToFactoryConfig( config, context.messageValidationVisitor(), *this); const auto& header_config = @@ -1703,13 +1681,31 @@ name: accesslog InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.addCopy("test-header", "foo/bar"); EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); +} + +TEST_F(AccessLogImplTest, EmitTime) { + const std::string yaml = R"EOF( +name: accesslog +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + log_format: + text_format_source: + inline_string: "%EMIT_TIME%" + )EOF"; + + InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + EXPECT_CALL(*file_, write(_)); + + log->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); + + const std::string time_regex = "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$"; + EXPECT_TRUE(std::regex_match(output_.value(), std::regex(time_regex))); } /** @@ -1720,9 +1716,8 @@ class SampleExtensionFilter : public Filter { SampleExtensionFilter(uint32_t sample_rate) : sample_rate_(sample_rate) {} // AccessLog::Filter - bool evaluate(const StreamInfo::StreamInfo&, const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - AccessLogType) const override { + bool evaluate(const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo&) const override { if (current_++ == 0) { return true; } @@ -1745,7 +1740,7 @@ class SampleExtensionFilterFactory : public ExtensionFilterFactory { ~SampleExtensionFilterFactory() override = default; FilterPtr createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, - Server::Configuration::CommonFactoryContext& context) override { + Server::Configuration::FactoryContext& context) override { auto factory_config = Config::Utility::translateToFactoryConfig( config, context.messageValidationVisitor(), *this); @@ -1783,16 +1778,13 @@ name: accesslog InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); // For rate=5 expect 1st request to be recorded, 2nd-5th skipped, and 6th recorded. EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); for (int i = 0; i <= 3; ++i) { EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, UnregisteredExtensionFilter) { @@ -1851,13 +1843,11 @@ name: accesslog request_headers_.addCopy("log", "true"); stream_info_.setResponseCode(404); EXPECT_CALL(*file_, write(_)); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); request_headers_.remove("log"); EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, CelExtensionFilterExpressionError) { @@ -1877,8 +1867,7 @@ name: accesslog InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); EXPECT_CALL(*file_, write(_)).Times(0); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } TEST_F(AccessLogImplTest, CelExtensionFilterExpressionUnparsable) { diff --git a/test/common/access_log/access_log_manager_impl_test.cc b/test/common/access_log/access_log_manager_impl_test.cc index 3bacc57c0bd9..e3debf6d851a 100644 --- a/test/common/access_log/access_log_manager_impl_test.cc +++ b/test/common/access_log/access_log_manager_impl_test.cc @@ -233,6 +233,7 @@ TEST_F(AccessLogManagerImplTest, FlushCountsIOErrors) { } TEST_F(AccessLogManagerImplTest, ReopenFile) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) NiceMock* timer = new NiceMock(&dispatcher_); Sequence sq; @@ -275,12 +276,14 @@ TEST_F(AccessLogManagerImplTest, ReopenFile) { log_file->write("reopened"); timer->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 2)); EXPECT_TRUE(file_->waitForEventCount(file_->num_opens_, 2)); } // Test that the `reopen()` will trigger file reopen even if no data is waiting. TEST_F(AccessLogManagerImplTest, ReopenFileNoWrite) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) NiceMock* timer = new NiceMock(&dispatcher_); Sequence sq; @@ -299,6 +302,7 @@ TEST_F(AccessLogManagerImplTest, ReopenFileNoWrite) { log_file->write("before"); timer->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 1)); EXPECT_CALL(*file_, close_()) @@ -318,6 +322,7 @@ TEST_F(AccessLogManagerImplTest, ReopenFileNoWrite) { } TEST_F(AccessLogManagerImplTest, ReopenRetry) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) NiceMock* timer = new NiceMock(&dispatcher_); Sequence sq; @@ -384,6 +389,7 @@ TEST_F(AccessLogManagerImplTest, ReopenRetry) { log_file->write("after reopen"); timer->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) EXPECT_TRUE(file_->waitForEventCount(file_->num_writes_, 3)); waitForCounterEq("filesystem.reopen_failed", 2); waitForGaugeEq("filesystem.write_total_buffered", 0); diff --git a/test/common/buffer/buffer_fuzz.cc b/test/common/buffer/buffer_fuzz.cc index 53d9aebe7f44..bb6903d9c089 100644 --- a/test/common/buffer/buffer_fuzz.cc +++ b/test/common/buffer/buffer_fuzz.cc @@ -204,7 +204,7 @@ class StringBuffer : public Buffer::Instance { return absl::StartsWith(asStringView(), data); } - std::string toString() const override { return std::string(data_.data() + start_, size_); } + std::string toString() const override { return {data_.data() + start_, size_}; } size_t addFragments(absl::Span fragments) override { size_t total_size_to_write = 0; diff --git a/test/common/buffer/buffer_memory_account_test.cc b/test/common/buffer/buffer_memory_account_test.cc index b6b6a2019371..a447c62df901 100644 --- a/test/common/buffer/buffer_memory_account_test.cc +++ b/test/common/buffer/buffer_memory_account_test.cc @@ -526,7 +526,10 @@ TEST(WatermarkBufferFactoryTest, ShouldOnlyResetAllStreamsGreatThanOrEqualToProv EXPECT_CALL(stream_that_should_not_be_reset, resetStream(_)).Times(0); // Should call resetStream on all streams in bucket >= 1. - EXPECT_EQ(factory.resetAccountsGivenPressure(0.85), 2); + EXPECT_LOG_CONTAINS("warn", "resetting 2 streams in 2 buckets, 5 empty buckets", + EXPECT_EQ(factory.resetAccountsGivenPressure(0.85), 2)); + EXPECT_LOG_NOT_CONTAINS("warn", "resetting", + EXPECT_EQ(factory.resetAccountsGivenPressure(0.85), 0)); account_to_not_reset->credit(kMinimumBalanceToTrack); account_to_not_reset->clearDownstream(); diff --git a/test/common/buffer/owned_impl_test.cc b/test/common/buffer/owned_impl_test.cc index c575f5ad9719..85fbdcf7a61f 100644 --- a/test/common/buffer/owned_impl_test.cc +++ b/test/common/buffer/owned_impl_test.cc @@ -77,6 +77,44 @@ TEST_F(OwnedImplTest, AddBufferFragmentWithCleanup) { EXPECT_TRUE(release_callback_called_); } +TEST_F(OwnedImplTest, MoveBufferFragment) { + Buffer::OwnedImpl buffer1; + testing::MockFunction + release_callback_tracker; + std::string frag_input("a"); + BufferFragmentImpl frag(frag_input.c_str(), frag_input.size(), + release_callback_tracker.AsStdFunction()); + buffer1.addBufferFragment(frag); + + Buffer::OwnedImpl buffer2; + buffer2.move(buffer1); + + EXPECT_EQ(0, buffer1.length()); + EXPECT_EQ(1, buffer2.length()); + + EXPECT_CALL(release_callback_tracker, Call(_, _, _)); + buffer2.drain(buffer2.length()); +} + +TEST_F(OwnedImplTest, MoveBufferFragmentWithReleaseDrainTracker) { + Buffer::OwnedImpl buffer1; + testing::MockFunction + release_callback_tracker; + std::string frag_input("a"); + BufferFragmentImpl frag(frag_input.c_str(), frag_input.size(), + release_callback_tracker.AsStdFunction()); + buffer1.addBufferFragment(frag); + + Buffer::OwnedImpl buffer2; + buffer2.move(buffer1, true); + + EXPECT_EQ(0, buffer1.length()); + EXPECT_EQ(1, buffer2.length()); + + EXPECT_CALL(release_callback_tracker, Call(_, _, _)); + buffer2.drain(buffer2.length()); +} + TEST_F(OwnedImplTest, AddEmptyFragment) { char input[] = "hello world"; BufferFragmentImpl frag1(input, 11, [](const void*, size_t, const BufferFragmentImpl*) {}); @@ -667,10 +705,10 @@ TEST_F(OwnedImplTest, LinearizeDrainTracking) { testing::MockFunction done_tracker; EXPECT_CALL(tracker1, Call()); EXPECT_CALL(drain_tracker, Call(3 * LargeChunk + 108 * SmallChunk, 16384)); - EXPECT_CALL(release_callback_tracker, Call(_, _, _)); EXPECT_CALL(tracker2, Call()); - EXPECT_CALL(release_callback_tracker2, Call(_, _, _)); + EXPECT_CALL(release_callback_tracker, Call(_, _, _)); EXPECT_CALL(tracker3, Call()); + EXPECT_CALL(release_callback_tracker2, Call(_, _, _)); EXPECT_CALL(drain_tracker, Call(2 * LargeChunk + 107 * SmallChunk, 16384)); EXPECT_CALL(drain_tracker, Call(LargeChunk + 106 * SmallChunk, 16384)); EXPECT_CALL(tracker4, Call()); diff --git a/test/common/common/hash_test.cc b/test/common/common/hash_test.cc index e1eda27b517b..5b83d8d84599 100644 --- a/test/common/common/hash_test.cc +++ b/test/common/common/hash_test.cc @@ -11,6 +11,32 @@ TEST(Hash, xxHash) { EXPECT_EQ(17241709254077376921U, HashUtil::xxHash64("")); } +TEST(Hash, xxHash64Value) { + // Verifying against constants should protect against surprise hash behavior + // changes, and, when run on a test host with different endianness, should also + // verify that different endianness doesn't change the result. + EXPECT_EQ(11149811956558368074UL, HashUtil::xxHash64Value(1234567890123456789UL)); + EXPECT_EQ(5127252389447085590UL, HashUtil::xxHash64Value(-1234567890123456789L)); + EXPECT_EQ(14922725725041217620UL, HashUtil::xxHash64Value(1234567890U)); + EXPECT_EQ(12903803813495632273UL, HashUtil::xxHash64Value(-1234567890)); + EXPECT_EQ(6394838272449507810UL, HashUtil::xxHash64Value(1234567890.12345)); + EXPECT_EQ(16086425465732342325UL, HashUtil::xxHash64Value(-1234567890.12345f)); + // All NaN should be alike. + EXPECT_EQ(13464136223671999926UL, HashUtil::xxHash64Value(std::nan(""))); + EXPECT_EQ(13464136223671999926UL, HashUtil::xxHash64Value(std::nan("1"))); + EXPECT_EQ(13464136223671999926UL, HashUtil::xxHash64Value(std::nanf(""))); + EXPECT_EQ(13464136223671999926UL, HashUtil::xxHash64Value(std::nanf("1"))); + // All Inf should be alike. + EXPECT_EQ(16018703821796664527UL, + HashUtil::xxHash64Value(std::numeric_limits::infinity())); + EXPECT_EQ(16018703821796664527UL, + HashUtil::xxHash64Value(std::numeric_limits::infinity())); + + EXPECT_EQ(9962287286179718960UL, HashUtil::xxHash64Value(true)); + EXPECT_EQ(16804241149081757544UL, HashUtil::xxHash64Value(false)); + EXPECT_EQ(9486749600008296231UL, HashUtil::xxHash64Value(false, /*seed=*/42)); +} + TEST(Hash, xxHashWithVector) { absl::InlinedVector v{"foo", "bar"}; EXPECT_EQ(17745830980996999794U, HashUtil::xxHash64(absl::MakeSpan(v))); diff --git a/test/common/common/lock_guard_test.cc b/test/common/common/lock_guard_test.cc index be786f073883..ef91e2627e35 100644 --- a/test/common/common/lock_guard_test.cc +++ b/test/common/common/lock_guard_test.cc @@ -37,7 +37,7 @@ TEST_F(LockGuardTest, TestTryLockGuard) { if (lock.tryLock()) { // This test doesn't work, because a_mutex_ is guarded, and thread // annotations don't work with TryLockGuard. The macro is defined in - // include/envoy/thread/thread.h. + // envoy/thread/thread.h. DISABLE_TRYLOCKGUARD_ANNOTATION(EXPECT_EQ(1, ++a_)); // TryLockGuard does functionally work with unguarded variables. diff --git a/test/common/common/log_verbosity_update_test.cc b/test/common/common/log_verbosity_update_test.cc index 72a105c43229..db2609252c1a 100644 --- a/test/common/common/log_verbosity_update_test.cc +++ b/test/common/common/log_verbosity_update_test.cc @@ -41,10 +41,8 @@ TEST(FineGrainLog, updateCurrentFilePath) { getFineGrainLogContext().setFineGrainLogger(__FILE__, spdlog::level::info); FINE_GRAIN_LOG(debug, "Debug: you shouldn't see this message!"); - absl::string_view file_path = __FILE__; - file_path.remove_suffix(3); const std::pair update = - std::make_pair(file_path, static_cast(spdlog::level::debug)); + std::make_pair(__FILE__, static_cast(spdlog::level::debug)); getFineGrainLogContext().updateVerbositySetting({update}); FINE_GRAIN_LOG(debug, "Debug: now level is debug"); @@ -86,13 +84,13 @@ TEST(FineGrainLog, verbosityUpdatePriority) { const std::pair update = std::make_pair(file_path, static_cast(spdlog::level::debug)); getFineGrainLogContext().updateVerbositySetting({update}); + // This will also try to clear the verbosity update by changing the default level. getFineGrainLogContext().updateVerbosityDefaultLevel(spdlog::level::trace); - FINE_GRAIN_LOG(debug, "Debug: now level is debug"); - FINE_GRAIN_LOG(trace, "Trace: you should not see this message"); + FINE_GRAIN_LOG(trace, "Trace: you should see this message"); SpdLoggerSharedPtr p = getFineGrainLogContext().getFineGrainLogEntry(__FILE__); ASSERT_NE(p, nullptr); - EXPECT_EQ(p->level(), spdlog::level::debug); + EXPECT_EQ(p->level(), spdlog::level::trace); } TEST(FineGrainLog, updateBasename) { diff --git a/test/common/common/matchers_test.cc b/test/common/common/matchers_test.cc index a07b7a00df35..68da1f973a22 100644 --- a/test/common/common/matchers_test.cc +++ b/test/common/common/matchers_test.cc @@ -184,6 +184,29 @@ TEST(MetadataTest, MatchPresentValue) { EXPECT_FALSE(Envoy::Matchers::MetadataMatcher(matcher).match(metadata)); } +TEST(MetadataTest, MatchStringOrBoolValue) { + envoy::config::core::v3::Metadata metadata; + Envoy::Config::Metadata::mutableMetadataValue(metadata, "envoy.filter.a", "label") + .set_string_value("test"); + Envoy::Config::Metadata::mutableMetadataValue(metadata, "envoy.filter.b", "label") + .set_bool_value(true); + Envoy::Config::Metadata::mutableMetadataValue(metadata, "envoy.filter.c", "label") + .set_bool_value(false); + + envoy::type::matcher::v3::MetadataMatcher matcher; + matcher.add_path()->set_key("label"); + + auto* or_match = matcher.mutable_value()->mutable_or_match(); + or_match->add_value_matchers()->mutable_string_match()->set_exact("test"); + or_match->add_value_matchers()->set_bool_match(true); + matcher.set_filter("envoy.filter.a"); + EXPECT_TRUE(Envoy::Matchers::MetadataMatcher(matcher).match(metadata)); + matcher.set_filter("envoy.filter.b"); + EXPECT_TRUE(Envoy::Matchers::MetadataMatcher(matcher).match(metadata)); + matcher.set_filter("envoy.filter.c"); + EXPECT_FALSE(Envoy::Matchers::MetadataMatcher(matcher).match(metadata)); +} + // Helper function to retrieve the reference of an entry in a ListMatcher from a MetadataMatcher. envoy::type::matcher::v3::ValueMatcher* listMatchEntry(envoy::type::matcher::v3::MetadataMatcher* matcher) { diff --git a/test/common/common/utility_speed_test.cc b/test/common/common/utility_speed_test.cc index ce9d8c790330..153db7dc846b 100644 --- a/test/common/common/utility_speed_test.cc +++ b/test/common/common/utility_speed_test.cc @@ -22,7 +22,7 @@ static size_t CacheControlLength = sizeof(CacheControl) - 1; // NOLINT(namespace-envoy) -static void BM_AccessLogDateTimeFormatter(benchmark::State& state) { +static void bmAccessLogDateTimeFormatter(benchmark::State& state) { int outputBytes = 0; // Generate a sequence of times for which the delta between each successive @@ -38,16 +38,17 @@ static void BM_AccessLogDateTimeFormatter(benchmark::State& state) { // the AccessLogDateTimeFormatter implementation is optimized further, we // should precompute a sequence of input timestamps so the benchmark's own // overhead won't obscure changes in the speed of the code being benchmarked. + UNREFERENCED_PARAMETER(_); time += std::chrono::milliseconds(static_cast(distribution(prng))); outputBytes += Envoy::AccessLogDateTimeFormatter::fromTime(time).length(); } benchmark::DoNotOptimize(outputBytes); } -BENCHMARK(BM_AccessLogDateTimeFormatter); +BENCHMARK(bmAccessLogDateTimeFormatter); -// This benchmark is basically similar with the above BM_AccessLogDateTimeFormatter, the only +// This benchmark is basically similar with the above bmAccessLogDateTimeFormatter, the only // difference is the format string input for the Envoy::DateFormatter. -static void BM_DateTimeFormatterWithSubseconds(benchmark::State& state) { +static void bmDateTimeFormatterWithSubseconds(benchmark::State& state) { int outputBytes = 0; Envoy::SystemTime time(std::chrono::seconds(1522796769)); @@ -55,18 +56,19 @@ static void BM_DateTimeFormatterWithSubseconds(benchmark::State& state) { std::uniform_int_distribution distribution(-10, 20); Envoy::DateFormatter date_formatter("%Y-%m-%dT%H:%M:%s.%3f"); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); time += std::chrono::milliseconds(static_cast(distribution(prng))); outputBytes += date_formatter.fromTime(time).length(); } benchmark::DoNotOptimize(outputBytes); } -BENCHMARK(BM_DateTimeFormatterWithSubseconds); +BENCHMARK(bmDateTimeFormatterWithSubseconds); -// This benchmark is basically similar with the above BM_DateTimeFormatterWithSubseconds, the +// This benchmark is basically similar with the above bmDateTimeFormatterWithSubseconds, the // differences are: 1. the format string input is long with duplicated subseconds. 2. The purpose // is to test DateFormatter.parse() which is called in constructor. // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_DateTimeFormatterWithLongSubsecondsString(benchmark::State& state) { +static void bmDateTimeFormatterWithLongSubsecondsString(benchmark::State& state) { int outputBytes = 0; Envoy::SystemTime time(std::chrono::seconds(1522796769)); @@ -81,16 +83,17 @@ static void BM_DateTimeFormatterWithLongSubsecondsString(benchmark::State& state absl::StrAppend(&input, duplicate_input); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); Envoy::DateFormatter date_formatter(input); time += std::chrono::milliseconds(static_cast(distribution(prng))); outputBytes += date_formatter.fromTime(time).length(); } benchmark::DoNotOptimize(outputBytes); } -BENCHMARK(BM_DateTimeFormatterWithLongSubsecondsString); +BENCHMARK(bmDateTimeFormatterWithLongSubsecondsString); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_DateTimeFormatterWithoutSubseconds(benchmark::State& state) { +static void bmDateTimeFormatterWithoutSubseconds(benchmark::State& state) { int outputBytes = 0; Envoy::SystemTime time(std::chrono::seconds(1522796769)); @@ -98,53 +101,58 @@ static void BM_DateTimeFormatterWithoutSubseconds(benchmark::State& state) { std::uniform_int_distribution distribution(-10, 20); Envoy::DateFormatter date_formatter("%Y-%m-%dT%H:%M:%s"); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); time += std::chrono::milliseconds(static_cast(distribution(prng))); outputBytes += date_formatter.fromTime(time).length(); } benchmark::DoNotOptimize(outputBytes); } -BENCHMARK(BM_DateTimeFormatterWithoutSubseconds); +BENCHMARK(bmDateTimeFormatterWithoutSubseconds); -static void BM_RTrimStringView(benchmark::State& state) { +static void bmRTrimStringView(benchmark::State& state) { int accum = 0; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); absl::string_view text(TextToTrim, TextToTrimLength); text = Envoy::StringUtil::rtrim(text); accum += TextToTrimLength - text.size(); } benchmark::DoNotOptimize(accum); } -BENCHMARK(BM_RTrimStringView); +BENCHMARK(bmRTrimStringView); -static void BM_RTrimStringViewAlreadyTrimmed(benchmark::State& state) { +static void bmRTrimStringViewAlreadyTrimmed(benchmark::State& state) { int accum = 0; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); absl::string_view text(AlreadyTrimmed, AlreadyTrimmedLength); text = Envoy::StringUtil::rtrim(text); accum += AlreadyTrimmedLength - text.size(); } benchmark::DoNotOptimize(accum); } -BENCHMARK(BM_RTrimStringViewAlreadyTrimmed); +BENCHMARK(bmRTrimStringViewAlreadyTrimmed); -static void BM_RTrimStringViewAlreadyTrimmedAndMakeString(benchmark::State& state) { +static void bmRTrimStringViewAlreadyTrimmedAndMakeString(benchmark::State& state) { int accum = 0; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); absl::string_view text(AlreadyTrimmed, AlreadyTrimmedLength); std::string string_copy = std::string(Envoy::StringUtil::rtrim(text)); accum += AlreadyTrimmedLength - string_copy.size(); } benchmark::DoNotOptimize(accum); } -BENCHMARK(BM_RTrimStringViewAlreadyTrimmedAndMakeString); +BENCHMARK(bmRTrimStringViewAlreadyTrimmedAndMakeString); -static void BM_FindToken(benchmark::State& state) { +static void bmFindToken(benchmark::State& state) { const absl::string_view cache_control(CacheControl, CacheControlLength); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); RELEASE_ASSERT(Envoy::StringUtil::findToken(cache_control, ",", "no-transform"), ""); } } -BENCHMARK(BM_FindToken); +BENCHMARK(bmFindToken); static bool nextToken(absl::string_view& str, char delim, bool strip_whitespace, absl::string_view* token) { @@ -180,18 +188,20 @@ static bool findTokenWithoutSplitting(absl::string_view str, char delim, absl::s return false; } -static void BM_FindTokenWithoutSplitting(benchmark::State& state) { +static void bmFindTokenWithoutSplitting(benchmark::State& state) { const absl::string_view cache_control(CacheControl, CacheControlLength); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); RELEASE_ASSERT(findTokenWithoutSplitting(cache_control, ',', "no-transform", true), ""); } } -BENCHMARK(BM_FindTokenWithoutSplitting); +BENCHMARK(bmFindTokenWithoutSplitting); -static void BM_FindTokenValueNestedSplit(benchmark::State& state) { +static void bmFindTokenValueNestedSplit(benchmark::State& state) { const absl::string_view cache_control(CacheControl, CacheControlLength); absl::string_view max_age; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); for (absl::string_view token : Envoy::StringUtil::splitToken(cache_control, ",")) { auto name_value = Envoy::StringUtil::splitToken(token, "="); if ((name_value.size() == 2) && (Envoy::StringUtil::trim(name_value[0]) == "max-age")) { @@ -201,10 +211,11 @@ static void BM_FindTokenValueNestedSplit(benchmark::State& state) { RELEASE_ASSERT(max_age == "300", ""); } } -BENCHMARK(BM_FindTokenValueNestedSplit); +BENCHMARK(bmFindTokenValueNestedSplit); -static void BM_FindTokenValueSearchForEqual(benchmark::State& state) { +static void bmFindTokenValueSearchForEqual(benchmark::State& state) { for (auto _ : state) { + UNREFERENCED_PARAMETER(_); const absl::string_view cache_control(CacheControl, CacheControlLength); absl::string_view max_age; for (absl::string_view token : Envoy::StringUtil::splitToken(cache_control, ",")) { @@ -217,10 +228,11 @@ static void BM_FindTokenValueSearchForEqual(benchmark::State& state) { RELEASE_ASSERT(max_age == "300", ""); } } -BENCHMARK(BM_FindTokenValueSearchForEqual); +BENCHMARK(bmFindTokenValueSearchForEqual); -static void BM_FindTokenValueNoSplit(benchmark::State& state) { +static void bmFindTokenValueNoSplit(benchmark::State& state) { for (auto _ : state) { + UNREFERENCED_PARAMETER(_); absl::string_view cache_control(CacheControl, CacheControlLength); absl::string_view max_age; for (absl::string_view token; nextToken(cache_control, ',', true, &token);) { @@ -232,9 +244,9 @@ static void BM_FindTokenValueNoSplit(benchmark::State& state) { RELEASE_ASSERT(max_age == "300", ""); } } -BENCHMARK(BM_FindTokenValueNoSplit); +BENCHMARK(bmFindTokenValueNoSplit); -static void BM_RemoveTokensLong(benchmark::State& state) { +static void bmRemoveTokensLong(benchmark::State& state) { auto size = state.range(0); std::string input(size, ','); std::vector to_remove; @@ -250,14 +262,16 @@ static void BM_RemoveTokensLong(benchmark::State& state) { input.append(to_remove[i]); } for (auto _ : state) { + UNREFERENCED_PARAMETER(_); Envoy::StringUtil::removeTokens(input, ",", to_remove_set, ","); state.SetBytesProcessed(static_cast(state.iterations()) * input.size()); } } -BENCHMARK(BM_RemoveTokensLong)->Range(8, 8 << 10); +BENCHMARK(bmRemoveTokensLong)->Range(8, 8 << 10); -static void BM_IntervalSetInsert17(benchmark::State& state) { +static void bmIntervalSetInsert17(benchmark::State& state) { for (auto _ : state) { + UNREFERENCED_PARAMETER(_); Envoy::IntervalSetImpl interval_set; interval_set.insert(7, 10); interval_set.insert(-2, -1); @@ -278,28 +292,30 @@ static void BM_IntervalSetInsert17(benchmark::State& state) { interval_set.insert(24, 9223372036854775805UL); } } -BENCHMARK(BM_IntervalSetInsert17); +BENCHMARK(bmIntervalSetInsert17); -static void BM_IntervalSet4ToVector(benchmark::State& state) { +static void bmIntervalSet4ToVector(benchmark::State& state) { Envoy::IntervalSetImpl interval_set; interval_set.insert(7, 10); interval_set.insert(-2, -1); interval_set.insert(22, 23); interval_set.insert(8, 15); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); benchmark::DoNotOptimize(interval_set.toVector()); } } -BENCHMARK(BM_IntervalSet4ToVector); +BENCHMARK(bmIntervalSet4ToVector); -static void BM_IntervalSet50ToVector(benchmark::State& state) { +static void bmIntervalSet50ToVector(benchmark::State& state) { Envoy::IntervalSetImpl interval_set; for (size_t i = 0; i < 100; i += 2) { interval_set.insert(i, i + 1); } for (auto _ : state) { + UNREFERENCED_PARAMETER(_); benchmark::DoNotOptimize(interval_set.toVector()); } } -BENCHMARK(BM_IntervalSet50ToVector); +BENCHMARK(bmIntervalSet50ToVector); } // namespace Envoy diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 13aed4619e3d..08fb0d1208eb 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -77,7 +77,7 @@ envoy_cc_test( envoy_cc_test_library( name = "subscription_test_harness", - srcs = ["subscription_test_harness.h"], + hdrs = ["subscription_test_harness.h"], deps = [ "//source/common/config:utility_lib", "//test/mocks/stats:stats_mocks", @@ -138,8 +138,8 @@ envoy_cc_test( "//test/test_common:environment_lib", "//test/test_common:logging_lib", "//test/test_common:utility_lib", - "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", - "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_github_cncf_xds//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/v3:pkg_cc_proto", "@envoy_api//envoy/api/v2:pkg_cc_proto", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", diff --git a/test/common/config/xds_resource_test.cc b/test/common/config/xds_resource_test.cc index 0cbe0954743f..d6058df8da38 100644 --- a/test/common/config/xds_resource_test.cc +++ b/test/common/config/xds_resource_test.cc @@ -43,7 +43,7 @@ TEST(XdsResourceIdentifierTest, DecodeEncode) { XdsResourceIdentifier::EncodeOptions encode_options; encode_options.sort_context_params_ = true; for (const std::string& uri : uris) { - EXPECT_EQ(uri, XdsResourceIdentifier::encodeUrn(XdsResourceIdentifier::decodeUrn(uri), + EXPECT_EQ(uri, XdsResourceIdentifier::encodeUrn(XdsResourceIdentifier::decodeUrn(uri).value(), encode_options)); EXPECT_EQ(uri, XdsResourceIdentifier::encodeUrl(XdsResourceIdentifier::decodeUrl(uri), encode_options)); @@ -53,7 +53,8 @@ TEST(XdsResourceIdentifierTest, DecodeEncode) { // Corner cases around path-identifier encoding/decoding. TEST(XdsResourceIdentifierTest, PathDividerEscape) { { - const auto resource_name = XdsResourceIdentifier::decodeUrn("xdstp:///type/foo%2Fbar/baz"); + const auto resource_name = + XdsResourceIdentifier::decodeUrn("xdstp:///type/foo%2Fbar/baz").value(); EXPECT_EQ("foo/bar/baz", resource_name.id()); EXPECT_EQ("xdstp:///type/foo/bar/baz", XdsResourceIdentifier::encodeUrn(resource_name)); } @@ -66,7 +67,8 @@ TEST(XdsResourceIdentifierTest, PathDividerEscape) { // Validate that URN decoding behaves as expected component-wise. TEST(XdsResourceNameTest, DecodeSuccess) { - const auto resource_name = XdsResourceIdentifier::decodeUrn(EscapedUrnWithManyQueryParams); + const auto resource_name = + XdsResourceIdentifier::decodeUrn(EscapedUrnWithManyQueryParams).value(); EXPECT_EQ("f123%/?#o", resource_name.authority()); EXPECT_EQ("envoy.config.listener.v3.Listener", resource_name.resource_type()); EXPECT_EQ(resource_name.id(), "b%:?#[]ar//baz"); @@ -98,7 +100,7 @@ TEST(XdsResourceLocatorTest, DecodeSuccess) { // Validate that the URN decoding behaves with a near-empty xDS resource name. TEST(XdsResourceLocatorTest, DecodeEmpty) { const auto resource_name = - XdsResourceIdentifier::decodeUrn("xdstp:///envoy.config.listener.v3.Listener"); + XdsResourceIdentifier::decodeUrn("xdstp:///envoy.config.listener.v3.Listener").value(); EXPECT_TRUE(resource_name.authority().empty()); EXPECT_EQ("envoy.config.listener.v3.Listener", resource_name.resource_type()); EXPECT_TRUE(resource_name.id().empty()); @@ -119,33 +121,28 @@ TEST(XdsResourceNameTest, DecodeEmpty) { // Negative tests for URN decoding. TEST(XdsResourceNameTest, DecodeFail) { { - EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrn("foo://"), - XdsResourceIdentifier::DecodeException, - "foo:// does not have an xdstp: scheme"); + EXPECT_EQ(XdsResourceIdentifier::decodeUrn("foo://").status().message(), + "foo:// does not have an xdstp: scheme"); } { - EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrn("xdstp://foo"), - XdsResourceIdentifier::DecodeException, - "Resource type missing from /"); + EXPECT_EQ(XdsResourceIdentifier::decodeUrn("xdstp://foo").status().message(), + "Resource type missing from /"); } } // Negative tests for URL decoding. TEST(XdsResourceLocatorTest, DecodeFail) { { - EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrl("foo://"), - XdsResourceIdentifier::DecodeException, + EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrl("foo://"), EnvoyException, "foo:// does not have a xdstp:, http: or file: scheme"); } { - EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrl("xdstp://foo"), - XdsResourceIdentifier::DecodeException, + EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrl("xdstp://foo"), EnvoyException, "Resource type missing from /"); } { EXPECT_THROW_WITH_MESSAGE(XdsResourceIdentifier::decodeUrl("xdstp://foo/some-type#bar=baz"), - XdsResourceIdentifier::DecodeException, - "Unknown fragment component bar=baz"); + EnvoyException, "Unknown fragment component bar=baz"); } } diff --git a/test/common/conn_pool/conn_pool_base_test.cc b/test/common/conn_pool/conn_pool_base_test.cc index c750aa8e078a..29b0b11c4663 100644 --- a/test/common/conn_pool/conn_pool_base_test.cc +++ b/test/common/conn_pool/conn_pool_base_test.cc @@ -473,6 +473,19 @@ TEST_F(ConnPoolImplDispatcherBaseTest, NoAvailableStreams) { pool_.destructAllConnections(); } +// Verify that not fully connected active client calls +// idle callbacks upon destruction. +TEST_F(ConnPoolImplBaseTest, PoolIdleNotConnected) { + auto active_client = std::make_unique>(pool_, stream_limit_, + concurrent_streams_, false); + + testing::MockFunction idle_pool_callback; + EXPECT_CALL(idle_pool_callback, Call()); + pool_.addIdleCallbackImpl(idle_pool_callback.AsStdFunction()); + + pool_.drainConnectionsImpl(Envoy::ConnectionPool::DrainBehavior::DrainAndDelete); +} + // Remote close simulates the peer closing the connection. TEST_F(ConnPoolImplBaseTest, PoolIdleCallbackTriggeredRemoteClose) { EXPECT_CALL(dispatcher_, createTimer_(_)).Times(AnyNumber()); @@ -491,13 +504,13 @@ TEST_F(ConnPoolImplBaseTest, PoolIdleCallbackTriggeredRemoteClose) { pool_.onStreamClosed(*clients_.back(), false); // Now that the last connection is closed, while there are no requests, the pool becomes idle. + // idle_pool_callback should be called once. testing::MockFunction idle_pool_callback; EXPECT_CALL(idle_pool_callback, Call()); pool_.addIdleCallbackImpl(idle_pool_callback.AsStdFunction()); dispatcher_.clearDeferredDeleteList(); clients_.back()->onEvent(Network::ConnectionEvent::RemoteClose); - EXPECT_CALL(idle_pool_callback, Call()); pool_.drainConnectionsImpl(Envoy::ConnectionPool::DrainBehavior::DrainAndDelete); } @@ -519,13 +532,13 @@ TEST_F(ConnPoolImplBaseTest, PoolIdleCallbackTriggeredLocalClose) { pool_.onStreamClosed(*clients_.back(), false); // Now that the last connection is closed, while there are no requests, the pool becomes idle. + // idle_pool_callback should be called once. testing::MockFunction idle_pool_callback; EXPECT_CALL(idle_pool_callback, Call()); pool_.addIdleCallbackImpl(idle_pool_callback.AsStdFunction()); dispatcher_.clearDeferredDeleteList(); clients_.back()->onEvent(Network::ConnectionEvent::LocalClose); - EXPECT_CALL(idle_pool_callback, Call()); pool_.drainConnectionsImpl(Envoy::ConnectionPool::DrainBehavior::DrainAndDelete); } diff --git a/test/common/crypto/utility_test.cc b/test/common/crypto/utility_test.cc index 46eabc6b03cb..005c820ba86a 100644 --- a/test/common/crypto/utility_test.cc +++ b/test/common/crypto/utility_test.cc @@ -72,25 +72,22 @@ TEST(UtilityTest, TestImportPublicKey) { wrapper = Common::Crypto::Access::getTyped(*crypto_ptr); pkey = wrapper->getEVP_PKEY(); EXPECT_EQ(nullptr, pkey); + + EVP_PKEY* empty_pkey = EVP_PKEY_new(); + wrapper->setEVP_PKEY(empty_pkey); + pkey = wrapper->getEVP_PKEY(); + EXPECT_NE(nullptr, pkey); } TEST(UtilityTest, TestVerifySignature) { - auto key = "30820122300d06092a864886f70d01010105000382010f003082010a0282010100a7471266d01d160308d" - "73409c06f2e8d35c531c458d3e480e9f3191847d062ec5ccff7bc51e949d5f2c3540c189a4eca1e8633a6" - "2cf2d0923101c27e38013e71de9ae91a704849bff7fbe2ce5bf4bd666fd9731102a53193fe5a9a5a50644" - "ff8b1183fa897646598caad22a37f9544510836372b44c58c98586fb7144629cd8c9479592d996d32ff6d" - "395c0b8442ec5aa1ef8051529ea0e375883cefc72c04e360b4ef8f5760650589ca814918f678eee39b884" - "d5af8136a9630a6cc0cde157dc8e00f39540628d5f335b2c36c54c7c8bc3738a6b21acff815405afa28e5" - "183f550dac19abcf1145a7f9ced987db680e4a229cac75dee347ec9ebce1fc3dbbbb0203010001"; - auto hash_func = "sha256"; - auto signature = - "345ac3a167558f4f387a81c2d64234d901a7ceaa544db779d2f797b0ea4ef851b740905a63e2f4d5af42cee093a2" - "9c7155db9a63d3d483e0ef948f5ac51ce4e10a3a6606fd93ef68ee47b30c37491103039459122f78e1c7ea71a1a5" - "ea24bb6519bca02c8c9915fe8be24927c91812a13db72dbcb500103a79e8f67ff8cb9e2a631974e0668ab3977bf5" - "70a91b67d1b6bcd5dce84055f21427d64f4256a042ab1dc8e925d53a769f6681a873f5859693a7728fcbe95beace" - "1563b5ffbcd7c93b898aeba31421dafbfadeea50229c49fd6c445449314460f3d19150bd29a91333beaced557ed6" - "295234f7c14fa46303b7e977d2c89ba8a39a46a35f33eb07a332"; - auto data = "hello"; + auto key = "30820122300d06092a864886f70d01010105000382010f003082010a0282010100ba10ebe185465586093" + "228fb3b0093c560853b7ebf28497aefb9961a6cc886dd3f6d3278a93244fa5084a9c263bd57feb4ea1868" + "aa8a2718aa46708c803ce49318619982ba06a6615d24bb853c0fb85ebed833a802245e4518d4e2ba10da1" + "f22c732505433c558bed8895eb1e97cb5d65f821be9330143e93a738ef6896165879f692d75c2d7928e01" + "fd7fe601d16931bdd876c7b15b741e48546fe80db45df56e22ed2fa974ab937af7644d20834f41a61aeb9" + "a70d0248d274642b14ed6585892403bed8e03a9a12485ae44e3d39ab53e5bd70dee58476fb81860a18679" + "9429b71f79f204894cf21d31cc19118d547bb1b946532d080e074ec97e23667818490203010001"; + auto data = "hello\n"; Common::Crypto::CryptoObjectPtr crypto_ptr( Common::Crypto::UtilitySingleton::get().importPublicKey(Hex::decode(key))); @@ -98,30 +95,92 @@ TEST(UtilityTest, TestVerifySignature) { std::vector text(data, data + strlen(data)); - auto sig = Hex::decode(signature); - auto result = UtilitySingleton::get().verifySignature(hash_func, *crypto, sig, text); + // Map of hash function names and their respective signatures + std::map hashSignatures = { + {"sha1", + "9ed4cc60e8b2b51ff00b1cc06c628263476c8be6136510fc47e4668423c3492d8711489000b32163cd022661049" + "360fa0b8366692e41a4d4fb4237479694484012833ccc88938c1a471e33757c198b42623961d91cdf41ca01b375" + "002930b37a62377517cad297d5658e31610acaf79216a3d5c0afe4715dfe6e5bad3c20dac77bbbd2e7a4cb44172" + "abb620fe60b968426726ed25d2aed99abf9e8f705b7021e3448a78824e6982e9d14dbd0a317f45d42198f785f3b" + "0ca8e311695cedb4ce19626c246b8010a5de7b7978b8a3b56c1558f87bd658023e52b6e155c39594bae6e3cbf77" + "9d487a9ce3bffd7d8a2641f336771bec5c9d4a40dc8d4163fd2c1dd3b"}, // Signature for SHA-1 hash + // function + {"sha224", + "03813d50082dfb43444ebf47788e69271ebbfa17e64f7e7600ce761bd89ff459e21ecea6bc7de8396cfd80fe0ee" + "3d92967f0c467c930f7d0b1b1734e5d139ffaa5d84c5047cb38793b152ba8b284ec6d31e0b410b1e1a06ffda171" + "42c83b30593ac02a2f07f8e863ade752d23b2f41d56bd1ab6328c46de47233e2e2e4189e5bd3bce0b0428f485ff" + "e75f7343d89b376bd7dc2953467e63f5c1eb9279ca349fa74535d37e80f57216b8b73b0e67b32f0f18f41bae6a7" + "6e350dbc6188525eda1c79c0977bf433bb170d49de47985bc14a418d7a03d72eda740666dc05185fdcea6bb2914" + "d7bd0271bd06b3de72bc9db82d625799bf3441e2abff8fcd273efe6c7"}, // Signature for SHA-224 hash + // function + {"sha256", + "504c24addc615c01c915e3244b8248b470a41c82faa6a82526ddd8f21e197adae6c8b985e40960157d47f336d8b" + "a31ce4b3b1795379a7415af57daa757d3027e870b3c6644e749b583c51a16f9984afd39c909325d271d8d4c8d00" + "6288bd8f7945aa24a783613ecece95a9692b3b56dd1d831fc06d82eca40fd432a15a6cdb837d7ce378ac889c4ab" + "00b0c1f9c2be279952696b70c9ea2bb014d6f20b72ed4917904d5f24d5776058bd11121f3ed02e03c384cf42734" + "6b1d300867969f22e27aa9f0607344cc9d8e9a90802e97ac39af9324f60ddad24682e48424346dd5a53fe550370" + "bdf9f94dec8a80810edd648a33cc767f9e4328660e3ee1be8b47e9cfa"}, // Signature for SHA-256 hash + // function + {"sha384", + "91491000964d11f12ff658101f7c21ce8b37da6fff3abe836859c17e268858d77ee9c96e88ca86b36bca2b06472" + "a1f551d642bf056f1241d11d5b853e1458c2a9d86f9096e872c81480a06000346a61e51cb94e5174a98b9daacf5" + "204dd28e081c99a72066c406334a046ae5f3eb0e0eea86f0ae7eeb27d5dea245e850d05cc6c972f8249b8a4f018" + "6531735137a2e45f1f6410bf8e2382e95b57618802a0068ca197b2d8bcca53d6738e04b86ed9c69d45dad6d9bd7" + "be55596a719f12531d363e74c9d659738eaa50ab854869416f2b445f054aa2c1223c9edd223cbc5ac0d3582cb9b" + "5af494138bd6ace049e3ab326bb23fadd3dbcd74e9a3b372843f926ec"}, // Signature for SHA-384 hash + // function + {"sha512", + "5d001462d000c0aa23d931f6cce5def5f8472c7aaa0185cab87b256697b7a0c8fb6a4c9f84debf1b4ff3bf53213" + "0bcb25f724e09a74b5d5c915feb9c943a005ab879078b2fbcab0828e128ebfb7befee25d219bcd6cf1ad1f62b94" + "b460021eebc4c249e34219c71b4f526628976ecea8fb70e1166053da212747e8ba4b29cb91fa6541d53d3400a9d" + "34881a227e01eebf157104d84555c9e20320280723a72d3a724eba99f1fb14d59399321636ebfe7070d83d7b6b2" + "381fcdb683fb73e7796d36fe45dfb14a622c3426fe5bf69af9c24f9f1b30affad129b5f2b7dfa6fa384c73ad212" + "f414606882c3f9133d4702f487f9b08df8d0265fe5e8e12a11c6cb35c"}, // Signature for SHA-512 hash + // function + }; + + // Loop through each hash function and its signature + for (const auto& entry : hashSignatures) { + const std::string& hash_func = entry.first; + const std::string& signature = entry.second; + auto sig = Hex::decode(signature); + + auto result = UtilitySingleton::get().verifySignature(hash_func, *crypto, sig, text); + EXPECT_EQ(true, result.result_); + EXPECT_EQ("", result.error_message_); + } - EXPECT_EQ(true, result.result_); - EXPECT_EQ("", result.error_message_); + auto signature = + "504c24addc615c01c915e3244b8248b470a41c82faa6a82526ddd8f21e197adae6c8b985e40960157d47f336d8ba" + "31ce4b3b1795379a7415af57daa757d3027e870b3c6644e749b583c51a16f9984afd39c909325d271d8d4c8d0062" + "88bd8f7945aa24a783613ecece95a9692b3b56dd1d831fc06d82eca40fd432a15a6cdb837d7ce378ac889c4ab00b" + "0c1f9c2be279952696b70c9ea2bb014d6f20b72ed4917904d5f24d5776058bd11121f3ed02e03c384cf427346b1d" + "300867969f22e27aa9f0607344cc9d8e9a90802e97ac39af9324f60ddad24682e48424346dd5a53fe550370bdf9f" + "94dec8a80810edd648a33cc767f9e4328660e3ee1be8b47e9cfa"; + auto sig = Hex::decode(signature); - result = UtilitySingleton::get().verifySignature("unknown", *crypto, sig, text); + // Test an unknown hash function + auto result = UtilitySingleton::get().verifySignature("unknown", *crypto, sig, text); EXPECT_EQ(false, result.result_); EXPECT_EQ("unknown is not supported.", result.error_message_); + // Test with an empty crypto object auto empty_crypto = std::make_unique(); - result = UtilitySingleton::get().verifySignature(hash_func, *empty_crypto, sig, text); + result = UtilitySingleton::get().verifySignature("sha256", *empty_crypto, sig, text); EXPECT_EQ(false, result.result_); EXPECT_EQ("Failed to initialize digest verify.", result.error_message_); + // Test with incorrect data data = "baddata"; text = std::vector(data, data + strlen(data)); - result = UtilitySingleton::get().verifySignature(hash_func, *crypto, sig, text); + result = UtilitySingleton::get().verifySignature("sha256", *crypto, sig, text); EXPECT_EQ(false, result.result_); EXPECT_EQ("Failed to verify digest. Error code: 0", result.error_message_); + // Test with incorrect signature data = "hello"; text = std::vector(data, data + strlen(data)); - result = UtilitySingleton::get().verifySignature(hash_func, *crypto, Hex::decode("000000"), text); + result = UtilitySingleton::get().verifySignature("sha256", *crypto, Hex::decode("000000"), text); EXPECT_EQ(false, result.result_); EXPECT_EQ("Failed to verify digest. Error code: 0", result.error_message_); } diff --git a/test/common/filesystem/BUILD b/test/common/filesystem/BUILD index 82e28ebda60d..8de735cd126d 100644 --- a/test/common/filesystem/BUILD +++ b/test/common/filesystem/BUILD @@ -23,6 +23,7 @@ envoy_cc_test( deps = [ "//source/common/filesystem:directory_lib", "//test/test_common:environment_lib", + "//test/test_common:status_utility_lib", ], ) diff --git a/test/common/filesystem/directory_test.cc b/test/common/filesystem/directory_test.cc index 75f3715dc793..60d0fed65d5f 100644 --- a/test/common/filesystem/directory_test.cc +++ b/test/common/filesystem/directory_test.cc @@ -6,6 +6,7 @@ #include "source/common/filesystem/directory.h" #include "test/test_common/environment.h" +#include "test/test_common/status_utility.h" #include "test/test_common/utility.h" #include "absl/container/node_hash_set.h" @@ -15,6 +16,9 @@ namespace Envoy { namespace Filesystem { +using StatusHelpers::HasStatus; +using testing::HasSubstr; + // NOLINTNEXTLINE(readability-identifier-naming) void PrintTo(const DirectoryEntry& entry, std::ostream* os) { *os << "{name=" << entry.name_ << ", type=" << static_cast(entry.type_) << ", size="; @@ -240,6 +244,40 @@ TEST_F(DirectoryTest, DirectoryWithBrokenSymlink) { EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } +#ifndef WIN32 +// Test that removing a file while iterating continues to the next file. +TEST_F(DirectoryTest, FileDeletedWhileIterating) { + addFiles({"file", "file2"}); + DirectoryIteratorImpl dir_iterator(dir_path_); + EntrySet found; + TestEnvironment::removePath(dir_path_ + "/file"); + TestEnvironment::removePath(dir_path_ + "/file2"); + while (!(*dir_iterator).name_.empty()) { + found.insert(*dir_iterator); + ++dir_iterator; + } + // Test environment can potentially list files in any order, so if the + // first file was one of the temporary files it will still be in the + // set. The important thing is that at least one of them is *not* in + // the set, and we didn't crash. + EXPECT_THAT(found, testing::AnyOf( + EntrySet{ + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + }, + EntrySet{ + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"file", FileType::Regular, 0}, + }, + EntrySet{ + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"file1", FileType::Regular, 0}, + })); +} +#endif + // Test that we can list an empty directory TEST_F(DirectoryTest, DirectoryWithEmptyDirectory) { const EntrySet expected = { @@ -249,18 +287,16 @@ TEST_F(DirectoryTest, DirectoryWithEmptyDirectory) { EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } -// Test that the constructor throws an exception when given a non-existing path +// Test that the status is an error when given a non-existing path TEST(DirectoryIteratorImpl, NonExistingDir) { const std::string dir_path("some/non/existing/dir"); - + DirectoryIteratorImpl dir_iterator(dir_path); #ifdef WIN32 - EXPECT_THROW_WITH_MESSAGE( - DirectoryIteratorImpl dir_iterator(dir_path), EnvoyException, - fmt::format("unable to open directory {}: {}", dir_path, ERROR_PATH_NOT_FOUND)); + EXPECT_THAT(dir_iterator.status(), + HasStatus(absl::StatusCode::kUnknown, HasSubstr("unable to open directory"))); #else - EXPECT_THROW_WITH_MESSAGE( - DirectoryIteratorImpl dir_iterator(dir_path), EnvoyException, - fmt::format("unable to open directory {}: No such file or directory", dir_path)); + EXPECT_THAT(dir_iterator.status(), + HasStatus(absl::StatusCode::kNotFound, HasSubstr("unable to open directory"))); #endif } @@ -284,8 +320,8 @@ TEST_F(DirectoryTest, Fifo) { // instead by directly calling the private function. TEST_F(DirectoryTest, MakeEntryThrowsOnStatFailure) { Directory directory(dir_path_); - EXPECT_THROW_WITH_REGEX(directory.begin().makeEntry("foo"), EnvoyException, - "unable to stat file: '.*foo' .*"); + EXPECT_THAT(directory.begin().makeEntry("foo"), + HasStatus(absl::StatusCode::kNotFound, HasSubstr("unable to stat file"))); } #endif diff --git a/test/common/filesystem/filesystem_impl_test.cc b/test/common/filesystem/filesystem_impl_test.cc index f909d987cbb7..6163ab29b144 100644 --- a/test/common/filesystem/filesystem_impl_test.cc +++ b/test/common/filesystem/filesystem_impl_test.cc @@ -82,7 +82,7 @@ TEST_F(FileSystemImplTest, FileReadToEndSuccess) { const std::string data = "test string\ntest"; const std::string file_path = TestEnvironment::writeStringToFileForTest("test_envoy", data); - EXPECT_EQ(data, file_system_.fileReadToEnd(file_path)); + EXPECT_EQ(data, file_system_.fileReadToEnd(file_path).value()); } // Files are read into std::string; verify that all bytes (e.g., non-ascii characters) come back @@ -94,7 +94,7 @@ TEST_F(FileSystemImplTest, FileReadToEndSuccessBinary) { } const std::string file_path = TestEnvironment::writeStringToFileForTest("test_envoy", data); - const std::string read = file_system_.fileReadToEnd(file_path); + const std::string read = file_system_.fileReadToEnd(file_path).value(); const std::vector binary_read(read.begin(), read.end()); EXPECT_EQ(binary_read.size(), 256); for (unsigned i = 0; i < 256; i++) { @@ -102,10 +102,12 @@ TEST_F(FileSystemImplTest, FileReadToEndSuccessBinary) { } } -TEST_F(FileSystemImplTest, FileReadToEndDoesNotExist) { +TEST_F(FileSystemImplTest, FileReadToEndPathDoesNotExist) { unlink(TestEnvironment::temporaryPath("envoy_this_not_exist").c_str()); - EXPECT_THROW(file_system_.fileReadToEnd(TestEnvironment::temporaryPath("envoy_this_not_exist")), - EnvoyException); + EXPECT_THAT(file_system_.fileReadToEnd(TestEnvironment::temporaryPath("envoy_this_not_exist")) + .status() + .message(), + testing::StartsWith("Invalid path:")); } #ifndef WIN32 @@ -117,14 +119,14 @@ TEST_F(FileSystemImplTest, DISABLED_FileReadToEndNotReadable) { std::filesystem::permissions(file_path, std::filesystem::perms::owner_all, std::filesystem::perm_options::remove); - EXPECT_THROW(file_system_.fileReadToEnd(file_path), EnvoyException); + EXPECT_FALSE(file_system_.fileReadToEnd(file_path).status().ok()); } #endif TEST_F(FileSystemImplTest, FileReadToEndDenylisted) { - EXPECT_THROW(file_system_.fileReadToEnd("/dev/urandom"), EnvoyException); - EXPECT_THROW(file_system_.fileReadToEnd("/proc/cpuinfo"), EnvoyException); - EXPECT_THROW(file_system_.fileReadToEnd("/sys/block/sda/dev"), EnvoyException); + EXPECT_FALSE(file_system_.fileReadToEnd("/dev/urandom").status().ok()); + EXPECT_FALSE(file_system_.fileReadToEnd("/proc/cpuinfo").status().ok()); + EXPECT_FALSE(file_system_.fileReadToEnd("/sys/block/sda/dev").status().ok()); } #ifndef WIN32 @@ -143,36 +145,36 @@ TEST_F(FileSystemImplTest, CanonicalPathFail) { TEST_F(FileSystemImplTest, SplitPathFromFilename) { PathSplitResult result; - result = file_system_.splitPathFromFilename("/foo/bar/baz"); + result = file_system_.splitPathFromFilename("/foo/bar/baz").value(); EXPECT_EQ(result.directory_, "/foo/bar"); EXPECT_EQ(result.file_, "baz"); - result = file_system_.splitPathFromFilename("/foo/bar"); + result = file_system_.splitPathFromFilename("/foo/bar").value(); EXPECT_EQ(result.directory_, "/foo"); EXPECT_EQ(result.file_, "bar"); - result = file_system_.splitPathFromFilename("/foo"); + result = file_system_.splitPathFromFilename("/foo").value(); EXPECT_EQ(result.directory_, "/"); EXPECT_EQ(result.file_, "foo"); - result = file_system_.splitPathFromFilename("/"); + result = file_system_.splitPathFromFilename("/").value(); EXPECT_EQ(result.directory_, "/"); EXPECT_EQ(result.file_, ""); - EXPECT_THROW(file_system_.splitPathFromFilename("nopathdelimeter"), EnvoyException); + EXPECT_FALSE(file_system_.splitPathFromFilename("nopathdelimeter").ok()); #ifdef WIN32 - result = file_system_.splitPathFromFilename("c:\\foo/bar"); + result = file_system_.splitPathFromFilename("c:\\foo/bar").value(); EXPECT_EQ(result.directory_, "c:\\foo"); EXPECT_EQ(result.file_, "bar"); - result = file_system_.splitPathFromFilename("c:/foo\\bar"); + result = file_system_.splitPathFromFilename("c:/foo\\bar").value(); EXPECT_EQ(result.directory_, "c:/foo"); EXPECT_EQ(result.file_, "bar"); - result = file_system_.splitPathFromFilename("c:\\foo"); + result = file_system_.splitPathFromFilename("c:\\foo").value(); EXPECT_EQ(result.directory_, "c:\\"); EXPECT_EQ(result.file_, "foo"); - result = file_system_.splitPathFromFilename("c:foo"); + result = file_system_.splitPathFromFilename("c:foo").value(); EXPECT_EQ(result.directory_, "c:"); EXPECT_EQ(result.file_, "foo"); - result = file_system_.splitPathFromFilename("c:"); + result = file_system_.splitPathFromFilename("c:").value(); EXPECT_EQ(result.directory_, "c:"); EXPECT_EQ(result.file_, ""); - result = file_system_.splitPathFromFilename("\\\\?\\C:\\"); + result = file_system_.splitPathFromFilename("\\\\?\\C:\\").value(); EXPECT_EQ(result.directory_, "\\\\?\\C:\\"); EXPECT_EQ(result.file_, ""); #endif diff --git a/test/common/filter/config_discovery_impl_test.cc b/test/common/filter/config_discovery_impl_test.cc index 1071162b9553..7e1b1c2333fa 100644 --- a/test/common/filter/config_discovery_impl_test.cc +++ b/test/common/filter/config_discovery_impl_test.cc @@ -32,7 +32,6 @@ using testing::_; using testing::InSequence; using testing::Invoke; -using testing::Return; using testing::ReturnRef; namespace Envoy { @@ -50,13 +49,13 @@ class TestHttpFilterFactory : public TestFilterFactory, public Server::Configuration::NamedHttpFilterConfigFactory, public Server::Configuration::UpstreamHttpFilterConfigFactory { public: - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, Server::Configuration::FactoryContext&) override { created_ = true; return [](Http::FilterChainFactoryCallbacks&) -> void {}; } - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, Server::Configuration::UpstreamFactoryContext&) override { created_ = true; @@ -186,7 +185,7 @@ template (); @@ -290,7 +289,7 @@ class HttpFilterConfigDiscoveryImplTest } }; -// HTTP upstream filter test +// Upstream HTTP filter test class HttpUpstreamFilterConfigDiscoveryImplTest : public FilterConfigDiscoveryImplTest< NamedHttpFilterFactoryCb, Server::Configuration::UpstreamFactoryContext, @@ -324,7 +323,7 @@ class NetworkFilterConfigDiscoveryImplTest } }; -// Network upstream filter test +// Upstream network filter test class NetworkUpstreamFilterConfigDiscoveryImplTest : public FilterConfigDiscoveryImplTest< Network::FilterFactoryCb, Server::Configuration::UpstreamFactoryContext, @@ -655,9 +654,9 @@ TYPED_TEST(FilterConfigDiscoveryImplTestParameter, TerminalFilterInvalid) { } EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(config_discovery_test.callbacks_ - ->onConfigUpdate(decoded_resources.refvec_, response.version_info()) - .ok()), + config_discovery_test.callbacks_ + ->onConfigUpdate(decoded_resources.refvec_, response.version_info()) + .IgnoreError(), EnvoyException, "Error: terminal filter named foo of type envoy.test.filter must be the last filter " "in a " + diff --git a/test/common/formatter/command_extension.cc b/test/common/formatter/command_extension.cc index 6d90df87fa51..48eac9d69b5f 100644 --- a/test/common/formatter/command_extension.cc +++ b/test/common/formatter/command_extension.cc @@ -27,7 +27,7 @@ FormatterProviderPtr TestCommandParser::parse(const std::string& command, const CommandParserPtr TestCommandFactory::createCommandParserFromProto(const Protobuf::Message& message, - Server::Configuration::CommonFactoryContext&) { + Server::Configuration::GenericFactoryContext&) { // Cast the config message to the actual type to test that it was constructed properly. [[maybe_unused]] const auto config = dynamic_cast(message); return std::make_unique(); @@ -63,7 +63,7 @@ FormatterProviderPtr AdditionalCommandParser::parse(const std::string& command, } CommandParserPtr AdditionalCommandFactory::createCommandParserFromProto( - const Protobuf::Message& message, Server::Configuration::CommonFactoryContext&) { + const Protobuf::Message& message, Server::Configuration::GenericFactoryContext&) { // Cast the config message to the actual type to test that it was constructed properly. [[maybe_unused]] const auto config = dynamic_cast(message); return std::make_unique(); @@ -81,7 +81,7 @@ std::string AdditionalCommandFactory::name() const { return "envoy.formatter.Add CommandParserPtr FailCommandFactory::createCommandParserFromProto(const Protobuf::Message& message, - Server::Configuration::CommonFactoryContext&) { + Server::Configuration::GenericFactoryContext&) { // Cast the config message to the actual type to test that it was constructed properly. [[maybe_unused]] const auto config = dynamic_cast(message); return nullptr; diff --git a/test/common/formatter/command_extension.h b/test/common/formatter/command_extension.h index 614e1e5d39ba..ac922147c951 100644 --- a/test/common/formatter/command_extension.h +++ b/test/common/formatter/command_extension.h @@ -32,7 +32,7 @@ class TestCommandFactory : public CommandParserFactory { public: CommandParserPtr createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) override; + Server::Configuration::GenericFactoryContext&) override; std::set configTypes() override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override; @@ -60,7 +60,7 @@ class AdditionalCommandFactory : public CommandParserFactory { public: CommandParserPtr createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) override; + Server::Configuration::GenericFactoryContext&) override; std::set configTypes() override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override; @@ -70,7 +70,7 @@ class FailCommandFactory : public CommandParserFactory { public: CommandParserPtr createCommandParserFromProto(const Protobuf::Message&, - Server::Configuration::CommonFactoryContext&) override; + Server::Configuration::GenericFactoryContext&) override; std::set configTypes() override; ProtobufTypes::MessagePtr createEmptyConfigProto() override; std::string name() const override; diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index c32f48a2943f..8d32a486dfad 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -604,14 +604,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(18443))); - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("18443"))); - } - // Validate for IPv6 address address = Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 19443)}; @@ -620,14 +612,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(19443))); - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("19443"))); - } - // Validate for Pipe address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; stream_info.upstreamInfo()->setUpstreamLocalAddress(address); @@ -660,14 +644,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_EQ("443", upstream_format.formatWithContext({}, stream_info)); EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(443))); - - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("443"))); - } } { @@ -754,14 +730,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(8443))); - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("8443"))); - } - // Validate for IPv6 address address = Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv6Instance("::1", 9443)}; @@ -770,14 +738,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(9443))); - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("9443"))); - } - // Validate for Pipe address = Network::Address::InstanceConstSharedPtr{new Network::Address::PipeInstance("/foo")}; stream_info.downstream_connection_info_provider_->setLocalAddress(address); @@ -805,14 +765,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_EQ("0", upstream_format.formatWithContext({}, stream_info)); EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(0))); - - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("0"))); - } } { @@ -834,14 +786,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_EQ("63443", upstream_format.formatWithContext({}, stream_info)); EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(63443))); - - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.format_ports_as_numbers", "false"}}); - - EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("63443"))); - } } { @@ -919,6 +863,30 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::nullValue())); } + { + StreamInfoFormatter upstream_connection_pool_callback_duration_format( + "UPSTREAM_CONNECTION_POOL_READY_DURATION"); + EXPECT_EQ(absl::nullopt, + upstream_connection_pool_callback_duration_format.formatWithContext({}, stream_info)); + EXPECT_THAT( + upstream_connection_pool_callback_duration_format.formatValueWithContext({}, stream_info), + ProtoEq(ValueUtil::nullValue())); + } + + { + StreamInfoFormatter upstream_connection_pool_callback_duration_format( + "UPSTREAM_CONNECTION_POOL_READY_DURATION"); + EXPECT_CALL(time_system, monotonicTime) + .WillOnce(Return(MonotonicTime(std::chrono::nanoseconds(25000000)))); + upstream_timing.recordConnectionPoolCallbackLatency( + MonotonicTime(std::chrono::nanoseconds(10000000)), time_system); + + EXPECT_EQ("15", + upstream_connection_pool_callback_duration_format.formatWithContext({}, stream_info)); + EXPECT_THAT( + upstream_connection_pool_callback_duration_format.formatValueWithContext({}, stream_info), + ProtoEq(ValueUtil::numberValue(15.0))); + } } TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { @@ -4001,7 +3969,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { const std::string format = "%START_TIME(%E4n)%"; const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).WillOnce(Return(start_time)); - FormatterImpl formatter(format); + FormatterImpl formatter(format, false); EXPECT_EQ("%E4n", formatter.formatWithContext(formatter_context, stream_info)); } #endif diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index 01458712de95..a40a85d8fb8b 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -40,6 +40,7 @@ envoy_cc_test( "//test/mocks/upstream:cluster_priority_set_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) @@ -231,6 +232,7 @@ envoy_cc_benchmark_binary( "//test/mocks/upstream:cluster_priority_set_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/test/common/grpc/async_client_manager_benchmark.cc b/test/common/grpc/async_client_manager_benchmark.cc index f23a0aba54e5..ca316d777cdf 100644 --- a/test/common/grpc/async_client_manager_benchmark.cc +++ b/test/common/grpc/async_client_manager_benchmark.cc @@ -1,5 +1,6 @@ #include +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/grpc/async_client.h" @@ -28,7 +29,9 @@ class AsyncClientManagerImplTest { public: AsyncClientManagerImplTest() : api_(Api::createApiForTest()), stat_names_(scope_.symbolTable()), - async_client_manager_(cm_, tls_, test_time_.timeSystem(), *api_, stat_names_) {} + async_client_manager_( + cm_, tls_, test_time_.timeSystem(), *api_, stat_names_, + envoy::config::bootstrap::v3::Bootstrap::GrpcAsyncClientManagerConfig()) {} Upstream::MockClusterManager cm_; NiceMock tls_; @@ -47,6 +50,7 @@ void testGetOrCreateAsyncClientWithConfig(::benchmark::State& state) { grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); for (int i = 0; i < 1000; i++) { RawAsyncClientSharedPtr foo_client0 = async_client_man_test.async_client_manager_.getOrCreateRawAsyncClient( @@ -63,6 +67,7 @@ void testGetOrCreateAsyncClientWithHashConfig(::benchmark::State& state) { GrpcServiceConfigWithHashKey config_with_hash_key_a = GrpcServiceConfigWithHashKey(grpc_service); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); for (int i = 0; i < 1000; i++) { RawAsyncClientSharedPtr foo_client0 = async_client_man_test.async_client_manager_.getOrCreateRawAsyncClientWithHashKey( diff --git a/test/common/grpc/async_client_manager_impl_test.cc b/test/common/grpc/async_client_manager_impl_test.cc index 36dae3f1001e..4e89ac6f296b 100644 --- a/test/common/grpc/async_client_manager_impl_test.cc +++ b/test/common/grpc/async_client_manager_impl_test.cc @@ -1,5 +1,7 @@ +#include #include +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/grpc/async_client.h" @@ -18,6 +20,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +using envoy::config::bootstrap::v3::Bootstrap; using ::testing::Return; namespace Envoy { @@ -28,7 +31,7 @@ class RawAsyncClientCacheTest : public testing::Test { public: RawAsyncClientCacheTest() : api_(Api::createApiForTest(time_system_)), - dispatcher_(api_->allocateDispatcher("test_thread")), client_cache_(*dispatcher_) {} + dispatcher_(api_->allocateDispatcher("test_thread")) {} // advanceTimeAndRun moves the current time as requested, and then executes // all executable timers in a non-deterministic order. This mimics real-time behavior in @@ -42,99 +45,197 @@ class RawAsyncClientCacheTest : public testing::Test { } } + void initialize(std::chrono::milliseconds entry_timeout_interval) { + client_cache_ = std::make_unique( + *dispatcher_, entry_timeout_interval); + } + protected: Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; - AsyncClientManagerImpl::RawAsyncClientCache client_cache_; + std::unique_ptr client_cache_; }; -TEST_F(RawAsyncClientCacheTest, CacheEviction) { +// Test cache eviction time with 50000ms. +TEST_F(RawAsyncClientCacheTest, CacheEvictionMilliseconds) { + initialize(std::chrono::milliseconds(50000)); envoy::config::core::v3::GrpcService foo_service; foo_service.mutable_envoy_grpc()->set_cluster_name("foo"); GrpcServiceConfigWithHashKey config_with_hash_key(foo_service); RawAsyncClientSharedPtr foo_client = std::make_shared(); - client_cache_.setCache(config_with_hash_key, foo_client); + client_cache_->setCache(config_with_hash_key, foo_client); waitForSeconds(49); // Cache entry hasn't been evicted because it was created 49s ago. - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), foo_client.get()); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), foo_client.get()); waitForSeconds(49); // Cache entry hasn't been evicted because it was accessed 49s ago. - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), foo_client.get()); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), foo_client.get()); waitForSeconds(51); - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), nullptr); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), nullptr); +} + +// Test cache eviction time with 20s. +TEST_F(RawAsyncClientCacheTest, CacheEvictionWithSeconds) { + initialize(std::chrono::seconds(20)); + envoy::config::core::v3::GrpcService foo_service; + foo_service.mutable_envoy_grpc()->set_cluster_name("foo"); + GrpcServiceConfigWithHashKey config_with_hash_key(foo_service); + RawAsyncClientSharedPtr foo_client = std::make_shared(); + client_cache_->setCache(config_with_hash_key, foo_client); + waitForSeconds(19); + // Cache entry hasn't been evicted because it was created 19s ago. + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), foo_client.get()); + waitForSeconds(19); + // Cache entry hasn't been evicted because it was accessed 19s ago. + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), foo_client.get()); + waitForSeconds(21); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), nullptr); } +// Test multiple cache entries eviction behaviour. TEST_F(RawAsyncClientCacheTest, MultipleCacheEntriesEviction) { + initialize(std::chrono::milliseconds(50000)); envoy::config::core::v3::GrpcService grpc_service; RawAsyncClientSharedPtr foo_client = std::make_shared(); for (int i = 1; i <= 50; i++) { grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(i)); GrpcServiceConfigWithHashKey config_with_hash_key(grpc_service); - client_cache_.setCache(config_with_hash_key, foo_client); + client_cache_->setCache(config_with_hash_key, foo_client); } waitForSeconds(20); for (int i = 51; i <= 100; i++) { grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(i)); GrpcServiceConfigWithHashKey config_with_hash_key(grpc_service); - client_cache_.setCache(config_with_hash_key, foo_client); + client_cache_->setCache(config_with_hash_key, foo_client); } waitForSeconds(30); // Cache entries created 50s before have expired. for (int i = 1; i <= 50; i++) { grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(i)); GrpcServiceConfigWithHashKey config_with_hash_key(grpc_service); - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), nullptr); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), nullptr); } // Cache entries 30s before haven't expired. for (int i = 51; i <= 100; i++) { grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(i)); GrpcServiceConfigWithHashKey config_with_hash_key(grpc_service); - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), foo_client.get()); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), foo_client.get()); } } // Test the case when the eviction timer doesn't fire on time, getting the oldest entry that has // already expired but hasn't been evicted should succeed. TEST_F(RawAsyncClientCacheTest, GetExpiredButNotEvictedCacheEntry) { + initialize(std::chrono::milliseconds(50000)); envoy::config::core::v3::GrpcService foo_service; foo_service.mutable_envoy_grpc()->set_cluster_name("foo"); RawAsyncClientSharedPtr foo_client = std::make_shared(); GrpcServiceConfigWithHashKey config_with_hash_key(foo_service); - client_cache_.setCache(config_with_hash_key, foo_client); + client_cache_->setCache(config_with_hash_key, foo_client); time_system_.advanceTimeAsyncImpl(std::chrono::seconds(50)); // Cache entry hasn't been evicted because it is accessed before timer fire. - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), foo_client.get()); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), foo_client.get()); time_system_.advanceTimeAndRun(std::chrono::seconds(50), *dispatcher_, Event::Dispatcher::RunType::NonBlock); // Cache entry has been evicted because it is accessed after timer fire. - EXPECT_EQ(client_cache_.getCache(config_with_hash_key).get(), nullptr); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key).get(), nullptr); +} + +class RawAsyncClientCacheTestBusyLoop : public testing::Test { +public: + RawAsyncClientCacheTestBusyLoop() { + timer_ = new Event::MockTimer(); + EXPECT_CALL(dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb) { + return timer_; + })); + client_cache_ = std::make_unique( + dispatcher_, std::chrono::milliseconds(50000)); + EXPECT_CALL(*timer_, enableTimer(testing::Not(std::chrono::milliseconds(0)), _)) + .Times(testing::AtLeast(1)); + } + + void waitForMilliSeconds(int ms) { + for (int i = 0; i < ms; i++) { + time_system_.advanceTimeAndRun(std::chrono::milliseconds(1), dispatcher_, + Event::Dispatcher::RunType::NonBlock); + } + } + +protected: + Event::SimulatedTimeSystem time_system_; + NiceMock dispatcher_; + Event::MockTimer* timer_; + std::unique_ptr client_cache_; +}; + +TEST_F(RawAsyncClientCacheTestBusyLoop, MultipleCacheEntriesEvictionBusyLoop) { + envoy::config::core::v3::GrpcService grpc_service; + RawAsyncClientSharedPtr foo_client = std::make_shared(); + // two entries are added to the cache + for (int i = 1; i <= 2; i++) { + grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(i)); + GrpcServiceConfigWithHashKey config_with_hash_key(grpc_service); + client_cache_->setCache(config_with_hash_key, foo_client); + } + // waiting for 49.2 secs to make sure that for the entry which is not accessed, time to expire is + // less than 1 second, ~0.8 secs + waitForMilliSeconds(49200); + + // Access first cache entry to so that evictEntriesAndResetEvictionTimer() gets called. + // Since we are getting first entry, access time of first entry will be updated to current time. + grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(1)); + GrpcServiceConfigWithHashKey config_with_hash_key_1(grpc_service); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key_1).get(), foo_client.get()); + + // Verifying that though the time to expire for second entry ~0.8 sec, it is considered as expired + // to avoid the busy loop which could happen if timer gets enabled with 0(0.8 rounded off to 0) + // duration. + grpc_service.mutable_envoy_grpc()->set_cluster_name(std::to_string(2)); + GrpcServiceConfigWithHashKey config_with_hash_key_2(grpc_service); + EXPECT_EQ(client_cache_->getCache(config_with_hash_key_2).get(), nullptr); } class AsyncClientManagerImplTest : public testing::Test { public: AsyncClientManagerImplTest() - : api_(Api::createApiForTest()), stat_names_(scope_.symbolTable()), - async_client_manager_(cm_, tls_, test_time_.timeSystem(), *api_, stat_names_) {} + : api_(Api::createApiForTest(time_system_)), + dispatcher_(api_->allocateDispatcher("test_grpc_manager")), + stat_names_(scope_.symbolTable()) { + tls_.setDispatcher(dispatcher_.get()); + } + + void initialize(absl::optional config = absl::nullopt) { + if (config.has_value()) { + async_client_manager_ = std::make_unique( + cm_, tls_, time_system_, *api_, stat_names_, config.value()); + } else { + async_client_manager_ = std::make_unique( + cm_, tls_, time_system_, *api_, stat_names_, Bootstrap::GrpcAsyncClientManagerConfig()); + } + } Upstream::MockClusterManager cm_; NiceMock tls_; Stats::MockStore store_; Stats::MockScope& scope_{store_.mockScope()}; - DangerousDeprecatedTestTime test_time_; + Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; StatNames stat_names_; - AsyncClientManagerImpl async_client_manager_; + std::unique_ptr async_client_manager_; }; TEST_F(AsyncClientManagerImplTest, EnvoyGrpcOk) { + initialize(); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); - EXPECT_CALL(cm_, checkActiveStaticCluster("foo")).WillOnce(Return()); - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false); + EXPECT_CALL(cm_, checkActiveStaticCluster("foo")).WillOnce(Return(absl::OkStatus())); + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false); } TEST_F(AsyncClientManagerImplTest, GrpcServiceConfigWithHashKeyTest) { + initialize(); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); envoy::config::core::v3::GrpcService grpc_service_c; @@ -153,50 +254,126 @@ TEST_F(AsyncClientManagerImplTest, GrpcServiceConfigWithHashKeyTest) { config_with_hash_key_c.getPreComputedHash()); } +// Test the client cache time is correctly configured with seconds from the bootstrap proto. +TEST_F(AsyncClientManagerImplTest, RawAsyncClientCacheWithSecondsConfig) { + envoy::config::core::v3::GrpcService grpc_service; + grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); + + Bootstrap::GrpcAsyncClientManagerConfig aync_manager_config; + aync_manager_config.mutable_max_cached_entry_idle_duration()->MergeFrom( + ProtobufUtil::TimeUtil::SecondsToDuration(20)); + + initialize(aync_manager_config); + + RawAsyncClientSharedPtr foo_client_0 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + RawAsyncClientSharedPtr foo_client_1 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + EXPECT_EQ(foo_client_0.get(), foo_client_1.get()); + + time_system_.advanceTimeAndRun(std::chrono::seconds(19), *dispatcher_, + Event::Dispatcher::RunType::NonBlock); + + RawAsyncClientSharedPtr foo_client_2 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + EXPECT_EQ(foo_client_1.get(), foo_client_2.get()); + + // Here we want to test behavior with a specific sequence of events, where each timer + // fires within a simulated second of what was programmed. + for (int i = 0; i < 21; i++) { + time_system_.advanceTimeAndRun(std::chrono::seconds(1), *dispatcher_, + Event::Dispatcher::RunType::NonBlock); + } + + RawAsyncClientSharedPtr foo_client_3 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + EXPECT_NE(foo_client_2.get(), foo_client_3.get()); +} + +// Test the client cache time is correctly configured with milliseconds from the bootstrap proto. +TEST_F(AsyncClientManagerImplTest, RawAsyncClientCacheWithMilliConfig) { + envoy::config::core::v3::GrpcService grpc_service; + grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); + + Bootstrap::GrpcAsyncClientManagerConfig aync_manager_config; + aync_manager_config.mutable_max_cached_entry_idle_duration()->MergeFrom( + ProtobufUtil::TimeUtil::MillisecondsToDuration(30000)); + + initialize(aync_manager_config); + + RawAsyncClientSharedPtr foo_client_0 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + RawAsyncClientSharedPtr foo_client_1 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + EXPECT_EQ(foo_client_0.get(), foo_client_1.get()); + + time_system_.advanceTimeAndRun(std::chrono::milliseconds(29999), *dispatcher_, + Event::Dispatcher::RunType::NonBlock); + + RawAsyncClientSharedPtr foo_client_2 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + EXPECT_EQ(foo_client_1.get(), foo_client_2.get()); + + // Here we want to test behavior with a specific sequence of events, where each timer + // fires within a simulated second of what was programmed. + for (int i = 0; i < 31; i++) { + time_system_.advanceTimeAndRun(std::chrono::seconds(1), *dispatcher_, + Event::Dispatcher::RunType::NonBlock); + } + + RawAsyncClientSharedPtr foo_client_3 = + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); + EXPECT_NE(foo_client_2.get(), foo_client_3.get()); +} + TEST_F(AsyncClientManagerImplTest, RawAsyncClientCache) { + initialize(); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); // Use cache when runtime is enabled. RawAsyncClientSharedPtr foo_client0 = - async_client_manager_.getOrCreateRawAsyncClient(grpc_service, scope_, true); + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); RawAsyncClientSharedPtr foo_client1 = - async_client_manager_.getOrCreateRawAsyncClient(grpc_service, scope_, true); + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); EXPECT_EQ(foo_client0.get(), foo_client1.get()); // Get a different raw async client with different cluster config. grpc_service.mutable_envoy_grpc()->set_cluster_name("bar"); RawAsyncClientSharedPtr bar_client = - async_client_manager_.getOrCreateRawAsyncClient(grpc_service, scope_, true); + async_client_manager_->getOrCreateRawAsyncClient(grpc_service, scope_, true); EXPECT_NE(foo_client1.get(), bar_client.get()); } TEST_F(AsyncClientManagerImplTest, EnvoyGrpcInvalid) { + initialize(); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); EXPECT_CALL(cm_, checkActiveStaticCluster("foo")).WillOnce(Invoke([](const std::string&) { - throw EnvoyException("fake exception"); + return absl::InvalidArgumentError("failure"); })); EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, - "fake exception"); + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + "failure"); } TEST_F(AsyncClientManagerImplTest, GoogleGrpc) { + initialize(); EXPECT_CALL(scope_, createScope_("grpc.foo.")); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_google_grpc()->set_stat_prefix("foo"); #ifdef ENVOY_GOOGLE_GRPC - EXPECT_NE(nullptr, async_client_manager_.factoryForGrpcService(grpc_service, scope_, false)); + EXPECT_NE(nullptr, async_client_manager_->factoryForGrpcService(grpc_service, scope_, false)); #else EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, "Google C++ gRPC client is not linked"); #endif } TEST_F(AsyncClientManagerImplTest, GoogleGrpcIllegalCharsInKey) { + initialize(); EXPECT_CALL(scope_, createScope_("grpc.foo.")); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_google_grpc()->set_stat_prefix("foo"); @@ -207,16 +384,17 @@ TEST_F(AsyncClientManagerImplTest, GoogleGrpcIllegalCharsInKey) { #ifdef ENVOY_GOOGLE_GRPC EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, "Illegal characters in gRPC initial metadata header key: illegalcharacter;."); #else EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, "Google C++ gRPC client is not linked"); #endif } TEST_F(AsyncClientManagerImplTest, LegalGoogleGrpcChar) { + initialize(); EXPECT_CALL(scope_, createScope_("grpc.foo.")); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_google_grpc()->set_stat_prefix("foo"); @@ -226,15 +404,16 @@ TEST_F(AsyncClientManagerImplTest, LegalGoogleGrpcChar) { metadata.set_value("value"); #ifdef ENVOY_GOOGLE_GRPC - EXPECT_NE(nullptr, async_client_manager_.factoryForGrpcService(grpc_service, scope_, false)); + EXPECT_NE(nullptr, async_client_manager_->factoryForGrpcService(grpc_service, scope_, false)); #else EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, "Google C++ gRPC client is not linked"); #endif } TEST_F(AsyncClientManagerImplTest, GoogleGrpcIllegalCharsInValue) { + initialize(); EXPECT_CALL(scope_, createScope_("grpc.foo.")); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_google_grpc()->set_stat_prefix("foo"); @@ -245,21 +424,22 @@ TEST_F(AsyncClientManagerImplTest, GoogleGrpcIllegalCharsInValue) { #ifdef ENVOY_GOOGLE_GRPC EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, "Illegal ASCII value for gRPC initial metadata header key: legal-key."); #else EXPECT_THROW_WITH_MESSAGE( - async_client_manager_.factoryForGrpcService(grpc_service, scope_, false), EnvoyException, + async_client_manager_->factoryForGrpcService(grpc_service, scope_, false), EnvoyException, "Google C++ gRPC client is not linked"); #endif } TEST_F(AsyncClientManagerImplTest, EnvoyGrpcUnknownSkipClusterCheck) { + initialize(); envoy::config::core::v3::GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name("foo"); EXPECT_CALL(cm_, checkActiveStaticCluster(_)).Times(0); - ASSERT_NO_THROW(async_client_manager_.factoryForGrpcService(grpc_service, scope_, true)); + ASSERT_NO_THROW(async_client_manager_->factoryForGrpcService(grpc_service, scope_, true)); } } // namespace diff --git a/test/common/grpc/grpc_client_integration_test.cc b/test/common/grpc/grpc_client_integration_test.cc index b65121808489..8cf420e1e1a1 100644 --- a/test/common/grpc/grpc_client_integration_test.cc +++ b/test/common/grpc/grpc_client_integration_test.cc @@ -36,8 +36,9 @@ TEST_P(GrpcClientIntegrationTest, BasicStreamWithBytesMeter) { SKIP_IF_GRPC_CLIENT(ClientType::GoogleGrpc); // The check in this test is based on HTTP2 codec logic (i.e., including H2_FRAME_HEADER_SIZE). // Skip this test if default protocol of this integration test is no longer HTTP2. - if (fake_upstream_config_.upstream_protocol_ != Http::CodecType::HTTP2) + if (fake_upstream_config_.upstream_protocol_ != Http::CodecType::HTTP2) { return; + } initialize(); auto stream = createStream(empty_metadata_); @@ -109,8 +110,9 @@ TEST_P(GrpcClientIntegrationTest, MultiStreamWithBytesMeter) { SKIP_IF_GRPC_CLIENT(ClientType::GoogleGrpc); // The check in this test is based on HTTP2 codec logic (i.e., including H2_FRAME_HEADER_SIZE). // Skip this test if default protocol of this integration test is no longer HTTP2. - if (fake_upstream_config_.upstream_protocol_ != Http::CodecType::HTTP2) + if (fake_upstream_config_.upstream_protocol_ != Http::CodecType::HTTP2) { return; + } initialize(); auto stream_0 = createStream(empty_metadata_); auto stream_1 = createStream(empty_metadata_); diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index f42d7ed316d1..239f7846ea88 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -5,6 +5,7 @@ #include "source/common/http/codec_client.h" #include "source/common/http/exception.h" #include "source/common/network/listen_socket_impl.h" +#include "source/common/network/tcp_listener_impl.h" #include "source/common/network/utility.h" #include "source/common/stream_info/stream_info_impl.h" #include "source/common/upstream/upstream_impl.h" @@ -30,7 +31,7 @@ using testing::_; using testing::AtMost; -using testing::ByMove; +using testing::ByMove; // NOLINT(misc-unused-using-decls) using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; @@ -496,8 +497,11 @@ class CodecNetworkTest : public Event::TestUsingSimulatedTime, socket->connectionInfoProvider().localAddress(), source_address_, Network::Test::createRawBufferSocket(), nullptr, nullptr); NiceMock listener_config; - upstream_listener_ = dispatcher_->createListener(std::move(socket), listener_callbacks_, - runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + upstream_listener_ = std::make_unique( + *dispatcher_, api_->randomGenerator(), runtime_, std::move(socket), listener_callbacks_, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); client_connection_ = client_connection.get(); client_connection_->addConnectionCallbacks(client_callbacks_); diff --git a/test/common/http/codec_impl_corpus/h10_empty_hostname b/test/common/http/codec_impl_corpus/h10_empty_hostname new file mode 100644 index 000000000000..f541da572451 --- /dev/null +++ b/test/common/http/codec_impl_corpus/h10_empty_hostname @@ -0,0 +1,33 @@ +h1_settings { + server { + accept_http_10: true + default_host_for_http_10: "\000\000\000\000\000\000\000\000" + } +} +actions { + new_stream { + request_headers { + headers { + key: ":method" + value: "GET" + } + headers { + key: ":path" + value: "/" + } + } + end_stream: true + } +} +actions { + mutate { + offset: 5 + value: 48 + } +} +actions { + mutate { + offset: 48 + value: 48 + } +} diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 42dc4e692c01..6101a2ed88c9 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -66,6 +66,12 @@ Http1Settings fromHttp1Settings(const test::common::http::Http1ServerSettings& s h1_settings.accept_http_10_ = settings.accept_http_10(); h1_settings.default_host_for_http_10_ = settings.default_host_for_http_10(); + // If the server accepts a HTTP/1.0 then the default host must be valid. + if (h1_settings.accept_http_10_ && + !HeaderUtility::authorityIsValid(h1_settings.default_host_for_http_10_)) { + throw EnvoyException("Invalid Http1ServerSettings, HTTP/1.0 is enabled and " + "'default_host_for_http_10' has invalid hostname, skipping test."); + } return h1_settings; } @@ -371,20 +377,24 @@ class HttpStream : public LinkedObject { dispatcher = &context_.client_connection_.dispatcher_; } - // With this feature enabled for http2 we end up creating a schedulable - // callback the first time we re-enable reading as it's used to process - // the backed up data. + // With this feature enabled for http2 the codec may end up creating a + // schedulable callback the first time it re-enables reading as it's used + // to process the backed up data if there's any to process. if (Runtime::runtimeFeatureEnabled(Runtime::defer_processing_backedup_streams)) { - const bool expecting_schedulable_callback_creation = + const bool might_schedulable_callback_creation = http_protocol_ == Protocol::Http2 && state.read_disable_count_ == 0 && !disable && !state.created_schedulable_callback_; - if (expecting_schedulable_callback_creation) { + if (might_schedulable_callback_creation) { ASSERT(dispatcher != nullptr); state.created_schedulable_callback_ = true; - // The unique pointer of this object will be returned in createSchedulableCallback_ of - // dispatcher, so there is no risk of object leak. - new Event::MockSchedulableCallback(dispatcher); + ON_CALL(*dispatcher, createSchedulableCallback_(_)) + .WillByDefault(testing::Invoke([dispatcher](std::function cb) { + // The unique pointer of this object will be returned in + // createSchedulableCallback_ of dispatcher, so there is no risk of this object + // leaking. + return new Event::MockSchedulableCallback(dispatcher, cb); + })); } } diff --git a/test/common/http/codes_speed_test.cc b/test/common/http/codes_speed_test.cc index ec82b9546c54..06c5bb7d5e19 100644 --- a/test/common/http/codes_speed_test.cc +++ b/test/common/http/codes_speed_test.cc @@ -98,6 +98,7 @@ static void BM_AddResponsesRealSymtab(benchmark::State& state) { Envoy::Http::CodeUtilitySpeedTest context; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.addResponses(); } } @@ -108,6 +109,7 @@ static void BM_ResponseTimingRealSymtab(benchmark::State& state) { Envoy::Http::CodeUtilitySpeedTest context; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.responseTiming(); } } diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index e783f2a184df..d8ce2ffbfa45 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -527,7 +527,7 @@ class FuzzStream { // Similarly, local replies should always contain this. const auto status = Utility::getResponseStatusOrNullopt(*headers); // The only 1xx header that may be provided to encodeHeaders() is a 101 upgrade, - // guaranteed by the codec parsers. See include/envoy/http/filter.h. + // guaranteed by the codec parsers. See envoy/http/filter.h. if (!status.has_value() || (CodeUtility::is1xx(status.value()) && status.value() != enumToInt(Http::Code::SwitchingProtocols))) { headers->setReferenceKey(Headers::get().Status, "200"); @@ -594,11 +594,8 @@ using FuzzStreamPtr = std::unique_ptr; DEFINE_PROTO_FUZZER(const test::common::http::ConnManagerImplTestCase& input) { try { TestUtility::validate(input); - } catch (const ProtoValidationException& e) { - ENVOY_LOG_MISC(debug, "ProtoValidationException: {}", e.what()); - return; - } catch (const Envoy::ProtobufMessage::DeprecatedProtoFieldException& e) { - ENVOY_LOG_MISC(debug, "DeprecatedProtoFieldException: {}", e.what()); + } catch (const Envoy::EnvoyException& e) { + ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); return; } diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index beb5dad69afe..7bac6ac9027b 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2142,9 +2142,9 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLog) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([&](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([&](const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) { EXPECT_EQ(&decoder_->streamInfo(), &stream_info); EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); @@ -2208,12 +2208,12 @@ TEST_F(HttpConnectionManagerImplTest, TestFilterCanEnrichAccessLogs) { filter->callbacks_->streamInfo().setDynamicMetadata("metadata_key", metadata); })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - auto dynamic_meta = stream_info.dynamicMetadata().filter_metadata().at("metadata_key"); - EXPECT_EQ("value", dynamic_meta.fields().at("field").string_value()); - })); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + auto dynamic_meta = stream_info.dynamicMetadata().filter_metadata().at("metadata_key"); + EXPECT_EQ("value", dynamic_meta.fields().at("field").string_value()); + })); EXPECT_CALL(*codec_, dispatch(_)) .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { @@ -2252,9 +2252,9 @@ TEST_F(HttpConnectionManagerImplTest, TestRemoteDownstreamDisconnectAccessLog) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) { EXPECT_FALSE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE( @@ -2296,12 +2296,12 @@ TEST_F(HttpConnectionManagerImplTest, TestLocalDownstreamDisconnectAccessLog) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_EQ("downstream_local_disconnect(reason_for_local_close)", - stream_info.responseCodeDetails().value()); - })); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ("downstream_local_disconnect(reason_for_local_close)", + stream_info.responseCodeDetails().value()); + })); EXPECT_CALL(*codec_, dispatch(_)) .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { @@ -2338,16 +2338,16 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogWithTrailers) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_TRUE(stream_info.responseCode()); - EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().remoteAddress()); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().directRemoteAddress()); - EXPECT_NE(nullptr, stream_info.route()); - })); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_TRUE(stream_info.responseCode()); + EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().remoteAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().directRemoteAddress()); + EXPECT_NE(nullptr, stream_info.route()); + })); EXPECT_CALL(*codec_, dispatch(_)) .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { @@ -2391,9 +2391,9 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogWithInvalidRequest) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([this](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([this](const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) { EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(400)); EXPECT_EQ("missing_host_header", stream_info.responseCodeDetails().value()); @@ -2439,21 +2439,19 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogOnNewRequest) { flush_access_log_on_new_request_ = true; - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { // First call to log() is made when a new HTTP request has been received // On the first call it is expected that there is no response code. - EXPECT_EQ(AccessLog::AccessLogType::DownstreamStart, access_log_type); + EXPECT_EQ(AccessLog::AccessLogType::DownstreamStart, log_context.accessLogType()); EXPECT_FALSE(stream_info.responseCode()); })) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { // Second call to log() is made when filter is destroyed, so it is expected // that the response code is available and matches the response headers. - EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, access_log_type); + EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, log_context.accessLogType()); EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); @@ -2503,21 +2501,19 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogOnTunnelEstablished) { flush_log_on_tunnel_successfully_established_ = true; - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { // First call to log() is made when a new HTTP tunnel has been established. - EXPECT_EQ(access_log_type, + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::DownstreamTunnelSuccessfullyEstablished); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); })) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { // Second call to log() is made when the request is completed, so it is expected // that the response code is available and matches the response headers. - EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, access_log_type); + EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, log_context.accessLogType()); EXPECT_TRUE(stream_info.responseCode()); EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); @@ -2587,22 +2583,18 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) { Buffer::OwnedImpl fake_input("1234"); conn_manager_->onData(fake_input, false); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([&](const HeaderMap* request_headers, const HeaderMap* response_headers, - const HeaderMap*, const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { - EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, access_log_type); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([&](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, log_context.accessLogType()); EXPECT_EQ(&decoder_->streamInfo(), &stream_info); - EXPECT_THAT(request_headers, testing::NotNull()); - EXPECT_THAT(response_headers, testing::IsNull()); EXPECT_EQ(stream_info.requestComplete(), absl::nullopt); EXPECT_THAT(stream_info.getDownstreamBytesMeter()->bytesAtLastDownstreamPeriodicLog(), testing::IsNull()); })) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { - EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, access_log_type); + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(AccessLog::AccessLogType::DownstreamPeriodic, log_context.accessLogType()); EXPECT_EQ(stream_info.getDownstreamBytesMeter() ->bytesAtLastDownstreamPeriodicLog() ->wire_bytes_received, @@ -2614,14 +2606,11 @@ TEST_F(HttpConnectionManagerImplTest, TestPeriodicAccessLogging) { // Add additional bytes. response_encoder_.stream_.bytes_meter_->addWireBytesReceived(12); periodic_log_timer->invokeCallback(); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([&](const HeaderMap* request_headers, const HeaderMap* response_headers, - const HeaderMap*, const StreamInfo::StreamInfo& stream_info, - AccessLog::AccessLogType access_log_type) { - EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, access_log_type); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([&](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(AccessLog::AccessLogType::DownstreamEnd, log_context.accessLogType()); EXPECT_EQ(&decoder_->streamInfo(), &stream_info); - EXPECT_THAT(request_headers, testing::NotNull()); - EXPECT_THAT(response_headers, testing::NotNull()); EXPECT_THAT(stream_info.responseCodeDetails(), testing::Optional(testing::StrEq("details"))); EXPECT_THAT(stream_info.responseCode(), testing::Optional(200)); @@ -2733,17 +2722,17 @@ TEST_F(HttpConnectionManagerImplTest, TestAccessLogSsl) { return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_TRUE(stream_info.responseCode()); - EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().remoteAddress()); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().directRemoteAddress()); - EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().sslConnection()); - EXPECT_NE(nullptr, stream_info.route()); - })); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_TRUE(stream_info.responseCode()); + EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200)); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().localAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().remoteAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().directRemoteAddress()); + EXPECT_NE(nullptr, stream_info.downstreamAddressProvider().sslConnection()); + EXPECT_NE(nullptr, stream_info.route()); + })); EXPECT_CALL(*codec_, dispatch(_)) .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { @@ -2976,13 +2965,13 @@ TEST_F(HttpConnectionManagerImplTest, TestStreamIdleAccessLog) { std::string response_body; EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_TRUE(stream_info.responseCode()); - EXPECT_TRUE(stream_info.hasAnyResponseFlag()); - EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout)); - })); + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_TRUE(stream_info.responseCode()); + EXPECT_TRUE(stream_info.hasAnyResponseFlag()); + EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout)); + })); EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillOnce(Invoke([&](FilterChainManager& manager) -> bool { @@ -4341,6 +4330,9 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusWithDetailsAndResponseCodeAndServerNa } TEST_F(ProxyStatusTest, PopulateProxyStatusWithDetailsAndResponseCode) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "true"}}); proxy_status_config_ = std::make_unique(); proxy_status_config_->set_remove_details(false); proxy_status_config_->set_set_recommended_response_code(true); @@ -4353,13 +4345,16 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusWithDetailsAndResponseCode) { ASSERT_TRUE(altered_headers); ASSERT_TRUE(altered_headers->ProxyStatus()); EXPECT_EQ(altered_headers->getProxyStatusValue(), - "custom_server_name; error=connection_timeout; details=\"bar; UT\""); + "custom_server_name; error=http_response_timeout; details=\"bar; UT\""); // Changed from request, since set_recommended_response_code is true. Here, // 504 is the recommended response code for UpstreamRequestTimeout. EXPECT_EQ(altered_headers->getStatusValue(), "504"); } TEST_F(ProxyStatusTest, PopulateProxyStatusWithDetails) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "true"}}); proxy_status_config_ = std::make_unique(); proxy_status_config_->set_remove_details(false); proxy_status_config_->set_remove_connection_termination_details(false); @@ -4374,7 +4369,7 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusWithDetails) { ASSERT_TRUE(altered_headers); ASSERT_TRUE(altered_headers->ProxyStatus()); EXPECT_EQ(altered_headers->getProxyStatusValue(), - "custom_server_name; error=connection_timeout; details=\"bar; UT\""); + "custom_server_name; error=http_response_timeout; details=\"bar; UT\""); // Unchanged from request, since set_recommended_response_code is false. Here, // 504 would be the recommended response code for UpstreamRequestTimeout, EXPECT_NE(altered_headers->getStatusValue(), "504"); @@ -4382,6 +4377,9 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusWithDetails) { } TEST_F(ProxyStatusTest, PopulateProxyStatusWithoutDetails) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "true"}}); proxy_status_config_ = std::make_unique(); proxy_status_config_->set_remove_details(true); proxy_status_config_->set_set_recommended_response_code(false); @@ -4393,7 +4391,8 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusWithoutDetails) { ASSERT_TRUE(altered_headers); ASSERT_TRUE(altered_headers->ProxyStatus()); - EXPECT_EQ(altered_headers->getProxyStatusValue(), "custom_server_name; error=connection_timeout"); + EXPECT_EQ(altered_headers->getProxyStatusValue(), + "custom_server_name; error=http_response_timeout"); // Unchanged. EXPECT_EQ(altered_headers->getStatusValue(), "403"); // Since remove_details=true, we should not have "baz", the value of @@ -4402,6 +4401,9 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusWithoutDetails) { } TEST_F(ProxyStatusTest, PopulateProxyStatusAppendToPreviousValue) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "true"}}); proxy_status_config_ = std::make_unique(); proxy_status_config_->set_remove_details(false); @@ -4415,7 +4417,7 @@ TEST_F(ProxyStatusTest, PopulateProxyStatusAppendToPreviousValue) { ASSERT_TRUE(altered_headers->ProxyStatus()); // Expect to see the appended previous value: "SomeCDN; custom_server_name; ...". EXPECT_EQ(altered_headers->getProxyStatusValue(), - "SomeCDN, custom_server_name; error=connection_timeout; details=\"baz; UT\""); + "SomeCDN, custom_server_name; error=http_response_timeout; details=\"baz; UT\""); } } // namespace Http diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 5daf2b1a45e6..431bc6d843a8 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -1,3 +1,5 @@ +#include + #include "test/common/http/conn_manager_impl_test_base.h" #include "test/common/http/custom_header_extension.h" #include "test/test_common/logging.h" @@ -11,6 +13,7 @@ using testing::InvokeWithoutArgs; using testing::Mock; using testing::Ref; using testing::Return; +using testing::ReturnArg; using testing::ReturnRef; namespace Envoy { @@ -39,6 +42,57 @@ TEST_F(HttpConnectionManagerImplTest, ResponseBeforeRequestComplete) { decoder_filters_[0]->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); } +TEST_F(HttpConnectionManagerImplTest, ResponseBeforeRequestComplete10) { + EXPECT_CALL(*codec_, protocol()).WillRepeatedly(Return(Protocol::Http10)); + setup(false, "envoy-server-test"); + setupFilterChain(1, 0); + + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + startRequest(); + + EXPECT_CALL(response_encoder_, encodeHeaders(_, true)) + .WillOnce(Invoke([](const ResponseHeaderMap& headers, bool) -> void { + EXPECT_NE(nullptr, headers.Server()); + EXPECT_EQ("envoy-server-test", headers.getServerValue()); + })); + EXPECT_CALL(*decoder_filters_[0], onStreamComplete()); + EXPECT_CALL(*decoder_filters_[0], onDestroy()); + // When framing by connection close, by default Envoy will FlushWrite, no delay. + EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite, _)); + + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + decoder_filters_[0]->callbacks_->streamInfo().setResponseCodeDetails(""); + decoder_filters_[0]->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); +} + +TEST_F(HttpConnectionManagerImplTest, ResponseBeforeRequestComplete10NoOptimize) { + EXPECT_CALL(runtime_.snapshot_, getBoolean(_, _)).WillRepeatedly(Return(false)); + EXPECT_CALL(*codec_, protocol()).WillRepeatedly(Return(Protocol::Http10)); + setup(false, "envoy-server-test"); + setupFilterChain(1, 0); + + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + startRequest(); + + EXPECT_CALL(response_encoder_, encodeHeaders(_, true)) + .WillOnce(Invoke([](const ResponseHeaderMap& headers, bool) -> void { + EXPECT_NE(nullptr, headers.Server()); + EXPECT_EQ("envoy-server-test", headers.getServerValue()); + })); + EXPECT_CALL(*decoder_filters_[0], onStreamComplete()); + EXPECT_CALL(*decoder_filters_[0], onDestroy()); + // When framing by connection close, by default Envoy will FlushWrite, no delay but with a runtime + // override, it will still flush close. + EXPECT_CALL(filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWriteAndDelay, _)); + + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + decoder_filters_[0]->callbacks_->streamInfo().setResponseCodeDetails(""); + decoder_filters_[0]->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); +} + TEST_F(HttpConnectionManagerImplTest, DisconnectOnProxyConnectionDisconnect) { setup(false, "envoy-server-test"); @@ -173,9 +227,9 @@ TEST_F(HttpConnectionManagerImplTest, TestDownstreamProtocolErrorAccessLog) { access_logs_ = {handler}; setup(false, ""); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) { EXPECT_FALSE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DownstreamProtocolError)); @@ -206,9 +260,9 @@ TEST_F(HttpConnectionManagerImplTest, TestDownstreamProtocolErrorAfterHeadersAcc return true; })); - EXPECT_CALL(*handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*handler, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) { EXPECT_FALSE(stream_info.responseCode()); EXPECT_TRUE(stream_info.hasAnyResponseFlag()); EXPECT_TRUE(stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DownstreamProtocolError)); @@ -247,13 +301,13 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWriteAndDelay, _)); - EXPECT_CALL(*log_handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); - EXPECT_EQ("codec_error:too_many_outbound_frames", - stream_info.responseCodeDetails().value()); - })); + EXPECT_CALL(*log_handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); + EXPECT_EQ("codec_error:too_many_outbound_frames", + stream_info.responseCodeDetails().value()); + })); // Kick off the incoming data. Buffer::OwnedImpl fake_input("1234"); EXPECT_LOG_NOT_CONTAINS("warning", "downstream HTTP flood", @@ -282,12 +336,12 @@ TEST_F(HttpConnectionManagerImplTest, EnvoyOverloadError) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWriteAndDelay, _)); - EXPECT_CALL(*log_handler, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); - EXPECT_EQ("overload_error:Envoy_Overloaded", stream_info.responseCodeDetails().value()); - })); + EXPECT_CALL(*log_handler, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); + EXPECT_EQ("overload_error:Envoy_Overloaded", stream_info.responseCodeDetails().value()); + })); // Kick off the incoming data. Buffer::OwnedImpl fake_input("1234"); conn_manager_->onData(fake_input, false); @@ -1117,85 +1171,7 @@ TEST_F(HttpConnectionManagerImplTest, BlockRouteCacheTest) { filter->callbacks_->downstreamCallbacks()->setRoute(nullptr); EXPECT_EQ(filter->callbacks_->route().get(), mock_route_2.get()); }, - "Should never try to refresh or clear the route cache when it is blocked! " - "To temporarily ignore this new constraint, " - "set runtime flag " - "`envoy.reloadable_features.prohibit_route_refresh_after_response_headers_sent` " - "to `false`"); - - EXPECT_CALL(response_encoder_, encodeData(_, true)); - expectOnDestroy(); - - Buffer::OwnedImpl response_data("ok"); - filter->callbacks_->encodeData(response_data, true); -} - -TEST_F(HttpConnectionManagerImplTest, BlockRouteCacheTestWithRuntimeFeatureDisabled) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.prohibit_route_refresh_after_response_headers_sent", "false"}}); - - setup(false, ""); - - MockStreamDecoderFilter* filter = new NiceMock(); - EXPECT_CALL(filter_factory_, createFilterChain(_)) - .WillOnce(Invoke([&](FilterChainManager& manager) -> bool { - auto factory = createDecoderFilterFactoryCb(StreamDecoderFilterSharedPtr{filter}); - manager.applyFilterFactoryCb({}, factory); - return true; - })); - - auto mock_route_0 = std::make_shared>(); - EXPECT_CALL(*route_config_provider_.route_config_, route(_, _, _, _)) - .WillOnce(Return(mock_route_0)); - - EXPECT_CALL(*filter, decodeHeaders(_, true)) - .WillOnce(Invoke([](RequestHeaderMap& headers, bool) -> FilterHeadersStatus { - EXPECT_NE(nullptr, headers.ForwardedFor()); - EXPECT_EQ("http", headers.getForwardedProtoValue()); - return FilterHeadersStatus::StopIteration; - })); - - EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> Http::Status { - decoder_ = &conn_manager_->newStream(response_encoder_); - RequestHeaderMapPtr headers{ - new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - decoder_->decodeHeaders(std::move(headers), true); - return Http::okStatus(); - })); - - Buffer::OwnedImpl fake_input; - conn_manager_->onData(fake_input, false); - - filter->callbacks_->downstreamCallbacks()->clearRouteCache(); - auto mock_route_1 = std::make_shared>(); - - // Refresh cached route after cache is cleared. - EXPECT_CALL(*route_config_provider_.route_config_, route(_, _, _, _)) - .WillOnce(Return(mock_route_1)); - EXPECT_EQ(filter->callbacks_->route().get(), mock_route_1.get()); - - auto mock_route_2 = std::make_shared>(); - - // We can also set route directly. - filter->callbacks_->downstreamCallbacks()->setRoute(mock_route_2); - EXPECT_EQ(filter->callbacks_->route().get(), mock_route_2.get()); - - ResponseHeaderMapPtr response_headers{ - new TestResponseHeaderMapImpl{{":status", "200"}, {"content-length", "2"}}}; - - EXPECT_CALL(response_encoder_, encodeHeaders(_, false)); - filter->callbacks_->streamInfo().setResponseCodeDetails(""); - filter->callbacks_->encodeHeaders(std::move(response_headers), false, "details"); - - // The cached route will be cleared because the runtime feature is disabled. - filter->callbacks_->downstreamCallbacks()->clearRouteCache(); - EXPECT_CALL(*route_config_provider_.route_config_, route(_, _, _, _)).WillOnce(Return(nullptr)); - EXPECT_EQ(filter->callbacks_->route().get(), nullptr); - - // We can set route after response headers are sent because the runtime feature is disabled. - filter->callbacks_->downstreamCallbacks()->setRoute(mock_route_2); - EXPECT_EQ(filter->callbacks_->route().get(), mock_route_2.get()); + "Should never try to refresh or clear the route cache when it is blocked!"); EXPECT_CALL(response_encoder_, encodeData(_, true)); expectOnDestroy(); @@ -1553,11 +1529,11 @@ TEST_F(HttpConnectionManagerImplTest, HitFilterWatermarkLimits) { EXPECT_CALL(callbacks2, onBelowWriteBufferLowWatermark()).Times(0); encoder_filters_[1]->callbacks_->setEncoderBufferLimit((buffer_len + 1) * 2); - EXPECT_CALL(*log_handler_, log(_, _, _, _, _)) - .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_FALSE(stream_info.hasAnyResponseFlag()); - })); + EXPECT_CALL(*log_handler_, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_FALSE(stream_info.hasAnyResponseFlag()); + })); expectOnDestroy(); EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)).Times(2); @@ -3767,5 +3743,296 @@ TEST_F(HttpConnectionManagerImplTest, NoProxyProtocolAdded) { // Clean up. filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } + +// Validate that deferred streams are processed with a variety of +// headers, data, metadata, and trailers arriving in the same I/O cycle +TEST_F(HttpConnectionManagerImplTest, LimitWorkPerIOCycle) { + const int kRequestsSentPerIOCycle = 100; + EXPECT_CALL(runtime_.snapshot_, getInteger(_, _)).WillRepeatedly(ReturnArg<1>()); + // Process 1 request per I/O cycle + auto* deferred_request_callback = enableStreamsPerIoLimit(1); + setup(false, ""); + + // Store the basic request encoder during filter chain setup. + std::vector> decoder_filters; + int decode_headers_call_count = 0; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + int mod5 = i % 5; + std::shared_ptr filter(new NiceMock()); + + // Each 0th request is headers only + EXPECT_CALL(*filter, decodeHeaders(_, mod5 == 0 ? true : false)) + .WillRepeatedly(Invoke([&](RequestHeaderMap&, bool) -> FilterHeadersStatus { + ++decode_headers_call_count; + return FilterHeadersStatus::StopIteration; + })); + + // Each 1st request is headers and data only + // Each 2nd request is headers, data and trailers + if (mod5 == 1 || mod5 == 2) { + EXPECT_CALL(*filter, decodeData(_, mod5 == 1 ? true : false)) + .WillOnce(Return(FilterDataStatus::StopIterationNoBuffer)); + } + + // Each 3rd request is headers and trailers (no data) + if (mod5 == 2 || mod5 == 3) { + EXPECT_CALL(*filter, decodeTrailers(_)).WillOnce(Return(FilterTrailersStatus::StopIteration)); + } + + // Each 4th request is headers, metadata, and data. + if (mod5 == 4) { + EXPECT_CALL(*filter, decodeMetadata(_)).WillOnce(Return(FilterMetadataStatus::Continue)); + EXPECT_CALL(*filter, decodeData(_, true)) + .WillOnce(Return(FilterDataStatus::StopIterationNoBuffer)); + } + EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)); + decoder_filters.push_back(std::move(filter)); + } + + uint64_t random_value = 0; + EXPECT_CALL(random_, random()).WillRepeatedly(Invoke([&random_value]() { + return random_value++; + })); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .Times(kRequestsSentPerIOCycle) + .WillRepeatedly(Invoke([&decoder_filters](FilterChainManager& manager) -> bool { + static int index = 0; + int i = index++; + FilterFactoryCb factory([&decoder_filters, i](FilterChainFactoryCallbacks& callbacks) { + callbacks.addStreamDecoderFilter(decoder_filters[i]); + }); + manager.applyFilterFactoryCb({}, factory); + return true; + })); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)) + .Times(kRequestsSentPerIOCycle); + + std::vector> response_encoders(kRequestsSentPerIOCycle); + for (auto& encoder : response_encoders) { + EXPECT_CALL(encoder, getStream()).WillRepeatedly(ReturnRef(encoder.stream_)); + } + + EXPECT_CALL(*codec_, dispatch(_)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + decoder_ = &conn_manager_->newStream(response_encoders[i]); + + RequestHeaderMapPtr headers{new TestRequestHeaderMapImpl{ + {":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + + MetadataMapPtr metadata = std::make_unique(); + (*metadata)["key1"] = "value1"; + + RequestTrailerMapPtr trailers{ + new TestRequestTrailerMapImpl{{"key1", "value1"}, {"key2", "value2"}}}; + + Buffer::OwnedImpl data("data"); + + switch (i % 5) { + case 0: + decoder_->decodeHeaders(std::move(headers), true); + break; + case 1: + decoder_->decodeHeaders(std::move(headers), false); + decoder_->decodeData(data, true); + break; + case 2: + decoder_->decodeHeaders(std::move(headers), false); + decoder_->decodeData(data, false); + decoder_->decodeTrailers(std::move(trailers)); + break; + case 3: + decoder_->decodeHeaders(std::move(headers), false); + decoder_->decodeTrailers(std::move(trailers)); + break; + case 4: + decoder_->decodeHeaders(std::move(headers), false); + decoder_->decodeMetadata(std::move(metadata)); + decoder_->decodeData(data, true); + break; + } + } + + data.drain(4); + return Http::okStatus(); + })); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + EXPECT_TRUE(deferred_request_callback->enabled_); + // Only one request should go through the filter chain + ASSERT_EQ(decode_headers_call_count, 1); + + // Let other requests to go through the filter chain. Call expectations will fail + // if this is not the case. + int deferred_request_count = 0; + while (deferred_request_callback->enabled_) { + deferred_request_callback->invokeCallback(); + ++deferred_request_count; + } + + ASSERT_EQ(deferred_request_count, kRequestsSentPerIOCycle); + + for (auto& filter : decoder_filters) { + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + filter->callbacks_->streamInfo().setResponseCodeDetails(""); + filter->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); + } + + EXPECT_EQ(kRequestsSentPerIOCycle, stats_.named_.downstream_rq_2xx_.value()); + EXPECT_EQ(kRequestsSentPerIOCycle, listener_stats_.downstream_rq_2xx_.value()); + EXPECT_EQ(kRequestsSentPerIOCycle, stats_.named_.downstream_rq_completed_.value()); + EXPECT_EQ(kRequestsSentPerIOCycle, listener_stats_.downstream_rq_completed_.value()); +} + +TEST_F(HttpConnectionManagerImplTest, StreamDeferralPreservesOrder) { + EXPECT_CALL(runtime_.snapshot_, getInteger(_, _)).WillRepeatedly(ReturnArg<1>()); + // Process 1 request per I/O cycle + auto* deferred_request_callback = enableStreamsPerIoLimit(1); + setup(false, ""); + + std::vector> encoder_filters; + int expected_request_id = 0; + const Http::LowerCaseString request_id_header(absl::string_view("request-id")); + // Two requests are processed in 2 I/O reads + const int TotalRequests = 2 * 2; + for (int i = 0; i < TotalRequests; ++i) { + std::shared_ptr filter(new NiceMock()); + + EXPECT_CALL(*filter, decodeHeaders(_, true)) + .WillRepeatedly(Invoke([&](RequestHeaderMap& headers, bool) -> FilterHeadersStatus { + // Check that requests are decoded in expected order + int request_id = 0; + ASSERT(absl::SimpleAtoi(headers.get(request_id_header)[0]->value().getStringView(), + &request_id)); + ASSERT(request_id == expected_request_id); + ++expected_request_id; + return FilterHeadersStatus::StopIteration; + })); + + EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)); + encoder_filters.push_back(std::move(filter)); + } + + uint64_t random_value = 0; + EXPECT_CALL(random_, random()).WillRepeatedly(Invoke([&random_value]() { + return random_value++; + })); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .Times(TotalRequests) + .WillRepeatedly(Invoke([&encoder_filters](FilterChainManager& manager) -> bool { + static int index = 0; + int i = index++; + FilterFactoryCb factory([&encoder_filters, i](FilterChainFactoryCallbacks& callbacks) { + callbacks.addStreamDecoderFilter(encoder_filters[i]); + }); + manager.applyFilterFactoryCb({}, factory); + return true; + })); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(TotalRequests); + + std::vector> response_encoders(TotalRequests); + for (auto& encoder : response_encoders) { + EXPECT_CALL(encoder, getStream()).WillRepeatedly(ReturnRef(encoder.stream_)); + } + auto response_encoders_iter = response_encoders.begin(); + + int request_id = 0; + EXPECT_CALL(*codec_, dispatch(_)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> Http::Status { + // The second request should be deferred + for (int i = 0; i < 2; ++i) { + decoder_ = &conn_manager_->newStream(*response_encoders_iter); + ++response_encoders_iter; + + RequestHeaderMapPtr headers{ + new TestRequestHeaderMapImpl{{":authority", "host"}, + {":path", "/"}, + {":method", "GET"}, + {"request-id", absl::StrCat(request_id)}}}; + + ++request_id; + decoder_->decodeHeaders(std::move(headers), true); + } + + data.drain(4); + return Http::okStatus(); + })); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + EXPECT_TRUE(deferred_request_callback->enabled_); + // Only one request should go through the filter chain + ASSERT_EQ(expected_request_id, 1); + + // Test arrival of another request. New request is read from the socket before deferred callbacks. + Buffer::OwnedImpl fake_input2("1234"); + conn_manager_->onData(fake_input2, false); + + // No requests from the second read should go through as there are deferred stream present + ASSERT_EQ(expected_request_id, 1); + + // Let other requests to go through the filter chain. Call expectations will fail + // if this is not the case. + int deferred_request_count = 0; + while (deferred_request_callback->enabled_) { + deferred_request_callback->invokeCallback(); + ++deferred_request_count; + } + + ASSERT_EQ(deferred_request_count, TotalRequests); + + for (auto& filter : encoder_filters) { + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + filter->callbacks_->streamInfo().setResponseCodeDetails(""); + filter->callbacks_->encodeHeaders(std::move(response_headers), true, "details"); + } + + EXPECT_EQ(TotalRequests, stats_.named_.downstream_rq_2xx_.value()); + EXPECT_EQ(TotalRequests, listener_stats_.downstream_rq_2xx_.value()); + EXPECT_EQ(TotalRequests, stats_.named_.downstream_rq_completed_.value()); + EXPECT_EQ(TotalRequests, listener_stats_.downstream_rq_completed_.value()); +} + +TEST_F(HttpConnectionManagerImplTest, DownstreamTimingsRecordWhenRequestHeaderProcessingIsDone) { + setup(/*ssl=*/true, /*server_name=*/"", /*tracing=*/false); + + // Set up the codec. + Buffer::OwnedImpl fake_input("input"); + conn_manager_->createCodec(fake_input); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> Http::Status { + // Set time to 5ms before creating the stream + test_time_.timeSystem().setMonotonicTime(MonotonicTime(std::chrono::milliseconds(5))); + decoder_ = &conn_manager_->newStream(response_encoder_); + RequestHeaderMapPtr headers{ + new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + // Advanced time to 20ms before decoding headers. + test_time_.timeSystem().setMonotonicTime(MonotonicTime(std::chrono::milliseconds(20))); + decoder_->decodeHeaders(std::move(headers), /*end_stream=*/false); + return Http::okStatus(); + })); + + conn_manager_->onData(fake_input, /*end_stream=*/false); + + auto request_headers_done = + decoder_->streamInfo().downstreamTiming().lastDownstreamHeaderRxByteReceived(); + auto request_headers_done_millis = std::chrono::duration_cast( + request_headers_done.value() - decoder_->streamInfo().startTimeMonotonic()); + // Expect time to be 20ms-5ms = 15ms. + EXPECT_EQ(request_headers_done_millis, std::chrono::milliseconds(15)); + + // Clean up. + filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); +} + } // namespace Http } // namespace Envoy diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index 24b1e58d02c2..da6d8d8aa9d7 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -78,6 +78,7 @@ void HttpConnectionManagerImplMixin::setup(bool ssl, const std::string& server_n conn_manager_ = std::make_unique( *this, drain_close_, random_, http_context_, runtime_, local_info_, cluster_manager_, overload_manager_, test_time_.timeSystem()); + conn_manager_->initializeReadFilterCallbacks(filter_callbacks_); if (tracing) { @@ -115,7 +116,7 @@ void HttpConnectionManagerImplMixin::setupFilterChain(int num_decoder_filters, .WillOnce(Invoke([num_decoder_filters, num_encoder_filters, req, this](FilterChainManager& manager) -> bool { bool applied_filters = false; - if (log_handler_.get()) { + if (log_handler_) { auto factory = createLogHandlerFactoryCb(log_handler_); manager.applyFilterFactoryCb({}, factory); applied_filters = true; @@ -395,5 +396,23 @@ void HttpConnectionManagerImplMixin::expectUhvTrailerCheck( })); } +Event::MockSchedulableCallback* +HttpConnectionManagerImplMixin::enableStreamsPerIoLimit(uint32_t limit) { + EXPECT_CALL(runtime_.snapshot_, getInteger("http.max_requests_per_io_cycle", _)) + .WillOnce(Return(limit)); + + // Expect HCM to create and set schedulable callback + auto* deferred_request_callback = + new Event::MockSchedulableCallback(&filter_callbacks_.connection_.dispatcher_); + EXPECT_CALL(*deferred_request_callback, enabled()) + .WillRepeatedly( + Invoke([deferred_request_callback]() { return deferred_request_callback->enabled_; })); + EXPECT_CALL(*deferred_request_callback, scheduleCallbackNextIteration()) + .WillRepeatedly( + Invoke([deferred_request_callback]() { deferred_request_callback->enabled_ = true; })); + + return deferred_request_callback; +} + } // namespace Http } // namespace Envoy diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 76a051c8641e..ddccd0a77321 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -206,6 +206,8 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { // validate headers. void expectCheckWithDefaultUhv(); + Event::MockSchedulableCallback* enableStreamsPerIoLimit(uint32_t limit); + Envoy::Event::SimulatedTimeSystem test_time_; NiceMock route_config_provider_; std::shared_ptr route_config_{new NiceMock()}; diff --git a/test/common/http/conn_pool_grid_test.cc b/test/common/http/conn_pool_grid_test.cc index f14c02f9ef5d..a52a6545ec8e 100644 --- a/test/common/http/conn_pool_grid_test.cc +++ b/test/common/http/conn_pool_grid_test.cc @@ -909,7 +909,7 @@ TEST_F(ConnectivityGridTest, Http3FailedRecentlyThenFailsAgain) { } // namespace Envoy #include "test/mocks/server/transport_socket_factory_context.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" namespace Envoy { namespace Http { namespace { diff --git a/test/common/http/filter_chain_helper_test.cc b/test/common/http/filter_chain_helper_test.cc index dfeefcde2907..75483f21cd68 100644 --- a/test/common/http/filter_chain_helper_test.cc +++ b/test/common/http/filter_chain_helper_test.cc @@ -18,7 +18,7 @@ class MockFilterChainOptions : public FilterChainOptions { public: MockFilterChainOptions() = default; - MOCK_METHOD(bool, filterDisabled, (absl::string_view), (const)); + MOCK_METHOD(absl::optional, filterDisabled, (absl::string_view), (const)); }; TEST(FilterChainUtilityTest, CreateFilterChainForFactoriesWithRouteDisabled) { @@ -32,7 +32,7 @@ TEST(FilterChainUtilityTest, CreateFilterChainForFactoriesWithRouteDisabled) { Filter::NamedHttpFilterFactoryCb{"filter_type_name", [](FilterChainFactoryCallbacks&) {}}, name); - filter_factories.push_back(std::move(provider)); + filter_factories.push_back({std::move(provider), false}); } { @@ -44,9 +44,43 @@ TEST(FilterChainUtilityTest, CreateFilterChainForFactoriesWithRouteDisabled) { { - EXPECT_CALL(options, filterDisabled("filter_0")).WillOnce(Return(true)); - EXPECT_CALL(options, filterDisabled("filter_1")).WillOnce(Return(false)); - EXPECT_CALL(options, filterDisabled("filter_2")).WillOnce(Return(true)); + EXPECT_CALL(options, filterDisabled("filter_0")).WillOnce(Return(absl::make_optional(true))); + EXPECT_CALL(options, filterDisabled("filter_1")).WillOnce(Return(absl::make_optional(false))); + EXPECT_CALL(options, filterDisabled("filter_2")).WillOnce(Return(absl::nullopt)); + + // 'filter_1' and 'filter_2' should be added. + EXPECT_CALL(manager, applyFilterFactoryCb(_, _)).Times(2); + FilterChainUtility::createFilterChainForFactories(manager, options, filter_factories); + } +} + +TEST(FilterChainUtilityTest, CreateFilterChainForFactoriesWithRouteDisabledAndDefaultDisabled) { + NiceMock manager; + NiceMock options; + FilterChainUtility::FilterFactoriesList filter_factories; + + for (const auto& name : {"filter_0", "filter_1", "filter_2"}) { + auto provider = + std::make_unique>( + Filter::NamedHttpFilterFactoryCb{"filter_type_name", + [](FilterChainFactoryCallbacks&) {}}, + name); + filter_factories.push_back({std::move(provider), true}); + } + + { + // If empty filter chain options is provided, all filters should not be added because they are + // all disabled by default. + EXPECT_CALL(manager, applyFilterFactoryCb(_, _)).Times(0); + FilterChainUtility::createFilterChainForFactories(manager, Http::EmptyFilterChainOptions{}, + filter_factories); + } + + { + + EXPECT_CALL(options, filterDisabled("filter_0")).WillOnce(Return(absl::make_optional(true))); + EXPECT_CALL(options, filterDisabled("filter_1")).WillOnce(Return(absl::make_optional(false))); + EXPECT_CALL(options, filterDisabled("filter_2")).WillOnce(Return(absl::nullopt)); // Only filter_1 should be added. EXPECT_CALL(manager, applyFilterFactoryCb(_, _)); diff --git a/test/common/http/filter_manager_test.cc b/test/common/http/filter_manager_test.cc index f272680cedd3..b7a8de0e303e 100644 --- a/test/common/http/filter_manager_test.cc +++ b/test/common/http/filter_manager_test.cc @@ -28,6 +28,7 @@ using testing::Return; namespace Envoy { namespace Http { namespace { +using Protobuf::util::MessageDifferencer; class FilterManagerTest : public testing::Test { public: void initialize() { @@ -57,6 +58,19 @@ class FilterManagerTest : public testing::Test { }; } + void validateFilterStateData(const std::string& expected_name) { + ASSERT_TRUE(filter_manager_->streamInfo().filterState()->hasData( + LocalReplyFilterStateKey)); + auto fs_value = + filter_manager_->streamInfo().filterState()->getDataReadOnly( + LocalReplyFilterStateKey); + EXPECT_EQ(fs_value->serializeAsString(), expected_name); + + auto expected = std::make_unique(); + expected->set_value(expected_name); + EXPECT_TRUE(MessageDifferencer::Equals(*(fs_value->serializeAsProto()), *expected)); + } + std::unique_ptr filter_manager_; NiceMock filter_manager_callbacks_; NiceMock dispatcher_; @@ -69,6 +83,61 @@ class FilterManagerTest : public testing::Test { std::make_shared(StreamInfo::FilterState::LifeSpan::Connection); }; +TEST_F(FilterManagerTest, RequestHeadersOrResponseHeadersAccess) { + initialize(); + + auto decoder_filter = std::make_shared>(); + auto encoder_filter = std::make_shared>(); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillOnce(Invoke([&](FilterChainManager& manager) -> bool { + auto decoder_factory = createDecoderFilterFactoryCb(decoder_filter); + manager.applyFilterFactoryCb({}, decoder_factory); + auto encoder_factory = createEncoderFilterFactoryCb(encoder_filter); + manager.applyFilterFactoryCb({}, encoder_factory); + return true; + })); + filter_manager_->createFilterChain(); + + RequestHeaderMapPtr request_headers{ + new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + ResponseHeaderMapPtr response_headers{new TestResponseHeaderMapImpl{{":status", "200"}}}; + RequestTrailerMapPtr request_trailers{new TestRequestTrailerMapImpl{{"foo", "bar"}}}; + ResponseTrailerMapPtr response_trailers{new TestResponseTrailerMapImpl{{"foo", "bar"}}}; + ResponseHeaderMapPtr informational_headers{ + new TestResponseHeaderMapImpl{{":status", "100"}, {"foo", "bar"}}}; + + EXPECT_CALL(filter_manager_callbacks_, requestHeaders()) + .Times(2) + .WillRepeatedly(Return(makeOptRef(*request_headers))); + EXPECT_CALL(filter_manager_callbacks_, responseHeaders()) + .Times(2) + .WillRepeatedly(Return(makeOptRef(*response_headers))); + EXPECT_CALL(filter_manager_callbacks_, requestTrailers()) + .Times(2) + .WillRepeatedly(Return(makeOptRef(*request_trailers))); + EXPECT_CALL(filter_manager_callbacks_, responseTrailers()) + .Times(2) + .WillRepeatedly(Return(makeOptRef(*response_trailers))); + EXPECT_CALL(filter_manager_callbacks_, informationalHeaders()) + .Times(2) + .WillRepeatedly(Return(makeOptRef(*informational_headers))); + + EXPECT_EQ(decoder_filter->callbacks_->requestHeaders().ptr(), request_headers.get()); + EXPECT_EQ(decoder_filter->callbacks_->responseHeaders().ptr(), response_headers.get()); + EXPECT_EQ(decoder_filter->callbacks_->requestTrailers().ptr(), request_trailers.get()); + EXPECT_EQ(decoder_filter->callbacks_->responseTrailers().ptr(), response_trailers.get()); + EXPECT_EQ(decoder_filter->callbacks_->informationalHeaders().ptr(), informational_headers.get()); + + EXPECT_EQ(encoder_filter->callbacks_->requestHeaders().ptr(), request_headers.get()); + EXPECT_EQ(encoder_filter->callbacks_->responseHeaders().ptr(), response_headers.get()); + EXPECT_EQ(encoder_filter->callbacks_->requestTrailers().ptr(), request_trailers.get()); + EXPECT_EQ(encoder_filter->callbacks_->responseTrailers().ptr(), response_trailers.get()); + EXPECT_EQ(encoder_filter->callbacks_->informationalHeaders().ptr(), informational_headers.get()); + + filter_manager_->destroyFilters(); +} + // Verifies that the local reply persists the gRPC classification even if the request headers are // modified. TEST_F(FilterManagerTest, SendLocalReplyDuringDecodingGrpcClassiciation) { @@ -81,7 +150,7 @@ TEST_F(FilterManagerTest, SendLocalReplyDuringDecodingGrpcClassiciation) { headers.setContentType("text/plain"); filter->callbacks_->sendLocalReply(Code::InternalServerError, "", nullptr, absl::nullopt, - ""); + "details"); return FilterHeadersStatus::StopIteration; })); @@ -98,13 +167,14 @@ TEST_F(FilterManagerTest, SendLocalReplyDuringDecodingGrpcClassiciation) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto factory = createDecoderFilterFactoryCb(filter); - manager.applyFilterFactoryCb({}, factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, factory); return true; })); filter_manager_->createFilterChain(); filter_manager_->requestHeadersInitialized(); + EXPECT_CALL(local_reply_, rewrite(_, _, _, _, _, _)); EXPECT_CALL(filter_manager_callbacks_, setResponseHeaders_(_)) .WillOnce(Invoke([](auto& response_headers) { @@ -114,7 +184,11 @@ TEST_F(FilterManagerTest, SendLocalReplyDuringDecodingGrpcClassiciation) { EXPECT_CALL(filter_manager_callbacks_, resetIdleTimer()); EXPECT_CALL(filter_manager_callbacks_, encodeHeaders(_, _)); EXPECT_CALL(filter_manager_callbacks_, endStream()); + filter_manager_->decodeHeaders(*grpc_headers, true); + + validateFilterStateData("configName1"); + filter_manager_->destroyFilters(); } @@ -140,17 +214,17 @@ TEST_F(FilterManagerTest, SendLocalReplyDuringEncodingGrpcClassiciation) { EXPECT_CALL(*encoder_filter, encodeHeaders(_, true)) .WillRepeatedly(Invoke([&](auto&, bool) -> FilterHeadersStatus { encoder_filter->encoder_callbacks_->sendLocalReply(Code::InternalServerError, "", nullptr, - absl::nullopt, ""); + absl::nullopt, "details"); return FilterHeadersStatus::StopIteration; })); EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto decoder_factory = createDecoderFilterFactoryCb(decoder_filter); - manager.applyFilterFactoryCb({}, decoder_factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, decoder_factory); auto stream_factory = createStreamFilterFactoryCb(encoder_filter); - manager.applyFilterFactoryCb({}, stream_factory); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, stream_factory); return true; })); @@ -174,7 +248,11 @@ TEST_F(FilterManagerTest, SendLocalReplyDuringEncodingGrpcClassiciation) { })); EXPECT_CALL(filter_manager_callbacks_, encodeHeaders(_, _)); EXPECT_CALL(filter_manager_callbacks_, endStream()); + filter_manager_->decodeHeaders(*grpc_headers, true); + + validateFilterStateData("configName2"); + filter_manager_->destroyFilters(); } @@ -193,11 +271,11 @@ TEST_F(FilterManagerTest, OnLocalReply) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto decoder_factory = createDecoderFilterFactoryCb(decoder_filter); - manager.applyFilterFactoryCb({}, decoder_factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, decoder_factory); auto stream_factory = createStreamFilterFactoryCb(stream_filter); - manager.applyFilterFactoryCb({}, stream_factory); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, stream_factory); auto encoder_factory = createEncoderFilterFactoryCb(encoder_filter); - manager.applyFilterFactoryCb({}, encoder_factory); + manager.applyFilterFactoryCb({"configName3", "filterName3"}, encoder_factory); return true; })); @@ -231,10 +309,13 @@ TEST_F(FilterManagerTest, OnLocalReply) { // The reason for the response (in this case the reset) will still be tracked // but as no response is sent the response code will remain absent. + ASSERT_TRUE(filter_manager_->streamInfo().responseCodeDetails().has_value()); EXPECT_EQ(filter_manager_->streamInfo().responseCodeDetails().value(), "details"); EXPECT_FALSE(filter_manager_->streamInfo().responseCode().has_value()); + validateFilterStateData("configName1"); + filter_manager_->destroyFilters(); } @@ -253,11 +334,11 @@ TEST_F(FilterManagerTest, MultipleOnLocalReply) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto decoder_factory = createDecoderFilterFactoryCb(decoder_filter); - manager.applyFilterFactoryCb({}, decoder_factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, decoder_factory); auto stream_factory = createStreamFilterFactoryCb(stream_filter); - manager.applyFilterFactoryCb({}, stream_factory); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, stream_factory); auto encoder_factory = createEncoderFilterFactoryCb(encoder_filter); - manager.applyFilterFactoryCb({}, encoder_factory); + manager.applyFilterFactoryCb({"configName3", "filterName3"}, encoder_factory); return true; })); @@ -301,6 +382,8 @@ TEST_F(FilterManagerTest, MultipleOnLocalReply) { EXPECT_EQ(filter_manager_->streamInfo().responseCodeDetails().value(), "details2"); EXPECT_FALSE(filter_manager_->streamInfo().responseCode().has_value()); + validateFilterStateData("configName1"); + filter_manager_->destroyFilters(); } @@ -336,15 +419,20 @@ TEST_F(FilterManagerTest, SetAndGetUpstreamOverrideHost) { })); filter_manager_->createFilterChain(); - decoder_filter->callbacks_->setUpstreamOverrideHost("1.2.3.4"); + decoder_filter->callbacks_->setUpstreamOverrideHost(std::make_pair("1.2.3.4", true)); auto override_host = decoder_filter->callbacks_->upstreamOverrideHost(); - EXPECT_EQ(override_host.value(), "1.2.3.4"); + EXPECT_EQ(override_host.value().first, "1.2.3.4"); + EXPECT_TRUE(override_host.value().second); filter_manager_->destroyFilters(); }; -TEST_F(FilterManagerTest, GetRouteLevelFilterConfig) { +TEST_F(FilterManagerTest, GetRouteLevelFilterConfigAndEnableDowngrade) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.no_downgrade_to_canonical_name", "false"}}); + initialize(); std::shared_ptr decoder_filter(new NiceMock()); @@ -405,7 +493,7 @@ TEST_F(FilterManagerTest, GetRouteLevelFilterConfig) { filter_manager_->destroyFilters(); }; -TEST_F(FilterManagerTest, GetRouteLevelFilterConfigButDisabledDowngrade) { +TEST_F(FilterManagerTest, GetRouteLevelFilterConfig) { TestScopedRuntime scoped_runtime; scoped_runtime.mergeValues( {{"envoy.reloadable_features.no_downgrade_to_canonical_name", "true"}}); @@ -503,9 +591,9 @@ TEST_F(FilterManagerTest, MetadataContinueAll) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto decoder_factory = createStreamFilterFactoryCb(filter_1); - manager.applyFilterFactoryCb({"filter1", "filter1"}, decoder_factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, decoder_factory); decoder_factory = createStreamFilterFactoryCb(filter_2); - manager.applyFilterFactoryCb({"filter2", "filter2"}, decoder_factory); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, decoder_factory); return true; })); filter_manager_->createFilterChain(); @@ -575,9 +663,9 @@ TEST_F(FilterManagerTest, DecodeMetadataSendsLocalReply) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto factory = createStreamFilterFactoryCb(filter_1); - manager.applyFilterFactoryCb({"filter1", "filter1"}, factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, factory); factory = createStreamFilterFactoryCb(filter_2); - manager.applyFilterFactoryCb({"filter2", "filter2"}, factory); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, factory); return true; })); filter_manager_->createFilterChain(); @@ -607,6 +695,93 @@ TEST_F(FilterManagerTest, DecodeMetadataSendsLocalReply) { EXPECT_THAT(*filter_manager_->streamInfo().responseCodeDetails(), "bad_metadata"); + validateFilterStateData("configName1"); + + filter_manager_->destroyFilters(); +} + +TEST_F(FilterManagerTest, MetadataContinueAllFollowedByHeadersLocalReply) { + initialize(); + + std::shared_ptr filter_1(new NiceMock()); + + std::shared_ptr filter_2(new NiceMock()); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { + auto decoder_factory = createStreamFilterFactoryCb(filter_1); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, decoder_factory); + decoder_factory = createStreamFilterFactoryCb(filter_2); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, decoder_factory); + return true; + })); + filter_manager_->createFilterChain(); + + // Decode path: + EXPECT_CALL(*filter_1, decodeHeaders(_, _)).WillOnce(Return(FilterHeadersStatus::StopIteration)); + RequestHeaderMapPtr basic_headers{ + new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + ON_CALL(filter_manager_callbacks_, requestHeaders()) + .WillByDefault(Return(makeOptRef(*basic_headers))); + + filter_manager_->requestHeadersInitialized(); + filter_manager_->decodeHeaders(*basic_headers, false); + + EXPECT_CALL(*filter_1, decodeMetadata(_)).WillOnce(Return(FilterMetadataStatus::ContinueAll)); + MetadataMap map1 = {{"a", "b"}}; + MetadataMap map2 = {{"c", "d"}}; + EXPECT_CALL(*filter_2, decodeHeaders(_, _)).WillOnce([&]() { + filter_2->decoder_callbacks_->sendLocalReply(Code::InternalServerError, "bad_headers", nullptr, + absl::nullopt, "bad_headers"); + return FilterHeadersStatus::StopIteration; + }); + // filter_2 should never decode metadata. + EXPECT_CALL(*filter_2, decodeMetadata(_)).Times(0); + filter_manager_->decodeMetadata(map1); + filter_manager_->destroyFilters(); +} + +TEST_F(FilterManagerTest, MetadataContinueAllFollowedByHeadersLocalReplyRuntimeFlagOff) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.stop_decode_metadata_on_local_reply", "false"}}); + initialize(); + + std::shared_ptr filter_1(new NiceMock()); + + std::shared_ptr filter_2(new NiceMock()); + + EXPECT_CALL(filter_factory_, createFilterChain(_)) + .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { + auto decoder_factory = createStreamFilterFactoryCb(filter_1); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, decoder_factory); + decoder_factory = createStreamFilterFactoryCb(filter_2); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, decoder_factory); + return true; + })); + filter_manager_->createFilterChain(); + + // Decode path: + EXPECT_CALL(*filter_1, decodeHeaders(_, _)).WillOnce(Return(FilterHeadersStatus::StopIteration)); + RequestHeaderMapPtr basic_headers{ + new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + ON_CALL(filter_manager_callbacks_, requestHeaders()) + .WillByDefault(Return(makeOptRef(*basic_headers))); + + filter_manager_->requestHeadersInitialized(); + filter_manager_->decodeHeaders(*basic_headers, false); + + EXPECT_CALL(*filter_1, decodeMetadata(_)).WillOnce(Return(FilterMetadataStatus::ContinueAll)); + MetadataMap map1 = {{"a", "b"}}; + MetadataMap map2 = {{"c", "d"}}; + EXPECT_CALL(*filter_2, decodeHeaders(_, _)).WillOnce([&]() { + filter_2->decoder_callbacks_->sendLocalReply(Code::InternalServerError, "bad_headers", nullptr, + absl::nullopt, "bad_headers"); + return FilterHeadersStatus::StopIteration; + }); + // filter_2 decodes metadata, even though the decoder filter chain has been aborted. + EXPECT_CALL(*filter_2, decodeMetadata(_)); + filter_manager_->decodeMetadata(map1); filter_manager_->destroyFilters(); } @@ -619,9 +794,9 @@ TEST_F(FilterManagerTest, EncodeMetadataSendsLocalReply) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto factory = createStreamFilterFactoryCb(filter_1); - manager.applyFilterFactoryCb({"filter1", "filter1"}, factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, factory); factory = createStreamFilterFactoryCb(filter_2); - manager.applyFilterFactoryCb({"filter2", "filter2"}, factory); + manager.applyFilterFactoryCb({"configName2", "filterName2"}, factory); return true; })); filter_manager_->createFilterChain(); @@ -646,6 +821,8 @@ TEST_F(FilterManagerTest, EncodeMetadataSendsLocalReply) { MetadataMap map1 = {{"a", "b"}}; filter_2->decoder_callbacks_->encodeMetadata(std::make_unique(map1)); + validateFilterStateData("configName2"); + filter_manager_->destroyFilters(); } @@ -656,7 +833,7 @@ TEST_F(FilterManagerTest, IdleTimerResets) { EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillRepeatedly(Invoke([&](FilterChainManager& manager) -> bool { auto factory = createStreamFilterFactoryCb(filter_1); - manager.applyFilterFactoryCb({"filter1", "filter1"}, factory); + manager.applyFilterFactoryCb({"configName1", "filterName1"}, factory); return true; })); filter_manager_->createFilterChain(); diff --git a/test/common/http/hcm_router_fuzz_test.cc b/test/common/http/hcm_router_fuzz_test.cc index 2bc755e55c30..144e6ae991fa 100644 --- a/test/common/http/hcm_router_fuzz_test.cc +++ b/test/common/http/hcm_router_fuzz_test.cc @@ -359,6 +359,16 @@ class FuzzClusterManager { return nullptr; } + FuzzCluster* + selectClusterByThreadLocalCluster(Upstream::ThreadLocalCluster& thread_local_cluster) { + for (auto& cluster : clusters_) { + if (&cluster->tlc_ == &thread_local_cluster) { + return cluster.get(); + } + } + return nullptr; + } + void reset() { for (auto& cluster : clusters_) { cluster->reset(); @@ -395,15 +405,14 @@ class FuzzGenericConnPoolFactory : public Router::GenericConnPoolFactory { std::string name() const override { return "envoy.filters.connection_pools.http.generic"; } std::string category() const override { return "envoy.upstreams"; } Router::GenericConnPoolPtr - createGenericConnPool(Upstream::ThreadLocalCluster&, + createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, - absl::optional protocol, + Upstream::ResourcePriority, absl::optional protocol, Upstream::LoadBalancerContext*) const override { if (upstream_protocol != UpstreamProtocol::HTTP) { return nullptr; } - FuzzCluster* cluster = cluster_manager_.selectClusterByName(route_entry.clusterName()); + FuzzCluster* cluster = cluster_manager_.selectClusterByThreadLocalCluster(thread_local_cluster); if (cluster == nullptr) { return nullptr; } diff --git a/test/common/http/header_map_impl_test.cc b/test/common/http/header_map_impl_test.cc index 8cee2dd72fe2..e4d949680161 100644 --- a/test/common/http/header_map_impl_test.cc +++ b/test/common/http/header_map_impl_test.cc @@ -1051,67 +1051,6 @@ TEST(HeaderMapImplTest, ValidHeaderString) { EXPECT_FALSE(validHeaderString("abc\n")); } -TEST(HeaderMapImplTest, HttpTraceContextTest) { - { - TestRequestHeaderMapImpl request_headers; - - // Protocol. - EXPECT_EQ(request_headers.protocol(), ""); - request_headers.addCopy(Http::Headers::get().Protocol, "HTTP/x"); - EXPECT_EQ(request_headers.protocol(), "HTTP/x"); - - // Host. - EXPECT_EQ(request_headers.host(), ""); - request_headers.addCopy(Http::Headers::get().Host, "test.com:233"); - EXPECT_EQ(request_headers.host(), "test.com:233"); - - // Path. - EXPECT_EQ(request_headers.path(), ""); - request_headers.addCopy(Http::Headers::get().Path, "/anything"); - EXPECT_EQ(request_headers.path(), "/anything"); - - // Method. - EXPECT_EQ(request_headers.method(), ""); - request_headers.addCopy(Http::Headers::get().Method, Http::Headers::get().MethodValues.Options); - EXPECT_EQ(request_headers.method(), Http::Headers::get().MethodValues.Options); - } - - { - size_t size = 0; - TestRequestHeaderMapImpl request_headers{{"host", "foo"}, {"bar", "var"}, {"ok", "no"}}; - request_headers.forEach([&size](absl::string_view key, absl::string_view val) { - size += key.size(); - size += val.size(); - return true; - }); - // 'host' will be converted to ':authority'. - EXPECT_EQ(23, size); - EXPECT_EQ(23, request_headers.byteSize()); - } - - { - TestRequestHeaderMapImpl request_headers{{"host", "foo"}}; - EXPECT_EQ(request_headers.getByKey("host").value(), "foo"); - - request_headers.setByKey("trace_key", "trace_value"); - EXPECT_EQ(request_headers.getByKey("trace_key").value(), "trace_value"); - - std::string trace_ref_key = "trace_ref_key"; - request_headers.setByReferenceKey(trace_ref_key, "trace_value"); - auto* header_entry = request_headers.get(Http::LowerCaseString(trace_ref_key))[0]; - EXPECT_EQ(reinterpret_cast(trace_ref_key.data()), - reinterpret_cast(header_entry->key().getStringView().data())); - - std::string trace_ref_value = "trace_ref_key"; - request_headers.setByReference(trace_ref_key, trace_ref_value); - header_entry = request_headers.get(Http::LowerCaseString(trace_ref_key))[0]; - EXPECT_EQ(reinterpret_cast(trace_ref_key.data()), - reinterpret_cast(header_entry->key().getStringView().data())); - EXPECT_EQ(reinterpret_cast(trace_ref_value.data()), - reinterpret_cast(header_entry->value().getStringView().data())); - } -} - // Test the header map max limits are setup correctly. TEST(HeaderMapImplTest, HeaderMapMaxLimits) { auto request_header_default = RequestHeaderMapImpl::create(); diff --git a/test/common/http/header_mutation_test.cc b/test/common/http/header_mutation_test.cc index 74ce706bd0e7..6f2f22114da3 100644 --- a/test/common/http/header_mutation_test.cc +++ b/test/common/http/header_mutation_test.cc @@ -27,8 +27,7 @@ TEST(HeaderMutationsTest, BasicRemove) { {":authority", "host"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ("", headers.get_("flag-header")); EXPECT_EQ("", headers.get_("another-flag-header")); EXPECT_EQ("not-flag-header-value", headers.get_("not-flag-header")); @@ -75,8 +74,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); // Original 'flag-header' is removed and no new value is appended because there is no // 'another-flag-header'. @@ -91,8 +89,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); // Original 'flag-header' is removed and the new value is appended. EXPECT_EQ("another-flag-header-value", headers.get_("flag-header")); @@ -105,8 +102,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ(2, headers.get(Http::LowerCaseString("flag-header-2")).size()); } @@ -116,8 +112,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ(1, headers.get(Http::LowerCaseString("flag-header-2")).size()); } @@ -128,8 +123,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ(1, headers.get(Http::LowerCaseString("flag-header-3")).size()); } @@ -140,8 +134,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ(1, headers.get(Http::LowerCaseString("flag-header-3")).size()); EXPECT_EQ("flag-header-3-value-old", headers.get_("flag-header-3")); @@ -154,8 +147,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ(1, headers.get(Http::LowerCaseString("flag-header-4")).size()); EXPECT_EQ("flag-header-4-value", headers.get_("flag-header-4")); @@ -166,8 +158,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ(1, headers.get(Http::LowerCaseString("flag-header-4")).size()); EXPECT_EQ("flag-header-4-value", headers.get_("flag-header-4")); @@ -184,8 +175,7 @@ TEST(HeaderMutationsTest, AllOperations) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); // 'flag-header' is removed and new 'flag-header' is added. EXPECT_EQ("another-flag-header-value", headers.get_("flag-header")); @@ -221,8 +211,7 @@ TEST(HeaderMutationsTest, KeepEmptyValue) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); // Original 'flag-header' is removed and empty value is appended. EXPECT_EQ(2, headers.size()); @@ -253,8 +242,7 @@ TEST(HeaderMutationsTest, BasicOrder) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ("", headers.get_("flag-header")); EXPECT_EQ(0, headers.get(Http::LowerCaseString("flag-header")).size()); } @@ -279,8 +267,7 @@ TEST(HeaderMutationsTest, BasicOrder) { {":method", "GET"}, }; - mutations.evaluateHeaders(headers, headers, *Http::StaticEmptyHeaders::get().response_headers, - stream_info); + mutations.evaluateHeaders(headers, {&headers}, stream_info); EXPECT_EQ("another-flag-header-value", headers.get_("flag-header")); } } diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index 1119ca4b1503..53a4e60e1c08 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -1148,16 +1148,16 @@ TEST(HeaderIsValidTest, IsConnect) { EXPECT_FALSE(HeaderUtility::isConnect(Http::TestRequestHeaderMapImpl{})); } -TEST(HeaderIsValidTest, IsConnectUdp) { - EXPECT_TRUE( - HeaderUtility::isConnectUdp(Http::TestRequestHeaderMapImpl{{"upgrade", "connect-udp"}})); +TEST(HeaderIsValidTest, IsConnectUdpRequest) { + EXPECT_TRUE(HeaderUtility::isConnectUdpRequest( + Http::TestRequestHeaderMapImpl{{"upgrade", "connect-udp"}})); // Should use case-insensitive comparison for the upgrade values. - EXPECT_TRUE( - HeaderUtility::isConnectUdp(Http::TestRequestHeaderMapImpl{{"upgrade", "CONNECT-UDP"}})); + EXPECT_TRUE(HeaderUtility::isConnectUdpRequest( + Http::TestRequestHeaderMapImpl{{"upgrade", "CONNECT-UDP"}})); // Extended CONNECT requests should be normalized to HTTP/1.1. - EXPECT_FALSE(HeaderUtility::isConnectUdp( + EXPECT_FALSE(HeaderUtility::isConnectUdpRequest( Http::TestRequestHeaderMapImpl{{":method", "CONNECT"}, {":protocol", "connect-udp"}})); - EXPECT_FALSE(HeaderUtility::isConnectUdp(Http::TestRequestHeaderMapImpl{})); + EXPECT_FALSE(HeaderUtility::isConnectUdpRequest(Http::TestRequestHeaderMapImpl{})); } TEST(HeaderIsValidTest, IsConnectResponse) { diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 60dc4ad1fc89..20e844510962 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -27,6 +27,7 @@ #include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -4473,43 +4474,6 @@ TEST_P(Http1ClientConnectionImplTest, NoContentLengthResponse) { } } -// Line folding (also called continuation line) is disallowed per RFC9112 Section 5.2. -TEST_P(Http1ServerConnectionImplTest, LineFolding) { - initialize(); - InSequence s; - - StrictMock decoder; - Http::ResponseEncoder* response_encoder = nullptr; - EXPECT_CALL(callbacks_, newStream(_, _)) - .WillOnce(Invoke([&](ResponseEncoder& encoder, bool) -> RequestDecoder& { - response_encoder = &encoder; - return decoder; - })); - - TestRequestHeaderMapImpl expected_headers{ - {":path", "/"}, {":method", "GET"}, {"foo", "folded value"}}; - if (parser_impl_ == Http1ParserImpl::BalsaParser) { - EXPECT_CALL(decoder, - sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error")); - } else { - EXPECT_CALL(decoder, decodeHeaders_(HeaderMapEqual(&expected_headers), true)); - } - - Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n" - "foo: \r\n" - " folded value\r\n\r\n"); - auto status = codec_->dispatch(buffer); - - if (parser_impl_ == Http1ParserImpl::BalsaParser) { - EXPECT_TRUE(isCodecProtocolError(status)); - EXPECT_FALSE(status.ok()); - EXPECT_EQ(status.message(), "http/1.1 protocol error: INVALID_HEADER_FORMAT"); - EXPECT_EQ("http1.codec_error", response_encoder->getStream().responseDetails()); - } else { - EXPECT_TRUE(status.ok()); - } -} - // Regression test for https://github.com/envoyproxy/envoy/issues/25458. TEST_P(Http1ServerConnectionImplTest, EmptyFieldName) { initialize(); @@ -4641,7 +4605,8 @@ TEST_P(Http1ServerConnectionImplTest, SeparatorInHeaderName) { // BalsaParser always rejects a header name with space. HttpParser only rejects // it in strict mode, which is disabled when ENVOY_ENABLE_UHV is defined. TEST_P(Http1ClientConnectionImplTest, SpaceInHeaderName) { - bool accept = parser_impl_ == Http1ParserImpl::HttpParser; + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) + bool accept = (parser_impl_ == Http1ParserImpl::HttpParser); #ifndef ENVOY_ENABLE_UHV accept = false; #endif @@ -4675,7 +4640,8 @@ TEST_P(Http1ClientConnectionImplTest, SpaceInHeaderName) { } TEST_P(Http1ServerConnectionImplTest, SpaceInHeaderName) { - bool accept = parser_impl_ == Http1ParserImpl::HttpParser; + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) + bool accept = (parser_impl_ == Http1ParserImpl::HttpParser); #ifndef ENVOY_ENABLE_UHV accept = false; #endif @@ -4862,5 +4828,109 @@ TEST_P(Http1ClientConnectionImplTest, InvalidCharacterInTrailerName) { EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); } +// When receiving header value with obsolete line folding, `obs-fold` should be replaced by SP. +// This is http-parser's behavior. BalsaParser does not support obsolete line folding and rejects +// such messages (also permitted by the specification). See RFC9110 Section 5.5: +// https://www.rfc-editor.org/rfc/rfc9110.html#name-field-values. +TEST_P(Http1ServerConnectionImplTest, ObsFold) { + // SPELLCHECKER(off) + initialize(); + + StrictMock decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + TestRequestHeaderMapImpl expected_headers{ + {":path", "/"}, + {":method", "GET"}, + {"multi-line-header", "foo bar\t\tbaz"}, + }; + EXPECT_CALL(decoder, decodeHeaders_(HeaderMapEqual(&expected_headers), true)); + + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\n" + "Multi-Line-Header: \r\n foo\r\n bar\r\n\t\tbaz\r\n" + "\r\n"); + auto status = codec_->dispatch(buffer); + EXPECT_TRUE(status.ok()); + // SPELLCHECKER(on) +} + +TEST_P(Http1ClientConnectionImplTest, ObsFold) { + // SPELLCHECKER(off) + initialize(); + + NiceMock response_decoder; + Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder); + TestRequestHeaderMapImpl headers{{":method", "GET"}, {":path", "/"}, {":authority", "host"}}; + EXPECT_TRUE(request_encoder.encodeHeaders(headers, true).ok()); + + TestRequestHeaderMapImpl expected_headers{ + {":status", "200"}, + {"multi-line-header", "foo bar\t\tbaz"}, + {"content-length", "0"}, + }; + EXPECT_CALL(response_decoder, decodeHeaders_(HeaderMapEqual(&expected_headers), true)); + + Buffer::OwnedImpl response("HTTP/1.1 200 OK\r\n" + "Multi-Line-Header: \r\n foo\r\n bar\r\n\t\tbaz\r\n" + "Content-Length: 0\r\n" + "\r\n"); + + auto status = codec_->dispatch(response); + EXPECT_TRUE(status.ok()); + // SPELLCHECKER(on) +} + +TEST_P(Http1ServerConnectionImplTest, ValueWithNullCharacter) { + initialize(); + + StrictMock decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_CALL(decoder, sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, + "http1.invalid_characters")); + } else { + EXPECT_CALL(decoder, + sendLocalReply(Http::Code::BadRequest, "Bad Request", _, _, "http1.codec_error")); + } + + Buffer::OwnedImpl buffer(absl::StrCat("GET / HTTP/1.1\r\n" + "key: value has ", + absl::string_view("\0", 1), + "null character\r\n" + "\r\n")); + auto status = codec_->dispatch(buffer); + EXPECT_FALSE(status.ok()); + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_EQ(status.message(), "http/1.1 protocol error: header value contains invalid chars"); + } else { + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); + } +} + +TEST_P(Http1ClientConnectionImplTest, ValueWithNullCharacter) { + initialize(); + + NiceMock response_decoder; + Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder); + TestRequestHeaderMapImpl headers{{":method", "GET"}, {":path", "/"}, {":authority", "host"}}; + EXPECT_TRUE(request_encoder.encodeHeaders(headers, true).ok()); + + Buffer::OwnedImpl response(absl::StrCat("HTTP/1.1 200 OK\r\n" + "key: value has ", + absl::string_view("\0", 1), + "null character\r\n" + "Content-Length: 0\r\n" + "\r\n")); + + auto status = codec_->dispatch(response); + EXPECT_FALSE(status.ok()); + if (parser_impl_ == Http1ParserImpl::BalsaParser) { + EXPECT_EQ(status.message(), "http/1.1 protocol error: header value contains invalid chars"); + } else { + EXPECT_EQ(status.message(), "http/1.1 protocol error: HPE_INVALID_HEADER_TOKEN"); + } +} + } // namespace Http } // namespace Envoy diff --git a/test/common/http/http1/http1_connection_fuzz_test.cc b/test/common/http/http1/http1_connection_fuzz_test.cc index af3888748f2e..9dae7af6c7b9 100644 --- a/test/common/http/http1/http1_connection_fuzz_test.cc +++ b/test/common/http/http1/http1_connection_fuzz_test.cc @@ -79,7 +79,7 @@ class Http1Harness { }; static std::unique_ptr harness; -static void reset_harness() { harness = nullptr; } +static void resetHarness() { harness = nullptr; } // Fuzzing strategy // Unconstrained fuzzing, rely on corpus for coverage @@ -89,7 +89,7 @@ DEFINE_FUZZER(const uint8_t* buf, size_t len) { Http1Settings server_settings = fromHttp1Settings(); Http1Settings client_settings = fromHttp1Settings(); harness = std::make_unique(server_settings, client_settings); - atexit(reset_harness); + atexit(resetHarness); } Buffer::OwnedImpl httpmsg; diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 35f902f8a194..6ccc25237dc2 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -180,6 +180,7 @@ envoy_cc_fuzz_test( deps = [ ":frame_replay_lib", "//test/common/http/http2:codec_impl_test_util", + "//test/test_common:test_runtime_lib", ], ) @@ -190,6 +191,7 @@ envoy_cc_fuzz_test( deps = [ ":frame_replay_lib", "//test/common/http/http2:codec_impl_test_util", + "//test/test_common:test_runtime_lib", ], ) diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 4a8c0e0a82a4..6eec4fc0b1c3 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -1645,6 +1645,7 @@ TEST_P(Http2CodecImplTest, ShouldRestoreCrashDumpInfoWhenHandlingDeferredProcess process_buffered_data_callback->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) EXPECT_THAT(ostream.contents(), HasSubstr("Http2::ConnectionImpl ")); EXPECT_THAT(ostream.contents(), HasSubstr("Dumping current stream:\n stream: \n ConnectionImpl::StreamImpl")); @@ -4016,6 +4017,7 @@ TEST_P(Http2CodecImplTest, InSequence seq; EXPECT_CALL(request_decoder_, decodeTrailers_(_)); process_buffered_data_callback->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) EXPECT_FALSE(process_buffered_data_callback->enabled_); } } @@ -4162,6 +4164,7 @@ TEST_P(Http2CodecImplTest, ChunksLargeBodyDuringDeferredProcessing) { EXPECT_CALL(request_decoder_, decodeData(_, true)); process_buffered_data_callback->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) EXPECT_FALSE(process_buffered_data_callback->enabled_); } } diff --git a/test/common/http/http2/frame_replay.cc b/test/common/http/http2/frame_replay.cc index 8a604884330f..c2755a683d40 100644 --- a/test/common/http/http2/frame_replay.cc +++ b/test/common/http/http2/frame_replay.cc @@ -12,8 +12,10 @@ namespace Http { namespace Http2 { FileFrame::FileFrame(absl::string_view path) : api_(Api::createApiForTest()) { - const std::string contents = api_->fileSystem().fileReadToEnd( - TestEnvironment::runfilesPath("test/common/http/http2/" + std::string(path))); + const std::string contents = api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath( + "test/common/http/http2/" + std::string(path))) + .value(); frame_.resize(contents.size()); contents.copy(reinterpret_cast(frame_.data()), frame_.size()); } diff --git a/test/common/http/http2/frame_replay_test.cc b/test/common/http/http2/frame_replay_test.cc index ac7d3e56ce79..abae3e024131 100644 --- a/test/common/http/http2/frame_replay_test.cc +++ b/test/common/http/http2/frame_replay_test.cc @@ -22,6 +22,13 @@ namespace Http { namespace Http2 { namespace { +bool skipForUhv() { +#ifdef ENVOY_ENABLE_UHV + return Runtime::runtimeFeatureEnabled("envoy.reloadable_features.http2_use_oghttp2"); +#else + return false; +#endif +} // For organizational purposes only. class RequestFrameCommentTest : public ::testing::Test {}; class ResponseFrameCommentTest : public ::testing::Test {}; @@ -191,6 +198,9 @@ TEST_F(ResponseFrameCommentTest, SimpleExamplePlain) { // https://httpwg.org/specs/rfc7540.html#rfc.section.10.3. We use a non-compressed frame with no // Huffman encoding to simplify. TEST_F(RequestFrameCommentTest, SingleByteNulCrLfInHeaderFrame) { + if (skipForUhv()) { + return; + } FileFrame header{"request_header_corpus/simple_example_plain"}; for (size_t offset = 0; offset < header.frame().size(); ++offset) { @@ -224,6 +234,9 @@ TEST_F(RequestFrameCommentTest, SingleByteNulCrLfInHeaderFrame) { // https://httpwg.org/specs/rfc7540.html#rfc.section.10.3. We use a non-compressed frame with no // Huffman encoding to simplify. TEST_F(ResponseFrameCommentTest, SingleByteNulCrLfInHeaderFrame) { + if (skipForUhv()) { + return; + } FileFrame header{"response_header_corpus/simple_example_plain"}; for (size_t offset = 0; offset < header.frame().size(); ++offset) { @@ -258,6 +271,9 @@ TEST_F(ResponseFrameCommentTest, SingleByteNulCrLfInHeaderFrame) { // CVE-2019-9900. See also https://httpwg.org/specs/rfc7540.html#rfc.section.10.3. We use a // non-compressed frame with no Huffman encoding to simplify. TEST_F(RequestFrameCommentTest, SingleByteNulCrLfInHeaderField) { + if (skipForUhv()) { + return; + } FileFrame header{"request_header_corpus/simple_example_plain"}; for (size_t offset = header.frame().size() - 11 /* foo: offset */; offset < header.frame().size(); @@ -296,6 +312,9 @@ TEST_F(RequestFrameCommentTest, SingleByteNulCrLfInHeaderField) { // CVE-2019-9900. See also https://httpwg.org/specs/rfc7540.html#rfc.section.10.3. We use a // non-compressed frame with no Huffman encoding to simplify. TEST_F(ResponseFrameCommentTest, SingleByteNulCrLfInHeaderField) { + if (skipForUhv()) { + return; + } FileFrame header{"response_header_corpus/simple_example_plain"}; for (size_t offset = header.frame().size() - 17 /* test: offset */; diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc index c5172938a804..319b3fc87380 100644 --- a/test/common/http/http2/http2_frame.cc +++ b/test/common/http/http2/http2_frame.cc @@ -51,12 +51,22 @@ Http2Frame::ResponseStatus Http2Frame::responseStatus() const { return ResponseStatus::Ok; case StaticHeaderIndex::Status404: return ResponseStatus::NotFound; + case StaticHeaderIndex::Status500: + return ResponseStatus::InternalServerError; default: break; } return ResponseStatus::Unknown; } +uint32_t Http2Frame::streamId() const { + if (empty() || size() <= HeaderSize) { + return 0; + } + return (uint32_t(data_[5]) << 24) + (uint32_t(data_[6]) << 16) + (uint32_t(data_[7]) << 8) + + uint32_t(data_[8]); +} + void Http2Frame::buildHeader(Type type, uint32_t payload_size, uint8_t flags, uint32_t stream_id) { data_.assign(payload_size + HeaderSize, 0); setPayloadSize(payload_size); @@ -341,7 +351,11 @@ Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host makeNetworkOrderStreamId(stream_index)); frame.appendStaticHeader(StaticHeaderIndex::MethodGet); frame.appendStaticHeader(StaticHeaderIndex::SchemeHttps); - frame.appendHeaderWithoutIndexing(StaticHeaderIndex::Path, path); + if (path.empty() || path == "/") { + frame.appendStaticHeader(StaticHeaderIndex::Path); + } else { + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::Path, path); + } frame.appendHeaderWithoutIndexing(StaticHeaderIndex::Authority, host); frame.adjustPayloadSize(); return frame; @@ -365,7 +379,11 @@ Http2Frame Http2Frame::makePostRequest(uint32_t stream_index, absl::string_view makeNetworkOrderStreamId(stream_index)); frame.appendStaticHeader(StaticHeaderIndex::MethodPost); frame.appendStaticHeader(StaticHeaderIndex::SchemeHttps); - frame.appendHeaderWithoutIndexing(StaticHeaderIndex::Path, path); + if (path.empty() || path == "/") { + frame.appendStaticHeader(StaticHeaderIndex::Path); + } else { + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::Path, path); + } frame.appendHeaderWithoutIndexing(StaticHeaderIndex::Authority, host); frame.adjustPayloadSize(); return frame; diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index 7fdf510ea256..2844c6bcebde 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -121,7 +121,7 @@ class Http2Frame { Http11Required }; - enum class ResponseStatus { Unknown, Ok, NotFound }; + enum class ResponseStatus { Unknown, Ok, NotFound, InternalServerError }; struct Header { Header(absl::string_view key, absl::string_view value) : key_(key), value_(value) {} @@ -226,6 +226,7 @@ class Http2Frame { return false; } ResponseStatus responseStatus() const; + uint32_t streamId() const; // Copy HTTP2 header. The `header` parameter must at least be HeaderSize long. // Allocates payload size based on the value in the header. @@ -239,7 +240,7 @@ class Http2Frame { if (data_.empty()) { return {}; } - return std::string(reinterpret_cast(data()), size()); + return {reinterpret_cast(data()), size()}; } uint32_t payloadSize() const; @@ -253,6 +254,16 @@ class Http2Frame { ConstIterator end() const { return data_.end(); } bool empty() const { return data_.empty(); } + void appendHeaderWithoutIndexing(const Header& header); + // This method updates payload length in the HTTP2 header based on the size of the data_ + void adjustPayloadSize() { + ASSERT(size() >= HeaderSize); + setPayloadSize(size() - HeaderSize); + } + // Headers are directly encoded + void appendStaticHeader(StaticHeaderIndex index); + void appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value); + private: void buildHeader(Type type, uint32_t payload_size = 0, uint8_t flags = 0, uint32_t stream_id = 0); void setPayloadSize(uint32_t size); @@ -269,18 +280,8 @@ class Http2Frame { std::copy(data.begin(), data.end(), data_.begin() + 9); } - // Headers are directly encoded - void appendStaticHeader(StaticHeaderIndex index); - void appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value); - void appendHeaderWithoutIndexing(const Header& header); void appendEmptyHeader(); - // This method updates payload length in the HTTP2 header based on the size of the data_ - void adjustPayloadSize() { - ASSERT(size() >= HeaderSize); - setPayloadSize(size() - HeaderSize); - } - DataContainer data_; }; diff --git a/test/common/http/http2/request_header_fuzz_test.cc b/test/common/http/http2/request_header_fuzz_test.cc index 626335c812ba..ccf5ea436588 100644 --- a/test/common/http/http2/request_header_fuzz_test.cc +++ b/test/common/http/http2/request_header_fuzz_test.cc @@ -7,6 +7,7 @@ #include "test/common/http/http2/codec_impl_test_util.h" #include "test/common/http/http2/frame_replay.h" #include "test/fuzz/fuzz_runner.h" +#include "test/test_common/test_runtime.h" namespace Envoy { namespace Http { @@ -27,6 +28,12 @@ void replay(const Frame& frame, ServerCodecFrameInjector& codec) { } DEFINE_FUZZER(const uint8_t* buf, size_t len) { +#ifdef ENVOY_ENABLE_UHV + // Temporarily disable oghttp2 for these fuzz tests. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); +#endif + // Create static objects. static ServerCodecFrameInjector codec; Frame frame; diff --git a/test/common/http/http2/response_header_fuzz_test.cc b/test/common/http/http2/response_header_fuzz_test.cc index 09f1995c2362..addb75d4e2c9 100644 --- a/test/common/http/http2/response_header_fuzz_test.cc +++ b/test/common/http/http2/response_header_fuzz_test.cc @@ -8,6 +8,7 @@ #include "test/common/http/http2/codec_impl_test_util.h" #include "test/common/http/http2/frame_replay.h" #include "test/fuzz/fuzz_runner.h" +#include "test/test_common/test_runtime.h" namespace Envoy { namespace Http { @@ -36,6 +37,12 @@ void replay(const Frame& frame, ClientCodecFrameInjector& codec) { } DEFINE_FUZZER(const uint8_t* buf, size_t len) { +#ifdef ENVOY_ENABLE_UHV + // Temporarily disable oghttp2 for these fuzz tests when UHV is enabled. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.http2_use_oghttp2", "false"}}); +#endif + ClientCodecFrameInjector codec; Frame frame; frame.assign(buf, buf + len); diff --git a/test/common/http/http3/conn_pool_test.cc b/test/common/http/http3/conn_pool_test.cc index d27e0b8fd92b..75f82c9336e5 100644 --- a/test/common/http/http3/conn_pool_test.cc +++ b/test/common/http/http3/conn_pool_test.cc @@ -2,7 +2,7 @@ #include #include "source/common/http/http3/conn_pool.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" #include "test/common/http/common.h" #include "test/common/upstream/utility.h" @@ -172,6 +172,7 @@ TEST_F(Http3ConnPoolImplTest, CreationAndNewStream) { })); EXPECT_CALL(*cluster_socket_option, setOption(_, _)).Times(3u); EXPECT_CALL(*socket_option_, setOption(_, _)).Times(3u); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) auto* async_connect_callback = new NiceMock(&dispatcher_); ConnectionPool::Cancellable* cancellable = pool_->newStream(decoder, callbacks, {/*can_send_early_data_=*/false, @@ -179,6 +180,7 @@ TEST_F(Http3ConnPoolImplTest, CreationAndNewStream) { EXPECT_NE(nullptr, cancellable); async_connect_callback->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) std::list& clients = Http3ConnPoolImplPeer::connectingClients(*pool_); EXPECT_EQ(1u, clients.size()); diff --git a/test/common/http/http_server_properties_cache_manager_test.cc b/test/common/http/http_server_properties_cache_manager_test.cc index 5f6ccede15de..f6b8fbdc317f 100644 --- a/test/common/http/http_server_properties_cache_manager_test.cc +++ b/test/common/http/http_server_properties_cache_manager_test.cc @@ -8,8 +8,6 @@ #include "gtest/gtest.h" -using testing::Return; - namespace Envoy { namespace Http { @@ -25,7 +23,8 @@ class HttpServerPropertiesCacheManagerTest : public testing::Test, options2_.mutable_max_entries()->set_value(max_entries2_); } void initialize() { - AlternateProtocolsData data(context_); + AlternateProtocolsData data(context_.server_factory_context_, + context_.messageValidationVisitor()); factory_ = std::make_unique( singleton_manager_, tls_, data); manager_ = factory_->get(); diff --git a/test/common/http/match_delegate/config_test.cc b/test/common/http/match_delegate/config_test.cc index bfd125148a5c..f29b62aa721e 100644 --- a/test/common/http/match_delegate/config_test.cc +++ b/test/common/http/match_delegate/config_test.cc @@ -21,7 +21,7 @@ struct TestFactory : public Envoy::Server::Configuration::NamedHttpFilterConfigF ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); } - Envoy::Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, Server::Configuration::FactoryContext&) override { return [](auto& callbacks) { @@ -96,7 +96,7 @@ TEST(MatchWrapper, WithMatcher) { EXPECT_NE(nullptr, dynamic_cast(filter.get())); })); EXPECT_CALL(factory_callbacks, addAccessLogHandler(testing::IsNull())); - cb(factory_callbacks); + cb.value()(factory_callbacks); } TEST(MatchWrapper, PerRouteConfig) { @@ -180,7 +180,7 @@ TEST(MatchWrapper, DEPRECATED_FEATURE_TEST(WithDeprecatedMatcher)) { EXPECT_NE(nullptr, dynamic_cast(filter.get())); })); EXPECT_CALL(factory_callbacks, addAccessLogHandler(testing::IsNull())); - cb(factory_callbacks); + cb.value()(factory_callbacks); } TEST(MatchWrapper, QueryParamMatcherYaml) { @@ -214,7 +214,7 @@ TEST(MatchWrapper, QueryParamMatcherYaml) { MatchDelegateConfig match_delegate_config; auto cb = match_delegate_config.createFilterFactoryFromProto(config, "", factory_context); - EXPECT_TRUE(cb); + EXPECT_TRUE(cb.value()); } TEST(MatchWrapper, WithMatcherInvalidDataInput) { @@ -248,7 +248,9 @@ TEST(MatchWrapper, WithMatcherInvalidDataInput) { MatchDelegateConfig match_delegate_config; EXPECT_THROW_WITH_REGEX( - match_delegate_config.createFilterFactoryFromProto(config, "", factory_context), + match_delegate_config.createFilterFactoryFromProto(config, "", factory_context) + .status() + .IgnoreError(), EnvoyException, "requirement violation while creating match tree: INVALID_ARGUMENT: data input typeUrl " "type.googleapis.com/envoy.type.matcher.v3.HttpResponseHeaderMatchInput not permitted " diff --git a/test/common/http/match_delegate/match_delegate_integration_test.cc b/test/common/http/match_delegate/match_delegate_integration_test.cc index dca1a61d2c7c..082fda0e1752 100644 --- a/test/common/http/match_delegate/match_delegate_integration_test.cc +++ b/test/common/http/match_delegate/match_delegate_integration_test.cc @@ -39,7 +39,7 @@ class MatchDelegateInegrationTest : public testing::TestWithParammutable_typed_per_filter_config()->insert( - MapPair("envoy.filters.http.match_delegate", cfg_any)); + MapPair("match_delegate_filter", cfg_any)); }); initialize(); auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); diff --git a/test/common/http/utility_fuzz_test.cc b/test/common/http/utility_fuzz_test.cc index dc311df0bf07..6c529b35642b 100644 --- a/test/common/http/utility_fuzz_test.cc +++ b/test/common/http/utility_fuzz_test.cc @@ -18,8 +18,8 @@ DEFINE_PROTO_FUZZER(const test::common::http::UtilityTestCase& input) { } switch (input.utility_selector_case()) { case test::common::http::UtilityTestCase::kParseQueryString: { - // TODO(dio): Add the case when using parseAndDecodeQueryString(). - Http::Utility::parseQueryString(input.parse_query_string()); + Http::Utility::QueryParamsMulti::parseQueryString(input.parse_query_string()); + Http::Utility::QueryParamsMulti::parseAndDecodeQueryString(input.parse_query_string()); break; } case test::common::http::UtilityTestCase::kParseCookieValue: { @@ -58,9 +58,12 @@ DEFINE_PROTO_FUZZER(const test::common::http::UtilityTestCase& input) { } case test::common::http::UtilityTestCase::kParseParameters: { const auto& parse_parameters = input.parse_parameters(); - // TODO(dio): Add a case when doing parse_parameters with decode_params flag true. - Http::Utility::parseParameters(parse_parameters.data(), parse_parameters.start(), - /*decode_params*/ false); + Http::Utility::QueryParamsMulti::parseParameters(parse_parameters.data(), + parse_parameters.start(), + /*decode_params*/ false); + Http::Utility::QueryParamsMulti::parseParameters(parse_parameters.data(), + parse_parameters.start(), + /*decode_params*/ true); break; } case test::common::http::UtilityTestCase::kFindQueryString: { diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 70d87052dab8..384ba03471d7 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -51,117 +51,79 @@ void sendLocalReplyTestHelper(const bool& is_reset, StreamDecoderFilterCallbacks } // namespace TEST(HttpUtility, parseQueryString) { - using vec = std::vector; - using map = absl::btree_map; + using Vec = std::vector; + using Map = absl::btree_map; auto input = "/hello"; - EXPECT_EQ(Utility::QueryParams(), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams(), Utility::parseAndDecodeQueryString(input)); - EXPECT_EQ(map{}, Utility::QueryParamsMulti::parseQueryString(input).data()); - EXPECT_EQ(map{}, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); + EXPECT_EQ(Map{}, Utility::QueryParamsMulti::parseQueryString(input).data()); + EXPECT_EQ(Map{}, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?"; - EXPECT_EQ(Utility::QueryParams(), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams(), Utility::parseAndDecodeQueryString(input)); - EXPECT_EQ(map{}, Utility::QueryParamsMulti::parseQueryString(input).data()); - EXPECT_EQ(map{}, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); + EXPECT_EQ(Map{}, Utility::QueryParamsMulti::parseQueryString(input).data()); + EXPECT_EQ(Map{}, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello"; - EXPECT_EQ(Utility::QueryParams({{"hello", ""}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello", ""}}), Utility::parseAndDecodeQueryString(input)); - auto expected = map{{"hello", vec{""}}}; + auto expected = Map{{"hello", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello%26"; - EXPECT_EQ(Utility::QueryParams({{"hello%26", ""}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello&", ""}}), Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello%26", vec{""}}}; + expected = Map{{"hello%26", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{{"hello&", vec{""}}}; + expected = Map{{"hello&", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello=world"; - EXPECT_EQ(Utility::QueryParams({{"hello", "world"}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello", "world"}}), Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello", vec{"world"}}}; + expected = Map{{"hello", Vec{"world"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello="; - EXPECT_EQ(Utility::QueryParams({{"hello", ""}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello", ""}}), Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello", vec{""}}}; + expected = Map{{"hello", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello%26="; - EXPECT_EQ(Utility::QueryParams({{"hello%26", ""}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello&", ""}}), Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello%26", vec{""}}}; + expected = Map{{"hello%26", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{{"hello&", vec{""}}}; + expected = Map{{"hello&", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello=&"; - EXPECT_EQ(Utility::QueryParams({{"hello", ""}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello", ""}}), Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello", vec{""}}}; + expected = Map{{"hello", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello%26=&"; - EXPECT_EQ(Utility::QueryParams({{"hello%26", ""}}), Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello&", ""}}), Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello%26", vec{""}}}; + expected = Map{{"hello%26", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{{"hello&", vec{""}}}; + expected = Map{{"hello&", Vec{""}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?hello=&hello2=world2"; - EXPECT_EQ(Utility::QueryParams({{"hello", ""}, {"hello2", "world2"}}), - Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"hello", ""}, {"hello2", "world2"}}), - Utility::parseAndDecodeQueryString(input)); - expected = map{{"hello", vec{""}}, {"hello2", vec{"world2"}}}; + expected = Map{{"hello", Vec{""}}, {"hello2", Vec{"world2"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/logging?name=admin&level=trace"; - EXPECT_EQ(Utility::QueryParams({{"name", "admin"}, {"level", "trace"}}), - Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"name", "admin"}, {"level", "trace"}}), - Utility::parseAndDecodeQueryString(input)); - expected = map{{"name", vec{"admin"}}, {"level", vec{"trace"}}}; + expected = Map{{"name", Vec{"admin"}}, {"level", Vec{"trace"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?param_value_has_encoded_ampersand=a%26b"; - EXPECT_EQ(Utility::QueryParams({{"param_value_has_encoded_ampersand", "a%26b"}}), - Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"param_value_has_encoded_ampersand", "a&b"}}), - Utility::parseAndDecodeQueryString(input)); - expected = map{{"param_value_has_encoded_ampersand", vec{"a%26b"}}}; + expected = Map{{"param_value_has_encoded_ampersand", Vec{"a%26b"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{{"param_value_has_encoded_ampersand", vec{"a&b"}}}; + expected = Map{{"param_value_has_encoded_ampersand", Vec{"a&b"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?params_has_encoded_%26=a%26b&ok=1"; - EXPECT_EQ(Utility::QueryParams({{"params_has_encoded_%26", "a%26b"}, {"ok", "1"}}), - Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"params_has_encoded_&", "a&b"}, {"ok", "1"}}), - Utility::parseAndDecodeQueryString(input)); - expected = map{{"params_has_encoded_%26", vec{"a%26b"}}, {"ok", vec{"1"}}}; + expected = Map{{"params_has_encoded_%26", Vec{"a%26b"}}, {"ok", Vec{"1"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{{"params_has_encoded_&", vec{"a&b"}}, {"ok", vec{"1"}}}; + expected = Map{{"params_has_encoded_&", Vec{"a&b"}}, {"ok", Vec{"1"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); input = "/hello?params_%xy_%%yz=%xy%%yz"; - EXPECT_EQ(Utility::QueryParams({{"params_%xy_%%yz", "%xy%%yz"}}), - Utility::parseQueryString(input)); - EXPECT_EQ(Utility::QueryParams({{"params_%xy_%%yz", "%xy%%yz"}}), - Utility::parseAndDecodeQueryString(input)); - expected = map{{"params_%xy_%%yz", vec{"%xy%%yz"}}}; + expected = Map{{"params_%xy_%%yz", Vec{"%xy%%yz"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); @@ -169,32 +131,21 @@ TEST(HttpUtility, parseQueryString) { // https://github.com/envoyproxy/envoy/issues/10926#issuecomment-651085261. input = "/stats?filter=%28cluster.upstream_%28rq_total%7Crq_time_sum%7Crq_time_count%7Crq_time_" "bucket%7Crq_xx%7Crq_complete%7Crq_active%7Ccx_active%29%29%7C%28server.version%29"; - EXPECT_EQ( - Utility::QueryParams( - {{"filter", - "%28cluster.upstream_%28rq_total%7Crq_time_sum%7Crq_time_count%7Crq_time_" - "bucket%7Crq_xx%7Crq_complete%7Crq_active%7Ccx_active%29%29%7C%28server.version%29"}}), - Utility::parseQueryString(input)); - EXPECT_EQ( - Utility::QueryParams( - {{"filter", "(cluster.upstream_(rq_total|rq_time_sum|rq_time_count|rq_time_bucket|rq_xx|" - "rq_complete|rq_active|cx_active))|(server.version)"}}), - Utility::parseAndDecodeQueryString(input)); - expected = map{ + expected = Map{ {"filter", - vec{"%28cluster.upstream_%28rq_total%7Crq_time_sum%7Crq_time_count%7Crq_time_" + Vec{"%28cluster.upstream_%28rq_total%7Crq_time_sum%7Crq_time_count%7Crq_time_" "bucket%7Crq_xx%7Crq_complete%7Crq_active%7Ccx_active%29%29%7C%28server.version%29"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{ - {"filter", vec{"(cluster.upstream_(rq_total|rq_time_sum|rq_time_count|rq_time_bucket|rq_xx|" + expected = Map{ + {"filter", Vec{"(cluster.upstream_(rq_total|rq_time_sum|rq_time_count|rq_time_bucket|rq_xx|" "rq_complete|rq_active|cx_active))|(server.version)"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); // Requests with repeating keys input = "/foo?a=1&b=2&a=3%264&a=5"; - expected = map{{"a", vec{"1", "3%264", "5"}}, {"b", vec{"2"}}}; + expected = Map{{"a", Vec{"1", "3%264", "5"}}, {"b", Vec{"2"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseQueryString(input).data()); - expected = map{{"a", vec{"1", "3&4", "5"}}, {"b", vec{"2"}}}; + expected = Map{{"a", Vec{"1", "3&4", "5"}}, {"b", Vec{"2"}}}; EXPECT_EQ(expected, Utility::QueryParamsMulti::parseAndDecodeQueryString(input).data()); } @@ -222,64 +173,29 @@ TEST(HttpUtility, stripQueryString) { TEST(HttpUtility, replaceQueryString) { // Replace with nothing auto params = Utility::QueryParamsMulti(); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/"), Utility::QueryParams()), "/"); EXPECT_EQ(params.replaceQueryString(HeaderString("/")), "/"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/?"), Utility::QueryParams()), "/"); EXPECT_EQ(params.replaceQueryString(HeaderString("/?")), "/"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/?x=0"), Utility::QueryParams()), "/"); EXPECT_EQ(params.replaceQueryString(HeaderString("/?x=0")), "/"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/a"), Utility::QueryParams()), "/a"); EXPECT_EQ(params.replaceQueryString(HeaderString("/a")), "/a"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/a/"), Utility::QueryParams()), "/a/"); EXPECT_EQ(params.replaceQueryString(HeaderString("/a/")), "/a/"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/a/?y=5"), Utility::QueryParams()), "/a/"); EXPECT_EQ(params.replaceQueryString(HeaderString("/a/?y=5")), "/a/"); // Replace with x=1 params = Utility::QueryParamsMulti::parseQueryString("/?x=1"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/"), Utility::QueryParams({{"x", "1"}})), - "/?x=1"); EXPECT_EQ(params.replaceQueryString(HeaderString("/")), "/?x=1"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/?"), Utility::QueryParams({{"x", "1"}})), - "/?x=1"); EXPECT_EQ(params.replaceQueryString(HeaderString("/?")), "/?x=1"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/?x=0"), Utility::QueryParams({{"x", "1"}})), - "/?x=1"); EXPECT_EQ(params.replaceQueryString(HeaderString("/?x=0")), "/?x=1"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/a?x=0"), Utility::QueryParams({{"x", "1"}})), - "/a?x=1"); EXPECT_EQ(params.replaceQueryString(HeaderString("/a?x=0")), "/a?x=1"); - EXPECT_EQ( - Utility::replaceQueryString(HeaderString("/a/?x=0"), Utility::QueryParams({{"x", "1"}})), - "/a/?x=1"); EXPECT_EQ(params.replaceQueryString(HeaderString("/a/?x=0")), "/a/?x=1"); // More replacements params = Utility::QueryParamsMulti::parseQueryString("/?x=1&z=3"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/foo"), - Utility::QueryParams({{"x", "1"}, {"z", "3"}})), - "/foo?x=1&z=3"); EXPECT_EQ(params.replaceQueryString(HeaderString("/foo")), "/foo?x=1&z=3"); params = Utility::QueryParamsMulti::parseQueryString("/?x=1&y=5"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/foo?z=2"), - Utility::QueryParams({{"x", "1"}, {"y", "5"}})), - "/foo?x=1&y=5"); EXPECT_EQ(params.replaceQueryString(HeaderString("/foo?z=2")), "/foo?x=1&y=5"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/foo?y=9"), - Utility::QueryParams({{"x", "1"}, {"y", "5"}})), - "/foo?x=1&y=5"); EXPECT_EQ(params.replaceQueryString(HeaderString("/foo?y=9")), "/foo?x=1&y=5"); // More path components - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/foo/bar?"), - Utility::QueryParams({{"x", "1"}, {"y", "5"}})), - "/foo/bar?x=1&y=5"); EXPECT_EQ(params.replaceQueryString(HeaderString("/foo/bar?")), "/foo/bar?x=1&y=5"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/foo/bar?y=9&a=b"), - Utility::QueryParams({{"x", "1"}, {"y", "5"}})), - "/foo/bar?x=1&y=5"); EXPECT_EQ(params.replaceQueryString(HeaderString("/foo/bar?y=9&a=b")), "/foo/bar?x=1&y=5"); params = Utility::QueryParamsMulti::parseQueryString("/?a=b&x=1&y=5"); - EXPECT_EQ(Utility::replaceQueryString(HeaderString("/foo/bar?y=11&z=7"), - Utility::QueryParams({{"a", "b"}, {"x", "1"}, {"y", "5"}})), - "/foo/bar?a=b&x=1&y=5"); EXPECT_EQ(params.replaceQueryString(HeaderString("/foo/bar?y=11&z=7")), "/foo/bar?a=b&x=1&y=5"); // Repeating keys params = Utility::QueryParamsMulti::parseQueryString("/?a=b&x=1&a=5"); @@ -298,6 +214,9 @@ TEST(HttpUtility, testQueryParamModification) { EXPECT_EQ(params.toString(), "?a=1&a=2&b=3&c=4"); params.overwrite("b", "foo"); EXPECT_EQ(params.toString(), "?a=1&a=2&b=foo&c=4"); + EXPECT_EQ(params.getFirstValue("a").value(), "1"); + EXPECT_EQ(params.getFirstValue("b").value(), "foo"); + EXPECT_FALSE(params.getFirstValue("d").has_value()); params.remove("b"); EXPECT_EQ(params.toString(), "?a=1&a=2&c=4"); params.overwrite("a", "bar"); @@ -1285,13 +1204,6 @@ TEST(HttpUtility, TestPrepareHeaders) { EXPECT_EQ("dns.name", message->headers().getHostValue()); } -TEST(HttpUtility, QueryParamsToString) { - EXPECT_EQ("", Utility::queryParamsToString(Utility::QueryParams({}))); - EXPECT_EQ("?a=1", Utility::queryParamsToString(Utility::QueryParams({{"a", "1"}}))); - EXPECT_EQ("?a=1&b=2", - Utility::queryParamsToString(Utility::QueryParams({{"a", "1"}, {"b", "2"}}))); -} - TEST(HttpUtility, ResetReasonToString) { EXPECT_EQ("local connection failure", Utility::resetReasonToString(Http::StreamResetReason::LocalConnectionFailure)); diff --git a/test/common/io/io_uring_impl_test.cc b/test/common/io/io_uring_impl_test.cc index 86ab4a317c50..303c870bf3f9 100644 --- a/test/common/io/io_uring_impl_test.cc +++ b/test/common/io/io_uring_impl_test.cc @@ -16,7 +16,7 @@ class TestRequest : public Request { public: explicit TestRequest(int& data) : Request(RequestType::Read, mock_io_uring_socket_), data_(data) {} - ~TestRequest() { data_ = -1; } + ~TestRequest() override { data_ = -1; } int& data_; MockIoUringSocket mock_io_uring_socket_; @@ -59,26 +59,26 @@ class IoUringImplParamTest : public IoUringImplTest, public testing::WithParamInterface> {}; -INSTANTIATE_TEST_SUITE_P(InvalidPrepareMethodParamsTest, IoUringImplParamTest, - testing::Values( - [](IoUring& uring, os_fd_t fd) -> IoUringResult { - return uring.prepareAccept(fd, nullptr, nullptr, nullptr); - }, - [](IoUring& uring, os_fd_t fd) -> IoUringResult { - auto address = - std::make_shared( - "test"); - return uring.prepareConnect(fd, address, nullptr); - }, - [](IoUring& uring, os_fd_t fd) -> IoUringResult { - return uring.prepareReadv(fd, nullptr, 0, 0, nullptr); - }, - [](IoUring& uring, os_fd_t fd) -> IoUringResult { - return uring.prepareWritev(fd, nullptr, 0, 0, nullptr); - }, - [](IoUring& uring, os_fd_t fd) -> IoUringResult { - return uring.prepareClose(fd, nullptr); - })); +INSTANTIATE_TEST_SUITE_P( + InvalidPrepareMethodParamsTest, IoUringImplParamTest, + testing::Values( + [](IoUring& uring, os_fd_t fd) -> IoUringResult { + return uring.prepareAccept(fd, nullptr, nullptr, nullptr); + }, + [](IoUring& uring, os_fd_t fd) -> IoUringResult { + auto address = std::make_shared("test"); + return uring.prepareConnect(fd, address, nullptr); + }, + [](IoUring& uring, os_fd_t fd) -> IoUringResult { + return uring.prepareReadv(fd, nullptr, 0, 0, nullptr); + }, + [](IoUring& uring, os_fd_t fd) -> IoUringResult { + return uring.prepareWritev(fd, nullptr, 0, 0, nullptr); + }, + [](IoUring& uring, os_fd_t fd) -> IoUringResult { return uring.prepareClose(fd, nullptr); }, + [](IoUring& uring, os_fd_t fd) -> IoUringResult { + return uring.prepareShutdown(fd, 0, nullptr); + })); TEST_P(IoUringImplParamTest, InvalidParams) { os_fd_t fd; diff --git a/test/common/io/io_uring_worker_impl_integration_test.cc b/test/common/io/io_uring_worker_impl_integration_test.cc index fb404ff76fc1..ce415930607f 100644 --- a/test/common/io/io_uring_worker_impl_integration_test.cc +++ b/test/common/io/io_uring_worker_impl_integration_test.cc @@ -17,7 +17,9 @@ namespace { class IoUringSocketTestImpl : public IoUringSocketEntry { public: - IoUringSocketTestImpl(os_fd_t fd, IoUringWorkerImpl& parent) : IoUringSocketEntry(fd, parent) {} + IoUringSocketTestImpl(os_fd_t fd, IoUringWorkerImpl& parent) + : IoUringSocketEntry( + fd, parent, [](uint32_t) {}, false) {} void onAccept(Request* req, int32_t result, bool injected) override { IoUringSocketEntry::onAccept(req, result, injected); @@ -64,6 +66,10 @@ class IoUringSocketTestImpl : public IoUringSocketEntry { void cleanupForTest() { cleanup(); } + void write(Buffer::Instance&) override {} + uint64_t write(const Buffer::RawSlice*, uint64_t) override { return 0; } + void shutdown(int) override {} + int32_t accept_result_{-1}; bool is_accept_injected_completion_{false}; int32_t connect_result_{-1}; @@ -84,7 +90,7 @@ class IoUringSocketTestImpl : public IoUringSocketEntry { class IoUringWorkerTestImpl : public IoUringWorkerImpl { public: IoUringWorkerTestImpl(IoUringPtr io_uring_instance, Event::Dispatcher& dispatcher) - : IoUringWorkerImpl(std::move(io_uring_instance), dispatcher) {} + : IoUringWorkerImpl(std::move(io_uring_instance), 8192, 1000, dispatcher) {} IoUringSocket& addTestSocket(os_fd_t fd) { return addSocket(std::make_unique(fd, *this)); @@ -268,6 +274,609 @@ TEST_F(IoUringWorkerIntegrationTest, MergeInjection) { socket.cleanupForTest(); } +TEST_F(IoUringWorkerIntegrationTest, ServerSocketRead) { + initialize(); + createServerListenerAndClientSocket(); + + absl::optional result = absl::nullopt; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + server_socket_, + [&socket, &result](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(absl::nullopt, socket->getReadParam()); + result = socket->getReadParam()->result_; + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Write data through client socket. + std::string write_data = "hello world"; + Api::OsSysCallsSingleton::get().write(client_socket_, write_data.data(), write_data.size()); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + EXPECT_EQ(result.value(), write_data.length()); + + socket->close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketReadError) { + initialize(); + + absl::optional result = absl::nullopt; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + -1, + [&socket, &result](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(absl::nullopt, socket->getReadParam()); + result = socket->getReadParam()->result_; + socket->close(false); + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + EXPECT_EQ(result.value(), -EBADF); + + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketRemoteClose) { + initialize(); + createServerListenerAndClientSocket(); + + absl::optional result = absl::nullopt; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + server_socket_, + [&socket, &result](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(absl::nullopt, socket->getReadParam()); + result = socket->getReadParam()->result_; + socket->close(false); + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Close the client socket to trigger an error. + Api::OsSysCallsSingleton::get().close(client_socket_); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + EXPECT_EQ(result.value(), 0); + + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +// Verify that enables a socket will continue reading its remaining data. +TEST_F(IoUringWorkerIntegrationTest, ServerSocketDisable) { + initialize(); + createServerListenerAndClientSocket(); + + absl::optional result = absl::nullopt; + OptRef socket; + bool drained = false; + socket = io_uring_worker_->addServerSocket( + server_socket_, + [&socket, &result, &drained](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(absl::nullopt, socket->getReadParam()); + result = socket->getReadParam()->result_; + if (!drained) { + socket->getReadParam()->buf_.drain(5); + drained = true; + } + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Write data through client socket. + std::string write_data = "hello world"; + Api::OsSysCallsSingleton::get().write(client_socket_, write_data.data(), write_data.size()); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + EXPECT_EQ(result.value(), write_data.length()); + + // Emulate enable behavior. + result.reset(); + socket->disableRead(); + socket->enableRead(); + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + EXPECT_EQ(result.value(), write_data.length() - 5); + + socket->close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketWrite) { + initialize(); + createServerListenerAndClientSocket(); + + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Waiting for the server socket sending the data. + std::string write_data = "hello world"; + struct Buffer::RawSlice slice; + slice.mem_ = write_data.data(); + slice.len_ = write_data.length(); + socket.write(&slice, 1); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Read data from client socket. + struct iovec read_iov; + auto read_buf = std::make_unique(20); + read_iov.iov_base = read_buf.get(); + read_iov.iov_len = 20; + auto size = Api::OsSysCallsSingleton::get().readv(client_socket_, &read_iov, 1).return_value_; + EXPECT_EQ(write_data.size(), size); + + // Test another write interface. + Buffer::OwnedImpl buffer; + buffer.add(write_data); + socket.write(buffer); + EXPECT_EQ(buffer.length(), 0); + + size = Api::OsSysCallsSingleton::get().readv(client_socket_, &read_iov, 1).return_value_; + EXPECT_EQ(write_data.size(), size); + + socket.close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketWriteError) { + initialize(); + + absl::optional result = absl::nullopt; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + -1, + [&socket, &result](uint32_t events) { + ASSERT(events == Event::FileReadyType::Write); + EXPECT_NE(absl::nullopt, socket->getWriteParam()); + result = socket->getWriteParam()->result_; + socket->close(false); + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + socket->disableRead(); + // Waiting for the server socket sending the data. + std::string write_data = "hello world"; + struct Buffer::RawSlice slice; + slice.mem_ = write_data.data(); + slice.len_ = write_data.length(); + socket->write(&slice, 1); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + EXPECT_EQ(result.value(), -EBADF); + + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketWriteTimeout) { + initialize(); + createServerListenerAndClientSocket(); + + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Fill peer socket receive buffer. + std::string write_data(10000000, 'a'); + struct Buffer::RawSlice slice; + slice.mem_ = write_data.data(); + slice.len_ = write_data.length(); + // The following line may partially complete since write buffer is full. + socket.write(&slice, 1); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + // The following will block in iouring. + socket.write(&slice, 1); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Continuously sending the data in server socket until timeout. + socket.close(false); + runToClose(server_socket_); + + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketShutdownAfterWrite) { + initialize(); + createServerListenerAndClientSocket(); + + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Waiting for the server socket sending the data. + std::string write_data = "hello world"; + struct Buffer::RawSlice slice; + slice.mem_ = write_data.data(); + slice.len_ = write_data.length(); + socket.write(&slice, 1); + socket.shutdown(SHUT_WR); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Read data from client socket. + struct iovec read_iov; + auto read_buf = std::make_unique(20); + read_iov.iov_base = read_buf.get(); + read_iov.iov_len = 20; + auto size = Api::OsSysCallsSingleton::get().readv(client_socket_, &read_iov, 1).return_value_; + EXPECT_EQ(write_data.size(), size); + + socket.close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseAfterShutdown) { + initialize(); + createServerListenerAndClientSocket(); + + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + socket.shutdown(SHUT_WR); + + socket.close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseAfterShutdownWrite) { + initialize(); + createServerListenerAndClientSocket(); + + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Waiting for the server socket sending the data. + std::string write_data = "hello world"; + struct Buffer::RawSlice slice; + slice.mem_ = write_data.data(); + slice.len_ = write_data.length(); + socket.write(&slice, 1); + socket.shutdown(SHUT_WR); + socket.close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + + // Read data from client socket. + struct iovec read_iov; + auto read_buf = std::make_unique(20); + read_iov.iov_base = read_buf.get(); + read_iov.iov_len = 20; + auto size = Api::OsSysCallsSingleton::get().readv(client_socket_, &read_iov, 1).return_value_; + EXPECT_EQ(write_data.size(), size); + + size = Api::OsSysCallsSingleton::get().readv(client_socket_, &read_iov, 1).return_value_; + EXPECT_EQ(0, size); + + cleanup(); +} + +// This tests the case when the socket disabled, then a remote close happened. +// In this case, we should deliver this close event if the enable_close_event is true. +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseAfterDisabledWithEnableCloseEvent) { + initialize(); + createServerListenerAndClientSocket(); + + bool is_closed = false; + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, + [&is_closed](uint32_t events) { + ASSERT(events == Event::FileReadyType::Closed); + is_closed = true; + }, + false); + socket.enableCloseEvent(true); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + // Waiting for the server socket sending the data. + socket.disableRead(); + + Api::OsSysCallsSingleton::get().close(client_socket_); + + while (!is_closed) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + cleanup(); +} + +// This tests the case when the socket disabled, then a remote close happened. +// Different from the previous cast, in the test the client socket will first write and then +// close. +TEST_F(IoUringWorkerIntegrationTest, ServerSocketReadAndCloseAfterDisabledWithEnableCloseEvent) { + initialize(); + createServerListenerAndClientSocket(); + + bool is_closed = false; + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, + [&is_closed](uint32_t events) { + ASSERT(events == Event::FileReadyType::Closed); + is_closed = true; + }, + true); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + // Waiting for the server socket sending the data. + socket.disableRead(); + + // Write data through client socket. + std::string write_data = "hello world"; + Api::OsSysCallsSingleton::get().write(client_socket_, write_data.data(), write_data.size()); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + Api::OsSysCallsSingleton::get().close(client_socket_); + + while (!is_closed) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + cleanup(); +} + +// This tests the case when the socket disabled, then a remote close happened. +// In this case, we should deliver this remote close by read event if enable_close_event +// is false. +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseWithoutEnableCloseEvent) { + initialize(); + createServerListenerAndClientSocket(); + + bool is_closed = false; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + server_socket_, + [&socket, &is_closed](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(socket->getReadParam(), absl::nullopt); + EXPECT_EQ(socket->getReadParam()->result_, 0); + is_closed = true; + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + Api::OsSysCallsSingleton::get().close(client_socket_); + + while (!is_closed) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + cleanup(); +} + +// This tests the case the socket is disabled, and the close event isn't listened. Then +// a remote close happened, then deliver the remote close by write event. +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseAfterDisabledWithoutEnableCloseEvent) { + initialize(); + createServerListenerAndClientSocket(); + + bool is_closed = false; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + server_socket_, + [&socket, &is_closed](uint32_t events) { + ASSERT(events == Event::FileReadyType::Write); + EXPECT_NE(socket->getWriteParam(), absl::nullopt); + EXPECT_EQ(socket->getWriteParam()->result_, 0); + is_closed = true; + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + socket->disableRead(); + + Api::OsSysCallsSingleton::get().close(client_socket_); + + while (!is_closed) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseWithAnyRequest) { + initialize(); + createServerListenerAndClientSocket(); + + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Disable the socket, then it won't submit any new request. + socket.disableRead(); + // Write data through client socket, then it will consume the existing request. + std::string write_data = "hello world"; + Api::OsSysCallsSingleton::get().write(client_socket_, write_data.data(), write_data.size()); + + // Running the event loop, to let the iouring worker process the read request. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + // Close the socket now, it expected the socket will be close directly without cancel. + socket.close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +// This test ensures that a write after the client reset the connection +// will trigger a closed event. +TEST_F(IoUringWorkerIntegrationTest, ServerSocketWriteWithClientRST) { + initialize(); + createServerListenerAndClientSocket(); + bool expected_read = false; + bool expected_closed = false; + auto& socket = io_uring_worker_->addServerSocket( + server_socket_, + [&expected_read, &expected_closed](uint32_t events) { + if (expected_read) { + EXPECT_TRUE(events | Event::FileReadyType::Read); + expected_read = false; + return; + } + if (expected_closed) { + EXPECT_TRUE(events | Event::FileReadyType::Closed); + expected_closed = false; + return; + } + // It shouldn't reach here. + EXPECT_TRUE(false); + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + Api::OsSysCallsSingleton::get().shutdown(client_socket_, SHUT_WR); + expected_read = true; + // Running the event loop, to let the iouring worker process the read request. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_FALSE(expected_read); + + struct linger sl; + sl.l_onoff = 1; /* non-zero value enables linger option in kernel */ + sl.l_linger = 0; /* timeout interval in seconds */ + setsockopt(client_socket_, SOL_SOCKET, SO_LINGER, &sl, sizeof(sl)); + Api::OsSysCallsSingleton::get().close(client_socket_); + + expected_closed = true; + std::string write_data = "hello"; + Buffer::OwnedImpl buffer; + buffer.add(write_data); + socket.write(buffer); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_FALSE(expected_closed); + + // Close the socket now, it expected the socket will be close directly without cancel. + socket.close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, AddServerSocketWithBuffer) { + initialize(); + createServerListenerAndClientSocket(); + + absl::optional result = absl::nullopt; + OptRef socket; + std::string data = "hello"; + Buffer::OwnedImpl buffer; + buffer.add(data); + socket = io_uring_worker_->addServerSocket( + server_socket_, buffer, + [&socket, &result](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(absl::nullopt, socket->getReadParam()); + result = socket->getReadParam()->result_; + }, + false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + EXPECT_EQ(result.value(), data.length()); + + socket->close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketCloseButKeepFDOpen) { + initialize(); + createServerListenerAndClientSocket(); + + OptRef socket; + socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) {}, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + socket->close(true); + while (io_uring_worker_->getSockets().size() != 0) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + // Ensure the server socket is still open. + std::string write_data = "hello world"; + auto rc = + Api::OsSysCallsSingleton::get().write(server_socket_, write_data.data(), write_data.size()); + EXPECT_EQ(rc.return_value_, write_data.length()); + + cleanup(); +} + +TEST_F(IoUringWorkerIntegrationTest, ServerSocketUpdateFileEventCb) { + initialize(); + createServerListenerAndClientSocket(); + + absl::optional result = absl::nullopt; + OptRef socket; + socket = io_uring_worker_->addServerSocket( + server_socket_, [](uint32_t) { EXPECT_TRUE(false); }, false); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 1); + + socket->setFileReadyCb([&socket, &result](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + EXPECT_NE(absl::nullopt, socket->getReadParam()); + result = socket->getReadParam()->result_; + }); + // Write data through client socket. + std::string write_data = "hello world"; + Api::OsSysCallsSingleton::get().write(client_socket_, write_data.data(), write_data.size()); + + // Waiting for the server socket receive the data. + while (!result.has_value()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + EXPECT_EQ(result.value(), write_data.length()); + + socket->close(false); + runToClose(server_socket_); + EXPECT_EQ(io_uring_worker_->getSockets().size(), 0); + cleanup(); +} + } // namespace } // namespace Io } // namespace Envoy diff --git a/test/common/io/io_uring_worker_impl_test.cc b/test/common/io/io_uring_worker_impl_test.cc index 870e7db6bbe2..313d70e35873 100644 --- a/test/common/io/io_uring_worker_impl_test.cc +++ b/test/common/io/io_uring_worker_impl_test.cc @@ -1,3 +1,5 @@ +#include + #include "source/common/io/io_uring_worker_impl.h" #include "source/common/network/address_impl.h" @@ -20,14 +22,20 @@ namespace { class IoUringSocketTestImpl : public IoUringSocketEntry { public: - IoUringSocketTestImpl(os_fd_t fd, IoUringWorkerImpl& parent) : IoUringSocketEntry(fd, parent) {} + IoUringSocketTestImpl(os_fd_t fd, IoUringWorkerImpl& parent) + : IoUringSocketEntry( + fd, parent, [](uint32_t) {}, false) {} void cleanupForTest() { cleanup(); } + + void write(Buffer::Instance&) override {} + uint64_t write(const Buffer::RawSlice*, uint64_t) override { return 0; } + void shutdown(int) override {} }; class IoUringWorkerTestImpl : public IoUringWorkerImpl { public: IoUringWorkerTestImpl(IoUringPtr io_uring_instance, Event::Dispatcher& dispatcher) - : IoUringWorkerImpl(std::move(io_uring_instance), dispatcher) {} + : IoUringWorkerImpl(std::move(io_uring_instance), 8192, 1000, dispatcher) {} IoUringSocket& addTestSocket(os_fd_t fd) { return addSocket(std::make_unique(fd, *this)); @@ -38,6 +46,83 @@ class IoUringWorkerTestImpl : public IoUringWorkerImpl { void submitForTest() { submit(); } }; +// TODO (soulxu): This is only for test coverage, we suppose to have correct +// implementation to handle the request submit failed. +TEST(IoUringWorkerImplTest, SubmitRequestsFailed) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, createFileEvent_(_, _, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read)); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + + os_fd_t fd; + SET_SOCKET_INVALID(fd); + auto& io_uring_socket = worker.addTestSocket(fd); + + EXPECT_CALL(mock_io_uring, prepareReadv(fd, _, _, _, _)) + .WillOnce(Return(IoUringResult::Ok)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, prepareReadv(fd, _, _, _, _)) + .WillOnce(Return(IoUringResult::Failed)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + delete worker.submitReadRequest(io_uring_socket); + + Buffer::OwnedImpl buf; + auto slices = buf.getRawSlices(); + EXPECT_CALL(mock_io_uring, prepareWritev(fd, _, _, _, _)) + .WillOnce(Return(IoUringResult::Ok)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, prepareWritev(fd, _, _, _, _)) + .WillOnce(Return(IoUringResult::Failed)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + delete worker.submitWriteRequest(io_uring_socket, slices); + + EXPECT_CALL(mock_io_uring, prepareCancel(_, _)) + .WillOnce(Return(IoUringResult::Ok)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, prepareCancel(_, _)) + .WillOnce(Return(IoUringResult::Failed)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + delete worker.submitCancelRequest(io_uring_socket, nullptr); + + EXPECT_CALL(mock_io_uring, prepareClose(fd, _)) + .WillOnce(Return(IoUringResult::Ok)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, prepareClose(fd, _)) + .WillOnce(Return(IoUringResult::Failed)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + delete worker.submitCloseRequest(io_uring_socket); + + EXPECT_CALL(mock_io_uring, prepareShutdown(fd, _, _)) + .WillOnce(Return(IoUringResult::Ok)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, prepareShutdown(fd, _, _)) + .WillOnce(Return(IoUringResult::Failed)) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + delete worker.submitShutdownRequest(io_uring_socket, SHUT_WR); + + EXPECT_EQ(fd, io_uring_socket.fd()); + EXPECT_EQ(1, worker.getSockets().size()); + EXPECT_CALL(mock_io_uring, removeInjectedCompletion(fd)); + EXPECT_CALL(dispatcher, deferredDelete_); + dynamic_cast(worker.getSockets().front().get())->cleanupForTest(); + EXPECT_EQ(0, worker.getNumOfSockets()); + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); +} + TEST(IoUringWorkerImplTest, CleanupSocket) { Event::MockDispatcher dispatcher; IoUringPtr io_uring_instance = std::make_unique(); @@ -94,6 +179,448 @@ TEST(IoUringWorkerImplTest, DelaySubmit) { EXPECT_CALL(dispatcher, clearDeferredDeleteList()); } +// This tests ensure the write request won't be override by an injected completion. +TEST(IoUringWorkerImplTest, ServerSocketInjectAfterWrite) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + Event::FileReadyCb file_event_callback; + + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read)) + .WillOnce( + DoAll(SaveArg<1>(&file_event_callback), ReturnNew>())); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + + os_fd_t fd = 11; + SET_SOCKET_INVALID(fd); + + Request* read_req = nullptr; + // The read request added by server socket constructor. + EXPECT_CALL(mock_io_uring, prepareReadv(fd, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + auto& io_uring_socket = worker.addServerSocket( + fd, [](uint32_t) {}, false); + + // Add a write request. + std::string data = "Hello"; + Buffer::OwnedImpl buf1; + buf1.add(data); + Request* write_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareWritev(fd, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&write_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + io_uring_socket.write(buf1); + + // Fake an injected completion. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&io_uring_socket](const CompletionCb& cb) { + auto* req = new Request(Request::RequestType::Write, io_uring_socket); + cb(req, -EAGAIN, true); + })); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + Buffer::OwnedImpl buf2; + buf2.add(data); + + // Add another write request to ensure the incomplete request is still + // there, so the new write request won't be submitted. + EXPECT_CALL(mock_io_uring, prepareWritev(fd, _, _, _, _)).Times(0); + EXPECT_CALL(mock_io_uring, submit()).Times(0).RetiresOnSaturation(); + io_uring_socket.write(buf2); + + // Start the close process. + Request* cancel_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareCancel(_, _)) + .WillOnce(DoAll(SaveArg<1>(&cancel_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + EXPECT_CALL(dispatcher, createTimer_(_)).WillOnce(ReturnNew>()); + io_uring_socket.close(false); + + // Finish the read, cancel and write request, then expect the close request submitted. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&read_req, &cancel_req, &write_req](const CompletionCb& cb) { + cb(read_req, -EAGAIN, false); + cb(cancel_req, 0, false); + cb(write_req, -EAGAIN, false); + })); + Request* close_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareClose(_, _)) + .WillOnce(DoAll(SaveArg<1>(&close_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + // After the close request finished, the socket will be cleanup. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&close_req](const CompletionCb& cb) { cb(close_req, 0, false); })); + EXPECT_CALL(mock_io_uring, removeInjectedCompletion(fd)); + EXPECT_CALL(dispatcher, deferredDelete_); + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + EXPECT_EQ(0, worker.getSockets().size()); +} + +// This tests ensure the read request won't be override by an injected completion. +TEST(IoUringWorkerImplTest, ServerSocketInjectAfterRead) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + Event::FileReadyCb file_event_callback; + + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read)) + .WillOnce( + DoAll(SaveArg<1>(&file_event_callback), ReturnNew>())); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + + os_fd_t fd = 11; + SET_SOCKET_INVALID(fd); + + // The read request added by server socket constructor. + Request* read_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareReadv(fd, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + auto& io_uring_socket = worker.addServerSocket( + fd, [](uint32_t) {}, false); + + // Fake an injected completion. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&io_uring_socket](const CompletionCb& cb) { + auto* req = new Request(Request::RequestType::Write, io_uring_socket); + cb(req, -EAGAIN, true); + })); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + // When close the socket, expect there still have a incomplete read + // request, so it has to cancel the request first. + Request* cancel_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareCancel(_, _)) + .WillOnce(DoAll(SaveArg<1>(&cancel_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + io_uring_socket.close(false); + + // Finish the read and cancel request, then expect the close request submitted. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&read_req, &cancel_req](const CompletionCb& cb) { + cb(read_req, -EAGAIN, false); + cb(cancel_req, 0, false); + })); + Request* close_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareClose(_, _)) + .WillOnce(DoAll(SaveArg<1>(&close_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + // After the close request finished, the socket will be cleanup. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&close_req](const CompletionCb& cb) { cb(close_req, 0, false); })); + EXPECT_CALL(mock_io_uring, removeInjectedCompletion(fd)); + EXPECT_CALL(dispatcher, deferredDelete_); + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + EXPECT_EQ(0, worker.getSockets().size()); +} + +TEST(IoUringWorkerImplTest, CloseAllSocketsWhenDestruction) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + Event::FileReadyCb file_event_callback; + + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read)) + .WillOnce( + DoAll(SaveArg<1>(&file_event_callback), ReturnNew>())); + + std::unique_ptr worker = + std::make_unique(std::move(io_uring_instance), dispatcher); + + os_fd_t fd = 11; + SET_SOCKET_INVALID(fd); + + // The read request added by server socket constructor. + Request* read_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareReadv(fd, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + worker->addServerSocket( + fd, [](uint32_t) {}, false); + + // The IoUringWorker will close all the existing sockets. + Request* cancel_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareCancel(_, _)) + .WillOnce(DoAll(SaveArg<1>(&cancel_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + + // The IoUringWorker will wait for the socket closed. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&mock_io_uring, fd, &read_req, &cancel_req](const CompletionCb& cb) { + // When the cancel request is done, the close request will be submitted. + Request* close_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareClose(fd, _)) + .WillOnce(DoAll(SaveArg<1>(&close_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + + EXPECT_CALL(mock_io_uring, removeInjectedCompletion(fd)); + + // Fake the read request cancel completion. + cb(read_req, -ECANCELED, false); + + // Fake the cancel request is done. + cb(cancel_req, 0, false); + + // Fake the close request is done. + cb(close_req, 0, false); + })); + + EXPECT_CALL(dispatcher, deferredDelete_); + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + worker.reset(); +} + +TEST(IoUringWorkerImplTest, ServerCloseWithWriteRequestOnly) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + Event::FileReadyCb file_event_callback; + + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read)) + .WillOnce( + DoAll(SaveArg<1>(&file_event_callback), ReturnNew>())); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + + os_fd_t fd = 11; + SET_SOCKET_INVALID(fd); + + // The read request added by server socket constructor. + Request* read_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareReadv(fd, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + auto& io_uring_socket = worker.addServerSocket( + fd, [](uint32_t) {}, false); + + // Disable the socket, then there will be no new read request. + io_uring_socket.disableRead(); + // Fake the read request finish. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&read_req](const CompletionCb& cb) { cb(read_req, -EAGAIN, false); })); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + Request* write_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareWritev(fd, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&write_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + Buffer::OwnedImpl write_buf; + write_buf.add("Hello"); + io_uring_socket.write(write_buf); + + // Close the socket, but there is no read request, so cancel request won't + // be submitted. + EXPECT_CALL(dispatcher, createTimer_(_)).WillOnce(ReturnNew>()); + io_uring_socket.close(false); + + Request* close_req = nullptr; + // Finish the read and cancel request, then expect the close request submitted. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&mock_io_uring, &write_req, &close_req](const CompletionCb& cb) { + EXPECT_CALL(mock_io_uring, prepareClose(_, _)) + .WillOnce(DoAll(SaveArg<1>(&close_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + + cb(write_req, -EAGAIN, false); + })); + file_event_callback(Event::FileReadyType::Read); + + // After the close request finished, the socket will be cleanup. + EXPECT_CALL(mock_io_uring, forEveryCompletion(_)) + .WillOnce(Invoke([&close_req](const CompletionCb& cb) { cb(close_req, 0, false); })); + EXPECT_CALL(mock_io_uring, removeInjectedCompletion(fd)); + EXPECT_CALL(dispatcher, deferredDelete_); + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + file_event_callback(Event::FileReadyType::Read); + + EXPECT_EQ(0, worker.getSockets().size()); +} + +// Make sure that even the socket is disabled, that remote close can be handled. +TEST(IoUringWorkerImplTest, CloseDetected) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, createFileEvent_(_, _, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read)); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + + Request* read_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareReadv(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + IoUringServerSocket socket( + 0, worker, [](uint32_t events) { EXPECT_EQ(events, Event::FileReadyType::Closed); }, 0, true); + socket.enableRead(); + socket.disableRead(); + + // Consumes the first read request. + Request* read_req2 = nullptr; + EXPECT_CALL(mock_io_uring, prepareReadv(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req2), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + socket.onRead(read_req, 1, false); + socket.onRead(nullptr, 0, false); + + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + delete read_req; + delete read_req2; +} + +TEST(IoUringWorkerImplTest, AvoidDuplicatedCloseRequest) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, createFileEvent_(_, _, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read)); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + IoUringServerSocket socket( + 0, worker, [](uint32_t events) { EXPECT_EQ(events, Event::FileReadyType::Closed); }, 0, true); + + Request* close_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareClose(_, _)) + .WillOnce(DoAll(SaveArg<1>(&close_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + + socket.close(false); + socket.close(false); + + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + delete close_req; +} + +TEST(IoUringWorkerImplTest, NoOnWriteCallingBackInShutdownWriteSocketInjection) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, createFileEvent_(_, _, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read)); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + IoUringServerSocket socket( + 0, worker, [](uint32_t) {}, 0, false); + + // Shutdown and then shutdown completes. + EXPECT_CALL(mock_io_uring, submit()); + Request* shutdown_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareShutdown(socket.fd(), _, _)) + .WillOnce(DoAll(SaveArg<2>(&shutdown_req), Return(IoUringResult::Ok))); + socket.shutdown(SHUT_WR); + socket.onShutdown(shutdown_req, 0, false); + + // onWrite happens after the shutdown completed will not trigger calling back. + socket.onWrite(nullptr, 0, true); + + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + delete shutdown_req; +} + +TEST(IoUringWorkerImplTest, NoOnWriteCallingBackInCloseAfterShutdownWriteSocketInjection) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, createFileEvent_(_, _, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read)); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + IoUringServerSocket socket( + 0, worker, [](uint32_t) {}, 0, false); + + // Shutdown and then close. + EXPECT_CALL(mock_io_uring, submit()); + Request* shutdown_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareShutdown(socket.fd(), _, _)) + .WillOnce(DoAll(SaveArg<2>(&shutdown_req), Return(IoUringResult::Ok))); + socket.shutdown(SHUT_WR); + Request* close_req = nullptr; + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + socket.close(false); + + // onWrite happens after the close after shutdown will not trigger calling back. + socket.onWrite(nullptr, 0, true); + + delete shutdown_req; + delete close_req; +} + +TEST(IoUringWorkerImplTest, CloseKeepFDOpenAndSaveData) { + Event::MockDispatcher dispatcher; + IoUringPtr io_uring_instance = std::make_unique(); + MockIoUring& mock_io_uring = *dynamic_cast(io_uring_instance.get()); + EXPECT_CALL(mock_io_uring, registerEventfd()); + EXPECT_CALL(dispatcher, createFileEvent_(_, _, Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read)); + IoUringWorkerTestImpl worker(std::move(io_uring_instance), dispatcher); + + // The read request submitted. + Request* read_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareReadv(_, _, _, _, _)) + .WillOnce(DoAll(SaveArg<4>(&read_req), Return(IoUringResult::Ok))); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + auto& socket = worker.addServerSocket( + 0, [](uint32_t events) { EXPECT_EQ(events, Event::FileReadyType::Closed); }, false); + + // Close the socket, but keep the fd open. + Request* cancel_req = nullptr; + EXPECT_CALL(mock_io_uring, prepareCancel(_, _)) + .WillOnce(DoAll(SaveArg<1>(&cancel_req), Return(IoUringResult::Ok))) + .RetiresOnSaturation(); + EXPECT_CALL(mock_io_uring, submit()).Times(1).RetiresOnSaturation(); + Buffer::OwnedImpl buffer; + bool is_closed = false; + socket.close(true, [&is_closed](Buffer::Instance& read_buffer) { + // Expect the data is saved. + EXPECT_EQ(1, read_buffer.length()); + is_closed = true; + }); + + // consumes the cancel request. + socket.onCancel(cancel_req, 0, false); + + // Consumes the read request. + EXPECT_CALL(mock_io_uring, removeInjectedCompletion(0)); + EXPECT_CALL(dispatcher, deferredDelete_); + socket.onRead(read_req, 1, false); + EXPECT_TRUE(is_closed); + + EXPECT_CALL(dispatcher, clearDeferredDeleteList()); + delete read_req; + delete cancel_req; +} + } // namespace } // namespace Io } // namespace Envoy diff --git a/test/common/json/json_sanitizer_test.cc b/test/common/json/json_sanitizer_test.cc index 04402369f48a..23222b72930a 100644 --- a/test/common/json/json_sanitizer_test.cc +++ b/test/common/json/json_sanitizer_test.cc @@ -115,7 +115,7 @@ TEST_F(JsonSanitizerTest, SevenBitAscii) { TEST_F(JsonSanitizerTest, Utf8) { // reference; https://www.charset.org/utf-8 auto unicode = [](std::vector chars) -> std::string { - return std::string(reinterpret_cast(&chars[0]), chars.size()); + return {reinterpret_cast(&chars[0]), chars.size()}; }; sanitizeAndCheckAgainstProtobufJson(unicode({0xc2, 0xa2})); // Cent. diff --git a/test/common/json/utf8.cc b/test/common/json/utf8.cc index db30872dcb8b..b04404c37978 100644 --- a/test/common/json/utf8.cc +++ b/test/common/json/utf8.cc @@ -30,7 +30,7 @@ UnicodeSizePair decode(const uint8_t* bytes, uint32_t size) { unicode = bytes[0] & ~Utf8::Mask2Byte; unicode = (unicode << Utf8::Shift) | (bytes[1] & ~Utf8::ContinueMask); if (unicode < 0x80) { - return UnicodeSizePair(0, 0); + return {0, 0}; } consumed = 2; } else if (size >= 3 && (bytes[0] & Utf8::Mask3Byte) == Utf8::Pattern3Byte && @@ -40,7 +40,7 @@ UnicodeSizePair decode(const uint8_t* bytes, uint32_t size) { unicode = (unicode << Utf8::Shift) | (bytes[1] & ~Utf8::ContinueMask); unicode = (unicode << Utf8::Shift) | (bytes[2] & ~Utf8::ContinueMask); if (unicode < 0x800) { // 3-byte starts at 0x800 - return UnicodeSizePair(0, 0); + return {0, 0}; } consumed = 3; } else if (size >= 4 && (bytes[0] & Utf8::Mask4Byte) == Utf8::Pattern4Byte && @@ -59,11 +59,11 @@ UnicodeSizePair decode(const uint8_t* bytes, uint32_t size) { // But the current RFC3629 section 3 limits UTF-8 encoding through code // point 0x10FFFF, to match the limits of UTF-16. if (unicode < 0x10000 || unicode > 0x10ffff) { - return UnicodeSizePair(0, 0); + return {0, 0}; } consumed = 4; } - return UnicodeSizePair(unicode, consumed); + return {unicode, consumed}; } } // namespace Utf8 diff --git a/test/common/json/utf8.h b/test/common/json/utf8.h index d09a101af6e5..418d197dfa81 100644 --- a/test/common/json/utf8.h +++ b/test/common/json/utf8.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "absl/strings/string_view.h" namespace Envoy { diff --git a/test/extensions/listener_managers/listener_manager/BUILD b/test/common/listener_manager/BUILD similarity index 93% rename from test/extensions/listener_managers/listener_manager/BUILD rename to test/common/listener_manager/BUILD index c80b0bcee67d..24c62c127ce9 100644 --- a/test/extensions/listener_managers/listener_manager/BUILD +++ b/test/common/listener_manager/BUILD @@ -24,8 +24,8 @@ envoy_cc_test_library( deps = [ ":config_cc_proto", "//source/common/init:manager_lib", + "//source/common/listener_manager:listener_manager_lib", "//source/extensions/common/matcher:trie_matcher_lib", - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//test/mocks/init:init_mocks", "//test/mocks/matcher:matcher_mocks", "//test/mocks/network:network_mocks", @@ -54,6 +54,7 @@ envoy_cc_test( ":listener_manager_impl_test_lib", "//source/common/api:os_sys_calls_lib", "//source/common/config:metadata_lib", + "//source/common/listener_manager:active_raw_udp_listener_config", "//source/common/network:addr_family_aware_socket_option_lib", "//source/common/network:listen_socket_lib", "//source/common/network:socket_option_lib", @@ -65,7 +66,6 @@ envoy_cc_test( "//source/extensions/filters/listener/tls_inspector:config", "//source/extensions/filters/network/http_connection_manager:config", "//source/extensions/filters/network/tcp_proxy:config", - "//source/extensions/listener_managers/listener_manager:active_raw_udp_listener_config", "//source/extensions/matching/network/application_protocol:config", "//source/extensions/matching/network/common:inputs_lib", "//source/extensions/request_id/uuid:config", @@ -111,14 +111,14 @@ envoy_cc_test( deps = [ "//source/common/api:os_sys_calls_lib", "//source/common/config:metadata_lib", + "//source/common/listener_manager:filter_chain_manager_lib", + "//source/common/listener_manager:listener_manager_lib", "//source/common/network:addr_family_aware_socket_option_lib", "//source/common/network:listen_socket_lib", "//source/common/network:socket_option_lib", "//source/common/network:utility_lib", "//source/common/protobuf", "//source/extensions/filters/network/http_connection_manager:config", - "//source/extensions/listener_managers/listener_manager:filter_chain_manager_lib", - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//source/extensions/matching/network/common:inputs_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:config", @@ -144,8 +144,8 @@ envoy_cc_test( "//test/config/integration/certs", ], deps = [ + "//source/common/listener_manager:lds_api_lib", "//source/common/protobuf:utility_lib", - "//source/extensions/listener_managers/listener_manager:lds_api_lib", "//test/mocks/config:config_mocks", "//test/mocks/init:init_mocks", "//test/mocks/protobuf:protobuf_mocks", @@ -168,7 +168,7 @@ envoy_cc_benchmark_binary( ], deps = [ "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", - "//source/extensions/listener_managers/listener_manager:filter_chain_manager_lib", + "//source/common/listener_manager:filter_chain_manager_lib", "//test/test_common:environment_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:factory_context_mocks", diff --git a/test/extensions/listener_managers/listener_manager/config.proto b/test/common/listener_manager/config.proto similarity index 100% rename from test/extensions/listener_managers/listener_manager/config.proto rename to test/common/listener_manager/config.proto diff --git a/test/extensions/listener_managers/listener_manager/filter_chain_benchmark_test.cc b/test/common/listener_manager/filter_chain_benchmark_test.cc similarity index 99% rename from test/extensions/listener_managers/listener_manager/filter_chain_benchmark_test.cc rename to test/common/listener_manager/filter_chain_benchmark_test.cc index b32b9ff1534f..16f711af4e2b 100644 --- a/test/extensions/listener_managers/listener_manager/filter_chain_benchmark_test.cc +++ b/test/common/listener_manager/filter_chain_benchmark_test.cc @@ -7,8 +7,8 @@ #include "envoy/network/listen_socket.h" #include "envoy/protobuf/message_validator.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" #include "source/common/network/socket_impl.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" #include "test/benchmark/main.h" #include "test/mocks/network/mocks.h" diff --git a/test/extensions/listener_managers/listener_manager/filter_chain_manager_impl_test.cc b/test/common/listener_manager/filter_chain_manager_impl_test.cc similarity index 97% rename from test/extensions/listener_managers/listener_manager/filter_chain_manager_impl_test.cc rename to test/common/listener_manager/filter_chain_manager_impl_test.cc index b55c4d885406..1d42fc5e4ec8 100644 --- a/test/extensions/listener_managers/listener_manager/filter_chain_manager_impl_test.cc +++ b/test/common/listener_manager/filter_chain_manager_impl_test.cc @@ -11,14 +11,14 @@ #include "source/common/api/os_sys_calls_impl.h" #include "source/common/config/metadata.h" +#include "source/common/listener_manager/filter_chain_manager_impl.h" +#include "source/common/listener_manager/listener_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/io_socket_handle_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/network/socket_option_impl.h" #include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" -#include "source/extensions/listener_managers/listener_manager/filter_chain_manager_impl.h" -#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include "source/extensions/transport_sockets/tls/ssl_socket.h" #include "source/server/configuration_impl.h" @@ -273,8 +273,7 @@ TEST_P(FilterChainManagerImplTest, CreatedFilterChainFactoryContextHasIndependen EXPECT_CALL(not_a_draining_manager, drainClose).WillRepeatedly(Return(false)); Configuration::MockServerFactoryContext mock_server_context; EXPECT_CALL(mock_server_context, drainManager).WillRepeatedly(ReturnRef(not_a_draining_manager)); - EXPECT_CALL(parent_context_, getServerFactoryContext) - .WillRepeatedly(ReturnRef(mock_server_context)); + EXPECT_CALL(parent_context_, serverFactoryContext).WillRepeatedly(ReturnRef(mock_server_context)); EXPECT_FALSE(context0->drainDecision().drainClose()); EXPECT_FALSE(context1->drainDecision().drainClose()); diff --git a/test/extensions/listener_managers/listener_manager/lds_api_test.cc b/test/common/listener_manager/lds_api_test.cc similarity index 95% rename from test/extensions/listener_managers/listener_manager/lds_api_test.cc rename to test/common/listener_manager/lds_api_test.cc index 562b82212443..1482978d3192 100644 --- a/test/extensions/listener_managers/listener_manager/lds_api_test.cc +++ b/test/common/listener_manager/lds_api_test.cc @@ -4,8 +4,8 @@ #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/service/discovery/v3/discovery.pb.h" +#include "source/common/listener_manager/lds_api.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/listener_managers/listener_manager/lds_api.h" #include "test/mocks/config/mocks.h" #include "test/mocks/init/mocks.h" @@ -131,9 +131,8 @@ TEST_F(LdsApiTest, MisconfiguredListenerNameIsPresentInException) { EXPECT_CALL(init_watcher_, ready()); const auto decoded_resources = TestUtility::decodeResources({listener}); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(lds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, "Error adding/updating listener(s) invalid-listener: something is wrong\n"); + EXPECT_EQ(lds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "Error adding/updating listener(s) invalid-listener: something is wrong\n"); } TEST_F(LdsApiTest, EmptyListenersUpdate) { @@ -182,11 +181,9 @@ TEST_F(LdsApiTest, ListenerCreationContinuesEvenAfterException) { const auto decoded_resources = TestUtility::decodeResources({listener_0, listener_1, listener_2, listener_3}); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(lds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, - "Error adding/updating listener(s) invalid-listener-1: something is " - "wrong\ninvalid-listener-2: something else is wrong\n"); + EXPECT_EQ(lds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "Error adding/updating listener(s) invalid-listener-1: something is " + "wrong\ninvalid-listener-2: something else is wrong\n"); } // Validate onConfigUpdate throws EnvoyException with duplicate listeners. @@ -208,11 +205,9 @@ TEST_F(LdsApiTest, ValidateDuplicateListeners) { EXPECT_CALL(init_watcher_, ready()); const auto decoded_resources = TestUtility::decodeResources({listener, listener}); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(lds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, - "Error adding/updating listener(s) duplicate_listener: duplicate " - "listener duplicate_listener found\n"); + EXPECT_EQ(lds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "Error adding/updating listener(s) duplicate_listener: duplicate " + "listener duplicate_listener found\n"); } TEST_F(LdsApiTest, Basic) { diff --git a/test/extensions/listener_managers/listener_manager/listener_manager_impl_quic_only_test.cc b/test/common/listener_manager/listener_manager_impl_quic_only_test.cc similarity index 99% rename from test/extensions/listener_managers/listener_manager/listener_manager_impl_quic_only_test.cc rename to test/common/listener_manager/listener_manager_impl_quic_only_test.cc index 70903f4a18dd..23a7d340853b 100644 --- a/test/extensions/listener_managers/listener_manager/listener_manager_impl_quic_only_test.cc +++ b/test/common/listener_manager/listener_manager_impl_quic_only_test.cc @@ -2,10 +2,10 @@ #include "envoy/config/listener/v3/listener.pb.h" #if defined(ENVOY_ENABLE_QUIC) -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_server_transport_socket_factory.h" #endif -#include "test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h" +#include "test/common/listener_manager/listener_manager_impl_test.h" #include "test/server/utility.h" #include "test/test_common/threadsafe_singleton_injector.h" #include "test/mocks/network/mocks.h" diff --git a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc b/test/common/listener_manager/listener_manager_impl_test.cc similarity index 98% rename from test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc rename to test/common/listener_manager/listener_manager_impl_test.cc index 9bcac4d5a461..c7d883f53d05 100644 --- a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc +++ b/test/common/listener_manager/listener_manager_impl_test.cc @@ -1,4 +1,4 @@ -#include "test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h" +#include "test/common/listener_manager/listener_manager_impl_test.h" #include #include @@ -29,8 +29,8 @@ #include "source/extensions/filters/listener/tls_inspector/tls_inspector.h" #include "source/extensions/transport_sockets/tls/ssl_socket.h" -#include "test/extensions/listener_managers/listener_manager/config.pb.h" -#include "test/extensions/listener_managers/listener_manager/config.pb.validate.h" +#include "test/common/listener_manager/config.pb.h" +#include "test/common/listener_manager/config.pb.validate.h" #include "test/mocks/init/mocks.h" #include "test/mocks/matcher/mocks.h" #include "test/server/utility.h" @@ -45,7 +45,6 @@ namespace Envoy { namespace Server { namespace { -using testing::AtLeast; using testing::ByMove; using testing::InSequence; using testing::Return; @@ -131,7 +130,7 @@ class ListenerManagerImplForInPlaceFilterChainUpdateTest : public Event::Simulat void expectAddListener(const envoy::config::listener::v3::Listener& listener_proto, ListenerHandle*) { EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); addOrUpdateListener(listener_proto); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -152,7 +151,7 @@ class ListenerManagerImplForInPlaceFilterChainUpdateTest : public Event::Simulat new_socket = listener_factory_.socket_.get(); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, bind_type, _, 0)); } - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*worker_, stopListener(_, _, _)); EXPECT_CALL(*old_listener_handle->drain_manager_, startDrainSequence(_)); @@ -500,7 +499,7 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, UdpAddress) { EXPECT_TRUE(Protobuf::TextFormat::ParseFromString(proto_text, &listener_proto)); EXPECT_CALL(server_.api_.random_, uuid()); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(listener_factory_, createListenSocket(_, Network::Socket::Type::Datagram, _, ListenerComponentFactory::BindType::ReusePort, _, 0)) @@ -1153,7 +1152,7 @@ version_info: version2 nanos: 2000000 )EOF"); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*worker_, start(_, _)); manager_->startWorkers(guard_dog_, callback_.AsStdFunction()); worker_->callAddCompletion(); @@ -1265,7 +1264,7 @@ version_info: version4 nanos: 4000000 )EOF"); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_baz_different_address->target_.ready(); worker_->callAddCompletion(); @@ -1543,9 +1542,10 @@ filter_chains: {} // Start workers and capture ListenerImpl. Network::ListenerConfig* listener_config = nullptr; - EXPECT_CALL(*worker_, addListener(_, _, _, _)) - .WillOnce(Invoke([&listener_config](auto, Network::ListenerConfig& config, auto, - Runtime::Loader&) -> void { listener_config = &config; })) + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)) + .WillOnce(Invoke( + [&listener_config](auto, Network::ListenerConfig& config, auto, Runtime::Loader&, + Random::RandomGenerator&) -> void { listener_config = &config; })) .RetiresOnSaturation(); EXPECT_CALL(*worker_, start(_, _)); @@ -1569,7 +1569,7 @@ name: foo ListenerHandle* listener_foo_update1 = expectListenerOverridden(false); EXPECT_CALL(*listener_factory_.socket_, duplicate()); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); auto* timer = new Event::MockTimer(dynamic_cast(&server_.dispatcher())); EXPECT_CALL(*timer, enableTimer(_, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_update1_yaml))); @@ -1706,7 +1706,7 @@ version_info: version2 .value()); // Start workers. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*worker_, start(_, _)); manager_->startWorkers(guard_dog_, callback_.AsStdFunction()); // Validate that workers_started stat is still zero before workers set the status via @@ -1732,7 +1732,7 @@ version_info: version2 // removal. ListenerHandle* listener_foo_update2 = expectListenerCreate(false, true); EXPECT_CALL(*duplicated_socket, duplicate()); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*worker_, stopListener(_, _, _)); EXPECT_CALL(*listener_foo_update1->drain_manager_, startDrainSequence(_)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "version3", true)); @@ -1802,7 +1802,7 @@ filter_chains: {} ListenerHandle* listener_bar = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_bar_yaml), "version4", true)); EXPECT_EQ(2UL, manager_->listeners().size()); worker_->callAddCompletion(); @@ -1910,7 +1910,7 @@ name: baz checkStats(__LINE__, 3, 3, 0, 1, 2, 0, 0); // Finish initialization for baz which should make it active. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_baz_update1->target_.ready(); EXPECT_EQ(3UL, manager_->listeners().size()); worker_->callAddCompletion(); @@ -1943,7 +1943,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -2009,7 +2009,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -2330,7 +2330,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -2394,7 +2394,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); @@ -2418,7 +2418,7 @@ name: foo // Add foo again. ListenerHandle* listener_foo2 = expectListenerCreate(false, true); EXPECT_CALL(*listener_factory_.socket_, duplicate()); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 2, 0, 1, 0, 1, 1, 0); @@ -2456,7 +2456,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)).Times(2); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); @@ -2480,7 +2480,7 @@ name: foo // Add foo again. ListenerHandle* listener_foo2 = expectListenerCreate(false, true); EXPECT_CALL(*listener_factory_.socket_, duplicate()).Times(2); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 2, 0, 1, 0, 1, 1, 0); @@ -2517,7 +2517,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); @@ -2535,7 +2535,7 @@ name: foo // Add foo again. We should use the socket from draining. ListenerHandle* listener_foo2 = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 2, 0, 1, 0, 1, 1, 0); @@ -2571,7 +2571,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)).Times(2); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); @@ -2588,7 +2588,7 @@ name: foo ListenerHandle* listener_foo2 = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)).Times(2); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 2, 0, 1, 0, 1, 1, 0); @@ -2675,7 +2675,7 @@ bind_to_port: false return real_listener_factory.createListenSocket( address, socket_type, options, bind_type, creation_options, worker_index); })); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); @@ -2937,7 +2937,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); worker_->callAddCompletion(); checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); @@ -3008,7 +3008,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 2, 0, 1, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -3075,7 +3075,7 @@ traffic_direction: INBOUND auto foo_inbound_proto = parseListenerFromV3Yaml(listener_foo_yaml); EXPECT_TRUE(addOrUpdateListener(foo_inbound_proto)); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -3097,7 +3097,7 @@ traffic_direction: OUTBOUND EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); EXPECT_CALL(listener_foo_outbound->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_outbound_yaml))); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo_outbound->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(2UL, manager_->listeners().size()); @@ -3122,7 +3122,7 @@ traffic_direction: OUTBOUND ListenerHandle* listener_bar_outbound = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_bar_outbound_yaml))); EXPECT_EQ(3UL, manager_->listeners().size()); worker_->callAddCompletion(); @@ -3176,7 +3176,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -3225,7 +3225,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -3335,7 +3335,7 @@ bind_to_port: false "error adding listener: 'bar' has duplicate address '0.0.0.0:1234' as existing listener"); // Move foo to active and then try to add again. This should still fail. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); @@ -5679,10 +5679,11 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, Metadata) { server_.server_factory_context_->cluster_manager_.initializeClusters({"service_foo"}, {}); addOrUpdateListener(parseListenerFromV3Yaml(yaml)); ASSERT_NE(nullptr, listener_factory_context); - EXPECT_EQ("test_value", Config::Metadata::metadataValue( - &listener_factory_context->listenerMetadata(), "com.bar.foo", "baz") - .string_value()); - EXPECT_EQ(envoy::config::core::v3::INBOUND, listener_factory_context->direction()); + EXPECT_EQ("test_value", + Config::Metadata::metadataValue(&listener_factory_context->listenerInfo().metadata(), + "com.bar.foo", "baz") + .string_value()); + EXPECT_EQ(envoy::config::core::v3::INBOUND, listener_factory_context->listenerInfo().direction()); #endif } @@ -5767,32 +5768,9 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilter) { ASSERT_TRUE(listener_factory_context != nullptr); ListenerFactoryContextBaseImpl& parent_context = static_cast(listener_factory_context)->parentFactoryContext(); - EXPECT_EQ(&listener_factory_context->timeSource(), &listener_factory_context->api().timeSource()); EXPECT_EQ(&listener_factory_context->initManager(), &listener.initManager()); - EXPECT_EQ(&listener_factory_context->lifecycleNotifier(), &server_.lifecycleNotifier()); - EXPECT_EQ(&listener_factory_context->messageValidationContext(), - &listener_factory_context->getServerFactoryContext().messageValidationContext()); - EXPECT_EQ(&listener_factory_context->mainThreadDispatcher(), - &parent_context.mainThreadDispatcher()); - EXPECT_EQ(&listener_factory_context->options(), &parent_context.options()); - EXPECT_EQ(&listener_factory_context->grpcContext(), &parent_context.grpcContext()); - EXPECT_EQ(listener_factory_context->healthCheckFailed(), parent_context.healthCheckFailed()); - EXPECT_EQ(&listener_factory_context->httpContext(), &parent_context.httpContext()); - EXPECT_EQ(&listener_factory_context->routerContext(), &parent_context.routerContext()); - EXPECT_EQ(&listener_factory_context->overloadManager(), &parent_context.overloadManager()); - EXPECT_EQ(listener_factory_context->admin().has_value(), parent_context.admin().has_value()); - EXPECT_EQ(&listener_factory_context->listenerTypedMetadata(), - &parent_context.listenerTypedMetadata()); - EXPECT_EQ(listener_factory_context->processContext().has_value(), - parent_context.processContext().has_value()); EXPECT_EQ(&listener_factory_context->getTransportSocketFactoryContext(), &parent_context.getTransportSocketFactoryContext()); - EXPECT_EQ(listener_factory_context->isQuicListener(), parent_context.isQuicListener()); - - // Unit test ListenerFactoryContextBaseImpl for coverage. - EXPECT_EQ(&parent_context.timeSource(), &listener_factory_context->api().timeSource()); - EXPECT_EQ(&parent_context.messageValidationContext(), &server_.messageValidationContext()); - EXPECT_EQ(&parent_context.lifecycleNotifier(), &server_.lifecycleNotifier()); Network::FilterChainFactory& filterChainFactory = listener.filterChainFactory(); Network::MockListenerFilterManager manager; @@ -5801,11 +5779,13 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilter) { EXPECT_CALL(os_sys_calls_, getsockopt_(_, _, _, _, _)).WillRepeatedly(Return(-1)); NiceMock callbacks; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::AcceptedSocketImpl socket(std::make_unique(), Network::Address::InstanceConstSharedPtr{ new Network::Address::Ipv4Instance("127.0.0.1", 1234)}, Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance("127.0.0.1", 5678)}); + new Network::Address::Ipv4Instance("127.0.0.1", 5678)}, + overload_state, false); EXPECT_CALL(callbacks, socket()).WillOnce(Invoke([&]() -> Network::ConnectionSocket& { return socket; @@ -5840,7 +5820,7 @@ class OriginalDstTestConfigFactory : public Configuration::NamedListenerFilterCo createListenerFilterFactoryFromProto(const Protobuf::Message&, const Network::ListenerFilterMatcherSharedPtr&, Configuration::ListenerFactoryContext& context) override { - return [traffic_direction = context.listenerConfig().direction()]( + return [traffic_direction = context.listenerInfo().direction()]( Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter(nullptr, std::make_unique(traffic_direction)); @@ -5885,10 +5865,11 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterOutbound) { Network::MockListenerFilterManager manager; NiceMock callbacks; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::AcceptedSocketImpl socket( std::make_unique(), std::make_unique("127.0.0.1", 1234), - std::make_unique("127.0.0.1", 5678)); + std::make_unique("127.0.0.1", 5678), overload_state, false); #ifdef WIN32 EXPECT_CALL(os_sys_calls_, ioctl(_, _, _, _, _, _, _)); @@ -5950,10 +5931,11 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilterStopsIteration) EXPECT_CALL(os_sys_calls_, ioctl(_, _, _, _, _, _, _)) .WillRepeatedly(testing::Return(Api::SysCallIntResult{-1, SOCKET_ERROR_NOT_SUP})); #endif + Server::ThreadLocalOverloadStateOptRef overload_state; Network::AcceptedSocketImpl socket( std::make_unique(), std::make_unique("127.0.0.1", 1234), - std::make_unique("127.0.0.1", 5678)); + std::make_unique("127.0.0.1", 5678), overload_state, false); EXPECT_CALL(callbacks, socket()).WillOnce(Invoke([&]() -> Network::ConnectionSocket& { return socket; @@ -5999,9 +5981,10 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterInbound) { NiceMock callbacks; auto io_handle = std::make_unique>(); + Server::ThreadLocalOverloadStateOptRef overload_state; Network::AcceptedSocketImpl socket( std::move(io_handle), std::make_unique("127.0.0.1", 1234), - std::make_unique("127.0.0.1", 5678)); + std::make_unique("127.0.0.1", 5678), overload_state, false); EXPECT_CALL(callbacks, socket()).WillOnce(Invoke([&]() -> Network::ConnectionSocket& { return socket; @@ -6040,7 +6023,7 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterIPv6) { createListenerFilterFactoryFromProto(const Protobuf::Message&, const Network::ListenerFilterMatcherSharedPtr&, Configuration::ListenerFactoryContext& context) override { - return [traffic_direction = context.listenerConfig().direction()]( + return [traffic_direction = context.listenerInfo().direction()]( Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter( nullptr, std::make_unique(traffic_direction)); @@ -6082,10 +6065,11 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterIPv6) { Network::MockListenerFilterManager manager; NiceMock callbacks; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::AcceptedSocketImpl socket( std::make_unique(), std::make_unique("::0001", 1234), - std::make_unique("::0001", 5678)); + std::make_unique("::0001", 5678), overload_state, false); EXPECT_CALL(callbacks, socket()).WillOnce(Invoke([&]() -> Network::ConnectionSocket& { return socket; @@ -6942,7 +6926,7 @@ per_connection_buffer_limit_bytes: 10 .value()); // Start workers. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*worker_, start(_, _)); manager_->startWorkers(guard_dog_, callback_.AsStdFunction()); // Validate that workers_started stat is still zero before workers set the status via @@ -6967,7 +6951,7 @@ per_connection_buffer_limit_bytes: 10 // Update foo. Should go into warming, have an immediate warming callback, and start immediate // removal. ListenerHandle* listener_foo_update2 = expectListenerCreate(false, true); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*worker_, stopListener(_, _, _)); EXPECT_CALL(*listener_foo_update1->drain_manager_, startDrainSequence(_)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "version3", true)); @@ -7025,7 +7009,7 @@ per_connection_buffer_limit_bytes: 10 )EOF"; ListenerHandle* listener_bar = expectListenerCreate(false, true); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_bar_yaml), "version4", true)); EXPECT_EQ(2UL, manager_->listeners().size()); worker_->callAddCompletion(); @@ -7118,7 +7102,7 @@ per_connection_buffer_limit_bytes: 10 checkStats(__LINE__, 3, 3, 0, 1, 2, 0, 0); // Finish initialization for baz which should make it active. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_baz_update1->target_.ready(); EXPECT_EQ(3UL, manager_->listeners().size()); worker_->callAddCompletion(); @@ -7154,7 +7138,7 @@ traffic_direction: INBOUND EXPECT_EQ(0, server_.stats_store_.counter("listener_manager.listener_in_place_updated").value()); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7216,7 +7200,7 @@ traffic_direction: INBOUND EXPECT_EQ(0, server_.stats_store_.counter("listener_manager.listener_in_place_updated").value()); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7285,7 +7269,7 @@ traffic_direction: INBOUND EXPECT_EQ(0, server_.stats_store_.counter("listener_manager.listener_in_place_updated").value()); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7314,7 +7298,7 @@ traffic_direction: INBOUND checkStats(__LINE__, 1, 1, 0, 1, 1, 0, 0); // Listener warmed up. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(*listener_foo, onDestroy()); listener_foo_update1->target_.ready(); worker_->callAddCompletion(); @@ -7346,7 +7330,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7376,7 +7360,7 @@ traffic_direction: INBOUND checkStats(__LINE__, 1, 1, 0, 1, 1, 0, 0); // The warmed up starts the drain timer. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(server_.options_, drainTime()).WillOnce(Return(std::chrono::seconds(600))); Event::MockTimer* filter_chain_drain_timer = new Event::MockTimer(&server_.dispatcher_); EXPECT_CALL(*filter_chain_drain_timer, enableTimer(std::chrono::milliseconds(600000), _)); @@ -7432,7 +7416,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7460,7 +7444,7 @@ traffic_direction: INBOUND checkStats(__LINE__, 1, 1, 0, 1, 1, 0, 0); // The warmed up starts the drain timer. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(server_.options_, drainTime()).WillOnce(Return(std::chrono::seconds(600))); Event::MockTimer* filter_chain_drain_timer = new Event::MockTimer(&server_.dispatcher_); EXPECT_CALL(*filter_chain_drain_timer, enableTimer(std::chrono::milliseconds(600000), _)); @@ -7479,6 +7463,68 @@ traffic_direction: INBOUND EXPECT_CALL(*listener_foo_update1, onDestroy()); } +TEST_P(ListenerManagerImplTest, SharedListenerInfoInInplaceUpdate) { + InSequence s; + + EXPECT_CALL(*worker_, start(_, _)); + manager_->startWorkers(guard_dog_, callback_.AsStdFunction()); + + // Add foo listener into warming. + const std::string listener_foo_yaml = R"EOF( +name: foo +traffic_direction: INBOUND +address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +filter_chains: +- filters: [] + )EOF"; + + ListenerHandle* listener_foo = expectListenerCreate(true, true); + EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); + EXPECT_CALL(listener_foo->target_, initialize()); + EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); + checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); + listener_foo->target_.ready(); + worker_->callAddCompletion(); + EXPECT_EQ(1UL, manager_->listeners().size()); + + const auto* listener_foo_info = &manager_->listeners()[0].get().listenerInfo(); + + // Update foo into warming. + const std::string listener_foo_update1_yaml = R"EOF( +name: foo +traffic_direction: INBOUND +address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +filter_chains: +- filters: + filter_chain_match: + destination_port: 1234 + )EOF"; + + ListenerHandle* listener_foo_update1 = expectListenerOverridden(true); + EXPECT_CALL(*listener_factory_.socket_, duplicate()); + EXPECT_CALL(listener_foo_update1->target_, initialize()); + EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_update1_yaml))); + EXPECT_EQ(1UL, manager_->listeners().size()); + EXPECT_EQ(1, server_.stats_store_.counter("listener_manager.listener_in_place_updated").value()); + checkStats(__LINE__, 1, 1, 0, 1, 1, 0, 0); + + const auto* listener_foo_update1_info = &manager_->listeners()[0].get().listenerInfo(); + + // The listener info should be the same because the listener info is shared between the + // new listener and the old listener when doing in place update. + EXPECT_EQ(listener_foo_info, listener_foo_update1_info); + + EXPECT_CALL(*listener_foo_update1, onDestroy()); + EXPECT_CALL(*listener_foo, onDestroy()); +} + TEST_P(ListenerManagerImplTest, ListenSocketFactoryIsClonedFromListenerDrainingFilterChain) { InSequence s; @@ -7502,7 +7548,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7530,7 +7576,7 @@ traffic_direction: INBOUND checkStats(__LINE__, 1, 1, 0, 1, 1, 0, 0); // The warmed up starts the drain timer. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(server_.options_, drainTime()).WillOnce(Return(std::chrono::seconds(600))); Event::MockTimer* filter_chain_drain_timer = new Event::MockTimer(&server_.dispatcher_); EXPECT_CALL(*filter_chain_drain_timer, enableTimer(std::chrono::milliseconds(600000), _)); @@ -7602,7 +7648,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -7635,7 +7681,7 @@ traffic_direction: INBOUND checkStats(__LINE__, 1, 1, 0, 1, 1, 0, 0); // The warmed up starts the drain timer. - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); EXPECT_CALL(server_.options_, drainTime()).WillOnce(Return(std::chrono::seconds(600))); Event::MockTimer* filter_chain_drain_timer = new Event::MockTimer(&server_.dispatcher_); EXPECT_CALL(*filter_chain_drain_timer, enableTimer(std::chrono::milliseconds(600000), _)); diff --git a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h b/test/common/listener_manager/listener_manager_impl_test.h similarity index 98% rename from test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h rename to test/common/listener_manager/listener_manager_impl_test.h index a872c64dc745..5ffff61ea8b9 100644 --- a/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h +++ b/test/common/listener_manager/listener_manager_impl_test.h @@ -7,10 +7,10 @@ #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/config/listener/v3/listener_components.pb.h" +#include "source/common/listener_manager/listener_impl.h" +#include "source/common/listener_manager/listener_manager_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/network/socket_option_impl.h" -#include "source/extensions/listener_managers/listener_manager/listener_impl.h" -#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include "source/server/configuration_impl.h" #include "test/mocks/network/mocks.h" @@ -398,7 +398,7 @@ class ListenerManagerImplTest : public testing::TestWithParam { EXPECT_CALL(listener_origin->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(origin))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_origin->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -441,7 +441,7 @@ class ListenerManagerImplTest : public testing::TestWithParam { EXPECT_CALL(listener_origin->target_, initialize()); EXPECT_TRUE(addOrUpdateListener(parseListenerFromV3Yaml(origin))); checkStats(__LINE__, 1, 0, 0, 1, 0, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _, _, _)); + EXPECT_CALL(*worker_, addListener(_, _, _, _, _)); listener_origin->target_.ready(); worker_->callAddCompletion(); EXPECT_EQ(1UL, manager_->listeners().size()); diff --git a/test/common/matcher/BUILD b/test/common/matcher/BUILD index 9b8f951e856f..d548f702026e 100644 --- a/test/common/matcher/BUILD +++ b/test/common/matcher/BUILD @@ -80,7 +80,7 @@ envoy_cc_test( "//test/mocks/server:factory_context_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:registry_lib", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/test/common/network/BUILD b/test/common/network/BUILD index 1ba3e6aa2067..6b433fcc8180 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -303,7 +303,7 @@ envoy_cc_test( envoy_cc_test_library( name = "socket_option_test", - srcs = ["socket_option_test.h"], + hdrs = ["socket_option_test.h"], deps = [ "//source/common/network:address_lib", "//source/common/network:socket_option_factory_lib", diff --git a/test/common/network/address_impl_speed_test.cc b/test/common/network/address_impl_speed_test.cc index 9eea883a3cf9..f110e6adbf89 100644 --- a/test/common/network/address_impl_speed_test.cc +++ b/test/common/network/address_impl_speed_test.cc @@ -15,6 +15,7 @@ static void ipv4InstanceCreate(benchmark::State& state) { static constexpr uint32_t Addr = 0xc00002ff; // From the RFC 5737 example range. addr.sin_addr.s_addr = htonl(Addr); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); Ipv4Instance address(&addr); benchmark::DoNotOptimize(address.ip()); } @@ -29,6 +30,7 @@ static void ipv6InstanceCreate(benchmark::State& state) { static const char* Addr = "2001:DB8::1234"; // From the RFC 3849 example range. inet_pton(AF_INET6, Addr, &addr.sin6_addr); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); Ipv6Instance address(addr); benchmark::DoNotOptimize(address.ip()); } diff --git a/test/common/network/cidr_range_test.cc b/test/common/network/cidr_range_test.cc index 6232e910ddfb..2938159c38ee 100644 --- a/test/common/network/cidr_range_test.cc +++ b/test/common/network/cidr_range_test.cc @@ -389,79 +389,82 @@ makeCidrRangeList(const std::vector>& ranges) { } TEST(IpListTest, Errors) { - { - EXPECT_THROW({ IpList list(makeCidrRangeList({{"foo", 0}})); }, EnvoyException); - } + { EXPECT_THROW(IpList::create(makeCidrRangeList({{"foo", 0}})).IgnoreError(), EnvoyException); } } TEST(IpListTest, SpecificAddressAllowed) { - IpList list(makeCidrRangeList({{"192.168.1.1", 24}})); + std::unique_ptr list = IpList::create(makeCidrRangeList({{"192.168.1.1", 24}})).value(); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.1.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.1.3"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.1.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("192.168.3.0"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("192.168.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.1.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.1.3"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.1.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("192.168.3.0"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("192.168.0.0"))); } TEST(IpListTest, Normal) { - IpList list(makeCidrRangeList({{"192.168.3.0", 24}, {"50.1.2.3", 32}, {"10.15.0.0", 16}})); - - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.3"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("192.168.2.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("192.168.4.0"))); - - EXPECT_TRUE(list.contains(Address::Ipv4Instance("50.1.2.3"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("50.1.2.2"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("50.1.2.4"))); - - EXPECT_TRUE(list.contains(Address::Ipv4Instance("10.15.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("10.15.90.90"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("10.15.255.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("10.14.255.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("10.16.0.0"))); - - EXPECT_FALSE(list.contains(Address::Ipv6Instance("::1"))); - EXPECT_FALSE(list.contains(Address::PipeInstance("foo"))); + std::unique_ptr list = + IpList::create(makeCidrRangeList({{"192.168.3.0", 24}, {"50.1.2.3", 32}, {"10.15.0.0", 16}})) + .value(); + + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.3"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("192.168.2.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("192.168.4.0"))); + + EXPECT_TRUE(list->contains(Address::Ipv4Instance("50.1.2.3"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("50.1.2.2"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("50.1.2.4"))); + + EXPECT_TRUE(list->contains(Address::Ipv4Instance("10.15.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("10.15.90.90"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("10.15.255.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("10.14.255.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("10.16.0.0"))); + + EXPECT_FALSE(list->contains(Address::Ipv6Instance("::1"))); + EXPECT_FALSE(list->contains(Address::PipeInstance("foo"))); } TEST(IpListTest, AddressVersionMix) { - IpList list(makeCidrRangeList({{"192.168.3.0", 24}, {"2001:db8:85a3::", 64}, {"::1", 128}})); - - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.3"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("192.168.2.255"))); - EXPECT_FALSE(list.contains(Address::Ipv4Instance("192.168.4.0"))); - - EXPECT_TRUE(list.contains(Address::Ipv6Instance("2001:db8:85a3::"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("2001:db8:85a3:0:1::"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("2001:db8:85a3::ffff:ffff:ffff:ffff"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("2001:db8:85a3::ffff"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("2001:db8:85a3::1"))); - EXPECT_FALSE(list.contains(Address::Ipv6Instance("2001:db8:85a3:1::"))); - EXPECT_FALSE(list.contains(Address::Ipv6Instance("2002:db8:85a3::"))); - - EXPECT_TRUE(list.contains(Address::Ipv6Instance("::1"))); - EXPECT_FALSE(list.contains(Address::Ipv6Instance("::"))); - - EXPECT_FALSE(list.contains(Address::PipeInstance("foo"))); + std::unique_ptr list = + IpList::create( + makeCidrRangeList({{"192.168.3.0", 24}, {"2001:db8:85a3::", 64}, {"::1", 128}})) + .value(); + + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.3"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("192.168.2.255"))); + EXPECT_FALSE(list->contains(Address::Ipv4Instance("192.168.4.0"))); + + EXPECT_TRUE(list->contains(Address::Ipv6Instance("2001:db8:85a3::"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("2001:db8:85a3:0:1::"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("2001:db8:85a3::ffff:ffff:ffff:ffff"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("2001:db8:85a3::ffff"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("2001:db8:85a3::1"))); + EXPECT_FALSE(list->contains(Address::Ipv6Instance("2001:db8:85a3:1::"))); + EXPECT_FALSE(list->contains(Address::Ipv6Instance("2002:db8:85a3::"))); + + EXPECT_TRUE(list->contains(Address::Ipv6Instance("::1"))); + EXPECT_FALSE(list->contains(Address::Ipv6Instance("::"))); + + EXPECT_FALSE(list->contains(Address::PipeInstance("foo"))); } TEST(IpListTest, MatchAny) { - IpList list(makeCidrRangeList({{"0.0.0.0", 0}})); + std::unique_ptr list = IpList::create(makeCidrRangeList({{"0.0.0.0", 0}})).value(); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.3"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.255"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.0.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("1.1.1.1"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.3"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.255"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.0.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("1.1.1.1"))); - EXPECT_FALSE(list.contains(Address::Ipv6Instance("::1"))); - EXPECT_FALSE(list.contains(Address::PipeInstance("foo"))); + EXPECT_FALSE(list->contains(Address::Ipv6Instance("::1"))); + EXPECT_FALSE(list->contains(Address::PipeInstance("foo"))); } TEST(IpListTest, MatchAnyImplicitPrefixLen) { @@ -470,35 +473,36 @@ TEST(IpListTest, MatchAnyImplicitPrefixLen) { cidrRange->set_address_prefix("0.0.0.0"); EXPECT_FALSE(cidrRange->has_prefix_len()); - IpList list(cidrRangeList); + std::unique_ptr list = IpList::create(cidrRangeList).value(); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.3"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.255"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.0.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("1.1.1.1"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.3"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.255"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.0.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("1.1.1.1"))); - EXPECT_FALSE(list.contains(Address::Ipv6Instance("::1"))); - EXPECT_FALSE(list.contains(Address::PipeInstance("foo"))); + EXPECT_FALSE(list->contains(Address::Ipv6Instance("::1"))); + EXPECT_FALSE(list->contains(Address::PipeInstance("foo"))); } TEST(IpListTest, MatchAnyAll) { - IpList list(makeCidrRangeList({{"0.0.0.0", 0}, {"::", 0}})); - - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.3"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.3.255"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.168.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("192.0.0.0"))); - EXPECT_TRUE(list.contains(Address::Ipv4Instance("1.1.1.1"))); - - EXPECT_TRUE(list.contains(Address::Ipv6Instance("::1"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("::"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("2001:db8:85a3::"))); - EXPECT_TRUE(list.contains(Address::Ipv6Instance("ffee::"))); - - EXPECT_FALSE(list.contains(Address::PipeInstance("foo"))); + std::unique_ptr list = + IpList::create(makeCidrRangeList({{"0.0.0.0", 0}, {"::", 0}})).value(); + + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.3"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.3.255"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.168.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("192.0.0.0"))); + EXPECT_TRUE(list->contains(Address::Ipv4Instance("1.1.1.1"))); + + EXPECT_TRUE(list->contains(Address::Ipv6Instance("::1"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("::"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("2001:db8:85a3::"))); + EXPECT_TRUE(list->contains(Address::Ipv6Instance("ffee::"))); + + EXPECT_FALSE(list->contains(Address::PipeInstance("foo"))); } } // namespace diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index e661d330bdbf..7f768eea8c21 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -19,6 +19,7 @@ #include "source/common/network/io_socket_handle_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/network/raw_buffer_socket.h" +#include "source/common/network/tcp_listener_impl.h" #include "source/common/network/utility.h" #include "source/common/runtime/runtime_impl.h" @@ -159,8 +160,11 @@ class ConnectionImplTestBase { } socket_ = std::make_shared(address); NiceMock listener_config; - listener_ = - dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + listener_ = std::make_unique( + *dispatcher_, api_->randomGenerator(), runtime_, socket_, listener_callbacks_, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); client_connection_ = std::make_unique( *dispatcher_, socket_->connectionInfoProvider().localAddress(), source_address_, createTransportSocket(), socket_options_, transport_socket_options_); @@ -1866,7 +1870,11 @@ TEST_P(ConnectionImplTest, BindFailureTest) { socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); NiceMock listener_config; - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + listener_ = std::make_unique( + *dispatcher_, api_->randomGenerator(), runtime_, socket_, listener_callbacks_, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); client_connection_ = dispatcher_->createClientConnection( socket_->connectionInfoProvider().localAddress(), source_address_, @@ -3416,8 +3424,11 @@ class ReadBufferLimitTest : public ConnectionImplTest { socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); NiceMock listener_config; - listener_ = - dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + listener_ = std::make_unique( + *dispatcher_, api_->randomGenerator(), runtime_, socket_, listener_callbacks_, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); client_connection_ = dispatcher_->createClientConnection( socket_->connectionInfoProvider().localAddress(), diff --git a/test/common/network/io_socket_handle_impl_test.cc b/test/common/network/io_socket_handle_impl_test.cc index 8e8da0e0b806..93a6ee74c9a8 100644 --- a/test/common/network/io_socket_handle_impl_test.cc +++ b/test/common/network/io_socket_handle_impl_test.cc @@ -24,47 +24,49 @@ namespace Network { namespace { TEST(IoSocketHandleImpl, TestIoSocketError) { - EXPECT_DEBUG_DEATH(IoSocketError(SOCKET_ERROR_AGAIN), - ".*assert failure: .* Details: Didn't use getIoSocketEagainInstance.*"); + EXPECT_DEBUG_DEATH(IoSocketError::create(SOCKET_ERROR_AGAIN), + ".*assert failure: .* Details: Didn't use getIoSocketEagainError.*"); + EXPECT_EQ(IoSocketError::IoErrorCode::Again, + IoSocketError::getIoSocketEagainError()->getErrorCode()); EXPECT_EQ(errorDetails(SOCKET_ERROR_AGAIN), - IoSocketError::getIoSocketEagainInstance()->getErrorDetails()); + IoSocketError::getIoSocketEagainError()->getErrorDetails()); - IoSocketError error2(SOCKET_ERROR_NOT_SUP); - EXPECT_EQ(IoSocketError::IoErrorCode::NoSupport, error2.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_NOT_SUP), error2.getErrorDetails()); + Api::IoErrorPtr error2 = IoSocketError::create(SOCKET_ERROR_NOT_SUP); + EXPECT_EQ(IoSocketError::IoErrorCode::NoSupport, error2->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_NOT_SUP), error2->getErrorDetails()); - IoSocketError error3(SOCKET_ERROR_AF_NO_SUP); - EXPECT_EQ(IoSocketError::IoErrorCode::AddressFamilyNoSupport, error3.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_AF_NO_SUP), error3.getErrorDetails()); + Api::IoErrorPtr error3 = IoSocketError::create(SOCKET_ERROR_AF_NO_SUP); + EXPECT_EQ(IoSocketError::IoErrorCode::AddressFamilyNoSupport, error3->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_AF_NO_SUP), error3->getErrorDetails()); - IoSocketError error4(SOCKET_ERROR_IN_PROGRESS); - EXPECT_EQ(IoSocketError::IoErrorCode::InProgress, error4.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_IN_PROGRESS), error4.getErrorDetails()); + Api::IoErrorPtr error4 = IoSocketError::create(SOCKET_ERROR_IN_PROGRESS); + EXPECT_EQ(IoSocketError::IoErrorCode::InProgress, error4->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_IN_PROGRESS), error4->getErrorDetails()); - IoSocketError error5(SOCKET_ERROR_PERM); - EXPECT_EQ(IoSocketError::IoErrorCode::Permission, error5.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_PERM), error5.getErrorDetails()); + Api::IoErrorPtr error5 = IoSocketError::create(SOCKET_ERROR_PERM); + EXPECT_EQ(IoSocketError::IoErrorCode::Permission, error5->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_PERM), error5->getErrorDetails()); - IoSocketError error6(SOCKET_ERROR_MSG_SIZE); - EXPECT_EQ(IoSocketError::IoErrorCode::MessageTooBig, error6.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_MSG_SIZE), error6.getErrorDetails()); + Api::IoErrorPtr error6 = IoSocketError::create(SOCKET_ERROR_MSG_SIZE); + EXPECT_EQ(IoSocketError::IoErrorCode::MessageTooBig, error6->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_MSG_SIZE), error6->getErrorDetails()); - IoSocketError error7(SOCKET_ERROR_INTR); - EXPECT_EQ(IoSocketError::IoErrorCode::Interrupt, error7.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_INTR), error7.getErrorDetails()); + Api::IoErrorPtr error7 = IoSocketError::create(SOCKET_ERROR_INTR); + EXPECT_EQ(IoSocketError::IoErrorCode::Interrupt, error7->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_INTR), error7->getErrorDetails()); - IoSocketError error8(SOCKET_ERROR_ADDR_NOT_AVAIL); - EXPECT_EQ(IoSocketError::IoErrorCode::AddressNotAvailable, error8.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_ADDR_NOT_AVAIL), error8.getErrorDetails()); + Api::IoErrorPtr error8 = IoSocketError::create(SOCKET_ERROR_ADDR_NOT_AVAIL); + EXPECT_EQ(IoSocketError::IoErrorCode::AddressNotAvailable, error8->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_ADDR_NOT_AVAIL), error8->getErrorDetails()); - IoSocketError error9(SOCKET_ERROR_CONNRESET); - EXPECT_EQ(IoSocketError::IoErrorCode::ConnectionReset, error9.getErrorCode()); - EXPECT_EQ(errorDetails(SOCKET_ERROR_CONNRESET), error9.getErrorDetails()); + Api::IoErrorPtr error9 = IoSocketError::create(SOCKET_ERROR_CONNRESET); + EXPECT_EQ(IoSocketError::IoErrorCode::ConnectionReset, error9->getErrorCode()); + EXPECT_EQ(errorDetails(SOCKET_ERROR_CONNRESET), error9->getErrorDetails()); // Random unknown error - IoSocketError error10(123); - EXPECT_EQ(IoSocketError::IoErrorCode::UnknownError, error10.getErrorCode()); - EXPECT_EQ(errorDetails(123), error10.getErrorDetails()); + Api::IoErrorPtr error10 = IoSocketError::create(123); + EXPECT_EQ(IoSocketError::IoErrorCode::UnknownError, error10->getErrorCode()); + EXPECT_EQ(errorDetails(123), error10->getErrorDetails()); } TEST(IoSocketHandleImpl, LastRoundTripTimeReturnsEmptyOptionalIfGetSocketFails) { diff --git a/test/common/network/lc_trie_speed_test.cc b/test/common/network/lc_trie_speed_test.cc index 7a524fe0f135..99eb7e6aede1 100644 --- a/test/common/network/lc_trie_speed_test.cc +++ b/test/common/network/lc_trie_speed_test.cc @@ -59,6 +59,7 @@ static void lcTrieConstruct(benchmark::State& state) { std::unique_ptr> trie; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); trie = std::make_unique>(inputs.tag_data_); } benchmark::DoNotOptimize(trie); @@ -71,6 +72,7 @@ static void lcTrieConstructNested(benchmark::State& state) { std::unique_ptr> trie; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); trie = std::make_unique>( inputs.tag_data_nested_prefixes_); } @@ -84,6 +86,8 @@ static void lcTrieConstructMinimal(benchmark::State& state) { std::unique_ptr> trie; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) trie = std::make_unique>(inputs.tag_data_minimal_); } benchmark::DoNotOptimize(trie); @@ -100,6 +104,7 @@ static void lcTrieLookup(benchmark::State& state) { static size_t i = 0; size_t output_tags = 0; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); i++; i %= address_inputs.addresses_.size(); output_tags += lc_trie->getData(address_inputs.addresses_[i]).size(); @@ -119,6 +124,7 @@ static void lcTrieLookupWithNestedPrefixes(benchmark::State& state) { static size_t i = 0; size_t output_tags = 0; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); i++; i %= address_inputs.addresses_.size(); output_tags += lc_trie_nested_prefixes->getData(address_inputs.addresses_[i]).size(); @@ -137,6 +143,7 @@ static void lcTrieLookupMinimal(benchmark::State& state) { static size_t i = 0; size_t output_tags = 0; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); i++; i %= address_inputs.addresses_.size(); output_tags += lc_trie_minimal->getData(address_inputs.addresses_[i]).size(); diff --git a/test/common/network/listener_filter_buffer_fuzz_test.cc b/test/common/network/listener_filter_buffer_fuzz_test.cc index 653e842d620c..a68726afb225 100644 --- a/test/common/network/listener_filter_buffer_fuzz_test.cc +++ b/test/common/network/listener_filter_buffer_fuzz_test.cc @@ -62,17 +62,15 @@ class ListenerFilterBufferFuzzer { // If the available is 0, then emulate an `EAGAIN`. if (append_data_size == 0) { EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - 0, Api::IoErrorPtr(IoSocketError::getIoSocketEagainInstance(), - IoSocketError::deleteIoError))))); + .WillOnce(Return( + ByMove(Api::IoCallUint64Result(0, IoSocketError::getIoSocketEagainError())))); } else { available_data_.insert(available_data_.end(), append_data_size, insert_value); EXPECT_CALL(io_handle_, recv).WillOnce([&](void* buffer, size_t length, int flags) { EXPECT_EQ(MSG_PEEK, flags); auto copy_size = std::min(length, available_data_.size()); ::memcpy(buffer, available_data_.data(), copy_size); - return Api::IoCallUint64Result(copy_size, - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(copy_size, Api::IoError::none()); }); drained_size_ = 0; } @@ -89,8 +87,7 @@ class ListenerFilterBufferFuzzer { EXPECT_EQ(drain_size, length); ::memcpy(buffer, available_data_.data(), drain_size); available_data_ = available_data_.substr(drain_size); - return Api::IoCallUint64Result(drain_size, - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(drain_size, Api::IoError::none()); }); } drained_size_ += drain_size; diff --git a/test/common/network/listener_filter_buffer_impl_test.cc b/test/common/network/listener_filter_buffer_impl_test.cc index b1b426868e2e..918c1a7e30bc 100644 --- a/test/common/network/listener_filter_buffer_impl_test.cc +++ b/test/common/network/listener_filter_buffer_impl_test.cc @@ -58,7 +58,7 @@ TEST_F(ListenerFilterBufferImplTest, Basic) { for (size_t i = 0; i < length / 2; i++) { buf[i] = 'a'; } - return Api::IoCallUint64Result(length / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length / 2, Api::IoError::none()); }); on_data_cb_ = [&](ListenerFilterBuffer& filter_buffer) { auto raw_buffer = filter_buffer.rawSlice(); @@ -78,7 +78,7 @@ TEST_F(ListenerFilterBufferImplTest, Basic) { for (size_t i = length / 2; i < length; i++) { buf[i] = 'b'; } - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length, Api::IoError::none()); }); on_data_cb_ = [&](ListenerFilterBuffer& filter_buffer) { @@ -98,9 +98,8 @@ TEST_F(ListenerFilterBufferImplTest, Basic) { bool is_closed = false; on_close_cb_ = [&](bool) { is_closed = true; }; EXPECT_CALL(io_handle_, recv) - .WillOnce(Return( - ByMove(Api::IoCallUint64Result(-1, Api::IoErrorPtr(new IoSocketError(SOCKET_ERROR_INTR), - IoSocketError::deleteIoError))))); + .WillOnce( + Return(ByMove(Api::IoCallUint64Result(-1, IoSocketError::create(SOCKET_ERROR_INTR))))); file_event_callback_(Event::FileReadyType::Read); EXPECT_TRUE(is_closed); @@ -108,17 +107,15 @@ TEST_F(ListenerFilterBufferImplTest, Basic) { is_closed = false; on_close_cb_ = [&](bool) { is_closed = true; }; EXPECT_CALL(io_handle_, recv) - .WillOnce(Return( - ByMove(Api::IoCallUint64Result(0, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(0, Api::IoError::none())))); file_event_callback_(Event::FileReadyType::Read); EXPECT_TRUE(is_closed); // On socket again. is_closed = false; EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove( - Api::IoCallUint64Result(0, Api::IoErrorPtr(IoSocketError::getIoSocketEagainInstance(), - IoSocketError::deleteIoError))))); + .WillOnce( + Return(ByMove(Api::IoCallUint64Result(0, IoSocketError::getIoSocketEagainError())))); file_event_callback_(Event::FileReadyType::Read); EXPECT_FALSE(is_closed); } @@ -134,7 +131,7 @@ TEST_F(ListenerFilterBufferImplTest, DrainData) { for (size_t i = 0; i < length / 2; i++) { buf[i] = 'a'; } - return Api::IoCallUint64Result(length / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length / 2, Api::IoError::none()); }); on_data_cb_ = [&](ListenerFilterBuffer& filter_buffer) { auto raw_buffer = filter_buffer.rawSlice(); @@ -157,16 +154,14 @@ TEST_F(ListenerFilterBufferImplTest, DrainData) { // expect to read the `drained_size` data EXPECT_EQ(drained_size, length); // only drain half data from the socket. - return Api::IoCallUint64Result(drained_size / 2, - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(drained_size / 2, Api::IoError::none()); }) .WillOnce([&](void*, size_t length, int flags) { // expect to read, not peek EXPECT_EQ(0, flags); // expect to read the `drained_size` data EXPECT_EQ(drained_size / 2, length); - return Api::IoCallUint64Result(drained_size / 2, - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(drained_size / 2, Api::IoError::none()); }); listener_buffer_->drain(drained_size); @@ -186,7 +181,7 @@ TEST_F(ListenerFilterBufferImplTest, DrainData) { for (uint64_t i = 0; i < length; i++) { buf[i] = 'b'; } - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length, Api::IoError::none()); }); on_data_cb_ = [&](ListenerFilterBuffer& filter_buffer) { auto raw_slice = filter_buffer.rawSlice(); @@ -210,7 +205,7 @@ TEST_F(ListenerFilterBufferImplTest, ResetCapacity) { for (size_t i = 0; i < length / 2; i++) { buf[i] = 'a'; } - return Api::IoCallUint64Result(length / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length / 2, Api::IoError::none()); }); on_data_cb_ = [&](ListenerFilterBuffer& filter_buffer) { auto raw_buffer = filter_buffer.rawSlice(); @@ -235,7 +230,7 @@ TEST_F(ListenerFilterBufferImplTest, ResetCapacity) { for (size_t i = 0; i < length; i++) { buf[i] = 'b'; } - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length, Api::IoError::none()); }); on_data_cb_ = [&](ListenerFilterBuffer& filter_buffer) { diff --git a/test/common/network/listener_impl_test.cc b/test/common/network/listener_impl_test.cc index e29da9219d2b..3342ffa20d77 100644 --- a/test/common/network/listener_impl_test.cc +++ b/test/common/network/listener_impl_test.cc @@ -39,8 +39,11 @@ static void errorCallbackTest(Address::IpVersion version) { Network::Test::getCanonicalLoopbackAddress(version)); Network::MockTcpListenerCallbacks listener_callbacks; NiceMock listener_config; - Network::ListenerPtr listener = - dispatcher->createListener(socket, listener_callbacks, runtime, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + Network::ListenerPtr listener = std::make_unique( + *dispatcher, api->randomGenerator(), runtime, socket, listener_callbacks, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); Network::ClientConnectionPtr client_connection = dispatcher->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -72,17 +75,20 @@ class TestTcpListenerImpl : public TcpListenerImpl { public: TestTcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random_generator, Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb, - bool bind_to_port, bool ignore_global_conn_limit) - : TestTcpListenerImpl(dispatcher, random_generator, runtime, std::move(socket), cb, - bind_to_port, ignore_global_conn_limit, - Network::DefaultMaxConnectionsToAcceptPerSocketEvent) {} + bool bind_to_port, bool ignore_global_conn_limit, + Server::ThreadLocalOverloadStateOptRef overload_state) + : TestTcpListenerImpl(dispatcher, random_generator, runtime, socket, cb, bind_to_port, + ignore_global_conn_limit, + Network::DefaultMaxConnectionsToAcceptPerSocketEvent, overload_state) {} TestTcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random_generator, Runtime::Loader& runtime, SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port, bool ignore_global_conn_limit, - uint32_t max_connections_to_accept_per_socket_event) - : TcpListenerImpl(dispatcher, random_generator, runtime, std::move(socket), cb, bind_to_port, - ignore_global_conn_limit, max_connections_to_accept_per_socket_event) {} + uint32_t max_connections_to_accept_per_socket_event, + Server::ThreadLocalOverloadStateOptRef overload_state) + : TcpListenerImpl(dispatcher, random_generator, runtime, socket, cb, bind_to_port, + ignore_global_conn_limit, max_connections_to_accept_per_socket_event, + overload_state) {} MOCK_METHOD(Address::InstanceConstSharedPtr, getLocalAddress, (os_fd_t fd)); }; @@ -99,13 +105,13 @@ TEST_P(TcpListenerImplTest, UseActualDst) { Network::MockTcpListenerCallbacks listener_callbacks1; Random::MockRandomGenerator random_generator; NiceMock runtime; - + Server::ThreadLocalOverloadStateOptRef overload_state; // Do not redirect since use_original_dst is false. Network::TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks1, true, false); + listener_callbacks1, true, false, overload_state); Network::MockTcpListenerCallbacks listener_callbacks2; Network::TestTcpListenerImpl listenerDst(dispatcherImpl(), random_generator, runtime, socketDst, - listener_callbacks2, false, false); + listener_callbacks2, false, false, overload_state); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -140,8 +146,11 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks listener_callbacks; NiceMock listener_config; - Network::ListenerPtr listener = dispatcher_->createListener( - socket, listener_callbacks, scoped_runtime.loader(), listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + Network::ListenerPtr listener = std::make_unique( + *dispatcher_, api_->randomGenerator(), scoped_runtime.loader(), socket, listener_callbacks, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); std::vector client_connections; std::vector server_connections; @@ -210,8 +219,11 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitListenerOptOut) { Network::MockTcpListenerCallbacks listener_callbacks; NiceMock listener_config; EXPECT_CALL(listener_config, ignoreGlobalConnLimit()).WillOnce(Return(true)); - Network::ListenerPtr listener = dispatcher_->createListener( - socket, listener_callbacks, scoped_runtime.loader(), listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + Network::ListenerPtr listener = std::make_unique( + *dispatcher_, api_->randomGenerator(), scoped_runtime.loader(), socket, listener_callbacks, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); std::vector client_connections; std::vector server_connections; @@ -256,9 +268,10 @@ TEST_P(TcpListenerImplTest, WildcardListenerUseActualDst) { Network::MockTcpListenerCallbacks listener_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; // Do not redirect since use_original_dst is false. Network::TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); auto local_dst_address = Network::Utility::getAddressWithPort( *Network::Test::getCanonicalLoopbackAddress(version_), @@ -301,10 +314,10 @@ TEST_P(TcpListenerImplTest, WildcardListenerIpv4Compat) { NiceMock runtime; ASSERT_TRUE(socket->connectionInfoProvider().localAddress()->ip()->isAnyAddress()); - + Server::ThreadLocalOverloadStateOptRef overload_state; // Do not redirect since use_original_dst is false. Network::TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); auto listener_address = Network::Utility::getAddressWithPort( *Network::Test::getCanonicalLoopbackAddress(version_), @@ -345,8 +358,9 @@ TEST_P(TcpListenerImplTest, DisableAndEnableListener) { MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); // When listener is disabled, the timer should fire before any connection is accepted. listener.disable(); @@ -388,8 +402,9 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionZero) { MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); listener.setRejectFraction(UnitFloat(0)); @@ -421,8 +436,9 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionIntermediate) { MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); listener.setRejectFraction(UnitFloat(0.5f)); @@ -488,8 +504,9 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionAll) { MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); listener.setRejectFraction(UnitFloat(1)); @@ -522,8 +539,9 @@ TEST_P(TcpListenerImplTest, LoadShedPointCanRejectConnection) { MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); Server::MockOverloadManager overload_manager; Server::MockLoadShedPoint accept_connection_point; @@ -563,8 +581,9 @@ TEST_P(TcpListenerImplTest, EachQueuedConnectionShouldQueryTheLoadShedPoint) { MockConnectionCallbacks connection_callbacks2; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, - listener_callbacks, true, false); + listener_callbacks, true, false, overload_state); Server::MockOverloadManager overload_manager; Server::MockLoadShedPoint accept_connection_point; @@ -627,10 +646,11 @@ TEST_P(TcpListenerImplTest, ShouldOnlyAcceptTheMaxNumberOfConnectionsConfiguredP MockTcpListenerCallbacks listener_callbacks; Random::MockRandomGenerator random_generator; NiceMock runtime; + Server::ThreadLocalOverloadStateOptRef overload_state; const uint32_t max_connections_to_accept_per_socket_event = 1; TestTcpListenerImpl listener(dispatcherImpl(), random_generator, runtime, socket, listener_callbacks, true, false, - max_connections_to_accept_per_socket_event); + max_connections_to_accept_per_socket_event, overload_state); // Create two client connections, they should get accepted. MockConnectionCallbacks connection_callbacks1; diff --git a/test/common/network/transport_socket_options_impl_test.cc b/test/common/network/transport_socket_options_impl_test.cc index bdf54a5198e8..122ba78f9217 100644 --- a/test/common/network/transport_socket_options_impl_test.cc +++ b/test/common/network/transport_socket_options_impl_test.cc @@ -6,6 +6,7 @@ #include "source/common/network/proxy_protocol_filter_state.h" #include "source/common/network/transport_socket_options_impl.h" #include "source/common/network/upstream_server_name.h" +#include "source/common/network/upstream_subject_alt_names.h" #include "source/common/stream_info/filter_state_impl.h" #include "gtest/gtest.h" @@ -18,6 +19,16 @@ class TransportSocketOptionsImplTest : public testing::Test { public: TransportSocketOptionsImplTest() : filter_state_(StreamInfo::FilterState::LifeSpan::FilterChain) {} + void setFilterStateObject(const std::string& key, const std::string& value) { + auto* factory = + Registry::FactoryRegistry::getFactory(key); + ASSERT_NE(nullptr, factory); + EXPECT_EQ(key, factory->name()); + auto object = factory->createFromBytes(value); + ASSERT_NE(nullptr, object); + filter_state_.setData(key, std::move(object), StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::FilterChain); + } protected: StreamInfo::FilterStateImpl filter_state_; @@ -128,6 +139,19 @@ TEST_F(TransportSocketOptionsImplTest, FilterStateNonHashable) { EXPECT_EQ(keys.size(), 0); } +TEST_F(TransportSocketOptionsImplTest, DynamicObjects) { + setFilterStateObject(UpstreamServerName::key(), "www.example.com"); + setFilterStateObject(ApplicationProtocols::key(), "h2,http/1.1"); + setFilterStateObject(UpstreamSubjectAltNames::key(), "www.example.com,example.com"); + auto transport_socket_options = TransportSocketOptionsUtility::fromFilterState(filter_state_); + EXPECT_EQ(absl::make_optional("www.example.com"), + transport_socket_options->serverNameOverride()); + std::vector http_alpns{"h2", "http/1.1"}; + EXPECT_EQ(http_alpns, transport_socket_options->applicationProtocolListOverride()); + std::vector sans{"www.example.com", "example.com"}; + EXPECT_EQ(sans, transport_socket_options->verifySubjectAltNameListOverride()); +} + } // namespace } // namespace Network } // namespace Envoy diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 17674b976046..18810ca7467a 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -607,14 +607,9 @@ TEST_P(UdpListenerImplTest, UdpGroBasic) { })) .WillRepeatedly(Return(Api::SysCallSizeResult{-1, EAGAIN})); - EXPECT_CALL(listener_callbacks_, onReadReady()); + EXPECT_CALL(listener_callbacks_, onReadReady()).WillOnce(Invoke([&]() { dispatcher_->exit(); })); EXPECT_CALL(listener_callbacks_, onData(_)) - .WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateRecvCallbackParams(data, client_data.size()); - - const std::string data_str = data.buffer_->toString(); - EXPECT_EQ(data_str, client_data[num_packets_received_by_listener_ - 1]); - })) + .Times(4u) .WillRepeatedly(Invoke([&](const UdpRecvData& data) -> void { validateRecvCallbackParams(data, client_data.size()); @@ -624,7 +619,6 @@ TEST_P(UdpListenerImplTest, UdpGroBasic) { EXPECT_CALL(listener_callbacks_, onWriteReady(_)).WillOnce(Invoke([&](const Socket& socket) { EXPECT_EQ(&socket.ioHandle(), &server_socket_->ioHandle()); - dispatcher_->exit(); })); dispatcher_->run(Event::Dispatcher::RunType::Block); diff --git a/test/common/network/utility_test.cc b/test/common/network/utility_test.cc index 527b0b39eb87..f4e3e3886d79 100644 --- a/test/common/network/utility_test.cc +++ b/test/common/network/utility_test.cc @@ -113,6 +113,16 @@ TEST(NetworkUtility, resolveUrl) { EXPECT_EQ("[a:b:c:d::]:0", Utility::resolveUrl("udp://[a:b:c:d::]:0")->asString()); } +TEST(NetworkUtility, urlFromDatagramAddress) { + // UDP and unix URLs should be reversible with resolveUrl and urlFromDatagramAddress. + std::vector urls{ + "udp://[::1]:1", "udp://[a:b:c:d::]:0", "udp://1.2.3.4:1234", "unix://foo", "unix://", + }; + for (const std::string& url : urls) { + EXPECT_EQ(url, Utility::urlFromDatagramAddress(*Utility::resolveUrl(url))); + } +} + TEST(NetworkUtility, socketTypeFromUrl) { EXPECT_FALSE(Utility::socketTypeFromUrl("foo").ok()); EXPECT_FALSE(Utility::socketTypeFromUrl("abc://foo").ok()); @@ -683,9 +693,7 @@ TEST(PortRangeListTest, Errors) { } } -static Address::Ipv4Instance makeFromPort(uint32_t port) { - return Address::Ipv4Instance("0.0.0.0", port); -} +static Address::Ipv4Instance makeFromPort(uint32_t port) { return {"0.0.0.0", port}; } TEST(PortRangeListTest, Normal) { { diff --git a/test/common/protobuf/BUILD b/test/common/protobuf/BUILD index 5870810eeff2..befefa24b657 100644 --- a/test/common/protobuf/BUILD +++ b/test/common/protobuf/BUILD @@ -1,5 +1,7 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_benchmark_test", + "envoy_cc_benchmark_binary", "envoy_cc_fuzz_test", "envoy_cc_test", "envoy_package", @@ -21,6 +23,20 @@ envoy_cc_test( ], ) +envoy_proto_library( + name = "deterministic_hash_test_proto", + srcs = ["deterministic_hash_test.proto"], +) + +envoy_cc_test( + name = "deterministic_hash_test", + srcs = ["deterministic_hash_test.cc"], + deps = [ + ":deterministic_hash_test_proto_cc_proto", + "//source/common/protobuf:deterministic_hash_lib", + ], +) + envoy_proto_library( name = "utility_test_protos", srcs = [ @@ -29,8 +45,8 @@ envoy_proto_library( "utility_test_message_field_wip.proto", ], deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/annotations/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/annotations/v3:pkg", ], ) @@ -67,8 +83,8 @@ envoy_cc_test( name = "proto_descriptor_test", srcs = ["proto_descriptor_test.cc"], deps = [ - "//bazel/cc_proto_descriptor_library:create_dynamic_message", - "//bazel/cc_proto_descriptor_library:text_format_transcoder", + "@envoy_api//bazel/cc_proto_descriptor_library:create_dynamic_message", + "@envoy_api//bazel/cc_proto_descriptor_library:text_format_transcoder", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto_descriptor", ], @@ -82,3 +98,19 @@ envoy_cc_fuzz_test( tags = ["no_fuzz"], deps = ["//source/common/protobuf:utility_lib"], ) + +envoy_cc_benchmark_binary( + name = "utility_speed_test", + srcs = ["utility_speed_test.cc"], + external_deps = ["benchmark"], + deps = [ + ":deterministic_hash_test_proto_cc_proto", + "//source/common/protobuf:utility_lib", + "//test/test_common:test_runtime_lib", + ], +) + +envoy_benchmark_test( + name = "utility_speed_test_benchmark_test", + benchmark_binary = "utility_speed_test", +) diff --git a/test/common/protobuf/deterministic_hash_test.cc b/test/common/protobuf/deterministic_hash_test.cc new file mode 100644 index 000000000000..f5ebf94a1806 --- /dev/null +++ b/test/common/protobuf/deterministic_hash_test.cc @@ -0,0 +1,503 @@ +#include "source/common/protobuf/deterministic_hash.h" + +#include "test/common/protobuf/deterministic_hash_test.pb.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace DeterministicProtoHash { + +TEST(HashTest, EmptyMessageHashDiffersByMessageType) { + deterministichashtest::Maps empty1; + deterministichashtest::SingleFields empty2; + EXPECT_NE(hash(empty1), hash(empty2)); +} + +TEST(HashTest, EmptyMessageHashMatches) { + deterministichashtest::Maps empty1, empty2; + EXPECT_EQ(hash(empty1), hash(empty2)); +} + +TEST(HashTest, MapWithRemovedValueBehavesTheSameAsEmptyMap) { + // This test is from an abundance of caution, to make sure that + // reflection can't be driven to traverse a map field with no values + // in it (there would be an ASSERT in that path.) + deterministichashtest::Maps empty1, empty2; + (*empty1.mutable_bool_string())[false] = "false"; + (*empty1.mutable_bool_string()).erase(false); + EXPECT_EQ(hash(empty1), hash(empty2)); +} + +TEST(HashTest, BoolKeyedMapIsInsertionOrderAgnostic) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_bool_string())[false] = "false"; + (*map1.mutable_bool_string())[true] = "true"; + (*map2.mutable_bool_string())[true] = "true"; + (*map2.mutable_bool_string())[false] = "false"; + EXPECT_EQ(hash(map1), hash(map2)); +} + +TEST(HashTest, StringKeyedMapIsInsertionOrderAgnostic) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_string_bool())["false"] = false; + (*map1.mutable_string_bool())["true"] = true; + (*map2.mutable_string_bool())["true"] = true; + (*map2.mutable_string_bool())["false"] = false; + EXPECT_EQ(hash(map1), hash(map2)); +} + +TEST(HashTest, Int32KeyedMapIsInsertionOrderAgnostic) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_int32_uint32())[-5] = 5; + (*map1.mutable_int32_uint32())[-8] = 8; + (*map2.mutable_int32_uint32())[-8] = 8; + (*map2.mutable_int32_uint32())[-5] = 5; + EXPECT_EQ(hash(map1), hash(map2)); +} + +TEST(HashTest, UInt32KeyedMapIsInsertionOrderAgnostic) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_uint32_int32())[5] = -5; + (*map1.mutable_uint32_int32())[8] = -8; + (*map2.mutable_uint32_int32())[8] = -8; + (*map2.mutable_uint32_int32())[5] = -5; + EXPECT_EQ(hash(map1), hash(map2)); +} + +TEST(HashTest, Int64KeyedMapIsInsertionOrderAgnostic) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_int64_uint64())[-5] = 5; + (*map1.mutable_int64_uint64())[-8] = 8; + (*map2.mutable_int64_uint64())[-8] = 8; + (*map2.mutable_int64_uint64())[-5] = 5; + EXPECT_EQ(hash(map1), hash(map2)); +} + +TEST(HashTest, UInt64KeyedMapIsInsertionOrderAgnostic) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_uint64_int64())[5] = -5; + (*map1.mutable_uint64_int64())[8] = -8; + (*map2.mutable_uint64_int64())[8] = -8; + (*map2.mutable_uint64_int64())[5] = -5; + EXPECT_EQ(hash(map1), hash(map2)); +} + +TEST(HashTest, MapWithSameKeysAndValuesPairedDifferentlyDoesNotMatch) { + deterministichashtest::Maps map1, map2; + (*map1.mutable_string_bool())["false"] = false; + (*map1.mutable_string_bool())["true"] = true; + (*map2.mutable_string_bool())["true"] = false; + (*map2.mutable_string_bool())["false"] = true; + EXPECT_NE(hash(map1), hash(map2)); +} + +TEST(HashTest, RecursiveMessageMatchesWhenSame) { + deterministichashtest::Recursion r1, r2; + r1.set_index(0); + r1.mutable_child()->set_index(1); + r2.set_index(0); + r2.mutable_child()->set_index(1); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RecursiveMessageDoesNotMatchWhenSameValuesInDifferentOrder) { + deterministichashtest::Recursion r1, r2; + r1.set_index(0); + r1.mutable_child()->set_index(1); + r2.set_index(1); + r2.mutable_child()->set_index(0); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RecursiveMessageDoesNotMatchWithDifferentDepth) { + deterministichashtest::Recursion r1, r2; + r1.set_index(0); + r1.mutable_child()->set_index(1); + r2.set_index(0); + r2.mutable_child()->set_index(1); + r2.mutable_child()->mutable_child(); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, MatchingRepeatedBoolsMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_bools(false); + r1.add_bools(true); + r2.add_bools(false); + r2.add_bools(true); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedBoolsDifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_bools(false); + r1.add_bools(true); + r2.add_bools(true); + r2.add_bools(false); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedBoolsDifferentLengthMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_bools(false); + r2.add_bools(false); + r2.add_bools(false); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedStringsMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_strings("foo"); + r1.add_strings("bar"); + r2.add_strings("foo"); + r2.add_strings("bar"); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedStringsDifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_strings("foo"); + r1.add_strings("bar"); + r2.add_strings("bar"); + r2.add_strings("foo"); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedBytesMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_byteses("\x01\x02\x03\x04"); + r1.add_byteses("\x01\x02\x03\x04\x05"); + r2.add_byteses("\x01\x02\x03\x04"); + r2.add_byteses("\x01\x02\x03\x04\x05"); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedBytesDifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_byteses("\x01\x02\x03\x04"); + r1.add_byteses("\x01\x02\x03\x04\x05"); + r2.add_byteses("\x01\x02\x03\x04\x05"); + r2.add_byteses("\x01\x02\x03\x04"); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedInt32Match) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_int32s(5); + r1.add_int32s(8); + r2.add_int32s(5); + r2.add_int32s(8); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedInt32DifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_int32s(5); + r1.add_int32s(8); + r2.add_int32s(8); + r2.add_int32s(5); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedUInt32Match) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_uint32s(5); + r1.add_uint32s(8); + r2.add_uint32s(5); + r2.add_uint32s(8); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedUInt32DifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_uint32s(5); + r1.add_uint32s(8); + r2.add_uint32s(8); + r2.add_uint32s(5); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedInt64Match) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_int64s(5); + r1.add_int64s(8); + r2.add_int64s(5); + r2.add_int64s(8); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedInt64DifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_int64s(5); + r1.add_int64s(8); + r2.add_int64s(8); + r2.add_int64s(5); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedUInt64Match) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_uint64s(5); + r1.add_uint64s(8); + r2.add_uint64s(5); + r2.add_uint64s(8); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedUInt64DifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_uint64s(5); + r1.add_uint64s(8); + r2.add_uint64s(8); + r2.add_uint64s(5); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedEnumMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_enums(deterministichashtest::FOO); + r1.add_enums(deterministichashtest::BAR); + r2.add_enums(deterministichashtest::FOO); + r2.add_enums(deterministichashtest::BAR); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedEnumDifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_enums(deterministichashtest::FOO); + r1.add_enums(deterministichashtest::BAR); + r2.add_enums(deterministichashtest::BAR); + r2.add_enums(deterministichashtest::FOO); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedDoubleMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_doubles(1.84); + r1.add_doubles(-4.88); + r2.add_doubles(1.84); + r2.add_doubles(-4.88); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedDoubleDifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_doubles(1.84); + r1.add_doubles(-4.88); + r2.add_doubles(-4.88); + r2.add_doubles(1.84); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedFloatMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_floats(1.84f); + r1.add_floats(-4.88f); + r2.add_floats(1.84f); + r2.add_floats(-4.88f); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedFloatDifferentOrderMismatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_floats(1.84f); + r1.add_floats(-4.88f); + r2.add_floats(-4.88f); + r2.add_floats(1.84f); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedMessageMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_messages()->set_index(1); + r1.add_messages()->set_index(2); + r2.add_messages()->set_index(1); + r2.add_messages()->set_index(2); + EXPECT_EQ(hash(r1), hash(r2)); +} + +TEST(HashTest, RepeatedMessageOrderMisMatch) { + deterministichashtest::RepeatedFields r1, r2; + r1.add_messages()->set_index(1); + r1.add_messages()->set_index(2); + r2.add_messages()->set_index(2); + r2.add_messages()->set_index(1); + EXPECT_NE(hash(r1), hash(r2)); +} + +TEST(HashTest, SingleInt32Match) { + deterministichashtest::SingleFields s1, s2; + s1.set_int32(5); + s2.set_int32(5); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleInt32Mismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_int32(5); + s2.set_int32(8); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleUInt32Match) { + deterministichashtest::SingleFields s1, s2; + s1.set_uint32(5); + s2.set_uint32(5); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleUInt32Mismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_uint32(5); + s2.set_uint32(8); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleInt64Match) { + deterministichashtest::SingleFields s1, s2; + s1.set_int64(5); + s2.set_int64(5); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleInt64Mismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_int64(5); + s2.set_int64(8); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleUInt64Match) { + deterministichashtest::SingleFields s1, s2; + s1.set_uint64(5); + s2.set_uint64(5); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleUInt64Mismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_uint64(5); + s2.set_uint64(8); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleBoolMatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_b(true); + s2.set_b(true); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleBoolMismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_b(true); + s2.set_b(false); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleStringMatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_string("true"); + s2.set_string("true"); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleStringMismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_string("true"); + s2.set_string("false"); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleBytesMatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_bytes("\x01\x02"); + s2.set_bytes("\x01\x02"); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleBytesMismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_bytes("\x01\x02"); + s2.set_bytes("\x01\x02\x03"); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleDoubleMatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_db(3.9); + s2.set_db(3.9); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleDoubleMismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_db(3.9); + s2.set_db(3.91); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleFloatMatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_f(3.9f); + s2.set_f(3.9f); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleFloatMismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_f(3.9f); + s2.set_f(3.91f); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleEnumMatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_e(deterministichashtest::FOO); + s2.set_e(deterministichashtest::FOO); + EXPECT_EQ(hash(s1), hash(s2)); +} + +TEST(HashTest, SingleEnumMismatch) { + deterministichashtest::SingleFields s1, s2; + s1.set_e(deterministichashtest::FOO); + s2.set_e(deterministichashtest::BAR); + EXPECT_NE(hash(s1), hash(s2)); +} + +TEST(HashTest, AnyWithUnknownTypeMatch) { + deterministichashtest::AnyContainer a1, a2; + a1.mutable_any()->set_type_url("invalid_type"); + a2.mutable_any()->set_type_url("invalid_type"); + EXPECT_EQ(hash(a1), hash(a2)); +} + +TEST(HashTest, AnyWithUnknownTypeMismatch) { + deterministichashtest::AnyContainer a1, a2; + a1.mutable_any()->set_type_url("invalid_type"); + a2.mutable_any()->set_type_url("different_invalid_type"); + EXPECT_NE(hash(a1), hash(a2)); +} + +TEST(HashTest, AnyWithKnownTypeMatch) { + deterministichashtest::AnyContainer a1, a2; + deterministichashtest::Recursion value; + value.set_index(1); + a1.mutable_any()->PackFrom(value); + a2.mutable_any()->PackFrom(value); + EXPECT_EQ(hash(a1), hash(a2)); +} + +TEST(HashTest, AnyWithKnownTypeMismatch) { + deterministichashtest::AnyContainer a1, a2; + deterministichashtest::Recursion value; + value.set_index(1); + a1.mutable_any()->PackFrom(value); + value.set_index(2); + a2.mutable_any()->PackFrom(value); + EXPECT_NE(hash(a1), hash(a2)); +} + +} // namespace DeterministicProtoHash +} // namespace Envoy diff --git a/test/common/protobuf/deterministic_hash_test.proto b/test/common/protobuf/deterministic_hash_test.proto new file mode 100644 index 000000000000..5f56d35d41b0 --- /dev/null +++ b/test/common/protobuf/deterministic_hash_test.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; + +package deterministichashtest; + +import "google/protobuf/any.proto"; + +enum FooEnum { + ZERO = 0; + FOO = 1; + BAR = 2; +} + +message Maps { + map bool_string = 1; + map string_bool = 2; + map int32_uint32 = 3; + map uint32_int32 = 4; + map int64_uint64 = 5; + map uint64_int64 = 6; +}; + +message Recursion { + Recursion child = 1; + uint32 index = 2; +}; + +message RepeatedFields { + repeated bool bools = 1; + repeated string strings = 2; + repeated int32 int32s = 3; + repeated uint32 uint32s = 4; + repeated int64 int64s = 5; + repeated uint64 uint64s = 6; + repeated bytes byteses = 7; + repeated double doubles = 8; + repeated float floats = 9; + repeated FooEnum enums = 10; + repeated Recursion messages = 11; +}; + +message SingleFields { + bool b = 1; + string string = 2; + int32 int32 = 3; + uint32 uint32 = 4; + int64 int64 = 5; + uint64 uint64 = 6; + bytes bytes = 7; + double db = 8; + float f = 9; + FooEnum e = 10; +}; + +message AnyContainer { + google.protobuf.Any any = 1; +}; diff --git a/test/common/protobuf/utility_speed_test.cc b/test/common/protobuf/utility_speed_test.cc new file mode 100644 index 000000000000..491d2f71ed5b --- /dev/null +++ b/test/common/protobuf/utility_speed_test.cc @@ -0,0 +1,85 @@ +// Note: this should be run with --compilation_mode=opt, and would benefit from a +// quiescent system with disabled cstate power management. + +#include "source/common/protobuf/utility.h" + +#include "test/common/protobuf/deterministic_hash_test.pb.h" +#include "test/test_common/test_runtime.h" + +#include "benchmark/benchmark.h" + +namespace Envoy { + +// NOLINT(namespace-envoy) + +static std::unique_ptr testProtoWithMaps() { + auto msg = std::make_unique(); + (*msg->mutable_bool_string())[false] = "abcdefghijklmnopqrstuvwxyz"; + (*msg->mutable_bool_string())[true] = "abcdefghijklmnopqrstuvwxyz"; + for (int i = 0; i < 100; i++) { + (*msg->mutable_string_bool())[absl::StrCat(i)] = true; + (*msg->mutable_int32_uint32())[i] = i; + (*msg->mutable_uint32_int32())[i] = i; + (*msg->mutable_uint64_int64())[i + 1000000000000L] = i + 1000000000000L; + (*msg->mutable_int64_uint64())[i + 1000000000000L] = i + 1000000000000L; + } + return msg; +} + +static std::unique_ptr testProtoWithRecursion() { + auto msg = std::make_unique(); + deterministichashtest::Recursion* p = msg.get(); + for (int i = 0; i < 100; i++) { + p->set_index(i); + p = p->mutable_child(); + } + return msg; +} + +static std::unique_ptr testProtoWithRepeatedFields() { + auto msg = std::make_unique(); + for (int i = 0; i < 100; i++) { + msg->add_bools(true); + msg->add_strings("abcdefghijklmnopqrstuvwxyz"); + msg->add_int32s(-12345); + msg->add_uint32s(12345); + msg->add_int64s(123456789012345L); + msg->add_uint64s(-123456789012345UL); + msg->add_byteses("abcdefghijklmnopqrstuvwxyz"); + msg->add_doubles(123456789.12345); + msg->add_floats(12345.12345); + msg->add_enums(deterministichashtest::FOO); + } + return msg; +} + +static void bmHashByTextFormat(benchmark::State& state, std::unique_ptr msg) { + TestScopedRuntime runtime; + runtime.mergeValues({{"envoy.restart_features.use_fast_protobuf_hash", "false"}}); + uint64_t hash = 0; + for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + hash += MessageUtil::hash(*msg); + } + benchmark::DoNotOptimize(hash); +} + +static void bmHashByDeterministicHash(benchmark::State& state, + std::unique_ptr msg) { + TestScopedRuntime runtime; + runtime.mergeValues({{"envoy.restart_features.use_fast_protobuf_hash", "true"}}); + uint64_t hash = 0; + for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + hash += MessageUtil::hash(*msg); + } + benchmark::DoNotOptimize(hash); +} +BENCHMARK_CAPTURE(bmHashByDeterministicHash, map, testProtoWithMaps()); +BENCHMARK_CAPTURE(bmHashByTextFormat, map, testProtoWithMaps()); +BENCHMARK_CAPTURE(bmHashByDeterministicHash, recursion, testProtoWithRecursion()); +BENCHMARK_CAPTURE(bmHashByTextFormat, recursion, testProtoWithRecursion()); +BENCHMARK_CAPTURE(bmHashByDeterministicHash, repeatedFields, testProtoWithRepeatedFields()); +BENCHMARK_CAPTURE(bmHashByTextFormat, repeatedFields, testProtoWithRepeatedFields()); + +} // namespace Envoy diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 0ed33236b0e7..80743b0d1185 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -175,6 +175,12 @@ TEST_F(ProtobufUtilityTest, MessageUtilHash) { ProtobufWkt::Struct s; (*s.mutable_fields())["ab"].set_string_value("fgh"); (*s.mutable_fields())["cde"].set_string_value("ij"); + ProtobufWkt::Struct s2; + (*s2.mutable_fields())["ab"].set_string_value("ij"); + (*s2.mutable_fields())["cde"].set_string_value("fgh"); + ProtobufWkt::Struct s3; + (*s3.mutable_fields())["ac"].set_string_value("fgh"); + (*s3.mutable_fields())["cdb"].set_string_value("ij"); ProtobufWkt::Any a1; a1.PackFrom(s); @@ -184,11 +190,26 @@ TEST_F(ProtobufUtilityTest, MessageUtilHash) { a2.set_value(Base64::decode("CgsKA2NkZRIEGgJpagoLCgJhYhIFGgNmZ2g=")); ProtobufWkt::Any a3 = a1; a3.set_value(Base64::decode("CgsKAmFiEgUaA2ZnaAoLCgNjZGUSBBoCaWo=")); - - EXPECT_EQ(MessageUtil::hash(a1), MessageUtil::hash(a2)); - EXPECT_EQ(MessageUtil::hash(a2), MessageUtil::hash(a3)); - EXPECT_NE(0, MessageUtil::hash(a1)); - EXPECT_NE(MessageUtil::hash(s), MessageUtil::hash(a1)); + ProtobufWkt::Any a4, a5; + a4.PackFrom(s2); + a5.PackFrom(s3); + + TestScopedRuntime runtime_; + for (std::string runtime_value : {"true", "false"}) { + // TODO(ravenblack): when the runtime flag is removed, keep the expects + // but remove the loop around them and the extra output. + runtime_.mergeValues({{"envoy.restart_features.use_fast_protobuf_hash", runtime_value}}); + EXPECT_EQ(MessageUtil::hash(a1), MessageUtil::hash(a2)) << runtime_value; + EXPECT_EQ(MessageUtil::hash(a2), MessageUtil::hash(a3)) << runtime_value; + EXPECT_NE(0, MessageUtil::hash(a1)) << runtime_value; + // Same keys and values but with the values in a different order should not have + // the same hash. + EXPECT_NE(MessageUtil::hash(a1), MessageUtil::hash(a4)) << runtime_value; + // Different keys with the values in the same order should not have the same hash. + EXPECT_NE(MessageUtil::hash(a1), MessageUtil::hash(a5)) << runtime_value; + // Struct without 'any' around it should not hash the same as struct inside 'any'. + EXPECT_NE(MessageUtil::hash(s), MessageUtil::hash(a1)) << runtime_value; + } } TEST_F(ProtobufUtilityTest, RepeatedPtrUtilDebugString) { @@ -1195,19 +1216,6 @@ TEST_F(ProtobufUtilityTest, SanitizeUTF8) { EXPECT_EQ(absl::string_view("valid_prefix!!valid_middle!valid_suffix"), sanitized); EXPECT_EQ(sanitized.length(), original.length()); } - - { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.service_sanitize_non_utf8_strings", "false"}}); - std::string original("valid_prefix"); - original.append(1, char(0xc3)); - original.append(1, char(0xc7)); - original.append("valid_suffix"); - - std::string non_sanitized = MessageUtil::sanitizeUtf8String(original); - EXPECT_EQ(non_sanitized, original); - } } TEST_F(ProtobufUtilityTest, KeyValueStruct) { @@ -1695,29 +1703,29 @@ TEST(DurationUtilTest, OutOfRange) { { ProtobufWkt::Duration duration; duration.set_seconds(-1); - EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); + EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), EnvoyException); } { ProtobufWkt::Duration duration; duration.set_nanos(-1); - EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); + EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), EnvoyException); } { ProtobufWkt::Duration duration; duration.set_nanos(1000000000); - EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); + EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), EnvoyException); } { ProtobufWkt::Duration duration; duration.set_seconds(Protobuf::util::TimeUtil::kDurationMaxSeconds + 1); - EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); + EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), EnvoyException); } { ProtobufWkt::Duration duration; constexpr int64_t kMaxInt64Nanoseconds = std::numeric_limits::max() / (1000 * 1000 * 1000); duration.set_seconds(kMaxInt64Nanoseconds + 1); - EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), DurationUtil::OutOfRangeException); + EXPECT_THROW(DurationUtil::durationToMilliseconds(duration), EnvoyException); } } @@ -1882,7 +1890,7 @@ TEST_F(DeprecatedFieldsTest, IndividualFieldDeprecatedEmitsCrash) { {"envoy.features.fail_on_any_deprecated_feature", "true"}, }); EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'"); EXPECT_EQ(0, runtime_deprecated_feature_use_.value()); EXPECT_EQ(0, deprecated_feature_seen_since_process_start_.value()); @@ -1893,7 +1901,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(IndividualFieldDisallowed)) envoy::test::deprecation_test::Base base; base.set_is_deprecated_fatal("foo"); EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); } @@ -1904,7 +1912,7 @@ TEST_F(DeprecatedFieldsTest, // Make sure this is set up right. EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); // The config will be rejected, so the feature will not be used. @@ -1927,7 +1935,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(IndividualFieldDisallowedWi // Make sure this is set up right. EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); // The config will be rejected, so the feature will not be used. @@ -1955,7 +1963,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(DisallowViaRuntime)) { {{"envoy.deprecated_features:envoy.test.deprecation_test.Base.is_deprecated", " false"}}); EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'"); // Verify that even when the enable_all_deprecated_features is enabled the @@ -1963,7 +1971,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(DisallowViaRuntime)) { mergeValues({{"envoy.features.enable_all_deprecated_features", "true"}}); EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'"); } @@ -1977,7 +1985,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(MixOfFatalAndWarnings)) { EXPECT_LOG_CONTAINS( "warning", "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'", { EXPECT_THROW_WITH_REGEX( - checkForDeprecation(base), Envoy::ProtobufMessage::DeprecatedProtoFieldException, + checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); }); } @@ -2069,16 +2077,14 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(RuntimeOverrideEnumDefault) {{"envoy.deprecated_features:envoy.test.deprecation_test.Base.DEPRECATED_DEFAULT", "false"}}); // Make sure this is set up right. - EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), - Envoy::ProtobufMessage::DeprecatedProtoFieldException, + EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), Envoy::EnvoyException, "Using the default now-deprecated value DEPRECATED_DEFAULT"); // Verify that even when the enable_all_deprecated_features is enabled the // enum is disallowed. mergeValues({{"envoy.features.enable_all_deprecated_features", "true"}}); - EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), - Envoy::ProtobufMessage::DeprecatedProtoFieldException, + EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), Envoy::EnvoyException, "Using the default now-deprecated value DEPRECATED_DEFAULT"); } @@ -2087,8 +2093,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(FatalEnum)) { envoy::test::deprecation_test::Base base; base.mutable_enum_container()->set_deprecated_enum( envoy::test::deprecation_test::Base::DEPRECATED_FATAL); - EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), - Envoy::ProtobufMessage::DeprecatedProtoFieldException, + EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated value DEPRECATED_FATAL"); mergeValues( @@ -2108,8 +2113,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(FatalEnumGlobalOverride)) { envoy::test::deprecation_test::Base base; base.mutable_enum_container()->set_deprecated_enum( envoy::test::deprecation_test::Base::DEPRECATED_FATAL); - EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), - Envoy::ProtobufMessage::DeprecatedProtoFieldException, + EXPECT_THROW_WITH_REGEX(checkForDeprecation(base), Envoy::EnvoyException, "Using deprecated value DEPRECATED_FATAL"); mergeValues({{"envoy.features.enable_all_deprecated_features", "true"}}); diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 57d96dfbec4b..0d9444d28b12 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -127,6 +127,7 @@ envoy_cc_test( "//test/mocks/http:http_mocks", "//test/mocks/http:stream_decoder_mock", "//test/mocks/network:network_mocks", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@com_github_google_quiche//:quic_core_http_spdy_session_lib", "@com_github_google_quiche//:quic_test_tools_qpack_qpack_test_utils_lib", @@ -148,6 +149,7 @@ envoy_cc_test( "//test/mocks/http:http_mocks", "//test/mocks/http:stream_decoder_mock", "//test/mocks/network:network_mocks", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@com_github_google_quiche//:quic_core_http_spdy_session_lib", "@com_github_google_quiche//:quic_test_tools_qpack_qpack_test_utils_lib", @@ -213,11 +215,11 @@ envoy_cc_test( ":test_proof_source_lib", ":test_utils_lib", "//source/common/http:utility_lib", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:udp_packet_writer_handler_lib", "//source/common/quic:active_quic_listener_lib", "//source/common/quic:envoy_quic_utils_lib", "//source/common/quic:udp_gso_batch_writer_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//source/extensions/quic/crypto_stream:envoy_quic_crypto_server_stream_lib", "//source/extensions/quic/proof_source:envoy_quic_proof_source_factory_impl_lib", "//source/server:configuration_lib", @@ -239,12 +241,13 @@ envoy_cc_test( ":test_proof_source_lib", ":test_utils_lib", "//envoy/stats:stats_macros", + "//source/common/listener_manager:connection_handler_lib", "//source/common/quic:envoy_quic_alarm_factory_lib", "//source/common/quic:envoy_quic_connection_helper_lib", "//source/common/quic:envoy_quic_dispatcher_lib", "//source/common/quic:envoy_quic_proof_source_lib", "//source/common/quic:envoy_quic_server_session_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", + "//source/common/quic:quic_server_transport_socket_factory_lib", "//source/extensions/quic/crypto_stream:envoy_quic_crypto_server_stream_lib", "//source/server:configuration_lib", "//test/mocks/event:event_mocks", @@ -361,6 +364,7 @@ envoy_cc_test( ], tags = ["nofips"], deps = [ + "//source/common/quic:quic_server_transport_socket_factory_lib", "//source/common/quic:quic_transport_socket_factory_lib", "//source/extensions/transport_sockets/tls:context_config_lib", "//test/mocks/server:transport_socket_factory_context_mocks", @@ -395,6 +399,7 @@ envoy_cc_test_library( tags = ["nofips"], deps = [ ":envoy_quic_h3_fuzz_proto_cc_proto", + "//source/common/common:assert_lib", "@com_github_google_quiche//:quic_test_tools_test_utils_lib", ], ) diff --git a/test/common/quic/active_quic_listener_test.cc b/test/common/quic/active_quic_listener_test.cc index ca063966ebeb..edf8b6873b12 100644 --- a/test/common/quic/active_quic_listener_test.cc +++ b/test/common/quic/active_quic_listener_test.cc @@ -6,6 +6,7 @@ #include "source/common/common/logger.h" #include "source/common/http/utility.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/network/socket_option_factory.h" #include "source/common/network/udp_packet_writer_handler_impl.h" @@ -14,7 +15,6 @@ #include "source/common/quic/envoy_quic_utils.h" #include "source/common/quic/udp_gso_batch_writer.h" #include "source/common/runtime/runtime_impl.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_server_stream.h" #include "source/extensions/quic/proof_source/envoy_quic_proof_source_factory_impl.h" #include "source/server/configuration_impl.h" diff --git a/test/common/quic/client_connection_factory_impl_test.cc b/test/common/quic/client_connection_factory_impl_test.cc index e8d8f0e6e1ac..8958dcf1d901 100644 --- a/test/common/quic/client_connection_factory_impl_test.cc +++ b/test/common/quic/client_connection_factory_impl_test.cc @@ -1,7 +1,7 @@ #include #include "source/common/quic/client_connection_factory_impl.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" #include "test/common/upstream/utility.h" #include "test/mocks/common.h" diff --git a/test/common/quic/envoy_quic_client_session_test.cc b/test/common/quic/envoy_quic_client_session_test.cc index 921c0237f117..357519bef002 100644 --- a/test/common/quic/envoy_quic_client_session_test.cc +++ b/test/common/quic/envoy_quic_client_session_test.cc @@ -78,7 +78,7 @@ class EnvoyQuicClientSessionTest : public testing::TestWithParam()), envoy_quic_session_(quic_config_, quic_version_, std::unique_ptr(quic_connection_), - quic::QuicServerId("example.com", 443, false), crypto_config_, nullptr, + quic::QuicServerId("example.com", 443, false), crypto_config_, *dispatcher_, /*send_buffer_limit*/ 1024 * 1024, crypto_stream_factory_, quic_stat_names_, {}, *store_.rootScope(), transport_socket_options_), diff --git a/test/common/quic/envoy_quic_client_stream_test.cc b/test/common/quic/envoy_quic_client_stream_test.cc index ef4caa13d87e..076216ae100e 100644 --- a/test/common/quic/envoy_quic_client_stream_test.cc +++ b/test/common/quic/envoy_quic_client_stream_test.cc @@ -8,6 +8,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/http/stream_decoder.h" #include "test/mocks/network/mocks.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -247,6 +248,37 @@ TEST_F(EnvoyQuicClientStreamTest, PostRequestAndResponse) { quic_stream_->OnStreamFrame(frame); } +TEST_F(EnvoyQuicClientStreamTest, PostRequestAndResponseWithMemSliceReleasor) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.quiche_use_mem_slice_releasor_api", "true"}}); + + EXPECT_EQ(absl::nullopt, quic_stream_->http1StreamEncoderOptions()); + const auto result = quic_stream_->encodeHeaders(request_headers_, false); + EXPECT_TRUE(result.ok()); + quic_stream_->encodeData(request_body_, false); + quic_stream_->encodeTrailers(request_trailers_); + + size_t offset = receiveResponse(response_body_, false); + EXPECT_CALL(stream_decoder_, decodeTrailers_(_)) + .WillOnce(Invoke([](const Http::ResponseTrailerMapPtr& headers) { + Http::LowerCaseString key1("key1"); + Http::LowerCaseString key2(":final-offset"); + EXPECT_EQ("value1", headers->get(key1)[0]->value().getStringView()); + EXPECT_TRUE(headers->get(key2).empty()); + })); + std::string more_response_body{"bbb"}; + EXPECT_CALL(stream_decoder_, decodeData(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool finished_reading) { + EXPECT_EQ(more_response_body, buffer.toString()); + EXPECT_EQ(false, finished_reading); + })); + std::string payload = absl::StrCat(bodyToHttp3StreamPayload(more_response_body), + spdyHeaderToHttp3StreamPayload(spdy_trailers_)); + quic::QuicStreamFrame frame(stream_id_, true, offset, payload); + quic_stream_->OnStreamFrame(frame); +} + TEST_F(EnvoyQuicClientStreamTest, PostRequestAndResponseWithAccounting) { EXPECT_EQ(absl::nullopt, quic_stream_->http1StreamEncoderOptions()); EXPECT_EQ(0, quic_stream_->bytesMeter()->wireBytesSent()); @@ -317,7 +349,7 @@ TEST_F(EnvoyQuicClientStreamTest, PostRequestAnd1xx) { size_t i = 0; // Receive several 10x headers, only the first 100 Continue header should be // delivered. - for (const std::string& status : {"100", "199", "100"}) { + for (const std::string status : {"100", "199", "100"}) { spdy::Http2HeaderBlock continue_header; continue_header[":status"] = status; continue_header["i"] = absl::StrCat("", i++); diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index 1746b0c24dda..e14c36d09d9b 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -2,6 +2,7 @@ #include +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/quic/envoy_quic_alarm_factory.h" #include "source/common/quic/envoy_quic_clock.h" @@ -9,8 +10,7 @@ #include "source/common/quic/envoy_quic_dispatcher.h" #include "source/common/quic/envoy_quic_server_session.h" #include "source/common/quic/envoy_quic_utils.h" -#include "source/common/quic/quic_transport_socket_factory.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" +#include "source/common/quic/quic_server_transport_socket_factory.h" #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_server_stream.h" #include "source/server/configuration_impl.h" diff --git a/test/common/quic/envoy_quic_h3_fuzz_helper.h b/test/common/quic/envoy_quic_h3_fuzz_helper.h index 755ce18bdb40..646602cefbd5 100644 --- a/test/common/quic/envoy_quic_h3_fuzz_helper.h +++ b/test/common/quic/envoy_quic_h3_fuzz_helper.h @@ -2,6 +2,8 @@ #include +#include "source/common/common/assert.h" + #include "test/common/quic/envoy_quic_h3_fuzz.pb.h" #include "quiche/quic/core/crypto/null_decrypter.h" diff --git a/test/common/quic/envoy_quic_proof_verifier_test.cc b/test/common/quic/envoy_quic_proof_verifier_test.cc index 04957d0e2905..f6f7423c9fce 100644 --- a/test/common/quic/envoy_quic_proof_verifier_test.cc +++ b/test/common/quic/envoy_quic_proof_verifier_test.cc @@ -380,6 +380,7 @@ ZCFbredVxDBZuoVsfrKPSQa407Jj1Q== } TEST_F(EnvoyQuicProofVerifierTest, VerifySubjectAltNameListOverrideFailure) { + // NOLINTNEXTLINE(modernize-make-shared) transport_socket_options_.reset(new Network::TransportSocketOptionsImpl("", {"non-example.com"})); configCertVerificationDetails(true); std::unique_ptr cert_view = diff --git a/test/common/quic/envoy_quic_server_stream_test.cc b/test/common/quic/envoy_quic_server_stream_test.cc index 5cdb1d9bb3a8..5f976e56ba57 100644 --- a/test/common/quic/envoy_quic_server_stream_test.cc +++ b/test/common/quic/envoy_quic_server_stream_test.cc @@ -15,6 +15,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/http/stream_decoder.h" #include "test/mocks/network/mocks.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -278,6 +279,21 @@ TEST_F(EnvoyQuicServerStreamTest, PostRequestAndResponse) { EXPECT_EQ(absl::nullopt, quic_stream_->http1StreamEncoderOptions()); receiveRequest(request_body_, true, request_body_.size() * 2); quic_stream_->encodeHeaders(response_headers_, /*end_stream=*/false); + + std::string response(18 * 1024, 'a'); + Buffer::OwnedImpl buffer(response); + quic_stream_->encodeData(buffer, false); + quic_stream_->encodeTrailers(response_trailers_); +} + +TEST_F(EnvoyQuicServerStreamTest, PostRequestAndResponseWithMemSliceReleasor) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.quiche_use_mem_slice_releasor_api", "true"}}); + + EXPECT_EQ(absl::nullopt, quic_stream_->http1StreamEncoderOptions()); + receiveRequest(request_body_, true, request_body_.size() * 2); + quic_stream_->encodeHeaders(response_headers_, /*end_stream=*/false); quic_stream_->encodeTrailers(response_trailers_); } @@ -845,7 +861,7 @@ TEST_F(EnvoyQuicServerStreamTest, StatsGathererLogsOnStreamDestruction) { EXPECT_GT(quic_stream_->statsGatherer()->bytesOutstanding(), 0); // Close the stream; incoming acks will no longer invoke the stats gatherer but // the stats gatherer should log on stream close despite not receiving final downstream ack. - EXPECT_CALL(*mock_logger, log(_, _, _, _, _)); + EXPECT_CALL(*mock_logger, log(_, _)); quic_stream_->resetStream(Http::StreamResetReason::LocalRefusedStreamReset); } diff --git a/test/common/quic/envoy_quic_utils_test.cc b/test/common/quic/envoy_quic_utils_test.cc index de24bbec1de6..6950b09777a6 100644 --- a/test/common/quic/envoy_quic_utils_test.cc +++ b/test/common/quic/envoy_quic_utils_test.cc @@ -24,7 +24,7 @@ TEST(EnvoyQuicUtilsTest, ConversionBetweenQuicAddressAndEnvoyAddress) { quic::QuicSocketAddress quic_uninitialized_addr; EXPECT_EQ(nullptr, quicAddressToEnvoyAddressInstance(quic_uninitialized_addr)); - for (const std::string& ip_str : {"fd00:0:0:1::1", "1.2.3.4"}) { + for (const std::string ip_str : {"fd00:0:0:1::1", "1.2.3.4"}) { quic::QuicIpAddress quic_ip; quic_ip.FromString(ip_str); quic::QuicSocketAddress quic_addr(quic_ip, 12345); diff --git a/test/common/quic/platform/BUILD b/test/common/quic/platform/BUILD index 03b4fed648e5..3c500de5c7bf 100644 --- a/test/common/quic/platform/BUILD +++ b/test/common/quic/platform/BUILD @@ -27,7 +27,6 @@ envoy_cc_test( tags = ["nofips"], deps = [ "//source/common/memory:stats_lib", - "//source/common/quic:spdy_server_push_utils_for_envoy_lib", "//source/common/quic/platform:quiche_flags_impl_lib", "//test/common/buffer:utility_lib", "//test/common/stats:stat_test_utility_lib", diff --git a/test/common/quic/platform/quic_platform_test.cc b/test/common/quic/platform/quic_platform_test.cc index 79a642eb2b36..7d0800bd241a 100644 --- a/test/common/quic/platform/quic_platform_test.cc +++ b/test/common/quic/platform/quic_platform_test.cc @@ -110,17 +110,9 @@ TEST_F(QuicPlatformTest, QuicClientStats) { TEST_F(QuicPlatformTest, QuicExpectBug) { auto bug = [](const char* error_message) { QUIC_BUG(bug_id) << error_message; }; - auto peer_bug = [](const char* error_message) { QUIC_PEER_BUG(bug_id) << error_message; }; EXPECT_QUIC_BUG(bug("bug one is expected"), "bug one"); EXPECT_QUIC_BUG(bug("bug two is expected"), "bug two"); -#ifdef NDEBUG - // The 3rd triggering in release mode should not be logged. - EXPECT_LOG_NOT_CONTAINS("error", "bug three", bug("bug three is expected")); -#else - EXPECT_QUIC_BUG(bug("bug three is expected"), "bug three"); -#endif - EXPECT_QUIC_PEER_BUG(peer_bug("peer_bug_1 is expected"), "peer_bug_1"); EXPECT_QUIC_PEER_BUG(peer_bug("peer_bug_2 is expected"), "peer_bug_2"); } diff --git a/test/common/quic/platform/quiche_test_output_impl.cc b/test/common/quic/platform/quiche_test_output_impl.cc index 0dc9207b3a38..fa8cbecf66ab 100644 --- a/test/common/quic/platform/quiche_test_output_impl.cc +++ b/test/common/quic/platform/quiche_test_output_impl.cc @@ -95,7 +95,7 @@ bool QuicheLoadTestOutputImpl(absl::string_view filename, std::string* data) { QUICHE_LOG(ERROR) << "Test output file does not exist: " << read_path; return false; } - *data = file_system.fileReadToEnd(read_path); + *data = file_system.fileReadToEnd(read_path).value(); return true; } diff --git a/test/common/quic/quic_transport_socket_factory_test.cc b/test/common/quic/quic_transport_socket_factory_test.cc index 89aa18174ab1..fe67629bcfb9 100644 --- a/test/common/quic/quic_transport_socket_factory_test.cc +++ b/test/common/quic/quic_transport_socket_factory_test.cc @@ -1,4 +1,5 @@ -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" +#include "source/common/quic/quic_server_transport_socket_factory.h" #include "test/mocks/server/transport_socket_factory_context.h" #include "test/mocks/ssl/mocks.h" diff --git a/test/common/quic/test_utils.h b/test/common/quic/test_utils.h index ebbf5e70306f..8a5a46971f89 100644 --- a/test/common/quic/test_utils.h +++ b/test/common/quic/test_utils.h @@ -200,7 +200,7 @@ class MockEnvoyQuicClientSession : public IsolatedStoreProvider, public EnvoyQui quic::QuicServerId("example.com", 443, false), std::make_shared( quic::test::crypto_test_utils::ProofVerifierForTesting()), - nullptr, dispatcher, send_buffer_limit, crypto_stream_factory, + dispatcher, send_buffer_limit, crypto_stream_factory, quic_stat_names_, {}, *stats_store_.rootScope(), nullptr) {} void Initialize() override { diff --git a/test/common/rds/rds_test.cc b/test/common/rds/rds_test.cc index eb2b50b9722d..3417aa67a857 100644 --- a/test/common/rds/rds_test.cc +++ b/test/common/rds/rds_test.cc @@ -175,9 +175,9 @@ virtual_hosts: null auto response = TestUtility::parseYaml(response_json); const auto decoded_resources = TestUtility::decodeResources(response); - EXPECT_TRUE(server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ - ->onConfigUpdate(decoded_resources.refvec_, response.version_info()) - .ok()); + THROW_IF_NOT_OK( + server_factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate( + decoded_resources.refvec_, response.version_info())); } NiceMock outer_init_manager_; diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 982a9c35cb87..23037518759d 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -453,7 +453,6 @@ envoy_cc_test( "//source/common/config:metadata_lib", "//source/common/http:header_utility_lib", "//source/common/network:address_lib", - "//source/common/router:header_formatter_lib", "//source/common/router:header_parser_lib", "//source/common/router:string_accessor_lib", "//source/common/stream_info:filter_state_lib", diff --git a/test/common/router/config_impl_headermap_benchmark_test.cc b/test/common/router/config_impl_headermap_benchmark_test.cc index b8281af1fecf..18c2092a8a2d 100644 --- a/test/common/router/config_impl_headermap_benchmark_test.cc +++ b/test/common/router/config_impl_headermap_benchmark_test.cc @@ -56,8 +56,8 @@ static void manyCountryRoutesLongHeaders(benchmark::State& state) { Api::ApiPtr api(Api::createApiForTest()); NiceMock factory_context; ON_CALL(factory_context, api()).WillByDefault(ReturnRef(*api)); - ConfigImpl config(proto_config, OptionalHttpFilters(), factory_context, - ProtobufMessage::getNullValidationVisitor(), true); + ConfigImpl config(proto_config, factory_context, ProtobufMessage::getNullValidationVisitor(), + true); const auto stream_info = NiceMock(); auto req_headers = Http::TestRequestHeaderMapImpl{ diff --git a/test/common/router/config_impl_speed_test.cc b/test/common/router/config_impl_speed_test.cc index 8ef1fadf2eed..f7b492215e43 100644 --- a/test/common/router/config_impl_speed_test.cc +++ b/test/common/router/config_impl_speed_test.cc @@ -93,7 +93,7 @@ static void bmRouteTableSize(benchmark::State& state, RouteMatch::PathSpecifierC ON_CALL(factory_context, api()).WillByDefault(ReturnRef(*api)); // Create router config. - ConfigImpl config(genRouteConfig(state, match_type), OptionalHttpFilters(), factory_context, + ConfigImpl config(genRouteConfig(state, match_type), factory_context, ProtobufMessage::getNullValidationVisitor(), true); for (auto _ : state) { // NOLINT diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 549700fd2615..810481e02f17 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -63,10 +63,9 @@ class TestConfigImpl : public ConfigImpl { public: TestConfigImpl(const envoy::config::route::v3::RouteConfiguration& config, Server::Configuration::ServerFactoryContext& factory_context, - bool validate_clusters_default, - const OptionalHttpFilters& optional_http_filters = OptionalHttpFilters()) - : ConfigImpl(config, optional_http_filters, factory_context, - ProtobufMessage::getNullValidationVisitor(), validate_clusters_default), + bool validate_clusters_default) + : ConfigImpl(config, factory_context, ProtobufMessage::getNullValidationVisitor(), + validate_clusters_default), config_(config) {} void setupRouteConfig(const Http::RequestHeaderMap& headers, uint64_t random_value) const { @@ -139,8 +138,8 @@ genHeaders(const std::string& host, const std::string& path, const std::string& } if (random_value_pair.has_value()) { - hdrs.setByKey(Envoy::Http::LowerCaseString(random_value_pair.value().first), - random_value_pair.value().second); + hdrs.setCopy(Envoy::Http::LowerCaseString(random_value_pair.value().first), + random_value_pair.value().second); } return hdrs; } @@ -2205,7 +2204,7 @@ most_specific_header_mutations_wins: true // Validate that we can't add :-prefixed or Host request headers. TEST_F(RouteMatcherTest, TestRequestHeadersToAddNoHostOrPseudoHeader) { - for (const std::string& header : + for (const std::string header : {":path", ":authority", ":method", ":scheme", ":status", ":protocol", "host"}) { const std::string yaml = fmt::format(R"EOF( virtual_hosts: @@ -2231,7 +2230,7 @@ TEST_F(RouteMatcherTest, TestRequestHeadersToAddNoHostOrPseudoHeader) { // Validate that we can't remove :-prefixed request headers. TEST_F(RouteMatcherTest, TestRequestHeadersToRemoveNoPseudoHeader) { - for (const std::string& header : + for (const std::string header : {":path", ":authority", ":method", ":scheme", ":status", ":protocol", "host"}) { const std::string yaml = fmt::format(R"EOF( virtual_hosts: @@ -2704,6 +2703,20 @@ TEST_F(RouteMatcherTest, QueryParamMatchedRouting) { config.route(headers, 0)->routeEntry()->clusterName()); } + { // Repeated parameter - match should only track the first, and match + Http::TestRequestHeaderMapImpl headers = + genHeaders("example.com", "/?debug3=foo&debug3=bar", "GET"); + EXPECT_EQ("local_service_with_string_match_query_parameter", + config.route(headers, 0)->routeEntry()->clusterName()); + } + + { // Repeated parameter - match should only track the first, and not match + Http::TestRequestHeaderMapImpl headers = + genHeaders("example.com", "/?debug3=bar&debug3=foo", "GET"); + EXPECT_EQ("local_service_without_query_parameters", + config.route(headers, 0)->routeEntry()->clusterName()); + } + { Http::TestRequestHeaderMapImpl headers = genHeaders("example.com", "/?debug=2", "GET"); EXPECT_EQ("local_service_with_valueless_query_parameter", @@ -3191,10 +3204,19 @@ TEST_F(RouterMatcherHashPolicyTest, HashQueryParameters) { nullptr)); } { - Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/foo?param=xyz", "GET"); - Router::RouteConstSharedPtr route = config().route(headers, 0); - EXPECT_TRUE(route->routeEntry()->hashPolicy()->generateHash(nullptr, headers, add_cookie_nop_, - nullptr)); + Http::TestRequestHeaderMapImpl headers1 = genHeaders("www.lyft.com", "/foo?param=xyz", "GET"); + Router::RouteConstSharedPtr route1 = config().route(headers1, 0); + auto val1 = route1->routeEntry()->hashPolicy()->generateHash(nullptr, headers1, add_cookie_nop_, + nullptr); + EXPECT_TRUE(val1); + + // Only the first appearance of the query parameter should be considered + Http::TestRequestHeaderMapImpl headers2 = + genHeaders("www.lyft.com", "/foo?param=xyz¶m=qwer", "GET"); + Router::RouteConstSharedPtr route2 = config().route(headers2, 0); + auto val2 = route1->routeEntry()->hashPolicy()->generateHash(nullptr, headers2, add_cookie_nop_, + nullptr); + EXPECT_EQ(val1, val2); } { Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/bar?param=xyz", "GET"); @@ -3372,13 +3394,13 @@ TEST_F(RouterMatcherHashPolicyTest, HashTerminal) { EXPECT_NE(hash_1, hash_2); } -TEST_F(RouterMatcherHashPolicyTest, InvalidHashPolicies) { +// Verify that invalid enums (which are now fatal) don't pass early config +// validate checks. +TEST_F(RouterMatcherHashPolicyTest, InvalidHashPoliciesInvalid) { { - auto hash_policy = firstRouteHashPolicy(); - EXPECT_EQ(envoy::config::route::v3::RouteAction::HashPolicy::PolicySpecifierCase:: - POLICY_SPECIFIER_NOT_SET, - hash_policy->policy_specifier_case()); - EXPECT_THROW(config(), EnvoyException); + auto* hash_policy = firstRouteHashPolicy(); + EXPECT_THROW(MessageUtil::validate(*hash_policy, ProtobufMessage::getStrictValidationVisitor()), + EnvoyException); } { auto route = route_config_.mutable_virtual_hosts(0)->mutable_routes(0)->mutable_route(); @@ -3388,7 +3410,8 @@ TEST_F(RouterMatcherHashPolicyTest, InvalidHashPolicies) { EXPECT_EQ(envoy::config::route::v3::RouteAction::HashPolicy::PolicySpecifierCase:: POLICY_SPECIFIER_NOT_SET, hash_policy->policy_specifier_case()); - EXPECT_THROW(config(), EnvoyException); + EXPECT_THROW(MessageUtil::validate(*route, ProtobufMessage::getStrictValidationVisitor()), + EnvoyException); } } @@ -6670,6 +6693,8 @@ TEST(NullConfigImplTest, All) { EXPECT_FALSE(config.usesVhds()); EXPECT_FALSE(config.mostSpecificHeaderMutationsWins()); EXPECT_EQ(0Ul, config.maxDirectResponseBodySizeBytes()); + config.metadata(); + config.typedMetadata(); } class BadHttpRouteConfigurationsTest : public testing::Test, public ConfigImplTestBase {}; @@ -7547,7 +7572,7 @@ TEST_F(ConfigUtilityTest, ParseDirectResponseBody) { route.mutable_direct_response()->mutable_body()->set_filename("missing_file"); EXPECT_THROW_WITH_MESSAGE( ConfigUtility::parseDirectResponseBody(route, *api_, MaxResponseBodySizeBytes), - EnvoyException, "response body file missing_file does not exist"); + EnvoyException, "file missing_file does not exist"); // The default max body size in bytes is 4096 (MaxResponseBodySizeBytes). const std::string body(MaxResponseBodySizeBytes + 1, '*'); @@ -7560,7 +7585,7 @@ TEST_F(ConfigUtilityTest, ParseDirectResponseBody) { // Change the max body size to 2048 (MaxResponseBodySizeBytes/2) bytes. auto filename = TestEnvironment::writeStringToFileForTest("body", body); route.mutable_direct_response()->mutable_body()->set_filename(filename); - expected_message = "response body file " + filename + " size is 4097 bytes; maximum is 2048"; + expected_message = "file " + filename + " size is 4097 bytes; maximum is 2048"; EXPECT_THROW_WITH_MESSAGE( ConfigUtility::parseDirectResponseBody(route, *api_, MaxResponseBodySizeBytes / 2), EnvoyException, expected_message); @@ -7577,6 +7602,26 @@ TEST_F(ConfigUtilityTest, ParseDirectResponseBody) { ConfigUtility::parseDirectResponseBody(route, *api_, MaxResponseBodySizeBytes + 2).size()); } +TEST_F(ConfigUtilityTest, ParseDirectResponseBodyWithEnv) { + constexpr uint64_t MaxResponseBodySizeBytes = 4096; + envoy::config::route::v3::Route route; + + const std::string body(MaxResponseBodySizeBytes + 1, '*'); + TestEnvironment::setEnvVar("ResponseBodyContents", body, 1); + Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar("ResponseBodyContents"); }); + + route.mutable_direct_response()->mutable_body()->set_environment_variable("ResponseBodyContents"); + + std::string expected_message("response body size is 4097 bytes; maximum is 4096"); + + EXPECT_THROW_WITH_MESSAGE( + ConfigUtility::parseDirectResponseBody(route, *api_, MaxResponseBodySizeBytes), + EnvoyException, expected_message); + EXPECT_EQ( + MaxResponseBodySizeBytes + 1, + ConfigUtility::parseDirectResponseBody(route, *api_, MaxResponseBodySizeBytes + 2).size()); +} + TEST_F(RouteConfigurationV2, RedirectCode) { const std::string yaml = R"EOF( virtual_hosts: @@ -7674,9 +7719,11 @@ TEST_F(RouteConfigurationV2, BrokenTypedMetadata) { TEST_F(RouteConfigurationV2, RouteConfigGetters) { const std::string yaml = R"EOF( name: foo +metadata: { filter_metadata: { com.bar.foo: { baz: test_config_value }, baz: {name: config_bluh} } } virtual_hosts: - name: bar domains: ["*"] + metadata: { filter_metadata: { com.bar.foo: { baz: test_vh_value }, baz: {name: vh_bluh} } } routes: - match: safe_regex: @@ -7716,6 +7763,26 @@ name: foo EXPECT_EQ("bar", symbol_table_->toString(route_entry->virtualHost().statName())); EXPECT_EQ("foo", route_entry->virtualHost().routeConfig().name()); + + // Get metadata of virtual host. + const auto& vh_metadata = route_entry->virtualHost().metadata(); + const auto& vh_typed_metadata = route_entry->virtualHost().typedMetadata(); + + EXPECT_EQ( + "test_vh_value", + Envoy::Config::Metadata::metadataValue(&vh_metadata, "com.bar.foo", "baz").string_value()); + EXPECT_NE(nullptr, vh_typed_metadata.get(baz_factory.name())); + EXPECT_EQ("vh_bluh", vh_typed_metadata.get(baz_factory.name())->name); + + // Get metadata of route configuration. + const auto& config_metadata = route_entry->virtualHost().routeConfig().metadata(); + const auto& config_typed_metadata = route_entry->virtualHost().routeConfig().typedMetadata(); + + EXPECT_EQ("test_config_value", + Envoy::Config::Metadata::metadataValue(&config_metadata, "com.bar.foo", "baz") + .string_value()); + EXPECT_NE(nullptr, config_typed_metadata.get(baz_factory.name())); + EXPECT_EQ("config_bluh", config_typed_metadata.get(baz_factory.name())->name); } TEST_F(RouteConfigurationV2, RouteTracingConfig) { @@ -9355,7 +9422,6 @@ TEST_F(RouteMatcherTest, MixedPathPatternMatch) { } TEST_F(RouteMatcherTest, PatternMatchRewriteSimple) { - const std::string yaml = R"EOF( virtual_hosts: - name: path_pattern @@ -9388,8 +9454,42 @@ TEST_F(RouteMatcherTest, PatternMatchRewriteSimple) { EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); } -TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameUnnamed) { +TEST_F(RouteMatcherTest, PatternMatchRewriteDoubleEqual) { + const std::string yaml = R"EOF( +virtual_hosts: + - name: path_pattern + domains: ["*"] + routes: + - match: + path_match_policy: + name: envoy.path.match.uri_template.uri_template_matcher + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + path_template: "/one/two/{code=*}/{loc=*}/{curr=*}/{tri=**}" + case_sensitive: false + route: + cluster: "path-pattern-cluster-one" + path_rewrite_policy: + name: envoy.path.rewrite.uri_template.uri_template_rewriter + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + path_template_rewrite: "/{code}/{loc}/{curr}/{tri}" + )EOF"; + NiceMock stream_info; + factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); + + Http::TestRequestHeaderMapImpl headers = + genHeaders("path.prefix.com", "/one/two/en/gb/ilp==/dGasdA/?key1=test1&key2=test2", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + EXPECT_EQ("/en/gb/ilp==/dGasdA/?key1=test1&key2=test2", + route->currentUrlPathAfterRewrite(headers)); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("/en/gb/ilp==/dGasdA/?key1=test1&key2=test2", headers.get_(Http::Headers::get().Path)); + EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); +} +TEST_F(RouteMatcherTest, PatternMatchRewriteTripleEqualVariable) { const std::string yaml = R"EOF( virtual_hosts: - name: path_pattern @@ -9400,7 +9500,7 @@ TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameUnnamed) { name: envoy.path.match.uri_template.uri_template_matcher typed_config: "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig - path_template: "/rest/{one}/{two}/**.m3u8" + path_template: "/one/{two}/{code===na/*}" case_sensitive: false route: cluster: "path-pattern-cluster-one" @@ -9408,30 +9508,56 @@ TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameUnnamed) { name: envoy.path.rewrite.uri_template.uri_template_rewriter typed_config: "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig - path_template_rewrite: "/rest/{one}/{two}" + path_template_rewrite: "/{code}/{two}" )EOF"; NiceMock stream_info; factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); Http::TestRequestHeaderMapImpl headers = - genHeaders("path.prefix.com", "/rest/one/two/song.m3u8", "GET"); + genHeaders("path.prefix.com", "/one/two/==na/three", "GET"); const RouteEntry* route = config.route(headers, 0)->routeEntry(); - EXPECT_EQ("/rest/one/two", route->currentUrlPathAfterRewrite(headers)); + EXPECT_EQ("/==na/three/two", route->currentUrlPathAfterRewrite(headers)); route->finalizeRequestHeaders(headers, stream_info, true); - EXPECT_EQ("/rest/one/two", headers.get_(Http::Headers::get().Path)); + EXPECT_EQ("/==na/three/two", headers.get_(Http::Headers::get().Path)); EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); +} - Http::TestRequestHeaderMapImpl headers_multi = - genHeaders("path.prefix.com", "/rest/one/two/item/another/song.m3u8", "GET"); - const RouteEntry* route_multi = config.route(headers_multi, 0)->routeEntry(); - EXPECT_EQ("/rest/one/two", route_multi->currentUrlPathAfterRewrite(headers_multi)); - route->finalizeRequestHeaders(headers_multi, stream_info, true); - EXPECT_EQ("/rest/one/two", headers_multi.get_(Http::Headers::get().Path)); - EXPECT_EQ("path.prefix.com", headers_multi.get_(Http::Headers::get().Host)); +TEST_F(RouteMatcherTest, PatternMatchRewriteDoubleEqualVariable) { + const std::string yaml = R"EOF( +virtual_hosts: + - name: path_pattern + domains: ["*"] + routes: + - match: + path_match_policy: + name: envoy.path.match.uri_template.uri_template_matcher + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + path_template: "/one/{two}/{code==na/*}" + case_sensitive: false + route: + cluster: "path-pattern-cluster-one" + path_rewrite_policy: + name: envoy.path.rewrite.uri_template.uri_template_rewriter + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + path_template_rewrite: "/{code}/{two}" + )EOF"; + NiceMock stream_info; + factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); + + Http::TestRequestHeaderMapImpl headers = + genHeaders("path.prefix.com", "/one/two/=na/three", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + EXPECT_EQ("/=na/three/two", route->currentUrlPathAfterRewrite(headers)); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("/=na/three/two", headers.get_(Http::Headers::get().Path)); + EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); } -TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameQueryParameters) { +TEST_F(RouteMatcherTest, PatternMatchRewriteDoubleEqualInWildcard) { const std::string yaml = R"EOF( virtual_hosts: @@ -9443,7 +9569,7 @@ TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameQueryParameters) { name: envoy.path.match.uri_template.uri_template_matcher typed_config: "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig - path_template: "/api/cart/item/{one}/**.m3u8" + path_template: "/one/two/{code=*}/{loc=*}/{curr=*}/{tri=**}" case_sensitive: false route: cluster: "path-pattern-cluster-one" @@ -9451,25 +9577,66 @@ TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameQueryParameters) { name: envoy.path.rewrite.uri_template.uri_template_rewriter typed_config: "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig - path_template_rewrite: "/{one}" + path_template_rewrite: "/{code}/{loc}/{curr}/{tri}" )EOF"; NiceMock stream_info; factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); - Http::TestRequestHeaderMapImpl headers = genHeaders( - "path.prefix.com", "/api/cart/item/one/song.m3u8?one=0&two=1&three=2&four=3&go=ls", "GET"); + Http::TestRequestHeaderMapImpl headers = + genHeaders("path.prefix.com", "/one/two/en/gb/ilp/dGasdA==/?key1=test1&key2=test2", "GET"); const RouteEntry* route = config.route(headers, 0)->routeEntry(); - EXPECT_EQ("/one?one=0&two=1&three=2&four=3&go=ls", route->currentUrlPathAfterRewrite(headers)); + EXPECT_EQ("/en/gb/ilp/dGasdA==/?key1=test1&key2=test2", + route->currentUrlPathAfterRewrite(headers)); route->finalizeRequestHeaders(headers, stream_info, true); - EXPECT_EQ("/one?one=0&two=1&three=2&four=3&go=ls", headers.get_(Http::Headers::get().Path)); + EXPECT_EQ("/en/gb/ilp/dGasdA==/?key1=test1&key2=test2", headers.get_(Http::Headers::get().Path)); EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); } -TEST_F(RouteMatcherTest, - DEPRECATED_FEATURE_TEST(PatternMatchWildcardFilenameQueryParametersTruncated)) { +TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameUnnamed) { + + const std::string yaml = R"EOF( +virtual_hosts: + - name: path_pattern + domains: ["*"] + routes: + - match: + path_match_policy: + name: envoy.path.match.uri_template.uri_template_matcher + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + path_template: "/rest/{one}/{two}/**.m3u8" + case_sensitive: false + route: + cluster: "path-pattern-cluster-one" + path_rewrite_policy: + name: envoy.path.rewrite.uri_template.uri_template_rewriter + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + path_template_rewrite: "/rest/{one}/{two}" + )EOF"; + NiceMock stream_info; + factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); + + Http::TestRequestHeaderMapImpl headers = + genHeaders("path.prefix.com", "/rest/one/two/song.m3u8", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + EXPECT_EQ("/rest/one/two", route->currentUrlPathAfterRewrite(headers)); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("/rest/one/two", headers.get_(Http::Headers::get().Path)); + EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); + + Http::TestRequestHeaderMapImpl headers_multi = + genHeaders("path.prefix.com", "/rest/one/two/item/another/song.m3u8", "GET"); + const RouteEntry* route_multi = config.route(headers_multi, 0)->routeEntry(); + EXPECT_EQ("/rest/one/two", route_multi->currentUrlPathAfterRewrite(headers_multi)); + route->finalizeRequestHeaders(headers_multi, stream_info, true); + EXPECT_EQ("/rest/one/two", headers_multi.get_(Http::Headers::get().Path)); + EXPECT_EQ("path.prefix.com", headers_multi.get_(Http::Headers::get().Host)); +} - mergeValues({{"envoy.reloadable_features.append_query_parameters_path_rewriter", "false"}}); +TEST_F(RouteMatcherTest, PatternMatchWildcardFilenameQueryParameters) { const std::string yaml = R"EOF( virtual_hosts: @@ -9498,9 +9665,9 @@ TEST_F(RouteMatcherTest, Http::TestRequestHeaderMapImpl headers = genHeaders( "path.prefix.com", "/api/cart/item/one/song.m3u8?one=0&two=1&three=2&four=3&go=ls", "GET"); const RouteEntry* route = config.route(headers, 0)->routeEntry(); - EXPECT_EQ("/one", route->currentUrlPathAfterRewrite(headers)); + EXPECT_EQ("/one?one=0&two=1&three=2&four=3&go=ls", route->currentUrlPathAfterRewrite(headers)); route->finalizeRequestHeaders(headers, stream_info, true); - EXPECT_EQ("/one", headers.get_(Http::Headers::get().Path)); + EXPECT_EQ("/one?one=0&two=1&three=2&four=3&go=ls", headers.get_(Http::Headers::get().Path)); EXPECT_EQ("path.prefix.com", headers.get_(Http::Headers::get().Host)); } @@ -9763,35 +9930,6 @@ TEST_F(RouteMatcherTest, PatternMatchConfigMissingVariable) { "/rest/{one}/{two}/{missing}"); } -TEST_F(RouteMatcherTest, PatternMatchInvalidVariableName) { - const std::string yaml = R"EOF( -virtual_hosts: - - name: path_pattern - domains: ["*"] - routes: - - match: - path_match_policy: - name: envoy.path.match.uri_template.uri_template_matcher - typed_config: - "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig - path_template: "/rest/{on==e}/{two}" - case_sensitive: false - route: - cluster: "path-pattern-cluster-one" - path_rewrite_policy: - name: envoy.path.rewrite.uri_template.uri_template_rewriter - typed_config: - "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig - path_template_rewrite: "/rest/{one}/{two}/{missing}" - )EOF"; - NiceMock stream_info; - factory_context_.cluster_manager_.initializeClusters({"path-pattern-cluster-one"}, {}); - - EXPECT_THROW_WITH_MESSAGE( - TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true), - EnvoyException, "path_match_policy.path_template /rest/{on==e}/{two} is invalid"); -} - TEST_F(RouteMatcherTest, PatternMatchWildcardMiddleThreePartVariableNamed) { const std::string yaml = R"EOF( virtual_hosts: @@ -10082,6 +10220,7 @@ TEST_F(RouteConfigurationV2, DefaultInternalRedirectPolicyIsSensible) { EXPECT_EQ(1, internal_redirect_policy.maxInternalRedirects()); EXPECT_TRUE(internal_redirect_policy.predicates().empty()); EXPECT_FALSE(internal_redirect_policy.isCrossSchemeRedirectAllowed()); + EXPECT_TRUE(internal_redirect_policy.responseHeadersToCopy().empty()); } TEST_F(RouteConfigurationV2, InternalRedirectPolicyDropsInvalidRedirectCode) { @@ -10149,6 +10288,31 @@ TEST_F(RouteConfigurationV2, InternalRedirectPolicyDropsInvalidRedirectCodeCause internal_redirect_policy.shouldRedirectForResponseCode(static_cast(200))); } +TEST_F(RouteConfigurationV2, InternalRedirectPolicyAcceptsResponseHeadersToPrserve) { + const std::string yaml = R"EOF( +virtual_hosts: + - name: regex + domains: [idle.lyft.com] + routes: + - match: + safe_regex: + regex: "/regex" + route: + cluster: some-cluster + internal_redirect_policy: + response_headers_to_copy: ["x-foo", "x-bar"] + )EOF"; + + factory_context_.cluster_manager_.initializeClusters({"some-cluster"}, {}); + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); + Http::TestRequestHeaderMapImpl headers = + genRedirectHeaders("idle.lyft.com", "/regex", true, false); + const auto& internal_redirect_policy = + config.route(headers, 0)->routeEntry()->internalRedirectPolicy(); + EXPECT_TRUE(internal_redirect_policy.enabled()); + EXPECT_EQ(2, internal_redirect_policy.responseHeadersToCopy().size()); +} + class PerFilterConfigsTest : public testing::Test, public ConfigImplTestBase { public: PerFilterConfigsTest() @@ -10161,8 +10325,8 @@ class PerFilterConfigsTest : public testing::Test, public ConfigImplTestBase { public: TestFilterConfig() : EmptyHttpFilterConfig("test.filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { PANIC("not implemented"); } ProtobufTypes::MessagePtr createEmptyRouteConfigProto() override { @@ -10186,8 +10350,8 @@ class PerFilterConfigsTest : public testing::Test, public ConfigImplTestBase { public: DefaultTestFilterConfig() : EmptyHttpFilterConfig("test.default.filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { PANIC("not implemented"); } ProtobufTypes::MessagePtr createEmptyRouteConfigProto() override { @@ -10221,11 +10385,8 @@ class PerFilterConfigsTest : public testing::Test, public ConfigImplTestBase { << "config value does not match expected for source: " << source; } - void - checkNoPerFilterConfig(const std::string& yaml, const std::string& route_config_name, - const OptionalHttpFilters& optional_http_filters = OptionalHttpFilters()) { - const TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true, - optional_http_filters); + void checkNoPerFilterConfig(const std::string& yaml, const std::string& route_config_name) { + const TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); const auto route = config.route(genHeaders("www.foo.com", "/", "GET"), 0); absl::InlinedVector traveled_cfg; @@ -10287,31 +10448,6 @@ TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerVirtualHo "configurations"); } -TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAnyWithCheckPerVirtualHost) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - const std::string yaml = R"EOF( -virtual_hosts: - - name: bar - domains: ["*"] - routes: - - match: { prefix: "/" } - route: { cluster: baz } - typed_per_filter_config: - test.default.filter: - "@type": type.googleapis.com/google.protobuf.Struct - value: - seconds: 123 -)EOF"; - - OptionalHttpFilters optional_http_filters = {"test.default.filter"}; - factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); - checkNoPerFilterConfig(yaml, "test.default.filter", optional_http_filters); -} - TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerRoute) { const std::string yaml = R"EOF( virtual_hosts: @@ -10334,31 +10470,6 @@ TEST_F(PerFilterConfigsTest, DefaultFilterImplementationAnyWithCheckPerRoute) { "configurations"); } -TEST_F(PerFilterConfigsTest, OptionalDefaultFilterImplementationAnyWithCheckPerRoute) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - const std::string yaml = R"EOF( -virtual_hosts: - - name: bar - domains: ["*"] - routes: - - match: { prefix: "/" } - route: { cluster: baz } - typed_per_filter_config: - test.default.filter: - "@type": type.googleapis.com/google.protobuf.Struct - value: - seconds: 123 -)EOF"; - - OptionalHttpFilters optional_http_filters = {"test.default.filter"}; - factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); - checkNoPerFilterConfig(yaml, "test.default.filter", optional_http_filters); -} - TEST_F(PerFilterConfigsTest, PerVirtualHostWithUnknownFilter) { const std::string yaml = R"EOF( virtual_hosts: @@ -10378,30 +10489,6 @@ TEST_F(PerFilterConfigsTest, PerVirtualHostWithUnknownFilter) { "'google.protobuf.BoolValue'"); } -TEST_F(PerFilterConfigsTest, PerVirtualHostWithOptionalUnknownFilter) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - const std::string yaml = R"EOF( -virtual_hosts: - - name: bar - domains: ["*"] - routes: - - match: { prefix: "/" } - route: { cluster: baz } - typed_per_filter_config: - filter.unknown: - "@type": type.googleapis.com/google.protobuf.BoolValue -)EOF"; - - factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); - OptionalHttpFilters optional_http_filters; - optional_http_filters.insert("filter.unknown"); - checkNoPerFilterConfig(yaml, "filter.unknown", optional_http_filters); -} - TEST_F(PerFilterConfigsTest, PerRouteWithUnknownFilter) { const std::string yaml = R"EOF( virtual_hosts: @@ -10421,30 +10508,6 @@ TEST_F(PerFilterConfigsTest, PerRouteWithUnknownFilter) { "'google.protobuf.BoolValue'"); } -TEST_F(PerFilterConfigsTest, PerRouteWithHcmOptionalUnknownFilterLegacy) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - const std::string yaml = R"EOF( -virtual_hosts: - - name: bar - domains: ["*"] - routes: - - match: { prefix: "/" } - route: { cluster: baz } - typed_per_filter_config: - filter.unknown: - "@type": type.googleapis.com/google.protobuf.BoolValue -)EOF"; - - factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); - OptionalHttpFilters optional_http_filters; - optional_http_filters.insert("filter.unknown"); - checkNoPerFilterConfig(yaml, "filter.unknown", optional_http_filters); -} - TEST_F(PerFilterConfigsTest, PerRouteWithHcmOptionalUnknownFilter) { const std::string yaml = R"EOF( virtual_hosts: @@ -10459,13 +10522,9 @@ TEST_F(PerFilterConfigsTest, PerRouteWithHcmOptionalUnknownFilter) { )EOF"; factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); - OptionalHttpFilters optional_http_filters; - optional_http_filters.insert("filter.unknown"); EXPECT_THROW_WITH_MESSAGE( - TestConfigImpl(parseRouteConfigurationFromYaml(yaml), factory_context_, true, - optional_http_filters), - EnvoyException, + TestConfigImpl(parseRouteConfigurationFromYaml(yaml), factory_context_, true), EnvoyException, "Didn't find a registered implementation for 'filter.unknown' with type URL: " "'google.protobuf.BoolValue'"); } @@ -10801,19 +10860,20 @@ TEST_F(PerFilterConfigsTest, RouteFilterDisabledTest) { const TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); const auto route1 = config.route(genHeaders("host1", "/route1", "GET"), 0); - EXPECT_FALSE(route1->filterDisabled("test.filter")); + EXPECT_FALSE(route1->filterDisabled("test.filter").value()); + EXPECT_EQ(route1->filterDisabled("unknown.filter"), absl::nullopt); const auto route2 = config.route(genHeaders("host1", "/route2", "GET"), 0); - EXPECT_TRUE(route2->filterDisabled("test.filter")); + EXPECT_TRUE(route2->filterDisabled("test.filter").value()); const auto route3 = config.route(genHeaders("host2", "/route3", "GET"), 0); - EXPECT_TRUE(route3->filterDisabled("test.filter")); + EXPECT_TRUE(route3->filterDisabled("test.filter").value()); const auto route4 = config.route(genHeaders("host2", "/route4", "GET"), 0); - EXPECT_FALSE(route4->filterDisabled("test.filter")); + EXPECT_FALSE(route4->filterDisabled("test.filter").value()); const auto route5 = config.route(genHeaders("host3", "/route5", "GET"), 0); - EXPECT_TRUE(route5->filterDisabled("test.filter")); + EXPECT_TRUE(route5->filterDisabled("test.filter").value()); } class RouteMatchOverrideTest : public testing::Test, public ConfigImplTestBase {}; diff --git a/test/common/router/header_formatter_speed_test.cc b/test/common/router/header_formatter_speed_test.cc index f5e1657ab1a3..d8b7b0b5d3d9 100644 --- a/test/common/router/header_formatter_speed_test.cc +++ b/test/common/router/header_formatter_speed_test.cc @@ -40,8 +40,7 @@ static void bmEvaluateHeaders(benchmark::State& state) { HeaderParserPtr header_parser = HeaderParser::configure(headers_to_add); for (auto _ : state) { // NOLINT: Silences warning about dead store - header_parser->evaluateHeaders(*request_header, *request_header, - *Http::StaticEmptyHeaders::get().response_headers, *stream_info); + header_parser->evaluateHeaders(*request_header, {request_header.get()}, *stream_info); } } diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 49198acd68d9..dbe46e5b11a8 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -10,7 +10,6 @@ #include "source/common/config/utility.h" #include "source/common/http/header_utility.h" #include "source/common/network/address_impl.h" -#include "source/common/router/header_formatter.h" #include "source/common/router/header_parser.h" #include "source/common/router/string_accessor_impl.h" #include "source/common/stream_info/filter_state_impl.h" @@ -720,8 +719,8 @@ response_headers_to_remove: ["x-nope"] const SystemTime start_time(std::chrono::microseconds(1522796769123456)); EXPECT_CALL(stream_info, startTime()).Times(7).WillRepeatedly(Return(start_time)); - resp_header_parser->evaluateHeaders(response_header_map, request_header_map, response_header_map, - stream_info); + resp_header_parser->evaluateHeaders(response_header_map, + {&request_header_map, &response_header_map}, stream_info); EXPECT_TRUE(response_header_map.has("x-client-ip")); EXPECT_TRUE(response_header_map.has("x-client-ip-port")); EXPECT_TRUE(response_header_map.has("x-request-start-multiple")); diff --git a/test/common/router/header_parser_fuzz_test.cc b/test/common/router/header_parser_fuzz_test.cc index 59b82bc080b0..baa71273a469 100644 --- a/test/common/router/header_parser_fuzz_test.cc +++ b/test/common/router/header_parser_fuzz_test.cc @@ -19,7 +19,7 @@ DEFINE_PROTO_FUZZER(const test::common::router::TestCase& input) { MockTimeSystem time_system_; std::unique_ptr test_stream_info = fromStreamInfo(input.stream_info(), time_system_); - parser->evaluateHeaders(request_header_map, request_header_map, response_header_map, + parser->evaluateHeaders(request_header_map, {&request_header_map, &response_header_map}, *test_stream_info); ENVOY_LOG_MISC(trace, "Success"); } catch (const EnvoyException& e) { diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 3a7304dfd867..3464c5e41575 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -37,7 +37,6 @@ using testing::InSequence; using testing::Invoke; using testing::Return; using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Router { @@ -181,36 +180,6 @@ stat_prefix: foo "'google.protobuf.Struct'"); } -TEST_F(RdsImplTest, RdsAndStaticWithHcmOptionalUnknownFilterPerVirtualHostConfig) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - const std::string config_yaml = R"EOF( -route_config: - virtual_hosts: - - name: bar - domains: ["*"] - routes: - - match: { prefix: "/" } - typed_per_filter_config: - filter.unknown: - "@type": type.googleapis.com/google.protobuf.Struct - value: - seconds: 123 -codec_type: auto -stat_prefix: foo -http_filters: -- name: filter.unknown - is_optional: true - )EOF"; - - RouteConfigProviderUtil::create(parseHttpConnectionManagerFromYaml(config_yaml), - server_factory_context_, validation_visitor_, outer_init_manager_, - "foo.", *route_config_provider_manager_); -} - TEST_F(RdsImplTest, RdsAndStaticWithOptionalUnknownFilterPerVirtualHostConfig) { const std::string config_yaml = R"EOF( route_config: @@ -395,76 +364,6 @@ TEST_F(RdsImplTest, UnknownFacotryForPerVirtualHostTypedConfig) { "'google.protobuf.Struct'"); } -// validate the optional unknown factory will be ignored for per virtualhost typed config. -TEST_F(RdsImplTest, HcmOptionalUnknownFacotryForPerVirtualHostTypedConfig) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - InSequence s; - const std::string config_yaml = R"EOF( -rds: - config_source: - api_config_source: - api_type: REST - cluster_names: - - foo_cluster - refresh_delay: 1s - route_config_name: foo_route_config -codec_type: auto -stat_prefix: foo -http_filters: -- name: filter.unknown - is_optional: true - )EOF"; - - setup(config_yaml); - - const std::string response1_json = R"EOF( -{ - "version_info": "1", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "foo_route_config", - "virtual_hosts": [ - { - "name": "integration", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/foo" - }, - "route": { - "cluster_header": ":authority" - } - } - ], - "typed_per_filter_config": { - "filter.unknown": { - "@type": "type.googleapis.com/google.protobuf.Struct" - } - } - } - ] - } - ] -} -)EOF"; - auto response1 = - TestUtility::parseYaml(response1_json); - const auto decoded_resources = - TestUtility::decodeResources(response1); - - EXPECT_CALL(init_watcher_, ready()); - EXPECT_TRUE( - rds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()).ok()); -} - // Validate the optional unknown factory will be ignored for per virtualhost typed config. TEST_F(RdsImplTest, OptionalUnknownFacotryForPerVirtualHostTypedConfig) { InSequence s; @@ -588,76 +487,6 @@ TEST_F(RdsImplTest, UnknownFacotryForPerRouteTypedConfig) { "'google.protobuf.Struct'"); } -// validate the optional unknown factory will be ignored for per route typed config. -TEST_F(RdsImplTest, OptionalUnknownFacotryForPerRouteTypedConfig) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - InSequence s; - const std::string config_yaml = R"EOF( -rds: - config_source: - api_config_source: - api_type: REST - cluster_names: - - foo_cluster - refresh_delay: 1s - route_config_name: foo_route_config -codec_type: auto -stat_prefix: foo -http_filters: -- name: filter.unknown - is_optional: true - )EOF"; - - setup(config_yaml); - - const std::string response1_json = R"EOF( -{ - "version_info": "1", - "resources": [ - { - "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - "name": "foo_route_config", - "virtual_hosts": [ - { - "name": "integration", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/foo" - }, - "route": { - "cluster_header": ":authority" - }, - "typed_per_filter_config": { - "filter.unknown": { - "@type": "type.googleapis.com/google.protobuf.Struct" - } - } - } - ], - } - ] - } - ] -} -)EOF"; - auto response1 = - TestUtility::parseYaml(response1_json); - const auto decoded_resources = - TestUtility::decodeResources(response1); - - EXPECT_CALL(init_watcher_, ready()); - EXPECT_TRUE( - rds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()).ok()); -} - // Validates behavior when the config is delivered but it fails PGV validation. // The invalid config won't affect existing valid config. TEST_F(RdsImplTest, FailureInvalidConfig) { @@ -710,13 +539,10 @@ TEST_F(RdsImplTest, FailureInvalidConfig) { const auto decoded_resources_2 = TestUtility::decodeResources(response2); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE( - rds_callbacks_->onConfigUpdate(decoded_resources_2.refvec_, response2.version_info()) - .ok()), - EnvoyException, - "Unexpected RDS configuration (expecting foo_route_config): " - "INVALID_NAME_FOR_route_config"); + EXPECT_EQ(rds_callbacks_->onConfigUpdate(decoded_resources_2.refvec_, response2.version_info()) + .message(), + "Unexpected RDS configuration (expecting foo_route_config): " + "INVALID_NAME_FOR_route_config"); // Verify that the config is still the old value. ASSERT_EQ(config_impl_pointer, rds_subscription->routeConfigProvider()->config()); @@ -875,7 +701,7 @@ TEST_F(RdsRouteConfigSubscriptionTest, CreatesNoopInitManager) { TestUtility::parseYaml( rds_config); const auto route_config_provider = route_config_provider_manager_->createRdsRouteConfigProvider( - rds, OptionalHttpFilters(), server_factory_context_, "stat_prefix", outer_init_manager_); + rds, server_factory_context_, "stat_prefix", outer_init_manager_); RdsRouteConfigSubscription& subscription = (dynamic_cast(route_config_provider.get()))->subscription(); init_watcher_.expectReady(); // The parent_init_target_ will call once. @@ -904,7 +730,7 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { rds_.set_route_config_name("foo_route_config"); rds_.mutable_config_source()->set_path("foo_path"); provider_ = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, OptionalHttpFilters(), server_factory_context_, "foo_prefix.", outer_init_manager_); + rds_, server_factory_context_, "foo_prefix.", outer_init_manager_); rds_callbacks_ = server_factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -963,8 +789,8 @@ name: foo server_factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); RouteConfigProviderPtr static_config = route_config_provider_manager_->createStaticRouteConfigProvider( - parseRouteConfigurationFromV3Yaml(config_yaml), OptionalHttpFilters(), - server_factory_context_, validation_visitor_); + parseRouteConfigurationFromV3Yaml(config_yaml), server_factory_context_, + validation_visitor_); message_ptr = server_factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"]( universal_name_matcher); const auto& route_config_dump2 = @@ -1116,7 +942,7 @@ name: foo_route_config RouteConfigProviderSharedPtr provider2 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, OptionalHttpFilters(), server_factory_context_, "foo_prefix", outer_init_manager_); + rds_, server_factory_context_, "foo_prefix", outer_init_manager_); // provider2 should have route config immediately after create EXPECT_TRUE(provider2->configInfo().has_value()); @@ -1133,7 +959,7 @@ name: foo_route_config rds2.mutable_config_source()->set_path("bar_path"); RouteConfigProviderSharedPtr provider3 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds2, OptionalHttpFilters(), server_factory_context_, "foo_prefix", outer_init_manager_); + rds2, server_factory_context_, "foo_prefix", outer_init_manager_); EXPECT_NE(provider3, provider_); EXPECT_TRUE(server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ ->onConfigUpdate(decoded_resources.refvec_, "provider3") @@ -1176,8 +1002,8 @@ TEST_F(RouteConfigProviderManagerImplTest, SameProviderOnTwoInitManager) { Init::ManagerImpl real_init_manager("real"); RouteConfigProviderSharedPtr provider2 = - route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, OptionalHttpFilters(), mock_factory_context2, "foo_prefix", real_init_manager); + route_config_provider_manager_->createRdsRouteConfigProvider(rds_, mock_factory_context2, + "foo_prefix", real_init_manager); EXPECT_FALSE(provider2->configInfo().has_value()); @@ -1226,11 +1052,10 @@ TEST_F(RouteConfigProviderManagerImplTest, OnConfigUpdateWrongSize) { envoy::config::route::v3::RouteConfiguration route_config; const auto decoded_resources = TestUtility::decodeResources({route_config, route_config}); EXPECT_CALL(init_watcher_, ready()); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ - ->onConfigUpdate(decoded_resources.refvec_, "") - .ok()), - EnvoyException, "Unexpected RDS resource length: 2"); + EXPECT_EQ(server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ + ->onConfigUpdate(decoded_resources.refvec_, "") + .message(), + "Unexpected RDS resource length: 2"); } // Regression test for https://github.com/envoyproxy/envoy/issues/7939 diff --git a/test/common/router/route_fuzz_test.cc b/test/common/router/route_fuzz_test.cc index a26031f9ac55..93d545f9b58e 100644 --- a/test/common/router/route_fuzz_test.cc +++ b/test/common/router/route_fuzz_test.cc @@ -148,7 +148,7 @@ DEFINE_PROTO_FUZZER(const test::common::router::RouteTestCase& input) { TestUtility::validate(input); const auto cleaned_route_config = cleanRouteConfig(input.config()); ENVOY_LOG_MISC(debug, "cleaned route config: {}", cleaned_route_config.DebugString()); - ConfigImpl config(cleaned_route_config, OptionalHttpFilters(), factory_context, + ConfigImpl config(cleaned_route_config, factory_context, ProtobufMessage::getNullValidationVisitor(), true); auto headers = Fuzz::fromHeaders(input.headers()); auto route = config.route(headers, stream_info, input.random_value()); diff --git a/test/common/router/router_ratelimit_test.cc b/test/common/router/router_ratelimit_test.cc index 236b674966b7..65eea8fbe583 100644 --- a/test/common/router/router_ratelimit_test.cc +++ b/test/common/router/router_ratelimit_test.cc @@ -82,8 +82,8 @@ class RateLimitConfiguration : public testing::Test { void setupTest(const std::string& yaml) { envoy::config::route::v3::RouteConfiguration route_config; TestUtility::loadFromYaml(yaml, route_config); - config_ = std::make_unique(route_config, OptionalHttpFilters(), factory_context_, - any_validation_visitor_, true); + config_ = + std::make_unique(route_config, factory_context_, any_validation_visitor_, true); stream_info_.downstream_connection_info_provider_->setRemoteAddress(default_remote_address_); } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 41ac79017118..743e761ea83b 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -71,9 +71,7 @@ class TestAccessLog : public AccessLog::Instance { public: explicit TestAccessLog(std::function func) : func_(func) {} - void log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo& info, - AccessLog::AccessLogType) override { + void log(const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& info) override { func_(info); } @@ -789,6 +787,53 @@ TEST_F(RouterTest, MaintenanceMode) { EXPECT_EQ(callbacks_.details(), "maintenance_mode"); } +// DropOverload pass test +TEST_F(RouterTest, DropOverloadPassed) { + EXPECT_CALL(cm_.thread_local_cluster_, dropOverload()).WillRepeatedly(Return(UnitFloat(0.3))); + EXPECT_CALL(random_, random()) + .WillRepeatedly(Return(0.5 * float(std::numeric_limits::max()))); + + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + router_->decodeHeaders(headers, true); + EXPECT_EQ(0U, cm_.thread_local_cluster_.cluster_.info_->load_report_stats_store_ + .counter("upstream_rq_drop_overload") + .value()); + EXPECT_EQ(0U, cm_.thread_local_cluster_.cluster_.info_->load_report_stats_store_ + .counter("upstream_rq_dropped") + .value()); + router_->onDestroy(); +} + +// DropOverload drop test +TEST_F(RouterTest, DropOverloadDropped) { + EXPECT_CALL(cm_.thread_local_cluster_, dropOverload()).WillRepeatedly(Return(UnitFloat(0.3))); + EXPECT_CALL(random_, random()) + .WillRepeatedly(Return(0.2 * float(std::numeric_limits::max()))); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "503"}, + {"content-length", "13"}, + {"content-type", "text/plain"}, + {"x-envoy-drop-overload", "true"}}; + EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); + EXPECT_CALL(callbacks_, encodeData(_, true)); + EXPECT_CALL(callbacks_.stream_info_, setResponseFlag(StreamInfo::ResponseFlag::DropOverLoad)); + + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + router_->decodeHeaders(headers, true); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->load_report_stats_store_ + .counter("upstream_rq_drop_overload") + .value()); + EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); + EXPECT_EQ(0U, + callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->load_report_stats_store_ + .counter("upstream_rq_dropped") + .value()); + EXPECT_EQ(callbacks_.details(), "drop_overload"); +} + TEST_F(RouterTest, ResponseCodeDetailsSetByUpstream) { NiceMock encoder1; Http::ResponseDecoder* response_decoder = nullptr; @@ -2077,7 +2122,6 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { expectPerTryTimerCreate(); expectResponseTimerCreate(); EXPECT_EQ(0U, router_->upstreamRequests().size()); - EXPECT_TRUE(router_->finalUpstreamRequest() == nullptr); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; HttpTestUtility::addDefaultHeaders(headers); @@ -2111,7 +2155,6 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { EXPECT_EQ(2U, callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); EXPECT_EQ(2U, router_->upstreamRequests().size()); - EXPECT_TRUE(router_->finalUpstreamRequest() == nullptr); // We should not have updated any stats yet because no requests have been // canceled @@ -2137,7 +2180,6 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { response_decoder1->decodeHeaders(std::move(response_headers), true); EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); EXPECT_EQ(0U, router_->upstreamRequests().size()); - EXPECT_FALSE(router_->finalUpstreamRequest() == nullptr); // TODO: Verify hedge stats here once they are implemented. } @@ -4184,6 +4226,10 @@ TEST_F(RouterTest, InternalRedirectAcceptedWithRequestBody) { EXPECT_CALL(callbacks_, addDecodedData(_, true)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, router_->decodeData(*body_data, true)); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); + EXPECT_CALL(callbacks_.downstream_callbacks_, clearRouteCache()); EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); @@ -4233,6 +4279,9 @@ TEST_F(RouterTest, InternalRedirectRejectedByPredicate) { .WillOnce(Return(std::vector({mock_predicate}))); EXPECT_CALL(*mock_predicate, acceptTargetRoute(_, _, _, _)).WillOnce(Return(false)); ON_CALL(*mock_predicate, name()).WillByDefault(Return("mock_predicate")); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); EXPECT_CALL(callbacks_, recreateStream(_)).Times(0); response_decoder_->decodeHeaders(std::move(redirect_headers_), true); @@ -4255,6 +4304,10 @@ TEST_F(RouterTest, HttpInternalRedirectSucceeded) { EXPECT_CALL(callbacks_.downstream_callbacks_, clearRouteCache()); EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); + response_decoder_->decodeHeaders(std::move(redirect_headers_), false); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_internal_redirect_succeeded_total") @@ -4281,6 +4334,9 @@ TEST_F(RouterTest, HttpInternalRedirectMatchedToDirectResponseSucceeded) { .WillRepeatedly(Return(&direct_response)); })); EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); response_decoder_->decodeHeaders(std::move(redirect_headers_), false); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ @@ -4304,6 +4360,10 @@ TEST_F(RouterTest, InternalRedirectStripsFragment) { EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); Http::ResponseHeaderMapPtr redirect_headers{new Http::TestResponseHeaderMapImpl{ {":status", "302"}, {"location", "http://www.foo.com/#fragment"}}}; + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); + response_decoder_->decodeHeaders(std::move(redirect_headers), false); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_internal_redirect_succeeded_total") @@ -4324,6 +4384,9 @@ TEST_F(RouterTest, InternalRedirectKeepsFragmentWithOveride) { EXPECT_CALL(callbacks_.downstream_callbacks_, clearRouteCache()); EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); Http::ResponseHeaderMapPtr redirect_headers{new Http::TestResponseHeaderMapImpl{ {":status", "302"}, {"location", "http://www.foo.com/#fragment"}}}; response_decoder_->decodeHeaders(std::move(redirect_headers), false); @@ -4348,6 +4411,9 @@ TEST_F(RouterTest, HttpsInternalRedirectSucceeded) { EXPECT_CALL(connection_, ssl()).WillOnce(Return(ssl_connection)); EXPECT_CALL(callbacks_.downstream_callbacks_, clearRouteCache()); EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); response_decoder_->decodeHeaders(std::move(redirect_headers_), false); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_internal_redirect_succeeded_total") @@ -4371,6 +4437,33 @@ TEST_F(RouterTest, CrossSchemeRedirectAllowedByPolicy) { .WillOnce(Return(true)); EXPECT_CALL(callbacks_.downstream_callbacks_, clearRouteCache()); EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); + const std::vector toCopy; + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); + response_decoder_->decodeHeaders(std::move(redirect_headers_), false); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("upstream_internal_redirect_succeeded_total") + .value()); + + // In production, the HCM recreateStream would have called this. + router_->onDestroy(); +} + +TEST_F(RouterTest, ResponseHeadersTCopyCopiesHeadersOrClears) { + enableRedirects(); + default_request_headers_.setCopy(Http::LowerCaseString("x-to-clear"), "value"); + + sendRequest(); + + redirect_headers_->setCopy(Http::LowerCaseString("x-to-copy"), "bar"); + + std::vector toCopy; + toCopy.emplace_back("x-to-clear"); + toCopy.emplace_back("x-to-copy"); + EXPECT_CALL(callbacks_.route_->route_entry_.internal_redirect_policy_, responseHeadersToCopy()) + .WillOnce(ReturnRef(toCopy)); + EXPECT_CALL(callbacks_.downstream_callbacks_, clearRouteCache()); + EXPECT_CALL(callbacks_, recreateStream(_)).WillOnce(Return(true)); response_decoder_->decodeHeaders(std::move(redirect_headers_), false); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_internal_redirect_succeeded_total") @@ -4378,6 +4471,11 @@ TEST_F(RouterTest, CrossSchemeRedirectAllowedByPolicy) { // In production, the HCM recreateStream would have called this. router_->onDestroy(); + EXPECT_TRUE(default_request_headers_.get(Http::LowerCaseString("x-to-clear")).empty()); + auto copyResult = default_request_headers_.get(Http::LowerCaseString("x-to-copy")); + EXPECT_FALSE(copyResult.empty()); + EXPECT_EQ(1, copyResult.size()); + EXPECT_EQ("bar", copyResult[0]->value().getStringView()); } namespace { @@ -4710,6 +4808,53 @@ TEST_P(RouterShadowingTest, NoShadowForConnect) { router_->onDestroy(); } +// If the shadow stream watermark callbacks are invoked in the Router filter destructor, +// it causes a potential use-after-free bug, as the FilterManger may have already been freed. +TEST_P(RouterShadowingTest, ShadowCallbacksNotCalledInDestructor) { + if (!streaming_shadow_) { + GTEST_SKIP(); + } + ShadowPolicyPtr policy = makeShadowPolicy("foo", "", "bar"); + callbacks_.route_->route_entry_.shadow_policies_.push_back(policy); + ON_CALL(callbacks_, streamId()).WillByDefault(Return(43)); + + NiceMock encoder; + Http::ResponseDecoder* response_decoder = nullptr; + expectNewStreamWithImmediateEncoder(encoder, &response_decoder, Http::Protocol::Http10); + + EXPECT_CALL( + runtime_.snapshot_, + featureEnabled("bar", testing::Matcher(Percent(0)), + 43)) + .WillOnce(Return(true)); + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + NiceMock foo_client; + NiceMock foo_request(&foo_client); + EXPECT_CALL(*shadow_writer_, streamingShadow_("foo", _, _)) + .WillOnce(Invoke([&](const std::string&, Http::RequestHeaderMapPtr&, + const Http::AsyncClient::RequestOptions& options) { + EXPECT_EQ(absl::optional(10), options.timeout); + EXPECT_TRUE(options.sampled_.value()); + return &foo_request; + })); + router_->decodeHeaders(headers, false); + + Buffer::InstancePtr body_data(new Buffer::OwnedImpl("hello")); + EXPECT_CALL(callbacks_, addDecodedData(_, _)).Times(0); + EXPECT_CALL(foo_request, sendData(BufferStringEqual("hello"), false)); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, router_->decodeData(*body_data, false)); + + // Guarantee that callbacks are invoked in onDestroy instead of destructor. + { + EXPECT_CALL(foo_request, removeWatermarkCallbacks()); + EXPECT_CALL(foo_request, cancel()); + router_->onDestroy(); + } + EXPECT_CALL(foo_request, removeWatermarkCallbacks()).Times(0); + EXPECT_CALL(foo_request, cancel()).Times(0); +} + TEST_F(RouterTest, AltStatName) { // Also test no upstream timeout here. EXPECT_CALL(callbacks_.route_->route_entry_, timeout()) @@ -4908,6 +5053,30 @@ TEST_F(RouterTest, PropagatesUpstreamFilterState) { "upstream data")); } +TEST_F(RouterTest, PropagatesShadowState) { + NiceMock encoder; + Http::ResponseDecoder* response_decoder = nullptr; + ON_CALL(callbacks_.stream_info_, isShadow()).WillByDefault(Return(true)); + + // This pattern helps ensure that we're actually invoking the callback. + bool shadow_state_verified = false; + router_->config().upstream_logs_.push_back(std::make_shared( + [&](const auto& stream_info) { shadow_state_verified = stream_info.isShadow(); })); + expectResponseTimerCreate(); + expectNewStreamWithImmediateEncoder(encoder, &response_decoder, Http::Protocol::Http10); + + Http::TestRequestHeaderMapImpl headers{}; + HttpTestUtility::addDefaultHeaders(headers); + router_->decodeHeaders(headers, true); + ASSERT_THAT(response_decoder, testing::NotNull()); + + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "200"}}); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); + EXPECT_TRUE(shadow_state_verified); +} + TEST_F(RouterTest, UpstreamSSLConnection) { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; @@ -5183,8 +5352,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { NiceMock route; EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(10), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); } @@ -5192,8 +5360,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { NiceMock route; EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5204,8 +5371,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { NiceMock route; EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "bad"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(10), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5217,8 +5383,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5231,8 +5396,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(5), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5245,8 +5409,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, true, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, true, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(5), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5260,8 +5423,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { .WillRepeatedly(Return(absl::optional(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, true, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, true, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(5), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5274,8 +5436,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { route.retry_policy_.per_try_timeout_ = std::chrono::milliseconds(7); EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(7), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5288,8 +5449,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { route.retry_policy_.per_try_timeout_ = std::chrono::milliseconds(10); EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(0))); Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(0), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(10), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5303,8 +5463,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(5), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5317,8 +5476,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { EXPECT_CALL(route, maxGrpcTimeout()) .WillRepeatedly(Return(absl::optional(0))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(0), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("grpc-timeout")); @@ -5328,8 +5486,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { EXPECT_CALL(route, maxGrpcTimeout()).WillRepeatedly(Return(absl::nullopt)); EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(10), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("grpc-timeout")); @@ -5340,8 +5497,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { .WillRepeatedly(Return(absl::optional(0))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "1000m"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(1000), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_EQ("1000m", headers.get_("grpc-timeout")); @@ -5352,8 +5508,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { .WillRepeatedly(Return(absl::optional(999))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "1000m"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(999), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_EQ("999m", headers.get_("grpc-timeout")); @@ -5364,8 +5519,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { .WillRepeatedly(Return(absl::optional(999))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "0m"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(999), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_EQ("999m", headers.get_("grpc-timeout")); @@ -5378,8 +5532,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { .WillRepeatedly(Return(absl::optional(10))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "100m"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(90), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); } @@ -5391,8 +5544,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { .WillRepeatedly(Return(absl::optional(10))); Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "1m"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(1), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); } @@ -5403,8 +5555,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "1000m"}, {"x-envoy-upstream-rq-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5418,8 +5569,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "1000m"}, {"x-envoy-upstream-rq-timeout-ms", "bad"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(1000), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5434,8 +5584,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { {"grpc-timeout", "1000m"}, {"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5451,8 +5600,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { {"grpc-timeout", "1000m"}, {"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(5), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5468,8 +5616,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { Http::TestRequestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "1000m"}, {"x-envoy-upstream-rq-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(7), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5486,8 +5633,7 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { {"grpc-timeout", "1000m"}, {"x-envoy-upstream-rq-timeout-ms", "15"}, {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, true, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, true, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(5), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5511,8 +5657,8 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { Http::TestRequestHeaderMapImpl headers{{"x-envoy-expected-rq-timeout-ms", "8"}}; // Make ingress envoy respect `x-envoy-expected-rq-timeout-ms` header. bool respect_expected_rq_timeout = true; - FilterUtility::TimeoutData timeout = FilterUtility::finalTimeout( - route, headers, true, false, false, respect_expected_rq_timeout); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, + respect_expected_rq_timeout); EXPECT_EQ(std::chrono::milliseconds(8), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_EQ("8", headers.get_("x-envoy-expected-rq-timeout-ms")); @@ -5525,8 +5671,8 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { {"x-envoy-upstream-rq-per-try-timeout-ms", "4"}}; // Make ingress envoy respect `x-envoy-expected-rq-timeout-ms` header. bool respect_expected_rq_timeout = true; - FilterUtility::TimeoutData timeout = FilterUtility::finalTimeout( - route, headers, true, false, false, respect_expected_rq_timeout); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, + respect_expected_rq_timeout); EXPECT_EQ(std::chrono::milliseconds(8), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(4), timeout.per_try_timeout_); EXPECT_EQ("4", headers.get_("x-envoy-expected-rq-timeout-ms")); @@ -5539,8 +5685,8 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { // Test that ingress envoy populates `x-envoy-expected-rq-timeout-ms` header if it has not been // set by egress envoy. bool respect_expected_rq_timeout = true; - FilterUtility::TimeoutData timeout = FilterUtility::finalTimeout( - route, headers, true, false, false, respect_expected_rq_timeout); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, + respect_expected_rq_timeout); EXPECT_EQ(std::chrono::milliseconds(8), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5555,8 +5701,8 @@ TEST(RouterFilterUtilityTest, FinalTimeout) { // Make envoy override `x-envoy-expected-rq-timeout-ms` header. // Test that ingress envoy sets `x-envoy-expected-rq-timeout-ms` header. bool respect_expected_rq_timeout = false; - FilterUtility::TimeoutData timeout = FilterUtility::finalTimeout( - route, headers, true, false, false, respect_expected_rq_timeout); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, + respect_expected_rq_timeout); EXPECT_EQ(std::chrono::milliseconds(8), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -5571,8 +5717,7 @@ TEST(RouterFilterUtilityTest, FinalTimeoutSupressEnvoyHeaders) { NiceMock route; EXPECT_CALL(route, timeout()).WillOnce(Return(std::chrono::milliseconds(10))); Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-timeout-ms", "15"}}; - FilterUtility::TimeoutData timeout = - FilterUtility::finalTimeout(route, headers, true, false, false, false); + TimeoutData timeout = FilterUtility::finalTimeout(route, headers, true, false, false, false); EXPECT_EQ(std::chrono::milliseconds(15), timeout.global_timeout_); EXPECT_EQ(std::chrono::milliseconds(0), timeout.per_try_timeout_); EXPECT_FALSE(headers.has("x-envoy-upstream-rq-timeout-ms")); @@ -6185,7 +6330,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(0); @@ -6197,7 +6342,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(0); @@ -6207,7 +6352,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(150); @@ -6217,7 +6362,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(150); @@ -6227,7 +6372,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(150); @@ -6237,7 +6382,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(0); @@ -6247,7 +6392,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(150); @@ -6257,7 +6402,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(150); @@ -6268,7 +6413,7 @@ TEST(RouterFilterUtilityTest, SetTimeoutHeaders) { { NiceMock route; Http::TestRequestHeaderMapImpl headers; - FilterUtility::TimeoutData timeout; + TimeoutData timeout; timeout.global_timeout_ = std::chrono::milliseconds(200); timeout.per_try_timeout_ = std::chrono::milliseconds(150); @@ -6361,10 +6506,11 @@ TEST_F(RouterTest, RequestWithUpstreamOverrideHost) { // `LoadBalancerContext` is called, `upstreamOverrideHost` of StreamDecoderFilterCallbacks will be // called to get address of upstream host that should be selected first. EXPECT_CALL(callbacks_, upstreamOverrideHost()) - .WillOnce(Return(absl::make_optional("1.2.3.4"))); + .WillOnce(Return(absl::make_optional( + std::make_pair("1.2.3.4", false)))); auto override_host = router_->overrideHostToSelect(); - EXPECT_EQ("1.2.3.4", override_host.value()); + EXPECT_EQ("1.2.3.4", override_host.value().first); Http::TestRequestHeaderMapImpl headers{{"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}}; HttpTestUtility::addDefaultHeaders(headers); diff --git a/test/common/router/router_upstream_filter_test.cc b/test/common/router/router_upstream_filter_test.cc index c5ebb0b5c38a..9a5f98f59e76 100644 --- a/test/common/router/router_upstream_filter_test.cc +++ b/test/common/router/router_upstream_filter_test.cc @@ -77,10 +77,15 @@ class RouterUpstreamFilterTest : public testing::Test { EXPECT_CALL(callbacks_.dispatcher_, popTrackedObject(_)).Times(testing::AnyNumber()); upstream_locality_.set_zone("to_az"); - context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, address()) + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); + ON_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, + address()) .WillByDefault(Return(host_address_)); - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, locality()) + ON_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, + locality()) .WillByDefault(ReturnRef(upstream_locality_)); router_->downstream_connection_.stream_info_.downstream_connection_info_provider_ ->setLocalAddress(host_address_); @@ -92,19 +97,21 @@ class RouterUpstreamFilterTest : public testing::Test { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) - .WillOnce(Invoke([&](Http::ResponseDecoder& decoder, - Http::ConnectionPool::Callbacks& callbacks, - const Http::ConnectionPool::Instance::StreamOptions&) - -> Http::ConnectionPool::Cancellable* { - response_decoder = &decoder; - EXPECT_CALL(encoder.stream_, connectionInfoProvider()) - .WillRepeatedly(ReturnRef(connection_info1_)); - callbacks.onPoolReady(encoder, - context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, - stream_info_, Http::Protocol::Http10); - return nullptr; - })); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_, + newStream(_, _, _)) + .WillOnce( + Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, + const Http::ConnectionPool::Instance::StreamOptions&) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + EXPECT_CALL(encoder.stream_, connectionInfoProvider()) + .WillRepeatedly(ReturnRef(connection_info1_)); + callbacks.onPoolReady(encoder, + context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.conn_pool_.host_, + stream_info_, Http::Protocol::Http10); + return nullptr; + })); Http::TestRequestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); @@ -116,7 +123,8 @@ class RouterUpstreamFilterTest : public testing::Test { Http::ResponseHeaderMapPtr response_headers(new Http::TestResponseHeaderMapImpl()); response_headers->setStatus(200); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_ + .host_->outlier_detector_, putHttpResponseCode(200)); // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) response_decoder->decodeHeaders(std::move(response_headers), true); diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index 43e255f9861a..616b86c2d312 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -53,7 +53,7 @@ name: accesslog envoy::config::accesslog::v3::AccessLog upstream_log; TestUtility::loadFromYaml(yaml, upstream_log); - return absl::optional(upstream_log); + return {upstream_log}; } } // namespace @@ -104,7 +104,7 @@ class RouterUpstreamLogTest : public testing::Test { upstream_log_options->mutable_upstream_log_flush_interval()->set_seconds(1); } if (upstream_log) { - ON_CALL(*context_.access_log_manager_.file_, write(_)) + ON_CALL(*context_.server_factory_context_.access_log_manager_.file_, write(_)) .WillByDefault( Invoke([&](absl::string_view data) { output_.push_back(std::string(data)); })); @@ -124,10 +124,15 @@ class RouterUpstreamLogTest : public testing::Test { EXPECT_CALL(callbacks_.dispatcher_, popTrackedObject(_)).Times(testing::AnyNumber()); upstream_locality_.set_zone("to_az"); - context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, address()) + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); + ON_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, + address()) .WillByDefault(Return(host_address_)); - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, locality()) + ON_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, + locality()) .WillByDefault(ReturnRef(upstream_locality_)); router_->downstream_connection_.stream_info_.downstream_connection_info_provider_ ->setLocalAddress(host_address_); @@ -155,19 +160,21 @@ class RouterUpstreamLogTest : public testing::Test { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) - .WillOnce(Invoke([&](Http::ResponseDecoder& decoder, - Http::ConnectionPool::Callbacks& callbacks, - const Http::ConnectionPool::Instance::StreamOptions&) - -> Http::ConnectionPool::Cancellable* { - response_decoder = &decoder; - EXPECT_CALL(encoder.stream_, connectionInfoProvider()) - .WillRepeatedly(ReturnRef(connection_info1_)); - callbacks.onPoolReady(encoder, - context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, - stream_info_, Http::Protocol::Http10); - return nullptr; - })); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_, + newStream(_, _, _)) + .WillOnce( + Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, + const Http::ConnectionPool::Instance::StreamOptions&) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + EXPECT_CALL(encoder.stream_, connectionInfoProvider()) + .WillRepeatedly(ReturnRef(connection_info1_)); + callbacks.onPoolReady(encoder, + context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.conn_pool_.host_, + stream_info_, Http::Protocol::Http10); + return nullptr; + })); expectResponseTimerCreate(); Http::TestRequestHeaderMapImpl headers(request_headers_init); @@ -181,7 +188,8 @@ class RouterUpstreamLogTest : public testing::Test { new Http::TestResponseHeaderMapImpl(response_headers_init)); response_headers->setStatus(response_code); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_ + .host_->outlier_detector_, putHttpResponseCode(response_code)); // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) response_decoder->decodeHeaders(std::move(response_headers), false); @@ -197,19 +205,21 @@ class RouterUpstreamLogTest : public testing::Test { NiceMock encoder1; Http::ResponseDecoder* response_decoder = nullptr; - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) - .WillOnce(Invoke([&](Http::ResponseDecoder& decoder, - Http::ConnectionPool::Callbacks& callbacks, - const Http::ConnectionPool::Instance::StreamOptions&) - -> Http::ConnectionPool::Cancellable* { - response_decoder = &decoder; - EXPECT_CALL(encoder1.stream_, connectionInfoProvider()) - .WillRepeatedly(ReturnRef(connection_info1_)); - callbacks.onPoolReady(encoder1, - context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, - stream_info_, Http::Protocol::Http10); - return nullptr; - })); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_, + newStream(_, _, _)) + .WillOnce( + Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, + const Http::ConnectionPool::Instance::StreamOptions&) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + EXPECT_CALL(encoder1.stream_, connectionInfoProvider()) + .WillRepeatedly(ReturnRef(connection_info1_)); + callbacks.onPoolReady(encoder1, + context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.conn_pool_.host_, + stream_info_, Http::Protocol::Http10); + return nullptr; + })); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -220,28 +230,31 @@ class RouterUpstreamLogTest : public testing::Test { router_->decodeHeaders(headers, true); router_->retry_state_->expectResetRetry(); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_ + .host_->outlier_detector_, putResult(Upstream::Outlier::Result::LocalOriginTimeout, _)); per_try_timeout_->invokeCallback(); // We expect this reset to kick off a new request. NiceMock encoder2; - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) - .WillOnce(Invoke([&](Http::ResponseDecoder& decoder, - Http::ConnectionPool::Callbacks& callbacks, - const Http::ConnectionPool::Instance::StreamOptions&) - -> Http::ConnectionPool::Cancellable* { - response_decoder = &decoder; - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); - EXPECT_CALL(encoder2.stream_, connectionInfoProvider()) - .WillRepeatedly(ReturnRef(connection_info2_)); - callbacks.onPoolReady(encoder2, - context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, - stream_info_, Http::Protocol::Http10); - return nullptr; - })); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_, + newStream(_, _, _)) + .WillOnce( + Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, + const Http::ConnectionPool::Instance::StreamOptions&) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); + EXPECT_CALL(encoder2.stream_, connectionInfoProvider()) + .WillRepeatedly(ReturnRef(connection_info2_)); + callbacks.onPoolReady(encoder2, + context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.conn_pool_.host_, + stream_info_, Http::Protocol::Http10); + return nullptr; + })); expectPerTryTimerCreate(); router_->retry_state_->callback_(); @@ -250,7 +263,8 @@ class RouterUpstreamLogTest : public testing::Test { .WillOnce(Return(RetryStatus::No)); Http::ResponseHeaderMapPtr response_headers( new Http::TestResponseHeaderMapImpl{{":status", "200"}}); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_ + .host_->outlier_detector_, putHttpResponseCode(200)); if (response_decoder != nullptr) { response_decoder->decodeHeaders(std::move(response_headers), true); @@ -460,7 +474,8 @@ TEST_F(RouterUpstreamLogTest, PeriodicLog) { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_, + newStream(_, _, _)) .WillOnce( Invoke([&](Http::ResponseDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks, const Http::ConnectionPool::Instance::StreamOptions&) @@ -470,7 +485,8 @@ TEST_F(RouterUpstreamLogTest, PeriodicLog) { .WillRepeatedly(ReturnRef(connection_info1_)); encoder.stream_.bytes_meter_ = std::make_shared(); callbacks.onPoolReady(encoder, - context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_, + context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.conn_pool_.host_, stream_info_, Http::Protocol::Http10); return nullptr; })); @@ -490,9 +506,11 @@ TEST_F(RouterUpstreamLogTest, PeriodicLog) { encoder.stream_.bytes_meter_->addWireBytesReceived(9); EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _)); - EXPECT_CALL(*mock_upstream_log_, log(_, _, _, _, AccessLog::AccessLogType::UpstreamPeriodic)) - .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*mock_upstream_log_, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::UpstreamPeriodic); + EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10); EXPECT_THAT(stream_info.getDownstreamBytesMeter()->bytesAtLastUpstreamPeriodicLog(), @@ -507,9 +525,11 @@ TEST_F(RouterUpstreamLogTest, PeriodicLog) { encoder.stream_.bytes_meter_->addWireBytesReceived(7); EXPECT_CALL(*periodic_log_flush_, enableTimer(_, _)); - EXPECT_CALL(*mock_upstream_log_, log(_, _, _, _, AccessLog::AccessLogType::UpstreamPeriodic)) - .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*mock_upstream_log_, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::UpstreamPeriodic); + EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10 + 8); EXPECT_EQ(stream_info.getDownstreamBytesMeter() ->bytesAtLastUpstreamPeriodicLog() @@ -529,11 +549,14 @@ TEST_F(RouterUpstreamLogTest, PeriodicLog) { callbacks_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(6); encoder.stream_.bytes_meter_->addWireBytesReceived(5); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.conn_pool_.host_->outlier_detector_, + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.conn_pool_ + .host_->outlier_detector_, putHttpResponseCode(200)); - EXPECT_CALL(*mock_upstream_log_, log(_, _, _, _, AccessLog::AccessLogType::UpstreamEnd)) - .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*mock_upstream_log_, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::UpstreamEnd); + EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10 + 8 + 6); EXPECT_EQ(stream_info.getDownstreamBytesMeter() ->bytesAtLastUpstreamPeriodicLog() diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 21bda9810dc8..3be2ef5c9ab2 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -331,7 +331,7 @@ TEST_F(InlineScopedRoutesTest, ConfigLoadAndDump) { class ScopedRdsTest : public ScopedRoutesTestBase { protected: - void setup(const OptionalHttpFilters optional_http_filters = OptionalHttpFilters()) { + void setup() { ON_CALL(server_factory_context_.cluster_manager_, adsMux()) .WillByDefault(Return(std::make_shared<::Envoy::Config::NullGrpcMuxImpl>())); @@ -400,8 +400,7 @@ name: foo_scoped_routes provider_ = config_provider_manager_->createXdsConfigProvider( scoped_routes_config.scoped_rds(), server_factory_context_, context_init_manager_, "foo.", ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), - scoped_routes_config.rds_config_source(), - optional_http_filters)); + scoped_routes_config.rds_config_source())); srds_subscription_ = server_factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -570,47 +569,6 @@ route_configuration_name: foo_routes .value()); } -// Test ignoring the optional unknown factory in the per-virtualhost typed config. -TEST_F(ScopedRdsTest, OptionalUnknownFactoryForPerVirtualHostTypedConfig) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"}}); - - OptionalHttpFilters optional_http_filters; - optional_http_filters.insert("filter.unknown"); - setup(optional_http_filters); - init_watcher_.expectReady(); - const std::string config_yaml = R"EOF( -name: foo_scope -route_configuration_name: foo_routes -key: - fragments: - - string_key: x-foo-key -)EOF"; - - const auto resource = parseScopedRouteConfigurationFromYaml(config_yaml); - const auto decoded_resources = TestUtility::decodeResources({resource}); - - context_init_manager_.initialize(init_watcher_); - EXPECT_TRUE(srds_subscription_->onConfigUpdate(decoded_resources.refvec_, "1").ok()); - - constexpr absl::string_view route_config_tmpl = R"EOF( - name: {} - virtual_hosts: - - name: test - domains: ["*"] - routes: - - match: {{ prefix: "/" }} - route: {{ cluster: bluh }} - typed_per_filter_config: - filter.unknown: - "@type": type.googleapis.com/google.protobuf.Struct -)EOF"; - - pushRdsConfig({"foo_routes"}, "111", route_config_tmpl); -} - // Tests that multiple uniquely named non-conflict resources are allowed in config updates. TEST_F(ScopedRdsTest, MultipleResourcesSotw) { setup(); @@ -1814,8 +1772,7 @@ name: foo_scoped_routes provider_ = config_provider_manager_->createXdsConfigProvider( scoped_routes_config.scoped_rds(), server_factory_context_, context_init_manager_, "foo.", ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), - scoped_routes_config.rds_config_source(), - OptionalHttpFilters())); + scoped_routes_config.rds_config_source())); srds_subscription_ = server_factory_context_.cluster_manager_.subscription_factory_.callbacks_; const std::string config_yaml = R"EOF( diff --git a/test/common/router/upstream_request_test.cc b/test/common/router/upstream_request_test.cc index b658fb092ce8..67a6b003cd36 100644 --- a/test/common/router/upstream_request_test.cc +++ b/test/common/router/upstream_request_test.cc @@ -81,6 +81,34 @@ TEST_F(UpstreamRequestTest, TestAccessors) { // UpstreamRequest is responsible for adding proper gRPC annotations to spans. TEST_F(UpstreamRequestTest, DecodeHeadersGrpcSpanAnnotations) { + envoy::extensions::filters::http::router::v3::Router router_proto; + router_config_ = std::make_unique( + pool_.add("prefix"), context_, ShadowWriterPtr(new MockShadowWriter()), router_proto); + EXPECT_CALL(router_filter_interface_, config()).WillRepeatedly(ReturnRef(*router_config_)); + + // Enable tracing in config. + router_filter_interface_.callbacks_.tracing_config_.spawn_upstream_span_ = true; + + // Set expectations on span. + auto* child_span = new NiceMock(); + EXPECT_CALL(router_filter_interface_.callbacks_.active_span_, spawnChild_) + .WillOnce(Return(child_span)); + EXPECT_CALL(*child_span, setTag).Times(AnyNumber()); + EXPECT_CALL(*child_span, setTag(Eq("grpc.status_code"), Eq("1"))); + EXPECT_CALL(*child_span, setTag(Eq("grpc.message"), Eq("failure"))); + + // System under test. + initialize(); + auto upgrade_headers = + std::make_unique(Http::TestResponseHeaderMapImpl( + {{":status", "200"}, {"grpc-status", "1"}, {"grpc-message", "failure"}})); + EXPECT_CALL(router_filter_interface_, onUpstreamHeaders(_, _, _, _)); + upstream_request_->decodeHeaders(std::move(upgrade_headers), false); +} + +// UpstreamRequest is responsible for adding proper gRPC annotations to spans. +TEST_F(UpstreamRequestTest, + DEPRECATED_FEATURE_TEST(DecodeHeadersGrpcSpanAnnotationsWithStartChildSpan)) { // Enable tracing in config. envoy::extensions::filters::http::router::v3::Router router_proto; router_proto.set_start_child_span(true); diff --git a/test/common/router/vhds_test.cc b/test/common/router/vhds_test.cc index 4b72efce6331..4b2f65be16c5 100644 --- a/test/common/router/vhds_test.cc +++ b/test/common/router/vhds_test.cc @@ -77,8 +77,8 @@ name: my_route } RouteConfigUpdatePtr makeRouteConfigUpdate(const envoy::config::route::v3::RouteConfiguration& rc) { - RouteConfigUpdatePtr config_update_info = std::make_unique( - proto_traits_, factory_context_, OptionalHttpFilters()); + RouteConfigUpdatePtr config_update_info = + std::make_unique(proto_traits_, factory_context_); config_update_info->onRdsUpdate(rc, "1"); return config_update_info; } diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index 9e92060f114e..7ea7109e68e9 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -197,9 +197,10 @@ class SdsRotationApiTest : public SdsApiTestBase { api_ = Api::createApiForTest(filesystem_); setupMocks(); EXPECT_CALL(filesystem_, splitPathFromFilename(_)) - .WillRepeatedly(Invoke([](absl::string_view path) -> Filesystem::PathSplitResult { - return Filesystem::fileSystemForTest().splitPathFromFilename(path); - })); + .WillRepeatedly( + Invoke([](absl::string_view path) -> absl::StatusOr { + return Filesystem::fileSystemForTest().splitPathFromFilename(path); + })); } Secret::MockSecretCallbacks secret_callback_; @@ -805,9 +806,8 @@ TEST_F(SdsApiTest, EmptyResource) { init_manager_.add(*sds_api.initTarget()); initialize(); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(subscription_factory_.callbacks_->onConfigUpdate({}, "").ok()), EnvoyException, - "Missing SDS resources for abc.com in onConfigUpdate()"); + EXPECT_EQ(subscription_factory_.callbacks_->onConfigUpdate({}, "").message(), + "Missing SDS resources for abc.com in onConfigUpdate()"); } // Validate that SdsApi throws exception if multiple secrets are passed to onConfigUpdate(). @@ -834,10 +834,13 @@ TEST_F(SdsApiTest, SecretUpdateWrongSize) { const auto decoded_resources = TestUtility::decodeResources({typed_secret, typed_secret}); initialize(); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE( - subscription_factory_.callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, "Unexpected SDS secrets length: 2"); + EXPECT_EQ( + subscription_factory_.callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "Unexpected SDS secrets length: 2"); + Protobuf::RepeatedPtrField unused; + EXPECT_EQ(subscription_factory_.callbacks_->onConfigUpdate(decoded_resources.refvec_, unused, "") + .message(), + "Unexpected SDS secrets length: 2"); } // Validate that SdsApi throws exception if secret name passed to onConfigUpdate() @@ -865,10 +868,9 @@ TEST_F(SdsApiTest, SecretUpdateWrongSecretName) { const auto decoded_resources = TestUtility::decodeResources({typed_secret}); initialize(); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE( - subscription_factory_.callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, "Unexpected SDS secret (expecting abc.com): wrong.name.com"); + EXPECT_EQ( + subscription_factory_.callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "Unexpected SDS secret (expecting abc.com): wrong.name.com"); } } // namespace diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index b6335026065e..cbd8e21277aa 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -77,7 +77,7 @@ name: "abc.com" )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); SecretManagerPtr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_EQ(secret_manager->findStaticTlsCertificateProvider("undefined"), nullptr); ASSERT_NE(secret_manager->findStaticTlsCertificateProvider("abc.com"), nullptr); @@ -111,11 +111,11 @@ TEST_F(SecretManagerImplTest, DuplicateStaticTlsCertificateSecret) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); SecretManagerPtr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_NE(secret_manager->findStaticTlsCertificateProvider("abc.com"), nullptr); - EXPECT_THROW_WITH_MESSAGE(secret_manager->addStaticSecret(secret_config), EnvoyException, - "Duplicate static TlsCertificate secret name abc.com"); + EXPECT_EQ(secret_manager->addStaticSecret(secret_config).message(), + "Duplicate static TlsCertificate secret name abc.com"); } // Validate that secret manager adds static certificate validation context secret successfully. @@ -130,7 +130,7 @@ TEST_F(SecretManagerImplTest, CertificateValidationContextSecretLoadSuccess) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_EQ(secret_manager->findStaticCertificateValidationContextProvider("undefined"), nullptr); ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr); @@ -158,11 +158,11 @@ TEST_F(SecretManagerImplTest, DuplicateStaticCertificateValidationContextSecret) )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); SecretManagerPtr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr); - EXPECT_THROW_WITH_MESSAGE(secret_manager->addStaticSecret(secret_config), EnvoyException, - "Duplicate static CertificateValidationContext secret name abc.com"); + EXPECT_EQ(secret_manager->addStaticSecret(secret_config).message(), + "Duplicate static CertificateValidationContext secret name abc.com"); } // Validate that secret manager adds static STKs secret successfully. @@ -181,7 +181,7 @@ name: "abc.com" SecretManagerPtr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_EQ(secret_manager->findStaticTlsSessionTicketKeysContextProvider("undefined"), nullptr); ASSERT_NE(secret_manager->findStaticTlsSessionTicketKeysContextProvider("abc.com"), nullptr); @@ -210,11 +210,11 @@ name: "abc.com" SecretManagerPtr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_NE(secret_manager->findStaticTlsSessionTicketKeysContextProvider("abc.com"), nullptr); - EXPECT_THROW_WITH_MESSAGE(secret_manager->addStaticSecret(secret_config), EnvoyException, - "Duplicate static TlsSessionTicketKeys secret name abc.com"); + EXPECT_EQ(secret_manager->addStaticSecret(secret_config).message(), + "Duplicate static TlsSessionTicketKeys secret name abc.com"); } // Validate that secret manager adds static generic secret successfully. @@ -230,7 +230,7 @@ name: "encryption_key" filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/aes_128_key" )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret); - secret_manager->addStaticSecret(secret); + EXPECT_TRUE(secret_manager->addStaticSecret(secret).ok()); ASSERT_EQ(secret_manager->findStaticGenericSecretProvider("undefined"), nullptr); ASSERT_NE(secret_manager->findStaticGenericSecretProvider("encryption_key"), nullptr); @@ -255,11 +255,11 @@ name: "encryption_key" filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/aes_128_key" )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret); - secret_manager->addStaticSecret(secret); + EXPECT_TRUE(secret_manager->addStaticSecret(secret).ok()); ASSERT_NE(secret_manager->findStaticGenericSecretProvider("encryption_key"), nullptr); - EXPECT_THROW_WITH_MESSAGE(secret_manager->addStaticSecret(secret), EnvoyException, - "Duplicate static GenericSecret secret name encryption_key"); + EXPECT_EQ(secret_manager->addStaticSecret(secret).message(), + "Duplicate static GenericSecret secret name encryption_key"); } // Validate that secret manager deduplicates dynamic TLS certificate secret provider. @@ -900,7 +900,7 @@ name: "abc.com" )EOF"; envoy::extensions::transport_sockets::tls::v3::Secret tls_cert_secret; TestUtility::loadFromYaml(TestEnvironment::substitute(tls_certificate), tls_cert_secret); - secret_manager->addStaticSecret(tls_cert_secret); + EXPECT_TRUE(secret_manager->addStaticSecret(tls_cert_secret).ok()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "abc.com.nopassword" tls_certificate: @@ -910,7 +910,7 @@ name: "abc.com.nopassword" inline_string: "DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY" )EOF"), tls_cert_secret); - secret_manager->addStaticSecret(tls_cert_secret); + EXPECT_TRUE(secret_manager->addStaticSecret(tls_cert_secret).ok()); const std::string expected_config_dump = R"EOF( static_secrets: - name: "abc.com.nopassword" @@ -972,7 +972,7 @@ name: "abc.com.validation" )EOF"; envoy::extensions::transport_sockets::tls::v3::Secret validation_secret; TestUtility::loadFromYaml(TestEnvironment::substitute(validation_context), validation_secret); - secret_manager->addStaticSecret(validation_secret); + EXPECT_TRUE(secret_manager->addStaticSecret(validation_secret).ok()); const std::string expected_config_dump = R"EOF( static_secrets: - name: "abc.com.validation" @@ -1021,7 +1021,7 @@ name: "abc.com.stek" )EOF"; envoy::extensions::transport_sockets::tls::v3::Secret stek_secret; TestUtility::loadFromYaml(TestEnvironment::substitute(stek_context), stek_secret); - secret_manager->addStaticSecret(stek_secret); + EXPECT_TRUE(secret_manager->addStaticSecret(stek_secret).ok()); const std::string expected_config_dump = R"EOF( static_secrets: - name: "abc.com.stek" @@ -1051,7 +1051,7 @@ name: "signing_key" )EOF"; envoy::extensions::transport_sockets::tls::v3::Secret typed_secret; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), typed_secret); - secret_manager->addStaticSecret(typed_secret); + EXPECT_TRUE(secret_manager->addStaticSecret(typed_secret).ok()); const std::string expected_config_dump = R"EOF( static_secrets: @@ -1147,7 +1147,7 @@ TEST_F(SecretManagerImplTest, DeprecatedSanMatcher) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); - secret_manager->addStaticSecret(secret_config); + EXPECT_TRUE(secret_manager->addStaticSecret(secret_config).ok()); ASSERT_EQ(secret_manager->findStaticCertificateValidationContextProvider("undefined"), nullptr); ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr); diff --git a/test/common/singleton/manager_impl_test.cc b/test/common/singleton/manager_impl_test.cc index 9b7b75e0866c..d1a8513fbd48 100644 --- a/test/common/singleton/manager_impl_test.cc +++ b/test/common/singleton/manager_impl_test.cc @@ -14,7 +14,8 @@ namespace { static void deathTestWorker() { ManagerImpl manager(Thread::threadFactoryForTest()); - manager.get("foo", [] { return nullptr; }); + manager.get( + "foo", [] { return nullptr; }, false); } TEST(SingletonManagerImplDeathTest, NotRegistered) { @@ -41,9 +42,11 @@ TEST(SingletonManagerImplTest, Basic) { ManagerImpl manager(Thread::threadFactoryForTest()); std::shared_ptr singleton = std::make_shared(); - EXPECT_EQ(singleton, manager.get("test_singleton", [singleton] { return singleton; })); + EXPECT_EQ(singleton, manager.get( + "test_singleton", [singleton] { return singleton; }, false)); EXPECT_EQ(1UL, singleton.use_count()); - EXPECT_EQ(singleton, manager.get("test_singleton", [] { return nullptr; })); + EXPECT_EQ(singleton, manager.get( + "test_singleton", [] { return nullptr; }, false)); EXPECT_CALL(*singleton, onDestroy()); singleton.reset(); @@ -57,7 +60,8 @@ TEST(SingletonManagerImplTest, NonConstructingGetTyped) { std::shared_ptr singleton = std::make_shared(); // Use a construct on first use getter. - EXPECT_EQ(singleton, manager.get("test_singleton", [singleton] { return singleton; })); + EXPECT_EQ(singleton, manager.get( + "test_singleton", [singleton] { return singleton; }, false)); // Now access should return the constructed singleton. EXPECT_EQ(singleton, manager.getTyped("test_singleton")); EXPECT_EQ(1UL, singleton.use_count()); @@ -66,6 +70,58 @@ TEST(SingletonManagerImplTest, NonConstructingGetTyped) { singleton.reset(); } +TEST(SingletonManagerImplTest, PinnedSingleton) { + + { + ManagerImpl manager(Thread::threadFactoryForTest()); + TestSingleton* singleton_ptr{}; + + // Register a singleton and get it. + auto singleton = manager.getTyped(SINGLETON_MANAGER_REGISTERED_NAME(test), + [&]() -> InstanceSharedPtr { + auto s = std::make_shared(); + singleton_ptr = s.get(); + return s; + }); + EXPECT_EQ(singleton, manager.getTyped(SINGLETON_MANAGER_REGISTERED_NAME(test))); + + EXPECT_CALL(*singleton_ptr, onDestroy()); + // Destroy all copies of the singleton shared pointer. + singleton.reset(); + + // The singleton should be destroyed now. + EXPECT_EQ(nullptr, manager.getTyped(SINGLETON_MANAGER_REGISTERED_NAME(test))); + } + + { + ManagerImpl manager(Thread::threadFactoryForTest()); + TestSingleton* singleton_ptr{}; + + // Register a pinned singleton and get it. + auto singleton = manager.getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(test), + [&]() -> InstanceSharedPtr { + auto s = std::make_shared(); + singleton_ptr = s.get(); + return s; + }, + true); + EXPECT_EQ(singleton, manager.getTyped(SINGLETON_MANAGER_REGISTERED_NAME(test))); + + auto* expected_value = singleton.get(); + + // Destroy all copies of the singleton shared pointer. + singleton.reset(); + + // The singleton should still be available. + EXPECT_EQ(expected_value, + manager.getTyped(SINGLETON_MANAGER_REGISTERED_NAME(test)).get()); + + // Destroy the singleton after the manager is destroyed. + EXPECT_CALL(*singleton_ptr, onDestroy()); + } +} + } // namespace } // namespace Singleton } // namespace Envoy diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 92883c81c898..f6e03ec3f376 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -202,11 +202,15 @@ envoy_cc_fuzz_test( ], ) +envoy_cc_test_library( + name = "make_elements_helper_lib", + hdrs = ["make_elements_helper.h"], +) + envoy_cc_benchmark_binary( name = "symbol_table_benchmark", srcs = [ "make_elements_helper.cc", - "make_elements_helper.h", "symbol_table_speed_test.cc", ], external_deps = [ @@ -214,6 +218,7 @@ envoy_cc_benchmark_binary( "benchmark", ], deps = [ + ":make_elements_helper_lib", ":stat_test_utility_lib", "//source/common/common:hash_lib", "//source/common/memory:stats_lib", diff --git a/test/common/stats/allocator_impl_test.cc b/test/common/stats/allocator_impl_test.cc index 38fcb17466b2..c07d07d15768 100644 --- a/test/common/stats/allocator_impl_test.cc +++ b/test/common/stats/allocator_impl_test.cc @@ -23,9 +23,7 @@ class AllocatorImplTest : public testing::Test { AllocatorImplTest() : pool_(symbol_table_), alloc_(symbol_table_) {} ~AllocatorImplTest() override { clearStorage(); } - StatNameStorage makeStatStorage(absl::string_view name) { - return StatNameStorage(name, symbol_table_); - } + StatNameStorage makeStatStorage(absl::string_view name) { return {name, symbol_table_}; } StatName makeStat(absl::string_view name) { return pool_.add(name); } diff --git a/test/common/stats/isolated_store_impl_test.cc b/test/common/stats/isolated_store_impl_test.cc index a1a5b787638d..7c0d577a923c 100644 --- a/test/common/stats/isolated_store_impl_test.cc +++ b/test/common/stats/isolated_store_impl_test.cc @@ -35,6 +35,7 @@ class StatsIsolatedStoreImplTest : public testing::Test { }; TEST_F(StatsIsolatedStoreImplTest, All) { + EXPECT_TRUE(store_->fixedTags().empty()); ScopeSharedPtr scope1 = scope_->createScope("scope1."); Counter& c1 = scope_->counterFromString("c1"); Counter& c2 = scope1->counterFromString("c2"); diff --git a/test/common/stats/stat_test_utility.h b/test/common/stats/stat_test_utility.h index 5ea1edb97441..be806e0767d1 100644 --- a/test/common/stats/stat_test_utility.h +++ b/test/common/stats/stat_test_utility.h @@ -152,6 +152,9 @@ class TestStore : public SymbolTableProvider, public IsolatedStoreImpl { // Returns whether the given histogram has recorded any value since it was // created. bool histogramRecordedValues(const std::string& name) const; + const TagVector& fixedTags() override { return fixed_tags_; } + + TagVector fixed_tags_; protected: ScopeSharedPtr makeScope(StatName name) override; diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index e74b52a80607..e8c95391e308 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -287,7 +287,7 @@ TEST_F(StatNameTest, TestSameValueOnPartialFree) { StatNameStorage stat_foobar_1("foo.bar", table_); SymbolVec stat_foobar_1_symbols = getSymbols(stat_foobar_1.statName()); stat_foobar_1.free(table_); - StatName stat_foobar_2(makeStat("foo.bar")); + StatName stat_foobar_2(makeStat("foo.bar")); // NOLINT(clang-analyzer-unix.Malloc) SymbolVec stat_foobar_2_symbols = getSymbols(stat_foobar_2); EXPECT_EQ(stat_foobar_1_symbols[0], @@ -337,7 +337,7 @@ TEST_F(StatNameTest, TestShrinkingExpectation) { size_t table_size_0 = table_.numSymbols(); auto make_stat_storage = [this](absl::string_view name) -> StatNameStorage { - return StatNameStorage(name, table_); + return {name, table_}; }; StatNameStorage stat_a(make_stat_storage("a")); @@ -362,7 +362,7 @@ TEST_F(StatNameTest, TestShrinkingExpectation) { stat_ace.free(table_); EXPECT_EQ(table_size_4, table_.numSymbols()); - stat_acd.free(table_); + stat_acd.free(table_); // NOLINT(clang-analyzer-unix.Malloc) EXPECT_EQ(table_size_3, table_.numSymbols()); stat_ac.free(table_); @@ -586,7 +586,7 @@ TEST_F(StatNameTest, MutexContentionOnExistingSymbols) { accesses.DecrementCount(); wait.wait(); - })); + })); // NOLINT(clang-analyzer-unix.Malloc) } creation.setReady(); creates.Wait(); @@ -744,7 +744,7 @@ TEST(SymbolTableTest, Memory) { }; symbol_table_mem_used = test_memory_usage(record_stat); for (StatNameStorage& name : names) { - name.free(table); + name.free(table); // NOLINT(clang-analyzer-unix.Malloc) } } diff --git a/test/common/stats/symbol_table_speed_test.cc b/test/common/stats/symbol_table_speed_test.cc index e98eefa89bd7..2e6f084a07de 100644 --- a/test/common/stats/symbol_table_speed_test.cc +++ b/test/common/stats/symbol_table_speed_test.cc @@ -41,6 +41,7 @@ static void bmCreateRace(benchmark::State& state) { access.wait(); for (int count = 0; count < 1000; ++count) { + // NOLINTNEXTLINE(clang-analyzer-unix.Malloc) Envoy::Stats::StatNameStorage second(stat_name_string, table); second.free(table); } diff --git a/test/common/stats/tag_extractor_impl_test.cc b/test/common/stats/tag_extractor_impl_test.cc index 2363f8e1dab5..f75f1ccfa0cb 100644 --- a/test/common/stats/tag_extractor_impl_test.cc +++ b/test/common/stats/tag_extractor_impl_test.cc @@ -430,6 +430,20 @@ TEST(TagExtractorTest, DefaultTagExtractors) { redis_prefix.name_ = tag_names.REDIS_PREFIX; redis_prefix.value_ = "my_redis_prefix"; regex_tester.testRegex("redis.my_redis_prefix.response", "redis.response", {redis_prefix}); + + // Dns Filter Prefix + Tag dns_filter_prefix; + dns_filter_prefix.name_ = tag_names.DNS_FILTER_PREFIX; + dns_filter_prefix.value_ = "my_dns_prefix"; + regex_tester.testRegex("dns_filter.my_dns_prefix.local_a_record_answers", + "dns_filter.local_a_record_answers", {dns_filter_prefix}); + + // Connection Limit Filter Prefix + Tag connection_limit_prefix; + connection_limit_prefix.name_ = tag_names.CONNECTION_LIMIT_PREFIX; + connection_limit_prefix.value_ = "my_connection_limit_prefix"; + regex_tester.testRegex("connection_limit.my_connection_limit_prefix.limited_connections", + "connection_limit.limited_connections", {connection_limit_prefix}); } TEST(TagExtractorTest, ExtAuthzTagExtractors) { diff --git a/test/common/stats/tag_producer_impl_test.cc b/test/common/stats/tag_producer_impl_test.cc index 33eb3c50df09..de0856791298 100644 --- a/test/common/stats/tag_producer_impl_test.cc +++ b/test/common/stats/tag_producer_impl_test.cc @@ -127,5 +127,24 @@ TEST_F(TagProducerTest, Fixed) { checkTags(tag_config, tags); } +// Test that fixed tags both from cli and from stats_config are returned from `fixedTags()`. +TEST_F(TagProducerTest, FixedTags) { + const TagVector tag_config{{"my-tag", "fixed"}}; + + auto& specifier = *stats_config_.mutable_stats_tags()->Add(); + specifier.set_tag_name("tag2"); + specifier.set_fixed_value("value2"); + + // This one isn't a fixed value so it won't be included. + addSpecifier("regex", "value"); + + TagProducerImpl producer{stats_config_, tag_config}; + const auto& tags = producer.fixedTags(); + EXPECT_THAT(tags, testing::UnorderedElementsAreArray(TagVector{ + {"my-tag", "fixed"}, + {"tag2", "value2"}, + })); +} + } // namespace Stats } // namespace Envoy diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 9cd079074338..7939224a8855 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -796,6 +796,7 @@ TEST_F(StatsThreadLocalStoreTest, ExtractAndAppendTagsFixedValue) { EXPECT_EQ("b", symbol_table_.toString(tags[0].second)); EXPECT_EQ("foo", symbol_table_.toString(tags[1].first)); EXPECT_EQ("bar", symbol_table_.toString(tags[1].second)); + EXPECT_THAT(store_->fixedTags(), UnorderedElementsAre(Tag{"foo", "bar"})); } TEST_F(StatsThreadLocalStoreTest, ExtractAndAppendTagsRegexValueNoMatch) { diff --git a/test/common/stream_info/BUILD b/test/common/stream_info/BUILD index e34276643ad3..89795b60e010 100644 --- a/test/common/stream_info/BUILD +++ b/test/common/stream_info/BUILD @@ -66,6 +66,7 @@ envoy_cc_test( deps = [ "//source/common/stream_info:utility_lib", "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", ], ) diff --git a/test/common/stream_info/utility_test.cc b/test/common/stream_info/utility_test.cc index ed7f6747b4f2..9e7dabfdf72a 100644 --- a/test/common/stream_info/utility_test.cc +++ b/test/common/stream_info/utility_test.cc @@ -4,6 +4,7 @@ #include "source/common/stream_info/utility.h" #include "test/mocks/stream_info/mocks.h" +#include "test/test_common/test_runtime.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -315,11 +316,14 @@ TEST(ProxyStatusErrorToString, TestAll) { } TEST(ProxyStatusFromStreamInfo, TestAll) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "true"}}); for (const auto& [response_flag, proxy_status_error] : std::vector>{ {ResponseFlag::FailedLocalHealthCheck, ProxyStatusError::DestinationUnavailable}, {ResponseFlag::NoHealthyUpstream, ProxyStatusError::DestinationUnavailable}, - {ResponseFlag::UpstreamRequestTimeout, ProxyStatusError::ConnectionTimeout}, + {ResponseFlag::UpstreamRequestTimeout, ProxyStatusError::HttpResponseTimeout}, {ResponseFlag::LocalReset, ProxyStatusError::ConnectionTimeout}, {ResponseFlag::UpstreamRemoteReset, ProxyStatusError::ConnectionTerminated}, {ResponseFlag::UpstreamConnectionFailure, ProxyStatusError::ConnectionRefused}, @@ -343,6 +347,16 @@ TEST(ProxyStatusFromStreamInfo, TestAll) { } } +TEST(ProxyStatusFromStreamInfo, TestUpstreamRequestTimeout) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "false"}}); + NiceMock stream_info; + ON_CALL(stream_info, hasResponseFlag(ResponseFlag::UpstreamRequestTimeout)) + .WillByDefault(Return(true)); + EXPECT_THAT(ProxyStatusUtils::fromStreamInfo(stream_info), ProxyStatusError::ConnectionTimeout); +} + } // namespace } // namespace StreamInfo } // namespace Envoy diff --git a/test/common/tcp/async_tcp_client_impl_test.cc b/test/common/tcp/async_tcp_client_impl_test.cc index 57f53cb3f3b8..20b6a8d7abbd 100644 --- a/test/common/tcp/async_tcp_client_impl_test.cc +++ b/test/common/tcp/async_tcp_client_impl_test.cc @@ -20,7 +20,7 @@ namespace Tcp { class AsyncTcpClientImplTest : public Event::TestUsingSimulatedTime, public testing::Test { public: - AsyncTcpClientImplTest() {} + AsyncTcpClientImplTest() = default; void setUpClient() { cluster_manager_.initializeClusters({"fake_cluster"}, {}); @@ -34,6 +34,7 @@ class AsyncTcpClientImplTest : public Event::TestUsingSimulatedTime, public test void expectCreateConnection(bool trigger_connected = true) { connection_ = new NiceMock(); Upstream::MockHost::MockCreateConnectionData conn_info; + connection_->streamInfo().setAttemptCount(1); conn_info.connection_ = connection_; conn_info.host_description_ = Upstream::makeTestHost( @@ -155,6 +156,26 @@ TEST_F(AsyncTcpClientImplTest, TestGetDispatcher) { ASSERT_FALSE(client_->connected()); } +TEST_F(AsyncTcpClientImplTest, TestGetStreamInfo) { + setUpClient(); + expectCreateConnection(); + EXPECT_TRUE(client_->getStreamInfo().has_value()); + EXPECT_EQ(1, client_->getStreamInfo()->attemptCount()); + EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + client_->close(Network::ConnectionCloseType::NoFlush); + ASSERT_FALSE(client_->connected()); +} + +TEST_F(AsyncTcpClientImplTest, TestGetStreamInfoNullOpt) { + setUpClient(); + expectCreateConnection(); + EXPECT_TRUE(client_->getStreamInfo().has_value()); + EXPECT_CALL(callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + client_->close(Network::ConnectionCloseType::NoFlush); + ASSERT_FALSE(client_->connected()); + ASSERT_FALSE(client_->getStreamInfo().has_value()); +} + TEST_F(AsyncTcpClientImplTest, TestTimingStats) { setUpClient(); expectCreateConnection(); diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index 2d90e65f0a19..fe46c79cdeac 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -1182,7 +1182,6 @@ TEST_F(TcpConnPoolImplTest, TestIdleTimeout) { c1.releaseConn(); conn_pool_->test_conns_[0].connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); - EXPECT_CALL(idle_callback, Call()); conn_pool_->drainConnections(Envoy::ConnectionPool::DrainBehavior::DrainAndDelete); EXPECT_CALL(*conn_pool_, onConnDestroyedForTest()); dispatcher_.clearDeferredDeleteList(); diff --git a/test/common/tcp_proxy/config_test.cc b/test/common/tcp_proxy/config_test.cc index e659c3ed763f..db12396ae82e 100644 --- a/test/common/tcp_proxy/config_test.cc +++ b/test/common/tcp_proxy/config_test.cc @@ -263,10 +263,10 @@ TEST(ConfigTest, WeightedClustersConfig) { Config config_obj(constructConfigFromYaml(yaml, factory_context)); NiceMock connection; - EXPECT_CALL(factory_context.api_.random_, random()).WillOnce(Return(0)); + EXPECT_CALL(factory_context.server_factory_context_.api_.random_, random()).WillOnce(Return(0)); EXPECT_EQ(std::string("cluster1"), config_obj.getRouteFromEntries(connection)->clusterName()); - EXPECT_CALL(factory_context.api_.random_, random()).WillOnce(Return(2)); + EXPECT_CALL(factory_context.server_factory_context_.api_.random_, random()).WillOnce(Return(2)); EXPECT_EQ(std::string("cluster2"), config_obj.getRouteFromEntries(connection)->clusterName()); } @@ -303,7 +303,7 @@ TEST(ConfigTest, WeightedClustersWithMetadataMatchConfig) { HashedValue hv1(v1), hv2(v2); NiceMock connection; - EXPECT_CALL(factory_context.api_.random_, random()).WillOnce(Return(0)); + EXPECT_CALL(factory_context.server_factory_context_.api_.random_, random()).WillOnce(Return(0)); const auto route = config_obj.getRouteFromEntries(connection); EXPECT_NE(nullptr, route); @@ -330,7 +330,7 @@ TEST(ConfigTest, WeightedClustersWithMetadataMatchConfig) { HashedValue hv3(v3), hv4(v4); NiceMock connection; - EXPECT_CALL(factory_context.api_.random_, random()).WillOnce(Return(2)); + EXPECT_CALL(factory_context.server_factory_context_.api_.random_, random()).WillOnce(Return(2)); const auto route = config_obj.getRouteFromEntries(connection); EXPECT_NE(nullptr, route); @@ -396,7 +396,7 @@ TEST(ConfigTest, WeightedClustersWithMetadataMatchAndTopLevelMetadataMatchConfig HashedValue hv1(v1), hv2(v2); NiceMock connection; - EXPECT_CALL(factory_context.api_.random_, random()).WillOnce(Return(0)); + EXPECT_CALL(factory_context.server_factory_context_.api_.random_, random()).WillOnce(Return(0)); const auto route = config_obj.getRouteFromEntries(connection); EXPECT_NE(nullptr, route); @@ -429,7 +429,7 @@ TEST(ConfigTest, WeightedClustersWithMetadataMatchAndTopLevelMetadataMatchConfig HashedValue hv3(v3), hv4(v4); NiceMock connection; - EXPECT_CALL(factory_context.api_.random_, random()).WillOnce(Return(2)); + EXPECT_CALL(factory_context.server_factory_context_.api_.random_, random()).WillOnce(Return(2)); const auto route = config_obj.getRouteFromEntries(connection); EXPECT_NE(nullptr, route); @@ -686,14 +686,16 @@ class TcpProxyNonDeprecatedConfigRoutingTest : public testing::Test { cluster: fake_cluster )EOF"; - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); config_ = std::make_shared(constructConfigFromYaml(yaml, factory_context_)); } void initializeFilter() { EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = std::make_unique(config_, + factory_context_.server_factory_context_.cluster_manager_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } @@ -714,7 +716,8 @@ TEST_F(TcpProxyNonDeprecatedConfigRoutingTest, ClusterNameSet) { std::make_shared("1.2.3.4", 9999)); // Expect filter to try to open a connection to specified cluster. - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); absl::optional cluster_info; EXPECT_CALL(connection_.stream_info_, setUpstreamClusterInfo(_)) @@ -733,18 +736,25 @@ TEST_F(TcpProxyNonDeprecatedConfigRoutingTest, ClusterNameSet) { class TcpProxyHashingTest : public testing::Test { public: void setup(const std::string& yaml) { - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); config_ = std::make_shared(constructConfigFromYaml(yaml, factory_context_)); } - void initializeFilter() { - EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); + void initializeFilter() { initializeFilter(filter_callbacks_, connection_); } - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); - filter_->initializeReadFilterCallbacks(filter_callbacks_); + void initializeFilter(Network::MockReadFilterCallbacks& filter_callbacks, + Network::MockConnection& connection) { + EXPECT_CALL(filter_callbacks, connection()).WillRepeatedly(testing::ReturnRef(connection)); + + filter_ = std::make_unique(config_, + factory_context_.server_factory_context_.cluster_manager_); + filter_->initializeReadFilterCallbacks(filter_callbacks); } - Event::TestTimeSystem& timeSystem() { return factory_context_.timeSystem(); } + Event::TestTimeSystem& timeSystem() { + return factory_context_.server_factory_context_.timeSystem(); + } NiceMock factory_context_; ConfigSharedPtr config_; @@ -767,26 +777,36 @@ TEST_F(TcpProxyHashingTest, HashWithSourceIp) { - source_ip: {} )EOF"; setup(yaml); - initializeFilter(); - // Ensure there is no remote address (MockStreamInfo sets one by default), and expect no hash. - connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(nullptr); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) - .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { - EXPECT_FALSE(context->computeHashKey().has_value()); - return absl::nullopt; - })); - filter_->onNewConnection(); + { + NiceMock filter_callbacks; + NiceMock mock_connection; + initializeFilter(filter_callbacks, mock_connection); + + // Ensure there is no remote address (MockStreamInfo sets one by default), and expect no hash. + mock_connection.stream_info_.downstream_connection_info_provider_->setRemoteAddress(nullptr); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) + .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { + EXPECT_FALSE(context->computeHashKey().has_value()); + return absl::nullopt; + })); + filter_->onNewConnection(); + } // Set remote address, and expect a hash. - connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - std::make_shared("1.2.3.4", 1111)); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) - .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { - EXPECT_TRUE(context->computeHashKey().has_value()); - return absl::nullopt; - })); - filter_->onNewConnection(); + { + initializeFilter(); + connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + std::make_shared("1.2.3.4", 1111)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) + .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { + EXPECT_TRUE(context->computeHashKey().has_value()); + return absl::nullopt; + })); + filter_->onNewConnection(); + } } // Test TCP proxy using filter state to hash. @@ -800,26 +820,35 @@ TEST_F(TcpProxyHashingTest, HashWithFilterState) { } )EOF"; setup(yaml); - initializeFilter(); - // Expect no hash when filter state is unset. - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) - .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { - EXPECT_FALSE(context->computeHashKey().has_value()); - return absl::nullopt; - })); - filter_->onNewConnection(); + { + NiceMock filter_callbacks; + NiceMock mock_connection; + initializeFilter(filter_callbacks, mock_connection); + // Expect no hash when filter state is unset. + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) + .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { + EXPECT_FALSE(context->computeHashKey().has_value()); + return absl::nullopt; + })); + filter_->onNewConnection(); + } // Set filter state, and expect HashableObj's hash is now used. - connection_.stream_info_.filter_state_->setData("foo", std::make_unique(), - StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::FilterChain); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) - .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { - EXPECT_EQ(31337, context->computeHashKey().value()); - return absl::nullopt; - })); - filter_->onNewConnection(); + { + initializeFilter(); + connection_.stream_info_.filter_state_->setData("foo", std::make_unique(), + StreamInfo::FilterState::StateType::ReadOnly, + StreamInfo::FilterState::LifeSpan::FilterChain); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) + .WillOnce(Invoke([](Upstream::ResourcePriority, Upstream::LoadBalancerContext* context) { + EXPECT_EQ(31337, context->computeHashKey().value()); + return absl::nullopt; + })); + filter_->onNewConnection(); + } } } // namespace diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index 130addf2c3e1..41a2ce805c99 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -64,7 +64,8 @@ class TcpProxyTest : public TcpProxyTestBase { void setup(uint32_t connections, bool set_redirect_records, const envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy& config) override { if (config.has_on_demand()) { - EXPECT_CALL(factory_context_.cluster_manager_, allocateOdCdsApi(_, _, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + allocateOdCdsApi(_, _, _)) .WillOnce( Invoke([this]() { return Upstream::MockOdCdsApiHandlePtr(mock_odcds_api_handle_); })); } @@ -95,7 +96,8 @@ class TcpProxyTest : public TcpProxyTestBase { { testing::InSequence sequence; for (uint32_t i = 0; i < connections; i++) { - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(Upstream::TcpPoolData([]() {}, &conn_pool_))) .RetiresOnSaturation(); EXPECT_CALL(conn_pool_, newConnection(_)) @@ -106,7 +108,8 @@ class TcpProxyTest : public TcpProxyTestBase { })) .RetiresOnSaturation(); } - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillRepeatedly(Return(absl::nullopt)); } @@ -129,7 +132,8 @@ class TcpProxyTest : public TcpProxyTestBase { ->addOption( Network::SocketOptionFactory::buildWFPRedirectRecordsOptions(*redirect_records)); } - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = std::make_unique(config_, + factory_context_.server_factory_context_.cluster_manager_); EXPECT_CALL(filter_callbacks_.connection_, enableHalfClose(true)); EXPECT_CALL(filter_callbacks_.connection_, readDisable(true)); filter_->initializeReadFilterCallbacks(filter_callbacks_); @@ -197,7 +201,8 @@ TEST_F(TcpProxyTest, HalfCloseProxy) { // Test with an explicitly configured upstream. TEST_F(TcpProxyTest, ExplicitFactory) { // Explicitly configure an HTTP upstream, to test factory creation. - auto& info = factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_; + auto& info = factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_; info->upstream_config_ = std::make_unique(); envoy::extensions::upstreams::tcp::generic::v3::GenericConnectionPoolProto generic_config; info->upstream_config_->mutable_typed_config()->PackFrom(generic_config); @@ -219,7 +224,8 @@ TEST_F(TcpProxyTest, ExplicitFactory) { // Test nothing bad happens if an invalid factory is configured. TEST_F(TcpProxyTest, BadFactory) { - auto& info = factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_; + auto& info = factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_; info->upstream_config_ = std::make_unique(); // The HTTP Generic connection pool is not a valid type for TCP upstreams. envoy::extensions::upstreams::http::generic::v3::GenericConnectionPoolProto generic_config; @@ -239,12 +245,13 @@ TEST_F(TcpProxyTest, BadFactory) { std::make_unique>()); ON_CALL(*upstream_hosts_.at(0), cluster()) - .WillByDefault( - ReturnPointee(factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_)); + .WillByDefault(ReturnPointee(factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.cluster_.info_)); EXPECT_CALL(*upstream_connections_.at(0), dispatcher()) .WillRepeatedly(ReturnRef(filter_callbacks_.connection_.dispatcher_)); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = + std::make_unique(config_, factory_context_.server_factory_context_.cluster_manager_); EXPECT_CALL(filter_callbacks_.connection_, enableHalfClose(true)); EXPECT_CALL(filter_callbacks_.connection_, readDisable(true)); filter_->initializeReadFilterCallbacks(filter_callbacks_); @@ -275,8 +282,14 @@ TEST_F(TcpProxyTest, UpstreamLocalDisconnect) { TEST_F(TcpProxyTest, UpstreamRemoteDisconnect) { setup(1); + timeSystem().advanceTimeWait(std::chrono::microseconds(20)); raiseEventUpstreamConnected(0); + const absl::optional upstream_connection_establishment_latency = + filter_->getStreamInfo().upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency(); + ASSERT_TRUE(upstream_connection_establishment_latency.has_value()); + EXPECT_EQ(std::chrono::microseconds(20), upstream_connection_establishment_latency.value()); + Buffer::OwnedImpl buffer("hello"); EXPECT_CALL(*upstream_connections_.at(0), write(BufferEqual(&buffer), false)); filter_->onData(buffer, false); @@ -296,13 +309,19 @@ TEST_F(TcpProxyTest, ConnectAttemptsUpstreamLocalFail) { setup(2, config); + timeSystem().advanceTimeWait(std::chrono::microseconds(10)); raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::LocalConnectionFailure); + timeSystem().advanceTimeWait(std::chrono::microseconds(40)); raiseEventUpstreamConnected(1); - EXPECT_EQ(0U, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("upstream_cx_connect_attempts_exceeded") + EXPECT_EQ(0U, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->stats_store_.counter("upstream_cx_connect_attempts_exceeded") .value()); EXPECT_EQ(2U, filter_->getStreamInfo().attemptCount().value()); + const absl::optional upstream_connection_establishment_latency = + filter_->getStreamInfo().upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency(); + ASSERT_TRUE(upstream_connection_establishment_latency.has_value()); + EXPECT_EQ(std::chrono::microseconds(50), upstream_connection_establishment_latency.value()); } // Make sure that the tcp proxy code handles reentrant calls to onPoolFailure. @@ -338,8 +357,8 @@ TEST_F(TcpProxyTest, ConnectAttemptsUpstreamRemoteFail) { raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::RemoteConnectionFailure); raiseEventUpstreamConnected(1); - EXPECT_EQ(0U, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("upstream_cx_connect_attempts_exceeded") + EXPECT_EQ(0U, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->stats_store_.counter("upstream_cx_connect_attempts_exceeded") .value()); } @@ -352,8 +371,8 @@ TEST_F(TcpProxyTest, ConnectAttemptsUpstreamTimeout) { raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::Timeout); raiseEventUpstreamConnected(1); - EXPECT_EQ(0U, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("upstream_cx_connect_attempts_exceeded") + EXPECT_EQ(0U, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->stats_store_.counter("upstream_cx_connect_attempts_exceeded") .value()); } @@ -375,9 +394,16 @@ TEST_F(TcpProxyTest, ConnectAttemptsLimit) { // Try both failure modes raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::Timeout); + timeSystem().advanceTimeWait(std::chrono::microseconds(10)); raiseEventUpstreamConnectFailed(1, ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + timeSystem().advanceTimeWait(std::chrono::microseconds(15)); raiseEventUpstreamConnectFailed(2, ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + const absl::optional upstream_connection_establishment_latency = + filter_->getStreamInfo().upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency(); + ASSERT_TRUE(upstream_connection_establishment_latency.has_value()); + EXPECT_EQ(std::chrono::microseconds(25), upstream_connection_establishment_latency.value()); + filter_.reset(); EXPECT_EQ(access_log_data_, "UF,URX"); } @@ -479,7 +505,7 @@ TEST_F(TcpProxyTest, UpstreamConnectTimeout) { TEST_F(TcpProxyTest, UpstreamClusterNotFound) { setup(0, accessLogConfig("%RESPONSE_FLAGS%")); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) .WillRepeatedly(Return(nullptr)); EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); @@ -517,7 +543,8 @@ TEST_F(TcpProxyTest, RouteWithMetadataMatch) { {Envoy::Config::MetadataFilters::get().ENVOY_LB, metadata_struct}); configure(config); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = + std::make_unique(config_, factory_context_.server_factory_context_.cluster_manager_); filter_->initializeReadFilterCallbacks(filter_callbacks_); EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); @@ -557,7 +584,8 @@ TEST_F(TcpProxyTest, WeightedClusterWithMetadataMatch) { k0: v0 )EOF"; - factory_context_.cluster_manager_.initializeThreadLocalClusters({"cluster1", "cluster2"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"cluster1", "cluster2"}); config_ = std::make_shared(constructConfigFromYaml(yaml, factory_context_)); ProtobufWkt::Value v0, v1, v2; @@ -566,15 +594,19 @@ TEST_F(TcpProxyTest, WeightedClusterWithMetadataMatch) { v2.set_string_value("v2"); HashedValue hv0(v0), hv1(v1), hv2(v2); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); - filter_->initializeReadFilterCallbacks(filter_callbacks_); - // Expect filter to try to open a connection to cluster1. { + NiceMock filter_callbacks; + filter_ = std::make_unique(config_, + factory_context_.server_factory_context_.cluster_manager_); + filter_->initializeReadFilterCallbacks(filter_callbacks); + Upstream::LoadBalancerContext* context; - EXPECT_CALL(factory_context_.api_.random_, random()).WillOnce(Return(0)); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()) + .WillOnce(Return(0)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(DoAll(SaveArg<1>(&context), Return(absl::nullopt))); EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); @@ -595,10 +627,17 @@ TEST_F(TcpProxyTest, WeightedClusterWithMetadataMatch) { // Expect filter to try to open a connection to cluster2. { + NiceMock filter_callbacks; + filter_ = std::make_unique(config_, + factory_context_.server_factory_context_.cluster_manager_); + filter_->initializeReadFilterCallbacks(filter_callbacks); + Upstream::LoadBalancerContext* context; - EXPECT_CALL(factory_context_.api_.random_, random()).WillOnce(Return(2)); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()) + .WillOnce(Return(2)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(DoAll(SaveArg<1>(&context), Return(absl::nullopt))); EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); @@ -632,12 +671,14 @@ TEST_F(TcpProxyTest, StreamInfoDynamicMetadata) { EXPECT_CALL(filter_callbacks_.connection_.stream_info_, dynamicMetadata()) .WillRepeatedly(ReturnRef(metadata)); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = + std::make_unique(config_, factory_context_.server_factory_context_.cluster_manager_); filter_->initializeReadFilterCallbacks(filter_callbacks_); Upstream::LoadBalancerContext* context; - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(DoAll(SaveArg<1>(&context), Return(absl::nullopt))); EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); @@ -669,7 +710,8 @@ TEST_F(TcpProxyTest, StreamInfoDynamicMetadataAndConfigMerged) { k1: from_config )EOF"; - factory_context_.cluster_manager_.initializeThreadLocalClusters({"cluster1"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"cluster1"}); config_ = std::make_shared(constructConfigFromYaml(yaml, factory_context_)); ProtobufWkt::Value v0, v1, v2; @@ -686,12 +728,14 @@ TEST_F(TcpProxyTest, StreamInfoDynamicMetadataAndConfigMerged) { EXPECT_CALL(filter_callbacks_.connection_.stream_info_, dynamicMetadata()) .WillRepeatedly(ReturnRef(metadata)); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = + std::make_unique(config_, factory_context_.server_factory_context_.cluster_manager_); filter_->initializeReadFilterCallbacks(filter_callbacks_); Upstream::LoadBalancerContext* context; - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(DoAll(SaveArg<1>(&context), Return(absl::nullopt))); EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); @@ -715,7 +759,8 @@ TEST_F(TcpProxyTest, StreamInfoDynamicMetadataAndConfigMerged) { TEST_F(TcpProxyTest, DisconnectBeforeData) { configure(defaultConfig()); - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = + std::make_unique(config_, factory_context_.server_factory_context_.cluster_manager_); filter_->initializeReadFilterCallbacks(filter_callbacks_); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -741,19 +786,26 @@ TEST_F(TcpProxyTest, UpstreamConnectFailure) { setup(1, accessLogConfig("%RESPONSE_FLAGS%")); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush, _)); + timeSystem().advanceTimeWait(std::chrono::microseconds(20)); raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + const absl::optional upstream_connection_establishment_latency = + filter_->getStreamInfo().upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency(); + ASSERT_TRUE(upstream_connection_establishment_latency.has_value()); + EXPECT_EQ(std::chrono::microseconds(20), upstream_connection_establishment_latency.value()); + filter_.reset(); EXPECT_EQ(access_log_data_, "UF,URX"); } TEST_F(TcpProxyTest, UpstreamConnectionLimit) { configure(accessLogConfig("%RESPONSE_FLAGS%")); - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( - 0, 0, 0, 0, 0); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->resetResourceManager(0, 0, 0, 0, 0); // setup sets up expectation for tcpConnForCluster but this test is expected to NOT call that - filter_ = std::make_unique(config_, factory_context_.cluster_manager_); + filter_ = + std::make_unique(config_, factory_context_.server_factory_context_.cluster_manager_); // The downstream connection closes if the proxy can't make an upstream connection. EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush, _)); filter_->initializeReadFilterCallbacks(filter_callbacks_); @@ -975,9 +1027,11 @@ TEST_F(TcpProxyTest, IntermediateLogEntry) { // The timer will be enabled cyclically. EXPECT_CALL(*flush_timer, enableTimer(std::chrono::milliseconds(1000), _)); filter_callbacks_.connection_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(10); - EXPECT_CALL(*mock_access_logger_, log(_, _, _, _, AccessLog::AccessLogType::TcpPeriodic)) - .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*mock_access_logger_, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::TcpPeriodic); + EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 10); EXPECT_THAT(stream_info.getDownstreamBytesMeter()->bytesAtLastDownstreamPeriodicLog(), testing::IsNull()); @@ -988,9 +1042,11 @@ TEST_F(TcpProxyTest, IntermediateLogEntry) { EXPECT_EQ(access_log_data_.value(), AccessLogType_Name(AccessLog::AccessLogType::TcpPeriodic)); filter_callbacks_.connection_.stream_info_.downstream_bytes_meter_->addWireBytesReceived(9); - EXPECT_CALL(*mock_access_logger_, log(_, _, _, _, AccessLog::AccessLogType::TcpPeriodic)) - .WillOnce(Invoke([](const Http::HeaderMap*, const Http::HeaderMap*, const Http::HeaderMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { + EXPECT_CALL(*mock_access_logger_, log(_, _)) + .WillOnce(Invoke([](const Formatter::HttpFormatterContext& log_context, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::TcpPeriodic); + EXPECT_EQ(stream_info.getDownstreamBytesMeter()->wireBytesReceived(), 19); EXPECT_EQ(stream_info.getDownstreamBytesMeter() ->bytesAtLastDownstreamPeriodicLog() @@ -1000,7 +1056,11 @@ TEST_F(TcpProxyTest, IntermediateLogEntry) { EXPECT_CALL(*flush_timer, enableTimer(std::chrono::milliseconds(1000), _)); flush_timer->invokeCallback(); - EXPECT_CALL(*mock_access_logger_, log(_, _, _, _, AccessLog::AccessLogType::TcpConnectionEnd)); + EXPECT_CALL(*mock_access_logger_, log(_, _)) + .WillOnce(Invoke( + [](const Formatter::HttpFormatterContext& log_context, const StreamInfo::StreamInfo&) { + EXPECT_EQ(log_context.accessLogType(), AccessLog::AccessLogType::TcpConnectionEnd); + })); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); filter_.reset(); @@ -1211,21 +1271,25 @@ TEST_F(TcpProxyTest, PickClusterOnUpstreamFailure) { config.mutable_max_connect_attempts()->set_value(2); // The random number lead into picking the first one in the weighted clusters. - EXPECT_CALL(factory_context_.api_.random_, random()).WillOnce(Return(0)); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster_0")) - .WillOnce(Return(&factory_context_.cluster_manager_.thread_local_cluster_)); + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()).WillOnce(Return(0)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster_0")) + .WillOnce( + Return(&factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_)); setup(1, config); // The random number lead into picking the second cluster. - EXPECT_CALL(factory_context_.api_.random_, random()).WillOnce(Return(1)); + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()).WillOnce(Return(1)); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster_1")) - .WillOnce(Return(&factory_context_.cluster_manager_.thread_local_cluster_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster_1")) + .WillOnce( + Return(&factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_)); raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::LocalConnectionFailure); - EXPECT_EQ(0U, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("upstream_cx_connect_attempts_exceeded") + EXPECT_EQ(0U, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->stats_store_.counter("upstream_cx_connect_attempts_exceeded") .value()); } @@ -1237,14 +1301,16 @@ TEST_F(TcpProxyTest, OnDemandCallbackStickToTheSelectedCluster) { mock_odcds_api_handle_ = Upstream::MockOdCdsApiHandle::create().release(); // The random number lead to select the first one in the weighted clusters. - EXPECT_CALL(factory_context_.api_.random_, random()).WillOnce(Return(0)); + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()).WillOnce(Return(0)); // The first cluster is requested 2 times. - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster_0")) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster_0")) // Invoked on new connection. Null is returned which would trigger on demand. .WillOnce(Return(nullptr)) // Invoked in the callback of on demand look up. The cluster is ready upon callback. - .WillOnce(Return(&factory_context_.cluster_manager_.thread_local_cluster_)) + .WillOnce( + Return(&factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_)) .RetiresOnSaturation(); Upstream::ClusterDiscoveryCallbackPtr cluster_discovery_callback; @@ -1258,15 +1324,16 @@ TEST_F(TcpProxyTest, OnDemandCallbackStickToTheSelectedCluster) { // When the on-demand look up callback is invoked, the target cluster should not change. // The behavior is verified by checking the random() which is used during cluster re-pick. - EXPECT_CALL(factory_context_.api_.random_, random()).Times(0); + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()).Times(0); std::invoke(*cluster_discovery_callback, Upstream::ClusterDiscoveryStatus::Available); // Start to raise connect failure. // random() is raised in the cluster pick. `fake_cluster_1` will be picked. - EXPECT_CALL(factory_context_.api_.random_, random()).WillOnce(Return(1)); + EXPECT_CALL(factory_context_.server_factory_context_.api_.random_, random()).WillOnce(Return(1)); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster_1")) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster_1")) // Invoked on connect attempt. Null is returned which would trigger on demand. .WillOnce(Return(nullptr)) .RetiresOnSaturation(); @@ -1277,8 +1344,8 @@ TEST_F(TcpProxyTest, OnDemandCallbackStickToTheSelectedCluster) { })); raiseEventUpstreamConnectFailed(0, ConnectionPool::PoolFailureReason::LocalConnectionFailure); - EXPECT_EQ(0U, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_ - .counter("upstream_cx_connect_attempts_exceeded") + EXPECT_EQ(0U, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->stats_store_.counter("upstream_cx_connect_attempts_exceeded") .value()); EXPECT_CALL(filter_callbacks_.connection_, close(_, _)); @@ -1311,7 +1378,8 @@ TEST_F(TcpProxyTest, OdcdsCancelIfConnectionClose) { // To trigger the on demand request, we enforce the first call to getThreadLocalCluster returning // no cluster. - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster")) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster")) .WillOnce(Return(nullptr)) .RetiresOnSaturation(); @@ -1332,11 +1400,14 @@ TEST_F(TcpProxyTest, OdcdsBasicDownstreamLocalClose) { // To trigger the on demand request, we enforce the first call to getThreadLocalCluster returning // no cluster. - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster")) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster")) .WillOnce(Return(nullptr)) - .WillOnce(Return(&factory_context_.cluster_manager_.thread_local_cluster_)) + .WillOnce( + Return(&factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_)) .RetiresOnSaturation(); + timeSystem().advanceTimeWait(std::chrono::microseconds(20)); Upstream::ClusterDiscoveryCallbackPtr cluster_discovery_callback; EXPECT_CALL(*mock_odcds_api_handle_, requestOnDemandClusterDiscovery("fake_cluster", _, _)) .WillOnce(Invoke([&](auto&&, auto&& cb, auto&&) { @@ -1346,7 +1417,14 @@ TEST_F(TcpProxyTest, OdcdsBasicDownstreamLocalClose) { setup(1, config); std::invoke(*cluster_discovery_callback, Upstream::ClusterDiscoveryStatus::Available); + timeSystem().advanceTimeWait(std::chrono::microseconds(10)); + raiseEventUpstreamConnected(0); + const absl::optional upstream_connection_establishment_latency = + filter_->getStreamInfo().upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency(); + ASSERT_TRUE(upstream_connection_establishment_latency.has_value()); + // OdCds resolution time isn't included in time to connect to upstream. + EXPECT_EQ(std::chrono::microseconds(10), upstream_connection_establishment_latency.value()); Buffer::OwnedImpl buffer("hello"); EXPECT_CALL(*upstream_connections_.at(0), write(BufferEqual(&buffer), _)); @@ -1365,7 +1443,8 @@ TEST_F(TcpProxyTest, OdcdsClusterMissingCauseConnectionClose) { auto config = onDemandConfig(); mock_odcds_api_handle_ = Upstream::MockOdCdsApiHandle::create().release(); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster")) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster")) .WillOnce(Return(nullptr)) .RetiresOnSaturation(); @@ -1381,6 +1460,11 @@ TEST_F(TcpProxyTest, OdcdsClusterMissingCauseConnectionClose) { EXPECT_CALL(filter_callbacks_.connection_, close(_, _)); std::invoke(*cluster_discovery_callback, Upstream::ClusterDiscoveryStatus::Missing); + + // No upstream connection was attempted, so no latency should be recorded. + const absl::optional upstream_connection_establishment_latency = + filter_->getStreamInfo().upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency(); + ASSERT_FALSE(upstream_connection_establishment_latency.has_value()); } // Test that upstream transport failure message is reflected in access logs. diff --git a/test/common/tcp_proxy/tcp_proxy_test_base.h b/test/common/tcp_proxy/tcp_proxy_test_base.h index 6f54989c2e7e..fa621fccf6b2 100644 --- a/test/common/tcp_proxy/tcp_proxy_test_base.h +++ b/test/common/tcp_proxy/tcp_proxy_test_base.h @@ -54,13 +54,13 @@ inline Config constructConfigFromYaml(const std::string& yaml, Server::Configuration::FactoryContext& context) { envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy tcp_proxy; TestUtility::loadFromYamlAndValidate(yaml, tcp_proxy); - return Config(tcp_proxy, context); + return {tcp_proxy, context}; } class TcpProxyTestBase : public testing::Test { public: TcpProxyTestBase() { - ON_CALL(*factory_context_.access_log_manager_.file_, write(_)) + ON_CALL(*factory_context_.server_factory_context_.access_log_manager_.file_, write(_)) .WillByDefault(SaveArg<0>(&access_log_data_)); ON_CALL(filter_callbacks_.connection_.stream_info_, setUpstreamClusterInfo(_)) .WillByDefault(Invoke([this](const Upstream::ClusterInfoConstSharedPtr& cluster_info) { @@ -68,7 +68,8 @@ class TcpProxyTestBase : public testing::Test { })); ON_CALL(filter_callbacks_.connection_.stream_info_, upstreamClusterInfo()) .WillByDefault(ReturnPointee(&upstream_cluster_)); - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); } ~TcpProxyTestBase() override { @@ -150,7 +151,9 @@ class TcpProxyTestBase : public testing::Test { return connection; } - Event::TestTimeSystem& timeSystem() { return factory_context_.timeSystem(); } + Event::TestTimeSystem& timeSystem() { + return factory_context_.server_factory_context_.timeSystem(); + } NiceMock factory_context_; ConfigSharedPtr config_; diff --git a/test/common/tcp_proxy/upstream_test.cc b/test/common/tcp_proxy/upstream_test.cc index 8df040e69489..7b27fecdf266 100644 --- a/test/common/tcp_proxy/upstream_test.cc +++ b/test/common/tcp_proxy/upstream_test.cc @@ -25,14 +25,14 @@ namespace TcpProxy { namespace { using envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy_TunnelingConfig; -template class HttpUpstreamTest : public testing::Test { +class HttpUpstreamTest : public testing::TestWithParam { public: HttpUpstreamTest() { EXPECT_CALL(encoder_, getStream()).Times(AnyNumber()); EXPECT_CALL(encoder_, encodeHeaders(_, false)); EXPECT_CALL(encoder_, http1StreamEncoderOptions()).Times(AnyNumber()); EXPECT_CALL(encoder_, enableTcpTunneling()).Times(AnyNumber()); - if (typeid(T) == typeid(Http1Upstream)) { + if (GetParam() == Http::CodecType::HTTP1) { ON_CALL(encoder_, http1StreamEncoderOptions()) .WillByDefault(Return(Http::Http1StreamEncoderOptionsOptRef(stream_encoder_options_))); } @@ -42,7 +42,8 @@ template class HttpUpstreamTest : public testing::Test { void setupUpstream() { config_ = std::make_unique(config_message_, context_); - upstream_ = std::make_unique(callbacks_, *this->config_, downstream_stream_info_); + upstream_ = std::make_unique(callbacks_, *this->config_, downstream_stream_info_, + GetParam()); upstream_->setRequestEncoder(encoder_, true); } @@ -56,13 +57,11 @@ template class HttpUpstreamTest : public testing::Test { NiceMock context_; }; -using testing::Types; +INSTANTIATE_TEST_SUITE_P(H1H2H3Codecs, HttpUpstreamTest, + ::testing::Values(Http::CodecType::HTTP1, Http::CodecType::HTTP2, + Http::CodecType::HTTP3)); -using Implementations = Types; - -TYPED_TEST_SUITE(HttpUpstreamTest, Implementations); - -TYPED_TEST(HttpUpstreamTest, WriteUpstream) { +TEST_P(HttpUpstreamTest, WriteUpstream) { this->setupUpstream(); EXPECT_CALL(this->encoder_, encodeData(BufferStringEqual("foo"), false)); Buffer::OwnedImpl buffer1("foo"); @@ -73,12 +72,12 @@ TYPED_TEST(HttpUpstreamTest, WriteUpstream) { this->upstream_->encodeData(buffer2, true); // New upstream with no encoder. - this->upstream_ = - std::make_unique(this->callbacks_, *this->config_, this->downstream_stream_info_); + this->upstream_ = std::make_unique(this->callbacks_, *this->config_, + this->downstream_stream_info_, GetParam()); this->upstream_->encodeData(buffer2, true); } -TYPED_TEST(HttpUpstreamTest, WriteDownstream) { +TEST_P(HttpUpstreamTest, WriteDownstream) { this->setupUpstream(); EXPECT_CALL(this->callbacks_, onUpstreamData(BufferStringEqual("foo"), false)); Buffer::OwnedImpl buffer1("foo"); @@ -89,21 +88,21 @@ TYPED_TEST(HttpUpstreamTest, WriteDownstream) { this->upstream_->responseDecoder().decodeData(buffer2, true); } -TYPED_TEST(HttpUpstreamTest, InvalidUpgradeWithEarlyFin) { +TEST_P(HttpUpstreamTest, InvalidUpgradeWithEarlyFin) { this->setupUpstream(); EXPECT_CALL(this->callbacks_, onEvent(_)); Http::ResponseHeaderMapPtr headers{new Http::TestResponseHeaderMapImpl{{":status", "200"}}}; this->upstream_->responseDecoder().decodeHeaders(std::move(headers), true); } -TYPED_TEST(HttpUpstreamTest, InvalidUpgradeWithNon200) { +TEST_P(HttpUpstreamTest, InvalidUpgradeWithNon200) { this->setupUpstream(); EXPECT_CALL(this->callbacks_, onEvent(_)); Http::ResponseHeaderMapPtr headers{new Http::TestResponseHeaderMapImpl{{":status", "301"}}}; this->upstream_->responseDecoder().decodeHeaders(std::move(headers), false); } -TYPED_TEST(HttpUpstreamTest, ReadDisable) { +TEST_P(HttpUpstreamTest, ReadDisable) { this->setupUpstream(); EXPECT_CALL(this->encoder_.stream_, readDisable(true)); EXPECT_TRUE(this->upstream_->readDisable(true)); @@ -112,31 +111,31 @@ TYPED_TEST(HttpUpstreamTest, ReadDisable) { EXPECT_TRUE(this->upstream_->readDisable(false)); // New upstream with no encoder. - this->upstream_ = - std::make_unique(this->callbacks_, *this->config_, this->downstream_stream_info_); + this->upstream_ = std::make_unique(this->callbacks_, *this->config_, + this->downstream_stream_info_, GetParam()); EXPECT_FALSE(this->upstream_->readDisable(true)); } -TYPED_TEST(HttpUpstreamTest, AddBytesSentCallbackForCoverage) { +TEST_P(HttpUpstreamTest, AddBytesSentCallbackForCoverage) { this->setupUpstream(); this->upstream_->addBytesSentCallback([&](uint64_t) { return true; }); } -TYPED_TEST(HttpUpstreamTest, DownstreamDisconnect) { +TEST_P(HttpUpstreamTest, DownstreamDisconnect) { this->setupUpstream(); EXPECT_CALL(this->encoder_.stream_, resetStream(Http::StreamResetReason::LocalReset)); EXPECT_CALL(this->callbacks_, onEvent(_)).Times(0); EXPECT_TRUE(this->upstream_->onDownstreamEvent(Network::ConnectionEvent::LocalClose) == nullptr); } -TYPED_TEST(HttpUpstreamTest, UpstreamReset) { +TEST_P(HttpUpstreamTest, UpstreamReset) { this->setupUpstream(); EXPECT_CALL(this->encoder_.stream_, resetStream(_)).Times(0); EXPECT_CALL(this->callbacks_, onEvent(_)); this->upstream_->onResetStream(Http::StreamResetReason::ConnectionTermination, ""); } -TYPED_TEST(HttpUpstreamTest, UpstreamWatermarks) { +TEST_P(HttpUpstreamTest, UpstreamWatermarks) { this->setupUpstream(); EXPECT_CALL(this->callbacks_, onAboveWriteBufferHighWatermark()); this->upstream_->onAboveWriteBufferHighWatermark(); @@ -151,7 +150,7 @@ class MockHttpConnPoolCallbacks : public HttpConnPool::Callbacks { MOCK_METHOD(void, onFailure, ()); }; -TYPED_TEST(HttpUpstreamTest, DownstreamDisconnectBeforeConnectResponse) { +TEST_P(HttpUpstreamTest, DownstreamDisconnectBeforeConnectResponse) { this->setupUpstream(); auto conn_pool_callbacks = std::make_unique(); auto conn_pool_callbacks_raw = conn_pool_callbacks.get(); @@ -161,7 +160,7 @@ TYPED_TEST(HttpUpstreamTest, DownstreamDisconnectBeforeConnectResponse) { EXPECT_TRUE(this->upstream_->onDownstreamEvent(Network::ConnectionEvent::LocalClose) == nullptr); } -TYPED_TEST(HttpUpstreamTest, OnSuccessCalledOnValidResponse) { +TEST_P(HttpUpstreamTest, OnSuccessCalledOnValidResponse) { this->setupUpstream(); auto conn_pool_callbacks = std::make_unique(); auto conn_pool_callbacks_raw = conn_pool_callbacks.get(); @@ -172,7 +171,7 @@ TYPED_TEST(HttpUpstreamTest, OnSuccessCalledOnValidResponse) { this->upstream_->responseDecoder().decodeHeaders(std::move(headers), false); } -TYPED_TEST(HttpUpstreamTest, OnFailureCalledOnInvalidResponse) { +TEST_P(HttpUpstreamTest, OnFailureCalledOnInvalidResponse) { this->setupUpstream(); auto conn_pool_callbacks = std::make_unique(); auto conn_pool_callbacks_raw = conn_pool_callbacks.get(); @@ -183,7 +182,7 @@ TYPED_TEST(HttpUpstreamTest, OnFailureCalledOnInvalidResponse) { this->upstream_->responseDecoder().decodeHeaders(std::move(headers), false); } -TYPED_TEST(HttpUpstreamTest, DumpsResponseDecoderWithoutAllocatingMemory) { +TEST_P(HttpUpstreamTest, DumpsResponseDecoderWithoutAllocatingMemory) { std::array buffer; OutputBufferStream ostream{buffer.data(), buffer.size()}; this->setupUpstream(); @@ -194,7 +193,7 @@ TYPED_TEST(HttpUpstreamTest, DumpsResponseDecoderWithoutAllocatingMemory) { EXPECT_THAT(ostream.contents(), EndsWith("has not implemented dumpState\n")); } -TYPED_TEST(HttpUpstreamTest, UpstreamTrailersMarksDoneReading) { +TEST_P(HttpUpstreamTest, UpstreamTrailersMarksDoneReading) { this->setupUpstream(); EXPECT_CALL(this->encoder_.stream_, resetStream(_)).Times(0); this->upstream_->doneWriting(); @@ -202,14 +201,14 @@ TYPED_TEST(HttpUpstreamTest, UpstreamTrailersMarksDoneReading) { this->upstream_->responseDecoder().decodeTrailers(std::move(trailers)); } -template class HttpUpstreamRequestEncoderTest : public testing::Test { +class HttpUpstreamRequestEncoderTest : public testing::TestWithParam { public: HttpUpstreamRequestEncoderTest() { EXPECT_CALL(encoder_, getStream()).Times(AnyNumber()); EXPECT_CALL(encoder_, http1StreamEncoderOptions()).Times(AnyNumber()); EXPECT_CALL(this->encoder_, enableTcpTunneling()).Times(AnyNumber()); - if (typeid(T) == typeid(Http1Upstream)) { + if (GetParam() == Http::CodecType::HTTP1) { ON_CALL(encoder_, http1StreamEncoderOptions()) .WillByDefault(Return(Http::Http1StreamEncoderOptionsOptRef(stream_encoder_options_))); is_http2_ = false; @@ -219,7 +218,8 @@ template class HttpUpstreamRequestEncoderTest : public testing::Tes void setupUpstream() { config_ = std::make_unique(config_message_, context_); - upstream_ = std::make_unique(callbacks_, *this->config_, this->downstream_stream_info_); + upstream_ = std::make_unique(callbacks_, *this->config_, + this->downstream_stream_info_, GetParam()); } void populateMetadata(envoy::config::core::v3::Metadata& metadata, const std::string& ns, @@ -242,9 +242,11 @@ template class HttpUpstreamRequestEncoderTest : public testing::Tes bool is_http2_ = true; }; -TYPED_TEST_SUITE(HttpUpstreamRequestEncoderTest, Implementations); +INSTANTIATE_TEST_SUITE_P(H1H2H3Codecs, HttpUpstreamRequestEncoderTest, + ::testing::Values(Http::CodecType::HTTP1, Http::CodecType::HTTP2, + Http::CodecType::HTTP3)); -TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoder) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoder) { this->setupUpstream(); std::unique_ptr expected_headers; expected_headers = Http::createHeaderMap({ @@ -256,7 +258,7 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoder) { this->upstream_->setRequestEncoder(this->encoder_, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderUsePost) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoderUsePost) { this->config_message_.set_use_post(true); this->setupUpstream(); std::unique_ptr expected_headers; @@ -275,7 +277,7 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderUsePost) { this->upstream_->setRequestEncoder(this->encoder_, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderUsePostWithCustomPath) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoderUsePostWithCustomPath) { this->config_message_.set_use_post(true); this->config_message_.set_post_path("/test"); this->setupUpstream(); @@ -295,14 +297,14 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderUsePostWithCustomPath) this->upstream_->setRequestEncoder(this->encoder_, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderConnectWithCustomPath) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoderConnectWithCustomPath) { this->config_message_.set_use_post(false); this->config_message_.set_post_path("/test"); EXPECT_THROW_WITH_MESSAGE(this->setupUpstream(), EnvoyException, "Can't set a post path when POST method isn't used"); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderHeaders) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoderHeaders) { auto* header = this->config_message_.add_headers_to_add(); auto* hdr = header->mutable_header(); hdr->set_key("header0"); @@ -335,7 +337,7 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderHeaders) { this->upstream_->setRequestEncoder(this->encoder_, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, ConfigReuse) { +TEST_P(HttpUpstreamRequestEncoderTest, ConfigReuse) { auto* header = this->config_message_.add_headers_to_add(); auto* hdr = header->mutable_header(); hdr->set_key("key"); @@ -365,12 +367,12 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, ConfigReuse) { this->upstream_->setRequestEncoder(this->encoder_, false); Http::MockRequestEncoder another_encoder; - auto another_upstream = - std::make_unique(this->callbacks_, *this->config_, this->downstream_stream_info_); + auto another_upstream = std::make_unique(this->callbacks_, *this->config_, + this->downstream_stream_info_, GetParam()); EXPECT_CALL(another_encoder, getStream()).Times(AnyNumber()); EXPECT_CALL(another_encoder, http1StreamEncoderOptions()).Times(AnyNumber()); EXPECT_CALL(another_encoder, enableTcpTunneling()).Times(AnyNumber()); - if (typeid(TypeParam) == typeid(Http1Upstream)) { + if (GetParam() == Http::CodecType::HTTP1) { ON_CALL(another_encoder, http1StreamEncoderOptions()) .WillByDefault( Return(Http::Http1StreamEncoderOptionsOptRef(this->stream_encoder_options_))); @@ -379,7 +381,7 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, ConfigReuse) { another_upstream->setRequestEncoder(another_encoder, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderHeadersWithDownstreamInfo) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoderHeadersWithDownstreamInfo) { auto* header = this->config_message_.add_headers_to_add(); auto* hdr = header->mutable_header(); hdr->set_key("header0"); @@ -412,8 +414,8 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, RequestEncoderHeadersWithDownstreamIn this->upstream_->setRequestEncoder(this->encoder_, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, - RequestEncoderHostnameWithDownstreamInfoRequestedServerName) { +TEST_P(HttpUpstreamRequestEncoderTest, + RequestEncoderHostnameWithDownstreamInfoRequestedServerName) { this->config_message_.set_hostname("%REQUESTED_SERVER_NAME%:443"); this->setupUpstream(); @@ -437,8 +439,7 @@ TYPED_TEST(HttpUpstreamRequestEncoderTest, this->upstream_->setRequestEncoder(this->encoder_, false); } -TYPED_TEST(HttpUpstreamRequestEncoderTest, - RequestEncoderHostnameWithDownstreamInfoDynamicMetadata) { +TEST_P(HttpUpstreamRequestEncoderTest, RequestEncoderHostnameWithDownstreamInfoDynamicMetadata) { this->config_message_.set_hostname("%DYNAMIC_METADATA(tunnel:address)%:443"); this->setupUpstream(); diff --git a/test/common/tracing/BUILD b/test/common/tracing/BUILD index 84dacc8b5136..84911ca50618 100644 --- a/test/common/tracing/BUILD +++ b/test/common/tracing/BUILD @@ -84,3 +84,14 @@ envoy_cc_test( "//source/common/tracing:tracer_config_lib", ], ) + +envoy_cc_test( + name = "trace_context_impl_test", + srcs = [ + "trace_context_impl_test.cc", + ], + deps = [ + "//source/common/tracing:http_tracer_lib", + "//source/common/tracing:trace_context_lib", + ], +) diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 93f992d7f8f8..b0d325833943 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -709,6 +709,58 @@ TEST_F(HttpConnManFinalizerImplTest, CustomTagOverwritesCommonTag) { config); } +TEST(HttpTraceContextTest, HttpTraceContextTest) { + { + Http::TestRequestHeaderMapImpl request_headers; + HttpTraceContext trace_context(request_headers); + + // Protocol. + EXPECT_EQ(trace_context.protocol(), ""); + request_headers.addCopy(Http::Headers::get().Protocol, "HTTP/x"); + EXPECT_EQ(trace_context.protocol(), "HTTP/x"); + + // Host. + EXPECT_EQ(trace_context.host(), ""); + request_headers.addCopy(Http::Headers::get().Host, "test.com:233"); + EXPECT_EQ(trace_context.host(), "test.com:233"); + + // Path. + EXPECT_EQ(trace_context.path(), ""); + request_headers.addCopy(Http::Headers::get().Path, "/anything"); + EXPECT_EQ(trace_context.path(), "/anything"); + + // Method. + EXPECT_EQ(trace_context.method(), ""); + request_headers.addCopy(Http::Headers::get().Method, Http::Headers::get().MethodValues.Options); + EXPECT_EQ(trace_context.method(), Http::Headers::get().MethodValues.Options); + + // Set. + trace_context.set("foo", "bar"); + EXPECT_EQ(request_headers.get_("foo"), "bar"); + + // Get. + EXPECT_EQ(trace_context.get("foo").value(), "bar"); + + // Remove. + trace_context.remove("foo"); + EXPECT_EQ(request_headers.get_("foo"), ""); + EXPECT_EQ(trace_context.get("foo"), absl::nullopt); + } + + { + size_t size = 0; + Http::TestRequestHeaderMapImpl request_headers{{"host", "foo"}, {"bar", "var"}, {"ok", "no"}}; + HttpTraceContext trace_context(request_headers); + trace_context.forEach([&size](absl::string_view key, absl::string_view val) { + size += key.size(); + size += val.size(); + return true; + }); + // 'host' will be converted to ':authority'. + EXPECT_EQ(23, size); + } +} + } // namespace } // namespace Tracing } // namespace Envoy diff --git a/test/common/tracing/trace_context_impl_test.cc b/test/common/tracing/trace_context_impl_test.cc new file mode 100644 index 000000000000..531112c53a4b --- /dev/null +++ b/test/common/tracing/trace_context_impl_test.cc @@ -0,0 +1,166 @@ +#include "source/common/http/header_map_impl.h" +#include "source/common/tracing/http_tracer_impl.h" +#include "source/common/tracing/trace_context_impl.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Tracing { +namespace { + +TEST(TraceContextHandlerTest, TraceContextHandlerGetTest) { + // The key will be lowercase. + { + TraceContextHandler handler("KEY"); + EXPECT_EQ("key", handler.key().get()); + } + + TraceContextHandler normal_key("key"); + TraceContextHandler inline_key("content-type"); // This key is inline key for HTTP. + + // Test get. + { + auto headers = Http::RequestHeaderMapImpl::create(); + headers->setContentType("text/plain"); + headers->addCopy(Http::LowerCaseString("key"), "value"); + + HttpTraceContext http_tracer_context(*headers); + TestTraceContextImpl trace_context{{"key", "value"}, {"content-type", "text/plain"}}; + + EXPECT_EQ("value", normal_key.get(trace_context).value()); + EXPECT_EQ("text/plain", inline_key.get(trace_context).value()); + + EXPECT_EQ("value", normal_key.get(http_tracer_context).value()); + EXPECT_EQ("text/plain", inline_key.get(http_tracer_context).value()); + } +} + +TEST(TraceContextHandlerTest, TraceContextHandlerSetTest) { + + TraceContextHandler normal_key("key"); + TraceContextHandler inline_key("content-type"); // This key is inline key for HTTP. + + // Test set. + { + auto headers = Http::RequestHeaderMapImpl::create(); + HttpTraceContext http_tracer_context(*headers); + + TestTraceContextImpl trace_context{}; + + normal_key.set(trace_context, "value"); + EXPECT_EQ("value", normal_key.get(trace_context).value()); + + inline_key.set(trace_context, "text/html"); + EXPECT_EQ("text/html", inline_key.get(trace_context).value()); + + normal_key.set(http_tracer_context, "value"); + EXPECT_EQ("value", normal_key.get(http_tracer_context).value()); + + inline_key.set(http_tracer_context, "text/html"); + EXPECT_EQ("text/html", inline_key.get(http_tracer_context).value()); + } +} + +TEST(TraceContextHandlerTest, TraceContextHandlerRemoveTest) { + + TraceContextHandler normal_key("key"); + TraceContextHandler inline_key("content-type"); // This key is inline key for HTTP. + + // Test remove. + { + auto headers = Http::RequestHeaderMapImpl::create(); + headers->setContentType("text/plain"); + headers->addCopy(Http::LowerCaseString("key"), "value"); + + HttpTraceContext http_tracer_context(*headers); + TestTraceContextImpl trace_context{{"key", "value"}, {"content-type", "text/plain"}}; + + normal_key.remove(trace_context); + EXPECT_FALSE(normal_key.get(trace_context).has_value()); + + inline_key.remove(trace_context); + EXPECT_FALSE(inline_key.get(trace_context).has_value()); + + normal_key.remove(http_tracer_context); + EXPECT_FALSE(normal_key.get(http_tracer_context).has_value()); + + inline_key.remove(http_tracer_context); + EXPECT_FALSE(inline_key.get(http_tracer_context).has_value()); + } +} + +TEST(TraceContextHandlerTest, TraceContextHandlerSetRefKeyTest) { + + TraceContextHandler normal_key("key"); + TraceContextHandler inline_key("content-type"); // This key is inline key for HTTP. + // Test setRefKey. + { + auto headers = Http::RequestHeaderMapImpl::create(); + HttpTraceContext http_tracer_context(*headers); + TestTraceContextImpl trace_context{}; + + normal_key.setRefKey(trace_context, "value"); + auto iter = trace_context.context_map_.find("key"); + // setRefKey make no sense for non-HTTP context. + EXPECT_NE(iter->first.data(), normal_key.key().get().data()); + + inline_key.setRefKey(trace_context, "text/html"); + auto iter2 = trace_context.context_map_.find("content-type"); + // setRefKey make no sense for non-HTTP context. + EXPECT_NE(iter2->first.data(), inline_key.key().get().data()); + + normal_key.setRefKey(http_tracer_context, "value"); + auto iter3 = headers->get(Http::LowerCaseString("key")); + // setRefKey make sense for HTTP context. + EXPECT_EQ(iter3[0]->key().getStringView().data(), normal_key.key().get().data()); + + inline_key.setRefKey(http_tracer_context, "text/html"); + auto iter4 = headers->get(Http::LowerCaseString("content-type")); + // Note, setRefKey make no sense for inline key of HTTP context because + // inline key is stored in central registry and won't be referenced. + EXPECT_NE(iter4[0]->key().getStringView().data(), inline_key.key().get().data()); + } +} + +TEST(TraceContextHandlerTest, TraceContextHandlerSetRefTest) { + + TraceContextHandler normal_key("key"); + TraceContextHandler inline_key("content-type"); // This key is inline key for HTTP. + // Test setRef + { + auto headers = Http::RequestHeaderMapImpl::create(); + + HttpTraceContext http_tracer_context(*headers); + TestTraceContextImpl trace_context{}; + + const std::string value = "value"; + const std::string text_html = "text/html"; + + normal_key.setRef(trace_context, value); + auto iter = trace_context.context_map_.find("key"); + // setRef make no sense for non-HTTP context. + EXPECT_NE(iter->first.data(), value.data()); + + inline_key.setRef(trace_context, text_html); + auto iter2 = trace_context.context_map_.find("content-type"); + // setRef make no sense for non-HTTP context. + EXPECT_NE(iter2->first.data(), text_html.data()); + + normal_key.setRef(http_tracer_context, value); + auto iter3 = headers->get(Http::LowerCaseString("key")); + // setRef make sense for HTTP context. + EXPECT_EQ(iter3[0]->key().getStringView().data(), normal_key.key().get().data()); + EXPECT_EQ(iter3[0]->value().getStringView().data(), value.data()); + + inline_key.setRef(http_tracer_context, text_html); + auto iter4 = headers->get(Http::LowerCaseString("content-type")); + // setRef make sense for inline key of HTTP context, the value will be referenced. + EXPECT_EQ(iter4[0]->value().getStringView().data(), text_html.data()); + } +} + +} // namespace +} // namespace Tracing +} // namespace Envoy diff --git a/test/common/tracing/tracer_impl_test.cc b/test/common/tracing/tracer_impl_test.cc index 6535d3756b99..e2cd10049b07 100644 --- a/test/common/tracing/tracer_impl_test.cc +++ b/test/common/tracing/tracer_impl_test.cc @@ -289,6 +289,7 @@ class TracerImplTest : public testing::Test { {"grpc-status", "14"}, {"grpc-message", "unavailable"}}; Http::TestResponseTrailerMapImpl response_trailers_; + Tracing::HttpTraceContext trace_context_{request_headers_}; NiceMock stream_info_; NiceMock local_info_; NiceMock config_; @@ -303,7 +304,7 @@ TEST_F(TracerImplTest, BasicFunctionalityNullSpan) { EXPECT_CALL(config_, operationName()).Times(2); const std::string operation_name = "ingress"; EXPECT_CALL(*driver_, startSpan_(_, _, _, operation_name, _)).WillOnce(Return(nullptr)); - tracer_->startSpan(config_, request_headers_, stream_info_, {Reason::Sampling, true}); + tracer_->startSpan(config_, trace_context_, stream_info_, {Reason::Sampling, true}); } TEST_F(TracerImplTest, BasicFunctionalityNodeSet) { @@ -316,7 +317,7 @@ TEST_F(TracerImplTest, BasicFunctionalityNodeSet) { EXPECT_CALL(*span, setTag(_, _)).Times(testing::AnyNumber()); EXPECT_CALL(*span, setTag(Eq(Tracing::Tags::get().NodeId), Eq("node_name"))); - tracer_->startSpan(config_, request_headers_, stream_info_, {Reason::Sampling, true}); + tracer_->startSpan(config_, trace_context_, stream_info_, {Reason::Sampling, true}); } TEST_F(TracerImplTest, ChildGrpcUpstreamSpanTest) { @@ -330,7 +331,7 @@ TEST_F(TracerImplTest, ChildGrpcUpstreamSpanTest) { EXPECT_CALL(*span, setTag(Eq(Tracing::Tags::get().NodeId), Eq("node_name"))); auto parent_span = - tracer_->startSpan(config_, request_headers_, stream_info_, {Reason::Sampling, true}); + tracer_->startSpan(config_, trace_context_, stream_info_, {Reason::Sampling, true}); NiceMock* second_span = new NiceMock(); @@ -376,8 +377,7 @@ TEST_F(TracerImplTest, MetadataCustomTagReturnsDefaultValue) { *testing_metadata.mutable_default_value() = "default_value"; MetadataCustomTag tag("testing", testing_metadata); StreamInfo::MockStreamInfo testing_info_; - Http::TestRequestHeaderMapImpl header_map_; - CustomTagContext context{&header_map_, testing_info_}; + CustomTagContext context{trace_context_, testing_info_}; EXPECT_EQ(tag.value(context), "default_value"); } diff --git a/test/common/tracing/tracer_manager_impl_test.cc b/test/common/tracing/tracer_manager_impl_test.cc index e9eeb4ec6da2..6520fc2919d0 100644 --- a/test/common/tracing/tracer_manager_impl_test.cc +++ b/test/common/tracing/tracer_manager_impl_test.cc @@ -24,7 +24,7 @@ namespace { class SampleDriver : public Driver { public: SpanPtr startSpan(const Config&, Tracing::TraceContext&, const StreamInfo::StreamInfo&, - const std::string&, const Tracing::Decision) override { + const std::string&, Tracing::Decision) override { return nullptr; } }; diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 53c8d7143542..12fe0396553a 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -69,6 +69,8 @@ envoy_cc_test( "//envoy/upstream:cluster_manager_interface", "//source/extensions/clusters/eds:eds_lib", "//source/extensions/clusters/static:static_cluster_lib", + "//source/extensions/load_balancing_policies/ring_hash:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//test/mocks/config:config_mocks", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", @@ -100,10 +102,16 @@ envoy_cc_test( "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/least_request:config", "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/random:config", "//source/extensions/load_balancing_policies/ring_hash:config", + "//source/extensions/load_balancing_policies/round_robin:config", + "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/network/dns_resolver/cares:config", "//source/extensions/transport_sockets/tls:config", + "//source/extensions/upstreams/http/generic:config", "//test/config:v2_link_hacks", "//test/integration/load_balancers:custom_lb_policy", "//test/mocks/matcher:matcher_mocks", @@ -245,9 +253,9 @@ envoy_cc_test( "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", "//test/mocks/event:event_mocks", - "//test/mocks/upstream:cluster_info_mocks", "//test/mocks/upstream:host_mocks", "//test/mocks/upstream:load_balancer_context_mock", + "//test/test_common:stats_utility_lib", "//test/test_common:test_runtime_lib", ], ) @@ -473,7 +481,6 @@ envoy_cc_benchmark_binary( "//source/common/upstream:upstream_lib", "//source/extensions/load_balancing_policies/maglev:config", "//source/extensions/load_balancing_policies/ring_hash:config", - "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "//test/mocks/upstream:cluster_info_mocks", "//test/test_common:printers_lib", @@ -488,33 +495,6 @@ envoy_benchmark_test( benchmark_binary = "load_balancer_benchmark", ) -envoy_cc_test( - name = "subset_lb_test", - srcs = ["subset_lb_test.cc"], - deps = [ - ":utility_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/network:utility_lib", - "//source/common/upstream:load_balancer_lib", - "//source/common/upstream:upstream_includes", - "//source/common/upstream:upstream_lib", - "//source/extensions/load_balancing_policies/subset:config", - "//test/mocks:common_lib", - "//test/mocks/access_log:access_log_mocks", - "//test/mocks/filesystem:filesystem_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/mocks/upstream:cluster_info_mocks", - "//test/mocks/upstream:host_mocks", - "//test/mocks/upstream:host_set_mocks", - "//test/mocks/upstream:load_balancer_context_mock", - "//test/mocks/upstream:load_balancer_mocks", - "//test/mocks/upstream:priority_set_mocks", - "//test/test_common:simulated_time_system_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "transport_socket_matcher_test", srcs = ["transport_socket_matcher_test.cc"], @@ -575,6 +555,13 @@ envoy_cc_test( "//test/mocks/upstream:priority_set_mocks", "//test/mocks/upstream:thread_aware_load_balancer_mocks", "//test/mocks/upstream:typed_load_balancer_factory_mocks", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/least_request:config", + "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/random:config", + "//source/extensions/load_balancing_policies/ring_hash:config", + "//source/extensions/load_balancing_policies/round_robin:config", + "//source/extensions/load_balancing_policies/subset:config", "//test/test_common:registry_lib", "//test/test_common:utility_lib", ] + envoy_select_enable_http3([ @@ -617,7 +604,6 @@ envoy_cc_test_library( "//source/common/stats:stats_lib", "//source/common/upstream:cluster_factory_lib", "//source/common/upstream:cluster_manager_lib", - "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:context_lib", "//test/common/stats:stat_test_utility_lib", @@ -660,6 +646,7 @@ envoy_cc_test( "//source/common/upstream:cluster_factory_lib", "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/integration/clusters:custom_static_cluster", @@ -804,6 +791,7 @@ envoy_cc_test( srcs = ["local_address_selector_integration_test.cc"], deps = [ "test_local_address_selector", + "//source/extensions/load_balancing_policies/round_robin:config", "//test/integration:http_integration_lib", "//test/integration:http_protocol_integration_lib", "//test/mocks/upstream:cluster_info_mocks", diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index d0ebb8bcaaa9..bb0d2287c39d 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -136,11 +136,9 @@ TEST_F(CdsApiImplTest, ValidateDuplicateClusters) { EXPECT_CALL(cm_, clusters()).WillRepeatedly(Return(makeClusterInfoMaps({}))); EXPECT_CALL(initialized_, ready()); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, - "Error adding/updating cluster(s) duplicate_cluster: duplicate cluster " - "duplicate_cluster found"); + EXPECT_EQ(cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "Error adding/updating cluster(s) duplicate_cluster: duplicate cluster " + "duplicate_cluster found"); } TEST_F(CdsApiImplTest, EmptyConfigUpdate) { @@ -249,9 +247,8 @@ TEST_F(CdsApiImplTest, ConfigUpdateAddsSecondClusterEvenIfFirstThrows) { expectAddToThrow("cluster_3", "Another exception"); const auto decoded_resources = TestUtility::decodeResources({cluster_1, cluster_2, cluster_3}); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, + EXPECT_EQ( + cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), "Error adding/updating cluster(s) cluster_1: An exception, cluster_3: Another exception"); } @@ -362,10 +359,8 @@ version_info: '0' EXPECT_CALL(initialized_, ready()); const auto decoded_resources = TestUtility::decodeResources(response1); - EXPECT_THROW( - EXPECT_TRUE( - cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()).ok()), - EnvoyException); + EXPECT_FALSE( + cds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()).ok()); EXPECT_EQ("", cds_->versionInfo()); } diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index adf91036c986..9da16fb69b90 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -9,10 +9,12 @@ #include "source/common/network/resolver_impl.h" #include "source/common/router/context_impl.h" #include "source/common/upstream/load_balancer_factory_base.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "source/extensions/transport_sockets/raw_buffer/config.h" #include "test/common/upstream/test_cluster_manager.h" #include "test/config/v2_link_hacks.h" +#include "test/mocks/config/mocks.h" #include "test/mocks/http/conn_pool.h" #include "test/mocks/matcher/mocks.h" #include "test/mocks/protobuf/mocks.h" @@ -49,6 +51,7 @@ class TcpPoolDataPeer { namespace { +using ::envoy::config::bootstrap::v3::Bootstrap; using ::testing::_; using ::testing::DoAll; using ::testing::InSequence; @@ -63,8 +66,8 @@ using ::testing::SaveArg; using namespace std::chrono_literals; -envoy::config::bootstrap::v3::Bootstrap parseBootstrapFromV3Yaml(const std::string& yaml) { - envoy::config::bootstrap::v3::Bootstrap bootstrap; +Bootstrap parseBootstrapFromV3Yaml(const std::string& yaml) { + Bootstrap bootstrap; TestUtility::loadFromYaml(yaml, bootstrap); return bootstrap; } @@ -84,6 +87,118 @@ void verifyCaresDnsConfigAndUnpack( typed_dns_resolver_config.typed_config().UnpackTo(&cares); } +// Helper to intercept calls to postThreadLocalClusterUpdate. +class MockLocalClusterUpdate { +public: + MOCK_METHOD(void, post, + (uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed)); +}; + +class MockLocalHostsRemoved { +public: + MOCK_METHOD(void, post, (const HostVector&)); +}; + +// Override postThreadLocalClusterUpdate so we can test that merged updates calls +// it with the right values at the right times. +class MockedUpdatedClusterManagerImpl : public TestClusterManagerImpl { +public: + using TestClusterManagerImpl::TestClusterManagerImpl; + + MockedUpdatedClusterManagerImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + ClusterManagerFactory& factory, Stats::Store& stats, + ThreadLocal::Instance& tls, Runtime::Loader& runtime, + const LocalInfo::LocalInfo& local_info, + AccessLog::AccessLogManager& log_manager, + Event::Dispatcher& main_thread_dispatcher, Server::Admin& admin, + ProtobufMessage::ValidationContext& validation_context, + Api::Api& api, MockLocalClusterUpdate& local_cluster_update, + MockLocalHostsRemoved& local_hosts_removed, + Http::Context& http_context, Grpc::Context& grpc_context, + Router::Context& router_context, Server::Instance& server) + : TestClusterManagerImpl(bootstrap, factory, stats, tls, runtime, local_info, log_manager, + main_thread_dispatcher, admin, validation_context, api, http_context, + grpc_context, router_context, server), + local_cluster_update_(local_cluster_update), local_hosts_removed_(local_hosts_removed) {} + +protected: + void postThreadLocalClusterUpdate(ClusterManagerCluster&, + ThreadLocalClusterUpdateParams&& params) override { + for (const auto& per_priority : params.per_priority_update_params_) { + local_cluster_update_.post(per_priority.priority_, per_priority.hosts_added_, + per_priority.hosts_removed_); + } + } + + void postThreadLocalRemoveHosts(const Cluster&, const HostVector& hosts_removed) override { + local_hosts_removed_.post(hosts_removed); + } + + MockLocalClusterUpdate& local_cluster_update_; + MockLocalHostsRemoved& local_hosts_removed_; +}; + +// A gRPC MuxFactory that returns a MockGrpcMux instance when trying to instantiate a mux for the +// `envoy.config_mux.grpc_mux_factory` type. This enables testing call expectations on the ADS gRPC +// mux. +class MockGrpcMuxFactory : public Config::MuxFactory { +public: + std::string name() const override { return "envoy.config_mux.grpc_mux_factory"; } + void shutdownAll() override {} + std::shared_ptr + create(std::unique_ptr&&, Event::Dispatcher&, Random::RandomGenerator&, + Stats::Scope&, const envoy::config::core::v3::ApiConfigSource&, + const LocalInfo::LocalInfo&, std::unique_ptr&&, + BackOffStrategyPtr&&, OptRef, + OptRef, bool) override { + return std::make_shared>(); + } +}; + +// A ClusterManagerImpl that overrides postThreadLocalClusterUpdate set up call expectations on the +// ADS gRPC mux. The ADS mux should not be started until the ADS cluster has been initialized. This +// solves the problem outlined in https://github.com/envoyproxy/envoy/issues/27702. +class UpdateOverrideClusterManagerImpl : public TestClusterManagerImpl { +public: + UpdateOverrideClusterManagerImpl(const Bootstrap& bootstrap, ClusterManagerFactory& factory, + Stats::Store& stats, ThreadLocal::Instance& tls, + Runtime::Loader& runtime, const LocalInfo::LocalInfo& local_info, + AccessLog::AccessLogManager& log_manager, + Event::Dispatcher& main_thread_dispatcher, Server::Admin& admin, + ProtobufMessage::ValidationContext& validation_context, + Api::Api& api, Http::Context& http_context, + Grpc::Context& grpc_context, Router::Context& router_context, + Server::Instance& server) + : TestClusterManagerImpl(bootstrap, factory, stats, tls, runtime, local_info, log_manager, + main_thread_dispatcher, admin, validation_context, api, http_context, + grpc_context, router_context, server) {} + +protected: + void postThreadLocalClusterUpdate(ClusterManagerCluster& cluster, + ThreadLocalClusterUpdateParams&& params) override { + int expected_start_call_count = 0; + if (cluster.cluster().info()->name() == "ads_cluster") { + // For the ADS cluster, we expect that the postThreadLocalClusterUpdate call below will + // invoke the ADS mux's start() method. Subsequent calls to postThreadLocalClusterUpdate() + // should not invoke the ADS mux's start() method. + if (!post_tls_update_for_ads_cluster_called_) { + expected_start_call_count = 1; + } + post_tls_update_for_ads_cluster_called_ = true; + } + + EXPECT_CALL(dynamic_cast(*adsMux()), start()) + .Times(expected_start_call_count); + + // The ClusterManagerImpl::postThreadLocalClusterUpdate method calls ads_mux_->start() if the + // cluster is used in ADS for EnvoyGrpc. + TestClusterManagerImpl::postThreadLocalClusterUpdate(cluster, std::move(params)); + } + +private: + bool post_tls_update_for_ads_cluster_called_ = false; +}; + class ClusterManagerImplTest : public testing::Test { public: ClusterManagerImplTest() @@ -91,13 +206,36 @@ class ClusterManagerImplTest : public testing::Test { router_context_(factory_.stats_.symbolTable()), registered_dns_factory_(dns_resolver_factory_) {} - virtual void create(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - cluster_manager_ = std::make_unique( + virtual void create(const Bootstrap& bootstrap) { + cluster_manager_ = TestClusterManagerImpl::createAndInit( bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, *factory_.api_, http_context_, grpc_context_, router_context_, server_); - cluster_manager_->setPrimaryClustersInitializedCb( - [this, bootstrap]() { cluster_manager_->initializeSecondaryClusters(bootstrap); }); + cluster_manager_->setPrimaryClustersInitializedCb([this, bootstrap]() { + THROW_IF_NOT_OK(cluster_manager_->initializeSecondaryClusters(bootstrap)); + }); + } + + void createWithBasicStaticCluster() { + const std::string yaml = R"EOF( + static_resources: + clusters: + - name: cluster_1 + connect_timeout: 0.250s + lb_policy: ROUND_ROBIN + type: STATIC + load_assignment: + cluster_name: cluster_1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + + create(parseBootstrapFromV3Yaml(yaml)); } void createWithLocalClusterUpdate(const bool enable_merge_window = true) { @@ -141,6 +279,15 @@ class ClusterManagerImplTest : public testing::Test { factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, *factory_.api_, local_cluster_update_, local_hosts_removed_, http_context_, grpc_context_, router_context_, server_); + THROW_IF_NOT_OK(cluster_manager_->init(bootstrap)); + } + + void createWithUpdateOverrideClusterManager(const Bootstrap& bootstrap) { + cluster_manager_ = std::make_unique( + bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, + factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, + *factory_.api_, http_context_, grpc_context_, router_context_, server_); + THROW_IF_NOT_OK(cluster_manager_->init(bootstrap)); } void checkStats(uint64_t added, uint64_t modified, uint64_t removed, uint64_t active, @@ -198,7 +345,7 @@ class ClusterManagerImplTest : public testing::Test { Registry::InjectFactory registered_dns_factory_; }; -envoy::config::bootstrap::v3::Bootstrap defaultConfig() { +Bootstrap defaultConfig() { const std::string yaml = R"EOF( static_resources: clusters: [] @@ -585,6 +732,122 @@ TEST_F(ClusterManagerImplTest, OutlierEventLog) { create(parseBootstrapFromV3Json(json)); } +TEST_F(ClusterManagerImplTest, AdsCluster) { + MockGrpcMuxFactory factory; + Registry::InjectFactory registry(factory); + + const std::string yaml = R"EOF( + dynamic_resources: + ads_config: + transport_api_version: V3 + api_type: GRPC + set_node_on_first_message_only: true + grpc_services: + envoy_grpc: + cluster_name: ads_cluster + static_resources: + clusters: + - name: ads_cluster + connect_timeout: 0.250s + type: static + lb_policy: round_robin + load_assignment: + cluster_name: ads_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + + createWithUpdateOverrideClusterManager(parseBootstrapFromV3Yaml(yaml)); +} + +TEST_F(ClusterManagerImplTest, AdsClusterStartsMuxOnlyOnce) { + MockGrpcMuxFactory factory; + Registry::InjectFactory registry(factory); + + const std::string yaml = R"EOF( + dynamic_resources: + ads_config: + transport_api_version: V3 + api_type: GRPC + set_node_on_first_message_only: true + grpc_services: + envoy_grpc: + cluster_name: ads_cluster + static_resources: + clusters: + - name: ads_cluster + connect_timeout: 0.250s + type: static + lb_policy: round_robin + load_assignment: + cluster_name: ads_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + + createWithUpdateOverrideClusterManager(parseBootstrapFromV3Yaml(yaml)); + + const std::string update_static_ads_cluster_yaml = R"EOF( + name: ads_cluster + connect_timeout: 0.250s + type: static + lb_policy: round_robin + load_assignment: + cluster_name: ads_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 12001 + )EOF"; + + // The static ads_cluster should not be updated by the ClusterManager (only dynamic clusters can + // be added or updated outside of the Bootstrap config). + EXPECT_FALSE(cluster_manager_->addOrUpdateCluster( + parseClusterFromV3Yaml(update_static_ads_cluster_yaml), "version2")); + EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated").value()); + EXPECT_EQ(1, factory_.stats_ + .gauge("cluster_manager.active_clusters", Stats::Gauge::ImportMode::NeverImport) + .value()); + EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_modified").value()); + + const std::string new_cluster_yaml = R"EOF( + name: new_cluster + connect_timeout: 0.250s + type: static + lb_policy: round_robin + load_assignment: + cluster_name: new_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 12001 + )EOF"; + + EXPECT_TRUE( + cluster_manager_->addOrUpdateCluster(parseClusterFromV3Yaml(new_cluster_yaml), "version1")); + EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated").value()); + EXPECT_EQ(2, factory_.stats_ + .gauge("cluster_manager.active_clusters", Stats::Gauge::ImportMode::NeverImport) + .value()); + EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_modified").value()); +} + TEST_F(ClusterManagerImplTest, NoSdsConfig) { const std::string yaml = R"EOF( static_resources: @@ -1227,7 +1490,7 @@ TEST_F(ClusterManagerImplTest, VerifyBufferLimits) { class ClusterManagerLifecycleTest : public ClusterManagerImplTest, public testing::WithParamInterface { protected: - void create(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) override { + void create(const Bootstrap& bootstrap) override { if (useDeferredCluster()) { auto bootstrap_with_deferred_cluster = bootstrap; bootstrap_with_deferred_cluster.mutable_cluster_manager() @@ -2643,6 +2906,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemove) { // drain callbacks, etc. dns_timer_->invokeCallback(); dns_callback(Network::DnsResolver::ResolutionStatus::Success, + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.3"})); factory_.tls_.shutdownThread(); } @@ -2864,6 +3128,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveWithTls) { // drain callbacks, etc. dns_timer_->invokeCallback(); dns_callback(Network::DnsResolver::ResolutionStatus::Success, + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.3"})); factory_.tls_.shutdownThread(); } @@ -3684,6 +3949,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveDefaultPriority) { // Remove the first host, this should lead to the cp being drained, without // crash. dns_timer_->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) dns_callback(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({})); factory_.tls_.shutdownThread(); @@ -3768,6 +4034,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolDestroyWithDraining) { // Remove the first host, this should lead to the cp being drained. dns_timer_->invokeCallback(); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) dns_callback(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({})); // The drained callback might get called when the CP is being destroyed. @@ -3822,10 +4089,15 @@ TEST_F(ClusterManagerImplTest, OriginalDstInitialization) { // Also tests that if hosts are added/removed between mergeable updates, delivery will // happen and the scheduled update will be cancelled. TEST_P(ClusterManagerLifecycleTest, MergedUpdates) { - createWithLocalClusterUpdate(); - // Ensure we see the right set of added/removed hosts on every call. EXPECT_CALL(local_cluster_update_, post(_, _, _)) + .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, + const HostVector& hosts_removed) -> void { + // 1st add of the 2 static localhost endpoints. + EXPECT_EQ(0, priority); + EXPECT_EQ(2, hosts_added.size()); + EXPECT_EQ(0, hosts_removed.size()); + })) .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) -> void { // 1st removal. @@ -3862,6 +4134,8 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdates) { .WillRepeatedly( Invoke([](const auto& hosts_removed) { EXPECT_EQ(1, hosts_removed.size()); })); + createWithLocalClusterUpdate(); + Event::MockTimer* timer = new NiceMock(&factory_.dispatcher_); Cluster& cluster = cluster_manager_->activeClusters().begin()->second; HostVectorSharedPtr hosts( @@ -3959,10 +4233,15 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdates) { // Tests that mergeable updates outside of a window get applied immediately. TEST_P(ClusterManagerLifecycleTest, MergedUpdatesOutOfWindow) { - createWithLocalClusterUpdate(); - // Ensure we see the right set of added/removed hosts on every call. EXPECT_CALL(local_cluster_update_, post(_, _, _)) + .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, + const HostVector& hosts_removed) -> void { + // 2 static host endpoints on Bootstrap's cluster. + EXPECT_EQ(0, priority); + EXPECT_EQ(2, hosts_added.size()); + EXPECT_EQ(0, hosts_removed.size()); + })) .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) -> void { // HC update, immediately delivered. @@ -3971,6 +4250,8 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdatesOutOfWindow) { EXPECT_EQ(0, hosts_removed.size()); })); + createWithLocalClusterUpdate(); + Cluster& cluster = cluster_manager_->activeClusters().begin()->second; HostVectorSharedPtr hosts( new HostVector(cluster.prioritySet().hostSetsPerPriority()[0]->hosts())); @@ -3995,6 +4276,14 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdatesOutOfWindow) { // Tests that mergeable updates inside of a window are not applied immediately. TEST_P(ClusterManagerLifecycleTest, MergedUpdatesInsideWindow) { + EXPECT_CALL(local_cluster_update_, post(_, _, _)) + .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, + const HostVector& hosts_removed) -> void { + EXPECT_EQ(0, priority); + EXPECT_EQ(2, hosts_added.size()); + EXPECT_EQ(0, hosts_removed.size()); + })); + createWithLocalClusterUpdate(); Cluster& cluster = cluster_manager_->activeClusters().begin()->second; @@ -4023,10 +4312,15 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdatesInsideWindow) { // Tests that mergeable updates outside of a window get applied immediately when // merging is disabled, and that the counters are correct. TEST_P(ClusterManagerLifecycleTest, MergedUpdatesOutOfWindowDisabled) { - createWithLocalClusterUpdate(false); - // Ensure we see the right set of added/removed hosts on every call. EXPECT_CALL(local_cluster_update_, post(_, _, _)) + .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, + const HostVector& hosts_removed) -> void { + // 2 static host endpoints on Bootstrap's cluster. + EXPECT_EQ(0, priority); + EXPECT_EQ(2, hosts_added.size()); + EXPECT_EQ(0, hosts_removed.size()); + })) .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) -> void { // HC update, immediately delivered. @@ -4035,6 +4329,8 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdatesOutOfWindowDisabled) { EXPECT_EQ(0, hosts_removed.size()); })); + createWithLocalClusterUpdate(false); + Cluster& cluster = cluster_manager_->activeClusters().begin()->second; HostVectorSharedPtr hosts( new HostVector(cluster.prioritySet().hostSetsPerPriority()[0]->hosts())); @@ -4056,33 +4352,40 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdatesOutOfWindowDisabled) { } TEST_P(ClusterManagerLifecycleTest, MergedUpdatesDestroyedOnUpdate) { - // We create the default cluster, although for this test we won't use it since - // we can only update dynamic clusters. - createWithLocalClusterUpdate(); - // Ensure we see the right set of added/removed hosts on every call, for the // dynamically added/updated cluster. EXPECT_CALL(local_cluster_update_, post(_, _, _)) .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) -> void { - // 1st add, when the cluster is added. + // 1st add, which adds 2 localhost static hosts at ports 11001 and 11002. + EXPECT_EQ(0, priority); + EXPECT_EQ(2, hosts_added.size()); + EXPECT_EQ(0, hosts_removed.size()); + })) + .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, + const HostVector& hosts_removed) -> void { + // 2nd add, when the dynamic `new_cluster` is added with static host 127.0.0.1:12001. EXPECT_EQ(0, priority); EXPECT_EQ(1, hosts_added.size()); EXPECT_EQ(0, hosts_removed.size()); })) .WillOnce(Invoke([](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) -> void { - // 1st removal. + // 1st removal of the dynamic `new_cluster`. EXPECT_EQ(0, priority); EXPECT_EQ(0, hosts_added.size()); EXPECT_EQ(1, hosts_removed.size()); })); EXPECT_CALL(local_hosts_removed_, post(_)).WillOnce(Invoke([](const auto& hosts_removed) { - // 1st removal. + // 1st removal of the dynamic `new_cluster`. EXPECT_EQ(1, hosts_removed.size()); })); + // We create the default cluster, although for this test we won't use it since + // we can only update dynamic clusters. + createWithLocalClusterUpdate(); + Event::MockTimer* timer = new NiceMock(&factory_.dispatcher_); // We can't used the bootstrap cluster, so add one dynamically. @@ -4195,7 +4498,7 @@ TEST_P(ClusterManagerLifecycleTest, MergedUpdatesDestroyedOnUpdate) { } TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsPassedToTcpConnPool) { - createWithLocalClusterUpdate(); + createWithBasicStaticCluster(); NiceMock context; auto to_create = new Tcp::ConnectionPool::MockInstance(); @@ -4212,12 +4515,12 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsPassedToTcpConnPool) { } TEST_F(ClusterManagerImplTest, SelectOverrideHostTestNoOverrideHost) { - createWithLocalClusterUpdate(); + createWithBasicStaticCluster(); NiceMock context; auto to_create = new Tcp::ConnectionPool::MockInstance(); - EXPECT_CALL(context, overrideHostToSelect()).WillOnce(Return(absl::nullopt)); + EXPECT_CALL(context, overrideHostToSelect()).Times(2).WillRepeatedly(Return(absl::nullopt)); EXPECT_CALL(factory_, allocateTcpConnPool_(_)).WillOnce(Return(to_create)); EXPECT_CALL(*to_create, addIdleCallback(_)); @@ -4228,17 +4531,18 @@ TEST_F(ClusterManagerImplTest, SelectOverrideHostTestNoOverrideHost) { } TEST_F(ClusterManagerImplTest, SelectOverrideHostTestWithOverrideHost) { - createWithLocalClusterUpdate(); + createWithBasicStaticCluster(); NiceMock context; auto to_create = new Tcp::ConnectionPool::MockInstance(); EXPECT_CALL(context, overrideHostToSelect()) - .WillRepeatedly(Return(absl::make_optional("127.0.0.1:11002"))); + .WillRepeatedly(Return(absl::make_optional( + std::make_pair("127.0.0.1:11001", false)))); EXPECT_CALL(factory_, allocateTcpConnPool_(_)) .WillOnce(testing::Invoke([&](HostConstSharedPtr host) { - EXPECT_EQ("127.0.0.1:11002", host->address()->asStringView()); + EXPECT_EQ("127.0.0.1:11001", host->address()->asStringView()); return to_create; })); EXPECT_CALL(*to_create, addIdleCallback(_)); @@ -4256,8 +4560,51 @@ TEST_F(ClusterManagerImplTest, SelectOverrideHostTestWithOverrideHost) { EXPECT_TRUE(opt_cp_3.has_value()); } +TEST_F(ClusterManagerImplTest, SelectOverrideHostTestWithNonExistingHost) { + createWithBasicStaticCluster(); + NiceMock context; + + auto to_create = new Tcp::ConnectionPool::MockInstance(); + + EXPECT_CALL(context, overrideHostToSelect()) + .WillRepeatedly(Return(absl::make_optional( + // Return non-existing host. Let the LB choose the host. + std::make_pair("127.0.0.2:12345", false)))); + + EXPECT_CALL(factory_, allocateTcpConnPool_(_)) + .WillOnce(testing::Invoke([&](HostConstSharedPtr host) { + EXPECT_THAT(host->address()->asStringView(), + testing::AnyOf("127.0.0.1:11001", "127.0.0.1:11002")); + return to_create; + })); + EXPECT_CALL(*to_create, addIdleCallback(_)); + + auto opt_cp = cluster_manager_->getThreadLocalCluster("cluster_1") + ->tcpConnPool(ResourcePriority::Default, &context); + EXPECT_TRUE(opt_cp.has_value()); +} + +TEST_F(ClusterManagerImplTest, SelectOverrideHostTestWithNonExistingHostStrict) { + createWithBasicStaticCluster(); + NiceMock context; + + EXPECT_CALL(context, overrideHostToSelect()) + .WillRepeatedly(Return(absl::make_optional( + // Return non-existing host and indicate strict mode. + // LB should not be allowed to choose host. + std::make_pair("127.0.0.2:12345", true)))); + + // Requested upstream host 127.0.0.2:12345 is not part of the cluster. + // Connection pool should not be created. + EXPECT_CALL(factory_, allocateTcpConnPool_(_)).Times(0); + + auto opt_cp = cluster_manager_->getThreadLocalCluster("cluster_1") + ->tcpConnPool(ResourcePriority::Default, &context); + EXPECT_FALSE(opt_cp.has_value()); +} + TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsPassedToConnPool) { - createWithLocalClusterUpdate(); + createWithBasicStaticCluster(); NiceMock context; Http::ConnectionPool::MockInstance* to_create = @@ -4274,51 +4621,60 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsPassedToConnPool) { } TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsUsedInConnPoolHash) { - createWithLocalClusterUpdate(); NiceMock context1; NiceMock context2; + createWithBasicStaticCluster(); + + // NOTE: many of these socket options are not available on Windows, so if changing the socket + // options used below, ensure they are available on Windows and Linux, as otherwise, the option + // won't affect the hash key (because if the socket option isn't available on the given platform, + // an empty SocketOptionImpl is used instead) and the test will end up using the same connection + // pool for different options, instead of different connection pool. + // See https://github.com/envoyproxy/envoy/issues/30360 for details. + Network::Socket::OptionsSharedPtr options1 = + Network::SocketOptionFactory::buildTcpKeepaliveOptions({}); + Network::Socket::OptionsSharedPtr options2 = + Network::SocketOptionFactory::buildIpPacketInfoOptions(); Http::ConnectionPool::MockInstance* to_create1 = new NiceMock(); Http::ConnectionPool::MockInstance* to_create2 = new NiceMock(); - Network::Socket::OptionsSharedPtr options1 = - Network::SocketOptionFactory::buildIpTransparentOptions(); - Network::Socket::OptionsSharedPtr options2 = - Network::SocketOptionFactory::buildSocketMarkOptions(3); - EXPECT_CALL(context1, upstreamSocketOptions()).WillRepeatedly(Return(options1)); - EXPECT_CALL(context2, upstreamSocketOptions()).WillRepeatedly(Return(options2)); + EXPECT_CALL(context1, upstreamSocketOptions()).WillOnce(Return(options1)); EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(to_create1)); - Http::ConnectionPool::Instance* cp1 = HttpPoolDataPeer::getPool( cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context1)); + EXPECT_NE(nullptr, cp1); + EXPECT_CALL(context2, upstreamSocketOptions()).WillOnce(Return(options2)); EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(to_create2)); Http::ConnectionPool::Instance* cp2 = HttpPoolDataPeer::getPool( cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context2)); + EXPECT_NE(nullptr, cp2); + // The different upstream options should lead to different hashKeys, thus different pools. + EXPECT_NE(cp1, cp2); + + EXPECT_CALL(context1, upstreamSocketOptions()).WillOnce(Return(options1)); Http::ConnectionPool::Instance* should_be_cp1 = HttpPoolDataPeer::getPool( cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context1)); - EXPECT_NE(nullptr, cp1); - EXPECT_NE(nullptr, cp2); + + EXPECT_CALL(context2, upstreamSocketOptions()).WillOnce(Return(options2)); Http::ConnectionPool::Instance* should_be_cp2 = HttpPoolDataPeer::getPool( cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context2)); - // The different upstream options should lead to different hashKeys, thus different pools. - EXPECT_NE(cp1, cp2); - // Reusing the same options should lead to the same connection pools. EXPECT_EQ(cp1, should_be_cp1); EXPECT_EQ(cp2, should_be_cp2); } TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsNullIsOkay) { - createWithLocalClusterUpdate(); + createWithBasicStaticCluster(); NiceMock context; Http::ConnectionPool::MockInstance* to_create = @@ -4334,7 +4690,7 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsNullIsOkay) { } TEST_F(ClusterManagerImplTest, HttpPoolDataForwardsCallsToConnectionPool) { - createWithLocalClusterUpdate(); + createWithBasicStaticCluster(); NiceMock context; Http::ConnectionPool::MockInstance* pool_mock = new Http::ConnectionPool::MockInstance(); @@ -4457,7 +4813,7 @@ class TestUpstreamNetworkFilterConfigFactory std::string name() const override { return "envoy.test.filter"; } }; -// Verify that configured upstream filters are added to client connections. +// Verify that configured upstream network filters are added to client connections. TEST_F(ClusterManagerImplTest, AddUpstreamFilters) { TestUpstreamNetworkFilterConfigFactory factory; Registry::InjectFactory registry( @@ -4516,9 +4872,11 @@ class MockClusterManagerCluster : public ClusterManagerCluster { ASSERT(!added_or_updated_); added_or_updated_ = true; } + bool requiredForAds() const override { return required_for_ads_; } NiceMock cluster_; bool added_or_updated_{}; + bool required_for_ads_{}; }; TEST_F(ClusterManagerInitHelperTest, ImmediateInitialize) { @@ -6001,11 +6359,11 @@ TEST_F(ClusterManagerImplTest, CheckActiveStaticCluster) { cluster_manager_->addOrUpdateCluster(parseClusterFromV3Yaml(added_via_api_yaml), "v1")); EXPECT_EQ(2, cluster_manager_->clusters().active_clusters_.size()); - EXPECT_NO_THROW(cluster_manager_->checkActiveStaticCluster("good")); - EXPECT_THROW_WITH_MESSAGE(cluster_manager_->checkActiveStaticCluster("nonexist"), EnvoyException, - "Unknown gRPC client cluster 'nonexist'"); - EXPECT_THROW_WITH_MESSAGE(cluster_manager_->checkActiveStaticCluster("added_via_api"), - EnvoyException, "gRPC client cluster 'added_via_api' is not static"); + EXPECT_TRUE(cluster_manager_->checkActiveStaticCluster("good").ok()); + EXPECT_EQ(cluster_manager_->checkActiveStaticCluster("nonexist").message(), + "Unknown gRPC client cluster 'nonexist'"); + EXPECT_EQ(cluster_manager_->checkActiveStaticCluster("added_via_api").message(), + "gRPC client cluster 'added_via_api' is not static"); } #ifdef WIN32 @@ -6050,7 +6408,7 @@ class PreconnectTest : public ClusterManagerImplTest { ReadyWatcher initialized; EXPECT_CALL(initialized, ready()); - envoy::config::bootstrap::v3::Bootstrap config = parseBootstrapFromV3Yaml(yaml); + Bootstrap config = parseBootstrapFromV3Yaml(yaml); if (ratio != 0) { config.mutable_static_resources() ->mutable_clusters(0) @@ -6144,7 +6502,8 @@ TEST_F(PreconnectTest, PreconnectOnWithOverrideHost) { NiceMock context; EXPECT_CALL(context, overrideHostToSelect()) - .WillRepeatedly(Return(absl::make_optional("127.0.0.1:80"))); + .WillRepeatedly(Return(absl::make_optional( + std::make_pair("127.0.0.1:80", false)))); // Only allocate connection pool once. EXPECT_CALL(factory_, allocateTcpConnPool_) diff --git a/test/common/upstream/default_local_address_selector_test.cc b/test/common/upstream/default_local_address_selector_test.cc index ba2d54cd381b..7653a4034ea1 100644 --- a/test/common/upstream/default_local_address_selector_test.cc +++ b/test/common/upstream/default_local_address_selector_test.cc @@ -18,9 +18,10 @@ namespace { TEST(ConfigTest, EmptyUpstreamAddresses) { DefaultUpstreamLocalAddressSelectorFactory factory; std::vector upstream_local_addresses; - EXPECT_THROW_WITH_MESSAGE( - factory.createLocalAddressSelector(upstream_local_addresses, absl::nullopt), EnvoyException, - "Bootstrap's upstream binding config has no valid source address."); + EXPECT_EQ(factory.createLocalAddressSelector(upstream_local_addresses, absl::nullopt) + .status() + .message(), + "Bootstrap's upstream binding config has no valid source address."); } TEST(ConfigTest, NullUpstreamAddress) { @@ -30,8 +31,8 @@ TEST(ConfigTest, NullUpstreamAddress) { UpstreamLocalAddress{nullptr, std::make_shared()}); // This should be exception free. UpstreamLocalAddressSelectorConstSharedPtr selector = - factory.createLocalAddressSelector(upstream_local_addresses, absl::nullopt); -} + factory.createLocalAddressSelector(upstream_local_addresses, absl::nullopt).value(); +} // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) } // namespace } // namespace Upstream diff --git a/test/common/upstream/deferred_cluster_initialization_test.cc b/test/common/upstream/deferred_cluster_initialization_test.cc index 30da60f9b69c..a9701fc77c25 100644 --- a/test/common/upstream/deferred_cluster_initialization_test.cc +++ b/test/common/upstream/deferred_cluster_initialization_test.cc @@ -62,12 +62,13 @@ class DeferredClusterInitializationTest : public testing::TestWithParam { router_context_(factory_.stats_.symbolTable()) {} void create(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - cluster_manager_ = std::make_unique( + cluster_manager_ = TestClusterManagerImpl::createAndInit( bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, *factory_.api_, http_context_, grpc_context_, router_context_, server_); - cluster_manager_->setPrimaryClustersInitializedCb( - [this, bootstrap]() { cluster_manager_->initializeSecondaryClusters(bootstrap); }); + cluster_manager_->setPrimaryClustersInitializedCb([this, bootstrap]() { + THROW_IF_NOT_OK(cluster_manager_->initializeSecondaryClusters(bootstrap)); + }); } ClusterType getStaticClusterType() const { @@ -516,6 +517,85 @@ TEST_P(EdsTest, ShouldMergeAddingHosts) { {1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000})); } +TEST_P(EdsTest, ShouldNotMergeAddingHostsForDifferentClustersWithSameName) { + const std::string bootstrap_yaml = R"EOF( + static_resources: + clusters: + - name: eds + connect_timeout: 0.250s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: bootstrap_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 60000 + )EOF"; + + auto bootstrap = parseBootstrapFromV3YamlEnableDeferredCluster(bootstrap_yaml); + create(bootstrap); + + EXPECT_CALL(factory_, create(_)) + .Times(2) + .WillRepeatedly( + testing::Invoke([this](Config::ConfigSubscriptionFactory::SubscriptionData& data) { + callbacks_ = &data.callbacks_; + return std::make_unique>(); + })); + + const std::string eds_cluster_yaml = R"EOF( + name: cluster_1 + connect_timeout: 0.25s + lb_policy: RING_HASH + eds_cluster_config: + service_name: fare + eds_config: + api_config_source: + api_type: REST + transport_api_version: V3 + cluster_names: + - eds + refresh_delay: 1s + )EOF"; + auto cluster = parseClusterFromV3Yaml(eds_cluster_yaml, getEdsClusterType()); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(cluster, "version1")); + + envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment; + cluster_load_assignment.set_cluster_name("fare"); + addEndpoint(cluster_load_assignment, 1000); + doOnConfigUpdateVerifyNoThrow(cluster_load_assignment); + + auto initiailization_instance = + cluster_manager_->clusterInitializationMap().find("cluster_1")->second; + EXPECT_NE(nullptr, initiailization_instance->load_balancer_factory_); + // RING_HASH lb policy requires Envoy re-create the load balancer when the cluster is updated. + EXPECT_TRUE(initiailization_instance->load_balancer_factory_->recreateOnHostChange()); + + // Update the cluster with a different lb policy. Now it's a different cluster and should + // not be merged. + cluster.set_lb_policy(::envoy::config::cluster::v3::Cluster::ROUND_ROBIN); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(cluster, "version2")); + + // Because the eds_service_name is the same, we can reuse the same load assignment here. + cluster_load_assignment.clear_endpoints(); + // Note we only add one endpoint to the priority 1 to the new cluster. + addEndpoint(cluster_load_assignment, 100, 1); + doOnConfigUpdateVerifyNoThrow(cluster_load_assignment); + + auto new_initialization_instance = + cluster_manager_->clusterInitializationMap().find("cluster_1")->second; + EXPECT_NE(initiailization_instance.get(), new_initialization_instance.get()); + + EXPECT_EQ(1, new_initialization_instance->per_priority_state_.at(1).hosts_added_.size()); + // Ensure the hosts_added_ is empty for priority 0. Because if unexpected merge happens, + // the hosts_added_ will be non-empty. + EXPECT_TRUE(!new_initialization_instance->per_priority_state_.contains(0) || + new_initialization_instance->per_priority_state_.at(0).hosts_added_.empty()); +} + // Test that removed hosts do not appear when initializing a deferred eds cluster. TEST_P(EdsTest, ShouldNotHaveRemovedHosts) { const std::string bootstrap_yaml = R"EOF( diff --git a/test/common/upstream/host_utility_test.cc b/test/common/upstream/host_utility_test.cc index 4130d55dd8d9..e0df8d442fd6 100644 --- a/test/common/upstream/host_utility_test.cc +++ b/test/common/upstream/host_utility_test.cc @@ -4,14 +4,18 @@ #include "test/common/upstream/utility.h" #include "test/mocks/common.h" -#include "test/mocks/upstream/cluster_info.h" #include "test/mocks/upstream/host.h" #include "test/mocks/upstream/load_balancer_context.h" +#include "test/test_common/stats_utility.h" #include "test/test_common/test_runtime.h" #include "gmock/gmock.h" +#include "gtest/gtest.h" +using ::testing::Contains; using ::testing::Return; +using ::testing::ReturnRef; +using ::testing::UnorderedElementsAreArray; namespace Envoy { namespace Upstream { @@ -171,206 +175,302 @@ TEST(HostUtilityTest, SelectOverrideHostTest) { EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, &context)); } - { - // The host map does not contain the expected host. - LoadBalancerContext::OverrideHost override_host{"1.2.3.4"}; - EXPECT_CALL(context, overrideHostToSelect()) - .WillOnce(Return(absl::make_optional(override_host))); - auto host_map = std::make_shared(); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); - } - { - auto mock_host = std::make_shared>(); - EXPECT_CALL(*mock_host, healthStatus()) - .WillRepeatedly(Return(envoy::config::core::v3::HealthStatus::UNHEALTHY)); - - LoadBalancerContext::OverrideHost override_host{"1.2.3.4"}; - EXPECT_CALL(context, overrideHostToSelect()) - .WillRepeatedly(Return(absl::make_optional(override_host))); - - auto host_map = std::make_shared(); - host_map->insert({"1.2.3.4", mock_host}); - EXPECT_EQ(mock_host, - HostUtility::selectOverrideHost(host_map.get(), UnhealthyStatus, &context)); - EXPECT_EQ(mock_host, - HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, &context)); - - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), DegradedStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), TimeoutStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), DrainingStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), UnknownStatus, &context)); + // Test overriding host in strict and non-strict mode. + for (const bool strict_mode : {false, true}) { + { + // The host map does not contain the expected host. + LoadBalancerContext::OverrideHost override_host{"1.2.3.4", strict_mode}; + EXPECT_CALL(context, overrideHostToSelect()) + .WillOnce(Return(absl::make_optional(override_host))); + auto host_map = std::make_shared(); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); + } + { + auto mock_host = std::make_shared>(); + EXPECT_CALL(*mock_host, healthStatus()) + .WillRepeatedly(Return(envoy::config::core::v3::HealthStatus::UNHEALTHY)); + + LoadBalancerContext::OverrideHost override_host{"1.2.3.4", strict_mode}; + EXPECT_CALL(context, overrideHostToSelect()) + .WillRepeatedly(Return(absl::make_optional(override_host))); + + auto host_map = std::make_shared(); + host_map->insert({"1.2.3.4", mock_host}); + + EXPECT_EQ(mock_host, + HostUtility::selectOverrideHost(host_map.get(), UnhealthyStatus, &context)); + EXPECT_EQ(mock_host, + HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, &context)); + + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), DegradedStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), TimeoutStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), DrainingStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), UnknownStatus, &context)); + } + { + auto mock_host = std::make_shared>(); + EXPECT_CALL(*mock_host, healthStatus()) + .WillRepeatedly(Return(envoy::config::core::v3::HealthStatus::DEGRADED)); + + LoadBalancerContext::OverrideHost override_host{"1.2.3.4", strict_mode}; + EXPECT_CALL(context, overrideHostToSelect()) + .WillRepeatedly(Return(absl::make_optional(override_host))); + + auto host_map = std::make_shared(); + host_map->insert({"1.2.3.4", mock_host}); + EXPECT_EQ(mock_host, + HostUtility::selectOverrideHost(host_map.get(), DegradedStatus, &context)); + EXPECT_EQ(mock_host, + HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, &context)); + + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); + EXPECT_EQ(nullptr, + HostUtility::selectOverrideHost(host_map.get(), UnhealthyStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), TimeoutStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), DrainingStatus, &context)); + EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), UnknownStatus, &context)); + } } - { - auto mock_host = std::make_shared>(); - EXPECT_CALL(*mock_host, healthStatus()) - .WillRepeatedly(Return(envoy::config::core::v3::HealthStatus::DEGRADED)); +} - LoadBalancerContext::OverrideHost override_host{"1.2.3.4"}; - EXPECT_CALL(context, overrideHostToSelect()) - .WillRepeatedly(Return(absl::make_optional(override_host))); +class PerEndpointMetricsTest : public testing::Test, public PerEndpointMetricsTestHelper { +public: + std::pair, + std::vector> + run() { + std::vector counters; + std::vector gauges; + HostUtility::forEachHostMetric( + cm_, + [&](Stats::PrimitiveCounterSnapshot&& metric) { counters.emplace_back(std::move(metric)); }, + [&](Stats::PrimitiveGaugeSnapshot&& metric) { gauges.emplace_back(std::move(metric)); }); + + return std::make_pair(counters, gauges); + } +}; + +template +std::vector metricNames(const std::vector& metrics) { + std::vector names; + names.reserve(metrics.size()); + for (const auto& metric : metrics) { + names.push_back(metric.name()); + } + return names; +} - auto host_map = std::make_shared(); - host_map->insert({"1.2.3.4", mock_host}); - EXPECT_EQ(mock_host, HostUtility::selectOverrideHost(host_map.get(), DegradedStatus, &context)); - EXPECT_EQ(mock_host, - HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, &context)); +template +std::vector> +metricNamesAndValues(const std::vector& metrics) { + std::vector> ret; + ret.reserve(metrics.size()); + for (const auto& metric : metrics) { + ret.push_back(std::make_pair(metric.name(), metric.value())); + } + return ret; +} - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), UnhealthyStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), TimeoutStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), DrainingStatus, &context)); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), UnknownStatus, &context)); +template +const MetricType& getMetric(absl::string_view name, const std::vector& metrics) { + for (const auto& metric : metrics) { + if (metric.name() == name) { + return metric; + } } + PANIC("not found"); } -TEST(HostUtilityTest, CreateOverrideHostStatusWithRuntimeFlagFlase) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.validate_detailed_override_host_statuses", "false"}}); +TEST_F(PerEndpointMetricsTest, Basic) { + makeCluster("mycluster", 1); + auto [counters, gauges] = run(); + + EXPECT_THAT(metricNamesAndValues(counters), + UnorderedElementsAreArray({ + std::make_pair("cluster.mycluster.endpoint.127.0.0.1_80.c1", 11), + std::make_pair("cluster.mycluster.endpoint.127.0.0.1_80.c2", 12), + })); + EXPECT_THAT(metricNamesAndValues(gauges), + UnorderedElementsAreArray({ + std::make_pair("cluster.mycluster.endpoint.127.0.0.1_80.g1", 13), + std::make_pair("cluster.mycluster.endpoint.127.0.0.1_80.g2", 14), + std::make_pair("cluster.mycluster.endpoint.127.0.0.1_80.healthy", 1), + })); +} - // Deprecated status that will be removed after runtime flag is removed. - static constexpr HostUtility::HostStatusSet UnhealthyStatus = - 1u << static_cast(Host::Health::Unhealthy); - static constexpr HostUtility::HostStatusSet DegradedStatus = - 1u << static_cast(Host::Health::Degraded); - static constexpr HostUtility::HostStatusSet HealthyStatus = - 1u << static_cast(Host::Health::Healthy); +// Warming clusters are not included +TEST_F(PerEndpointMetricsTest, Warming) { + makeCluster("mycluster", 1); + makeCluster("warming", 1, true /* warming */); - { - envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config; - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::UNKNOWN); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::HEALTHY); - EXPECT_EQ(HostUtility::createOverrideHostStatus(lb_config), HealthyStatus); - } - { - envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config; - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::UNHEALTHY); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::DRAINING); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::TIMEOUT); + auto [counters, gauges] = run(); - EXPECT_EQ(HostUtility::createOverrideHostStatus(lb_config), UnhealthyStatus); - } - { - envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config; - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::DEGRADED); - EXPECT_EQ(HostUtility::createOverrideHostStatus(lb_config), DegradedStatus); - } - { - envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config; - EXPECT_EQ(HostUtility::createOverrideHostStatus(lb_config), 0b110u); - } - { - envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config; - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::UNHEALTHY); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::DRAINING); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::TIMEOUT); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::UNKNOWN); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::HEALTHY); + EXPECT_THAT(metricNames(counters), + UnorderedElementsAreArray({"cluster.mycluster.endpoint.127.0.0.1_80.c1", + "cluster.mycluster.endpoint.127.0.0.1_80.c2"})); + EXPECT_THAT(metricNames(gauges), + UnorderedElementsAreArray({"cluster.mycluster.endpoint.127.0.0.1_80.g1", + "cluster.mycluster.endpoint.127.0.0.1_80.g2", + "cluster.mycluster.endpoint.127.0.0.1_80.healthy"})); +} - EXPECT_EQ(HostUtility::createOverrideHostStatus(lb_config), 0b101u); - } +TEST_F(PerEndpointMetricsTest, HealthyGaugeUnhealthy) { + auto& cluster = makeCluster("mycluster", 0); + auto& host = addHost(cluster); + EXPECT_CALL(host, coarseHealth()).WillOnce(Return(Host::Health::Unhealthy)); + auto [counters, gauges] = run(); + EXPECT_EQ(getMetric("cluster.mycluster.endpoint.127.0.0.1_80.healthy", gauges).value(), 0); +} - { - envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config; - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::UNHEALTHY); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::DRAINING); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::TIMEOUT); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::UNKNOWN); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::HEALTHY); - lb_config.mutable_override_host_status()->add_statuses( - ::envoy::config::core::v3::HealthStatus::DEGRADED); - EXPECT_EQ(HostUtility::createOverrideHostStatus(lb_config), 0b111u); - } +TEST_F(PerEndpointMetricsTest, HealthyGaugeDegraded) { + auto& cluster = makeCluster("mycluster", 0); + auto& host = addHost(cluster); + EXPECT_CALL(host, coarseHealth()).WillOnce(Return(Host::Health::Degraded)); + auto [counters, gauges] = run(); + EXPECT_EQ(getMetric("cluster.mycluster.endpoint.127.0.0.1_80.healthy", gauges).value(), 0); } -TEST(HostUtilityTest, SelectOverrideHostTestRuntimeFlagFlase) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.validate_detailed_override_host_statuses", "false"}}); +TEST_F(PerEndpointMetricsTest, MultipleClustersAndHosts) { + makeCluster("cluster1", 2); + makeCluster("cluster2", 3); + + auto [counters, gauges] = run(); + + EXPECT_THAT(metricNamesAndValues(counters), + UnorderedElementsAreArray({ + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.c1", 11), + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.c2", 12), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.c1", 21), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.c2", 22), + std::make_pair("cluster.cluster2.endpoint.127.0.0.3_80.c1", 31), + std::make_pair("cluster.cluster2.endpoint.127.0.0.3_80.c2", 32), + std::make_pair("cluster.cluster2.endpoint.127.0.0.4_80.c1", 41), + std::make_pair("cluster.cluster2.endpoint.127.0.0.4_80.c2", 42), + std::make_pair("cluster.cluster2.endpoint.127.0.0.5_80.c1", 51), + std::make_pair("cluster.cluster2.endpoint.127.0.0.5_80.c2", 52), + })); + EXPECT_THAT(metricNamesAndValues(gauges), + UnorderedElementsAreArray({ + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.g1", 13), + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.g2", 14), + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.healthy", 1), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.g1", 23), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.g2", 24), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.healthy", 1), + std::make_pair("cluster.cluster2.endpoint.127.0.0.3_80.g1", 33), + std::make_pair("cluster.cluster2.endpoint.127.0.0.3_80.g2", 34), + std::make_pair("cluster.cluster2.endpoint.127.0.0.3_80.healthy", 1), + std::make_pair("cluster.cluster2.endpoint.127.0.0.4_80.g1", 43), + std::make_pair("cluster.cluster2.endpoint.127.0.0.4_80.g2", 44), + std::make_pair("cluster.cluster2.endpoint.127.0.0.4_80.healthy", 1), + std::make_pair("cluster.cluster2.endpoint.127.0.0.5_80.g1", 53), + std::make_pair("cluster.cluster2.endpoint.127.0.0.5_80.g2", 54), + std::make_pair("cluster.cluster2.endpoint.127.0.0.5_80.healthy", 1), + })); +} - // Deprecated status that will be removed after runtime flag is removed. - static constexpr HostUtility::HostStatusSet UnhealthyStatus = - 1u << static_cast(Host::Health::Unhealthy); - static constexpr HostUtility::HostStatusSet DegradedStatus = - 1u << static_cast(Host::Health::Degraded); - static constexpr HostUtility::HostStatusSet HealthyStatus = - 1u << static_cast(Host::Health::Healthy); +TEST_F(PerEndpointMetricsTest, MultiplePriorityLevels) { + auto& cluster = makeCluster("cluster1", 1); + addHost(cluster, 2 /* non-default priority level */); + + auto [counters, gauges] = run(); + + EXPECT_THAT(metricNamesAndValues(counters), + UnorderedElementsAreArray({ + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.c1", 11), + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.c2", 12), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.c1", 21), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.c2", 22), + })); + EXPECT_THAT(metricNamesAndValues(gauges), + UnorderedElementsAreArray({ + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.g1", 13), + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.g2", 14), + std::make_pair("cluster.cluster1.endpoint.127.0.0.1_80.healthy", 1), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.g1", 23), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.g2", 24), + std::make_pair("cluster.cluster1.endpoint.127.0.0.2_80.healthy", 1), + })); +} - NiceMock context; +TEST_F(PerEndpointMetricsTest, Tags) { + auto& cluster = makeCluster("cluster1", 0); + auto& host1 = addHost(cluster); + std::string hostname = "host.example.com"; + EXPECT_CALL(host1, hostname()).WillOnce(ReturnRef(hostname)); + addHost(cluster); + + auto [counters, gauges] = run(); + + // Only the first host has a hostname, so only it has that tag. + EXPECT_THAT(getMetric("cluster.cluster1.endpoint.127.0.0.1_80.c1", counters).tags(), + UnorderedElementsAreArray({ + Stats::Tag{"envoy.cluster_name", "cluster1"}, + Stats::Tag{"envoy.endpoint_address", "127.0.0.1:80"}, + Stats::Tag{"envoy.endpoint_hostname", hostname}, + })); + + EXPECT_THAT(getMetric("cluster.cluster1.endpoint.127.0.0.2_80.c1", counters).tags(), + UnorderedElementsAreArray({ + Stats::Tag{"envoy.cluster_name", "cluster1"}, + Stats::Tag{"envoy.endpoint_address", "127.0.0.2:80"}, + })); +} - const HostUtility::HostStatusSet all_health_statuses = - UnhealthyStatus | DegradedStatus | HealthyStatus; +TEST_F(PerEndpointMetricsTest, FixedTags) { + auto& cluster = makeCluster("cluster1", 1); + Stats::TagVector fixed_tags{{"fixed1", "value1"}, {"fixed2", "value2"}}; + EXPECT_CALL(cluster.info_->stats_store_, fixedTags()).WillOnce(ReturnRef(fixed_tags)); - { - // No valid host map. - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(nullptr, all_health_statuses, &context)); - } - { - // No valid load balancer context. - auto host_map = std::make_shared(); - EXPECT_EQ(nullptr, - HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, nullptr)); - } - { - // No valid expected host. - EXPECT_CALL(context, overrideHostToSelect()).WillOnce(Return(absl::nullopt)); - auto host_map = std::make_shared(); - EXPECT_EQ(nullptr, - HostUtility::selectOverrideHost(host_map.get(), all_health_statuses, &context)); - } - { - // The host map does not contain the expected host. - LoadBalancerContext::OverrideHost override_host{"1.2.3.4"}; - EXPECT_CALL(context, overrideHostToSelect()) - .WillOnce(Return(absl::make_optional(override_host))); - auto host_map = std::make_shared(); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); - } - { - // The status of host is not as expected. - auto mock_host = std::make_shared>(); - EXPECT_CALL(*mock_host, coarseHealth()).WillOnce(Return(Host::Health::Unhealthy)); - - LoadBalancerContext::OverrideHost override_host{"1.2.3.4"}; - EXPECT_CALL(context, overrideHostToSelect()) - .WillOnce(Return(absl::make_optional(override_host))); + auto [counters, gauges] = run(); - auto host_map = std::make_shared(); - host_map->insert({"1.2.3.4", mock_host}); - EXPECT_EQ(nullptr, HostUtility::selectOverrideHost(host_map.get(), HealthyStatus, &context)); - } - { - // Get expected host. - auto mock_host = std::make_shared>(); - EXPECT_CALL(*mock_host, coarseHealth()).WillOnce(Return(Host::Health::Degraded)); + EXPECT_THAT(getMetric("cluster.cluster1.endpoint.127.0.0.1_80.c1", counters).tags(), + UnorderedElementsAreArray({ + Stats::Tag{"envoy.cluster_name", "cluster1"}, + Stats::Tag{"envoy.endpoint_address", "127.0.0.1:80"}, + Stats::Tag{"fixed1", "value1"}, + Stats::Tag{"fixed2", "value2"}, + })); +} - LoadBalancerContext::OverrideHost override_host{"1.2.3.4"}; - EXPECT_CALL(context, overrideHostToSelect()) - .WillOnce(Return(absl::make_optional(override_host))); +// Only clusters with the setting enabled produce metrics. +TEST_F(PerEndpointMetricsTest, Enabled) { + auto& disabled = makeCluster("disabled", 1); + auto& enabled = makeCluster("enabled", 1); + EXPECT_CALL(*disabled.info_, perEndpointStatsEnabled()).WillOnce(Return(false)); + EXPECT_CALL(*enabled.info_, perEndpointStatsEnabled()).WillOnce(Return(true)); + + auto [counters, gauges] = run(); + + EXPECT_THAT(metricNames(counters), + UnorderedElementsAreArray({"cluster.enabled.endpoint.127.0.0.2_80.c1", + "cluster.enabled.endpoint.127.0.0.2_80.c2"})); + EXPECT_THAT(metricNames(gauges), + UnorderedElementsAreArray({"cluster.enabled.endpoint.127.0.0.2_80.g1", + "cluster.enabled.endpoint.127.0.0.2_80.g2", + "cluster.enabled.endpoint.127.0.0.2_80.healthy"})); +} - auto host_map = std::make_shared(); - host_map->insert({"1.2.3.4", mock_host}); - EXPECT_EQ(mock_host, HostUtility::selectOverrideHost(host_map.get(), - HealthyStatus | DegradedStatus, &context)); - } +// Stats use observability name, and are sanitized. +TEST_F(PerEndpointMetricsTest, SanitizedObservabilityName) { + auto& cluster = makeCluster("notthisname", 1); + std::string name = "observability:name"; + EXPECT_CALL(*cluster.info_, observabilityName()).WillOnce(ReturnRef(name)); + + auto [counters, gauges] = run(); + + EXPECT_THAT(metricNames(counters), + UnorderedElementsAreArray({"cluster.observability_name.endpoint.127.0.0.1_80.c1", + "cluster.observability_name.endpoint.127.0.0.1_80.c2"})); + EXPECT_THAT( + metricNames(gauges), + UnorderedElementsAreArray({"cluster.observability_name.endpoint.127.0.0.1_80.g1", + "cluster.observability_name.endpoint.127.0.0.1_80.g2", + "cluster.observability_name.endpoint.127.0.0.1_80.healthy"})); + + EXPECT_THAT(getMetric("cluster.observability_name.endpoint.127.0.0.1_80.c1", counters).tags(), + Contains(Stats::Tag{"envoy.cluster_name", "observability_name"})); } } // namespace diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index c52d7b7a2c12..cc49c23f5594 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -9,7 +9,6 @@ #include "source/common/upstream/upstream_impl.h" #include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" #include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" -#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "test/benchmark/main.h" #include "test/common/upstream/utility.h" @@ -546,88 +545,6 @@ BENCHMARK(benchmarkMaglevLoadBalancerWeighted) ->Args({500, 95, 75, 25, 10000}) ->Unit(::benchmark::kMillisecond); -class SubsetLbTester : public BaseTester { -public: - SubsetLbTester(uint64_t num_hosts, bool single_host_per_subset) - : BaseTester(num_hosts, 0, 0, true /* attach metadata */) { - envoy::config::cluster::v3::Cluster::LbSubsetConfig subset_config; - subset_config.set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT); - auto* selector = subset_config.mutable_subset_selectors()->Add(); - selector->set_single_host_per_subset(single_host_per_subset); - *selector->mutable_keys()->Add() = std::string(metadata_key); - - subset_info_ = std::make_unique(subset_config); - auto child_lb_creator = std::make_unique( - LoadBalancerType::Random, absl::nullopt, absl::nullopt, absl::nullopt, absl::nullopt, - common_config_); - lb_ = std::make_unique(*subset_info_, std::move(child_lb_creator), - priority_set_, &local_priority_set_, stats_, - stats_scope_, runtime_, random_, simTime()); - - const HostVector& hosts = priority_set_.getOrCreateHostSet(0).hosts(); - ASSERT(hosts.size() == num_hosts); - orig_hosts_ = std::make_shared(hosts); - smaller_hosts_ = std::make_shared(hosts.begin() + 1, hosts.end()); - ASSERT(smaller_hosts_->size() + 1 == orig_hosts_->size()); - orig_locality_hosts_ = makeHostsPerLocality({*orig_hosts_}); - smaller_locality_hosts_ = makeHostsPerLocality({*smaller_hosts_}); - } - - // Remove a host and add it back. - void update() { - priority_set_.updateHosts(0, - HostSetImpl::partitionHosts(smaller_hosts_, smaller_locality_hosts_), - nullptr, {}, host_moved_, absl::nullopt); - priority_set_.updateHosts(0, HostSetImpl::partitionHosts(orig_hosts_, orig_locality_hosts_), - nullptr, host_moved_, {}, absl::nullopt); - } - - std::unique_ptr subset_info_; - std::unique_ptr lb_; - HostVectorConstSharedPtr orig_hosts_; - HostVectorConstSharedPtr smaller_hosts_; - HostsPerLocalitySharedPtr orig_locality_hosts_; - HostsPerLocalitySharedPtr smaller_locality_hosts_; - HostVector host_moved_; -}; - -void benchmarkSubsetLoadBalancerCreate(::benchmark::State& state) { - const bool single_host_per_subset = state.range(0); - const uint64_t num_hosts = state.range(1); - - if (benchmark::skipExpensiveBenchmarks() && num_hosts > 100) { - state.SkipWithError("Skipping expensive benchmark"); - return; - } - - for (auto _ : state) { // NOLINT: Silences warning about dead store - SubsetLbTester tester(num_hosts, single_host_per_subset); - } -} - -BENCHMARK(benchmarkSubsetLoadBalancerCreate) - ->Ranges({{false, true}, {50, 2500}}) - ->Unit(::benchmark::kMillisecond); - -void benchmarkSubsetLoadBalancerUpdate(::benchmark::State& state) { - const bool single_host_per_subset = state.range(0); - const uint64_t num_hosts = state.range(1); - if (benchmark::skipExpensiveBenchmarks() && num_hosts > 100) { - state.SkipWithError("Skipping expensive benchmark"); - return; - } - - SubsetLbTester tester(num_hosts, single_host_per_subset); - for (auto _ : state) { // NOLINT: Silences warning about dead store - tester.update(); - } -} - -BENCHMARK(benchmarkSubsetLoadBalancerUpdate) - ->Ranges({{false, true}, {50, 2500}}) - ->Unit(::benchmark::kMillisecond); - } // namespace } // namespace Upstream } // namespace Envoy diff --git a/test/common/upstream/load_balancer_simulation_test.cc b/test/common/upstream/load_balancer_simulation_test.cc index 22c1f175b44d..1ec475438a88 100644 --- a/test/common/upstream/load_balancer_simulation_test.cc +++ b/test/common/upstream/load_balancer_simulation_test.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -41,27 +42,58 @@ static HostSharedPtr newTestHost(Upstream::ClusterInfoConstSharedPtr cluster, envoy::config::core::v3::UNKNOWN, time_source)}; } -// Simulate weighted LR load balancer. -TEST(DISABLED_LeastRequestLoadBalancerWeightTest, Weight) { - const uint64_t num_hosts = 4; - const uint64_t weighted_subset_percent = 50; - const uint64_t weight = 2; // weighted_subset_percent of hosts will have this weight. - const uint64_t active_requests = 3; // weighted_subset_percent will have this active requests. +// Defines parameters for LeastRequestLoadBalancerWeightTest cases. +struct LRLBTestParams { + // The total number of hosts. + uint64_t num_hosts; + + // Number of hosts that are part of the subset. + uint64_t num_subset_hosts; + + // The weight assigned to each subset host. + uint64_t weight; + + // The number of active requests each subset host will be loaded with. + uint64_t active_request_count; + + // An unordered collection of expected selection probabilities for the hosts. The test will simply + // sort the expected and observed selection probabilities and verify each element is within some + // expected tolerance. Therefore, the vector does not need to be sorted. + std::vector expected_selection_probs; +}; + +void leastRequestLBWeightTest(LRLBTestParams params) { + constexpr uint64_t num_requests = 100000; + + // Observed selection probabilities must be within tolerance_pct of the expected to pass the test. + // The expected range is [0,100). + constexpr double tolerance_pct = 1.0; + + // Validate params. + ASSERT_GT(params.num_hosts, 0); + ASSERT_LE(params.num_subset_hosts, params.num_hosts); + ASSERT_GT(params.weight, 0); + ASSERT_EQ(params.expected_selection_probs.size(), params.num_hosts); + ASSERT_LT(tolerance_pct, 100); + ASSERT_GE(tolerance_pct, 0); - PrioritySetImpl priority_set; - std::shared_ptr info_{new NiceMock()}; NiceMock time_source_; HostVector hosts; - for (uint64_t i = 0; i < num_hosts; i++) { - const bool should_weight = i < num_hosts * (weighted_subset_percent / 100.0); - hosts.push_back(makeTestHost(info_, fmt::format("tcp://10.0.{}.{}:6379", i / 256, i % 256), - time_source_, should_weight ? weight : 1)); + absl::node_hash_map host_hits; + std::shared_ptr info{new NiceMock()}; + for (uint64_t i = 0; i < params.num_hosts; i++) { + const bool should_weight = i < params.num_subset_hosts; + auto hostPtr = makeTestHost(info, fmt::format("tcp://10.0.{}.{}:6379", i / 256, i % 256), + time_source_, should_weight ? params.weight : 1); + host_hits[hostPtr] = 0; + hosts.push_back(hostPtr); if (should_weight) { - hosts.back()->stats().rq_active_.set(active_requests); + hosts.back()->stats().rq_active_.set(params.active_request_count); } } HostVectorConstSharedPtr updated_hosts{new HostVector(hosts)}; HostsPerLocalitySharedPtr updated_locality_hosts{new HostsPerLocalityImpl(hosts)}; + PrioritySetImpl priority_set; priority_set.updateHosts( 0, updateHostsParams(updated_hosts, updated_locality_hosts, @@ -81,26 +113,82 @@ TEST(DISABLED_LeastRequestLoadBalancerWeightTest, Weight) { priority_set, nullptr, lb_stats, runtime, random, common_config, least_request_lb_config, *time_source}; - absl::node_hash_map host_hits; - const uint64_t total_requests = 100; - for (uint64_t i = 0; i < total_requests; i++) { + for (uint64_t i = 0; i < num_requests; i++) { host_hits[lb_.chooseHost(nullptr)]++; } - absl::node_hash_map weight_to_percent; + std::vector observed_pcts; for (const auto& host : host_hits) { - std::cout << fmt::format("url:{}, weight:{}, hits:{}, percent_of_total:{}\n", - host.first->address()->asString(), host.first->weight(), host.second, - (static_cast(host.second) / total_requests) * 100); - weight_to_percent[host.first->weight()] += - (static_cast(host.second) / total_requests) * 100; + observed_pcts.push_back((static_cast(host.second) / num_requests) * 100); } - for (const auto& weight : weight_to_percent) { - std::cout << fmt::format("weight:{}, percent:{}\n", weight.first, weight.second); + std::sort(observed_pcts.begin(), observed_pcts.end()); + std::sort(params.expected_selection_probs.begin(), params.expected_selection_probs.end()); + ASSERT_EQ(observed_pcts.size(), params.expected_selection_probs.size()); + for (uint64_t i = 0; i < observed_pcts.size(); i++) { + EXPECT_NEAR(params.expected_selection_probs[i], observed_pcts[i], tolerance_pct); } } +// Simulate weighted LR load balancer and verify expected selection probabilities. +TEST(LeastRequestLoadBalancerWeightTest, Weight) { + LRLBTestParams params; + + // No active requests or weight differences. This should look like uniform random LB. + params.num_hosts = 3; + params.num_subset_hosts = 1; + params.active_request_count = 0; + params.expected_selection_probs = {33.333, 33.333, 33.333}; + params.weight = 1; + leastRequestLBWeightTest(params); + + // Single host (out of 3) with lots of in-flight requests. Given that P2C will choose 2 hosts and + // take the one with higher weight, the only circumstance that the host with many in-flight + // requests will be picked is if P2C selects it twice. + params.num_hosts = 3; + params.num_subset_hosts = 1; + params.active_request_count = 10; + params.expected_selection_probs = {44.45, 44.45, 11.1}; + params.weight = 1; + leastRequestLBWeightTest(params); + + // Same as above, but with 2 hosts. The busy host will only be chosen if P2C picks it for both + // selections. + params.num_hosts = 2; + params.num_subset_hosts = 1; + params.active_request_count = 10; + params.expected_selection_probs = {25, 75}; + params.weight = 1; + leastRequestLBWeightTest(params); + + // Heterogeneous weights with no active requests. This should behave identically to weighted + // round-robin. + params.num_hosts = 2; + params.num_subset_hosts = 1; + params.active_request_count = 0; + params.expected_selection_probs = {66.66, 33.33}; + params.weight = 2; + leastRequestLBWeightTest(params); + + // Same as above, but we'll scale the subset's weight with active requests. With a default + // active_request_bias of 1.0, the subset host with a single active request will be cut in half, + // making both hosts have an identical weight. + params.num_hosts = 2; + params.num_subset_hosts = 1; + params.active_request_count = 1; + params.expected_selection_probs = {50, 50}; + params.weight = 2; + leastRequestLBWeightTest(params); + + // Same as above, but with 3 hosts. + params.num_hosts = 3; + params.num_subset_hosts = 1; + params.active_request_count = 1; + params.expected_selection_probs = {33.3, 33.3, 33.3}; + params.weight = 2; + leastRequestLBWeightTest(params); +} + /** * This test is for simulation only and should not be run as part of unit tests. */ diff --git a/test/common/upstream/load_stats_reporter_test.cc b/test/common/upstream/load_stats_reporter_test.cc index 429cea9b4182..1997e2d324c5 100644 --- a/test/common/upstream/load_stats_reporter_test.cc +++ b/test/common/upstream/load_stats_reporter_test.cc @@ -73,6 +73,12 @@ class LoadStatsReporterTest : public testing::Test { load_stats_reporter_->onReceiveMessage(std::move(response)); } + void setDropOverload(envoy::config::endpoint::v3::ClusterStats& cluster_stats, uint64_t count) { + auto* dropped_request = cluster_stats.add_dropped_requests(); + dropped_request->set_category("drop_overload"); + dropped_request->set_dropped_count(count); + } + Event::SimulatedTimeSystem time_system_; NiceMock cm_; Event::MockDispatcher dispatcher_; @@ -132,12 +138,14 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { deliverLoadStatsResponse({"foo"}); // Initial stats report for foo on timer tick. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(5); + foo_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(7); time_system_.setMonotonicTime(std::chrono::microseconds(4)); { envoy::config::endpoint::v3::ClusterStats foo_cluster_stats; foo_cluster_stats.set_cluster_name("foo"); foo_cluster_stats.set_cluster_service_name("bar"); foo_cluster_stats.set_total_dropped_requests(5); + setDropOverload(foo_cluster_stats, 7); foo_cluster_stats.mutable_load_report_interval()->MergeFrom( Protobuf::util::TimeUtil::MicrosecondsToDuration(1)); expectSendMessage({foo_cluster_stats}); @@ -148,6 +156,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { // Some traffic on foo/bar in between previous request and next response. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(5); // Start reporting on bar. time_system_.setMonotonicTime(std::chrono::microseconds(6)); @@ -155,6 +164,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { // Stats report foo/bar on timer tick. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(3); time_system_.setMonotonicTime(std::chrono::microseconds(28)); { envoy::config::endpoint::v3::ClusterStats foo_cluster_stats; @@ -166,6 +176,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { envoy::config::endpoint::v3::ClusterStats bar_cluster_stats; bar_cluster_stats.set_cluster_name("bar"); bar_cluster_stats.set_total_dropped_requests(1); + setDropOverload(bar_cluster_stats, 3); bar_cluster_stats.mutable_load_report_interval()->MergeFrom( Protobuf::util::TimeUtil::MicrosecondsToDuration(22)); expectSendMessage({bar_cluster_stats, foo_cluster_stats}); @@ -176,17 +187,20 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { // Some traffic on foo/bar in between previous request and next response. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(1); // Stop reporting on foo. deliverLoadStatsResponse({"bar"}); // Stats report for bar on timer tick. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(5); bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(5); + bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(7); time_system_.setMonotonicTime(std::chrono::microseconds(33)); { envoy::config::endpoint::v3::ClusterStats bar_cluster_stats; bar_cluster_stats.set_cluster_name("bar"); bar_cluster_stats.set_total_dropped_requests(6); + setDropOverload(bar_cluster_stats, 8); bar_cluster_stats.mutable_load_report_interval()->MergeFrom( Protobuf::util::TimeUtil::MicrosecondsToDuration(5)); expectSendMessage({bar_cluster_stats}); @@ -196,25 +210,31 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { // Some traffic on foo/bar in between previous request and next response. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + foo_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(8); bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(3); // Start tracking foo again, we should forget earlier history for foo. time_system_.setMonotonicTime(std::chrono::microseconds(43)); deliverLoadStatsResponse({"foo", "bar"}); // Stats report foo/bar on timer tick. foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + foo_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(9); bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1); + bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(4); time_system_.setMonotonicTime(std::chrono::microseconds(47)); { envoy::config::endpoint::v3::ClusterStats foo_cluster_stats; foo_cluster_stats.set_cluster_name("foo"); foo_cluster_stats.set_cluster_service_name("bar"); foo_cluster_stats.set_total_dropped_requests(1); + setDropOverload(foo_cluster_stats, 9); foo_cluster_stats.mutable_load_report_interval()->MergeFrom( Protobuf::util::TimeUtil::MicrosecondsToDuration(4)); envoy::config::endpoint::v3::ClusterStats bar_cluster_stats; bar_cluster_stats.set_cluster_name("bar"); bar_cluster_stats.set_total_dropped_requests(2); + setDropOverload(bar_cluster_stats, 7); bar_cluster_stats.mutable_load_report_interval()->MergeFrom( Protobuf::util::TimeUtil::MicrosecondsToDuration(14)); expectSendMessage({bar_cluster_stats, foo_cluster_stats}); diff --git a/test/common/upstream/od_cds_api_impl_test.cc b/test/common/upstream/od_cds_api_impl_test.cc index ffd925bb5234..4acab684f268 100644 --- a/test/common/upstream/od_cds_api_impl_test.cc +++ b/test/common/upstream/od_cds_api_impl_test.cc @@ -142,11 +142,9 @@ TEST_F(OdCdsApiImplTest, ValidateDuplicateClusters) { cluster_1.set_name("duplicate_cluster"); const auto decoded_resources = TestUtility::decodeResources({cluster_1, cluster_1}); - EXPECT_THROW_WITH_MESSAGE( - ASSERT_TRUE(odcds_callbacks_->onConfigUpdate(decoded_resources.refvec_, {}, "").ok()), - EnvoyException, - "Error adding/updating cluster(s) duplicate_cluster: duplicate cluster " - "duplicate_cluster found"); + ASSERT_EQ(odcds_callbacks_->onConfigUpdate(decoded_resources.refvec_, {}, "").message(), + "Error adding/updating cluster(s) duplicate_cluster: duplicate cluster " + "duplicate_cluster found"); } // Check that notifier gets a message about potentially missing cluster. diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index e8a7b294e9e2..2feaf7303dda 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -47,7 +47,8 @@ TEST(OutlierDetectorImplFactoryTest, NoDetector) { NiceMock random; EXPECT_EQ(nullptr, DetectorImplFactory::createForCluster(cluster, defaultStaticCluster("fake_cluster"), - dispatcher, runtime, nullptr, random)); + dispatcher, runtime, nullptr, random) + .value()); } TEST(OutlierDetectorImplFactoryTest, Detector) { @@ -59,7 +60,8 @@ TEST(OutlierDetectorImplFactoryTest, Detector) { NiceMock runtime; NiceMock random; EXPECT_NE(nullptr, DetectorImplFactory::createForCluster(cluster, fake_cluster, dispatcher, - runtime, nullptr, random)); + runtime, nullptr, random) + .value()); } class CallbackChecker { @@ -146,8 +148,10 @@ max_ejection_time: 400s envoy::config::cluster::v3::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); EXPECT_EQ(100UL, detector->config().intervalMs()); EXPECT_EQ(10000UL, detector->config().baseEjectionTimeMs()); @@ -179,8 +183,10 @@ base_ejection_time: 10s envoy::config::cluster::v3::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); EXPECT_EQ(100UL, detector->config().intervalMs()); EXPECT_EQ(10000UL, detector->config().baseEjectionTimeMs()); @@ -209,9 +215,10 @@ max_ejection_time: 3s envoy::config::cluster::v3::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); // Detector should reject the config. - ASSERT_THROW(DetectorImpl::create(cluster_, outlier_detection, dispatcher_, runtime_, - time_system_, event_logger_, random_), - EnvoyException); + ASSERT_FALSE(DetectorImpl::create(cluster_, outlier_detection, dispatcher_, runtime_, + time_system_, event_logger_, random_) + .status() + .ok()); } // Test verifies that legacy config without max_ejection_time value @@ -226,8 +233,10 @@ base_ejection_time: 400s envoy::config::cluster::v3::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); EXPECT_EQ(100UL, detector->config().intervalMs()); EXPECT_EQ(400000UL, detector->config().baseEjectionTimeMs()); @@ -242,7 +251,8 @@ TEST_F(OutlierDetectorImplTest, DestroyWithActive) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); loadRq(hosts_[0], 4, 500); @@ -274,7 +284,8 @@ TEST_F(OutlierDetectorImplTest, DestroyHostInUse) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); detector.reset(); @@ -293,7 +304,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); addHosts({"tcp://127.0.0.1:81"}); @@ -370,7 +382,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodesWithActiveHCUnejectHost) std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); addHosts({"tcp://127.0.0.1:81"}); @@ -476,8 +489,10 @@ successful_active_health_check_uneject_host: false EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)).Times(0); ON_CALL(cluster_, healthChecker()).WillByDefault(Return(health_checker.get())); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); addHosts({"tcp://127.0.0.1:81"}); @@ -534,7 +549,8 @@ TEST_F(OutlierDetectorImplTest, ConnectSuccessWithOptionalHTTP_OK) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Make sure that in non-split mode LOCAL_ORIGIN_CONNECT_SUCCESS with optional HTTP code 200 @@ -561,7 +577,8 @@ TEST_F(OutlierDetectorImplTest, ExternalOriginEventsNonSplit) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Make sure that EXT_ORIGIN_REQUEST_SUCCESS cancels EXT_ORIGIN_REQUEST_FAILED @@ -589,7 +606,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); addHosts({"tcp://127.0.0.1:81"}); @@ -668,7 +686,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingConsecutiveGatewayFailureRuntime, 0)) .WillByDefault(Return(true)); @@ -766,7 +785,8 @@ TEST_F(OutlierDetectorImplTest, TimeoutWithHttpCode) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Report several LOCAL_ORIGIN_TIMEOUT with optional Http code 500. Host should be ejected. @@ -839,7 +859,8 @@ TEST_F(OutlierDetectorImplTest, LargeNumberOfTimeouts) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingConsecutiveLocalOriginFailureRuntime, 100)) .WillByDefault(Return(true)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -894,7 +915,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingConsecutiveLocalOriginFailureRuntime, 100)) .WillByDefault(Return(true)); @@ -1009,7 +1031,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingConsecutiveGatewayFailureRuntime, 0)) .WillByDefault(Return(true)); @@ -1099,7 +1122,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowNonHttpCodesExternalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); addHosts({"tcp://127.0.0.1:81"}); @@ -1152,7 +1176,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Turn off 5xx detection to test SR detection in isolation. @@ -1252,7 +1277,8 @@ TEST_F(OutlierDetectorImplTest, ExternalOriginEventsWithSplit) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); for (auto i = 0; i < 100; i++) { hosts_[0]->outlierDetector().putResult(Result::ExtOriginRequestFailed); @@ -1286,7 +1312,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Turn off detecting consecutive local origin failures. @@ -1375,7 +1402,8 @@ TEST_F(OutlierDetectorImplTest, EmptySuccessRate) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); loadRq(hosts_, 200, 503); time_system_.setMonotonicTime(std::chrono::milliseconds(10000)); @@ -1399,7 +1427,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowFailurePercentageExternalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Turn off 5xx detection and SR detection to test failure percentage detection in isolation. @@ -1520,7 +1549,8 @@ TEST_F(OutlierDetectorImplTest, BasicFlowFailurePercentageLocalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Turn off 5xx detection and SR detection to test failure percentage detection in isolation. @@ -1619,7 +1649,8 @@ TEST_F(OutlierDetectorImplTest, RemoveWhileEjected) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); loadRq(hosts_[0], 4, 500); @@ -1649,7 +1680,8 @@ TEST_F(OutlierDetectorImplTest, Overflow) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); ON_CALL(runtime_.snapshot_, getInteger(MaxEjectionPercentRuntime, _)).WillByDefault(Return(60)); @@ -1678,7 +1710,8 @@ TEST_F(OutlierDetectorImplTest, NotEnforcing) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); loadRq(hosts_[0], 4, 503); @@ -1721,7 +1754,8 @@ TEST_F(OutlierDetectorImplTest, EjectionActiveValueIsAccountedWithoutMetricStora EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); ON_CALL(runtime_.snapshot_, getInteger(MaxEjectionPercentRuntime, _)).WillByDefault(Return(50)); @@ -1759,7 +1793,8 @@ TEST_F(OutlierDetectorImplTest, CrossThreadRemoveRace) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); loadRq(hosts_[0], 4, 500); @@ -1784,7 +1819,8 @@ TEST_F(OutlierDetectorImplTest, CrossThreadDestroyRace) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); loadRq(hosts_[0], 4, 500); @@ -1811,7 +1847,8 @@ TEST_F(OutlierDetectorImplTest, CrossThreadFailRace) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); loadRq(hosts_[0], 4, 500); @@ -1851,8 +1888,10 @@ max_ejection_time_jitter: 13s addHosts({"tcp://127.0.0.4:80"}); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1882,8 +1921,10 @@ max_ejection_time_jitter: 13s addHosts({"tcp://127.0.0.2:80"}); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection_, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); loadRq(hosts_[0], 5, 500); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); @@ -1896,7 +1937,8 @@ TEST_F(OutlierDetectorImplTest, Consecutive_5xxAlreadyEjected) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Cause a consecutive 5xx error. loadRq(hosts_[0], 4, 500); @@ -1934,7 +1976,8 @@ TEST_F(OutlierDetectorImplTest, EjectTimeBackoff) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Eject the node by consecutive 5xx errors. @@ -2088,6 +2131,159 @@ TEST_F(OutlierDetectorImplTest, EjectTimeBackoff) { EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); } +TEST_F(OutlierDetectorImplTest, EjectTimeBackoffTimeBasedDetection) { + // Setup base ejection time to 10 secs. + const uint64_t base_ejection_time = 10000; + ON_CALL(runtime_.snapshot_, getInteger(BaseEjectionTimeMsRuntime, _)) + .WillByDefault(Return(base_ejection_time)); + ON_CALL(runtime_.snapshot_, getInteger(MaxEjectionPercentRuntime, _)).WillByDefault(Return(100)); + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Turn off 5xx detection to test failure percentage in isolation. + ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingConsecutive5xxRuntime, 100)) + .WillByDefault(Return(false)); + ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingConsecutiveGatewayFailureRuntime, 100)) + .WillByDefault(Return(false)); + ON_CALL(runtime_.snapshot_, getInteger(FailurePercentageMinimumHostsRuntime, 5)) + .WillByDefault(Return(1)); + ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingSuccessRateRuntime, 100)) + .WillByDefault(Return(false)); + // Disable jitter. + ON_CALL(runtime_.snapshot_, getInteger(MaxEjectionTimeJitterMsRuntime, _)) + .WillByDefault(Return(0UL)); + // Turn on failure percentage detection. + ON_CALL(runtime_.snapshot_, featureEnabled(EnforcingFailurePercentageRuntime, 0)) + .WillByDefault(Return(true)); + + time_system_.setMonotonicTime(std::chrono::seconds(0)); + + // Simulate 100% failure. + // Expect non-enforcing logging to happen every time the consecutive_5xx_ counter + // gets saturated (every 5 times). + EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]), + _, envoy::data::cluster::v3::CONSECUTIVE_5XX, false)) + .Times(20); + loadRq(hosts_[0], 100, 500); + + uint32_t tick = 1; + // Invoke periodic timer. The node should be ejected. + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]), + _, envoy::data::cluster::v3::FAILURE_PERCENTAGE, true)); + interval_timer_->invokeCallback(); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Advance time to the next periodic timer tick. Node should be unejected. + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[0]))); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Keep replying with 5xx error codes. + EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]), + _, envoy::data::cluster::v3::CONSECUTIVE_5XX, false)) + .Times(20); + loadRq(hosts_[0], 100, 500); + + // Advance time to the next periodic timer tick. Node should be ejected. + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]), + _, envoy::data::cluster::v3::FAILURE_PERCENTAGE, true)); + interval_timer_->invokeCallback(); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // The node was ejected the second time in a row. The length of time the node + // should be ejected should be increased. + // Advance time to the next periodic timer tick. Node should stay ejected. + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + interval_timer_->invokeCallback(); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Advance time to next periodic timer tick. This time the node should be unejected + // (after 2 periods of base_ejection_time). + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[0]))); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Return success during the next two timer periods. This should decrement ejection time. + loadRq(hosts_[0], 100, 200); + + // Advance time to the next periodic time tick. The node should stay unejected. + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + loadRq(hosts_[0], 100, 200); + // Advance time to the next periodic time tick. The node should stay unejected. + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Return errors during the next period. + EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]), + _, envoy::data::cluster::v3::CONSECUTIVE_5XX, false)) + .Times(20); + loadRq(hosts_[0], 100, 500); + + // Advance time to next periodic timer tick. Node should be ejected. + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[0]), + _, envoy::data::cluster::v3::FAILURE_PERCENTAGE, true)); + interval_timer_->invokeCallback(); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Advance time to the next periodic timer tick. This time the node should be unejected + // only after ONE period of base_ejection_time (ejection time was reduced when the node + // did not fail for two timer periods). + tick++; + time_system_.setMonotonicTime(std::chrono::milliseconds(base_ejection_time * tick)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(base_ejection_time), _)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[0]))); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); +} + // Test that ejection time does not increase beyond maximum. // Test outline: // - max_ejection_time is 30 times longer than base_ejection_time. @@ -2107,7 +2303,8 @@ TEST_F(OutlierDetectorImplTest, MaxEjectTime) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Verify that maximum_ejection_time caps ejection time. @@ -2201,7 +2398,8 @@ TEST_F(OutlierDetectorImplTest, MaxEjectTimeNotAlligned) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Verify that maximum_ejection_time caps ejection time. @@ -2285,8 +2483,10 @@ max_ejection_time_jitter: 13s envoy::config::cluster::v3::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); EXPECT_EQ(100UL, detector->config().intervalMs()); EXPECT_EQ(10000UL, detector->config().baseEjectionTimeMs()); @@ -2304,8 +2504,10 @@ base_ejection_time: 10s envoy::config::cluster::v3::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100), _)); - std::shared_ptr detector(DetectorImpl::create( - cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_, random_)); + std::shared_ptr detector(DetectorImpl::create(cluster_, outlier_detection, + dispatcher_, runtime_, time_system_, + event_logger_, random_) + .value()); EXPECT_EQ(100UL, detector->config().intervalMs()); EXPECT_EQ(10000UL, detector->config().baseEjectionTimeMs()); @@ -2325,7 +2527,8 @@ TEST_F(OutlierDetectorImplTest, EjectionTimeJitterIsInRange) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Set the return value of random(). EXPECT_CALL(random_, random()).WillOnce(Return(123456789UL)); @@ -2355,7 +2558,8 @@ TEST_F(OutlierDetectorImplTest, EjectionTimeJitterIsZeroWhenNotConfigured) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create(cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, - event_logger_, random_)); + event_logger_, random_) + .value()); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); // Set the return value of random(). EXPECT_CALL(random_, random()).WillOnce(Return(1234567890UL)); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc deleted file mode 100644 index 6d48420a7eb4..000000000000 --- a/test/common/upstream/subset_lb_test.cc +++ /dev/null @@ -1,3199 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "envoy/config/cluster/v3/cluster.pb.h" -#include "envoy/config/core/v3/base.pb.h" - -#include "source/common/common/logger.h" -#include "source/common/config/metadata.h" -#include "source/common/upstream/upstream_impl.h" -#include "source/extensions/load_balancing_policies/subset/subset_lb.h" - -#include "test/common/upstream/utility.h" -#include "test/mocks/access_log/mocks.h" -#include "test/mocks/common.h" -#include "test/mocks/filesystem/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/upstream/cluster_info.h" -#include "test/mocks/upstream/host.h" -#include "test/mocks/upstream/host_set.h" -#include "test/mocks/upstream/load_balancer.h" -#include "test/mocks/upstream/load_balancer_context.h" -#include "test/mocks/upstream/priority_set.h" -#include "test/test_common/simulated_time_system.h" - -#include "absl/types/optional.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; - -namespace Envoy { -namespace Upstream { - -class SubsetLoadBalancerInternalStateTester { -public: - SubsetLoadBalancerInternalStateTester(std::shared_ptr lb) : lb_(lb) {} - - using MetadataVector = std::vector>; - - void testDescribeMetadata(std::string expected, const MetadataVector& metadata) { - const SubsetLoadBalancer::SubsetMetadata& subset_metadata(metadata); - EXPECT_EQ(expected, lb_.get()->describeMetadata(subset_metadata)); - } - - void validateLbTypeConfigs() const { - const auto* legacy_child_lb_creator = - dynamic_cast(lb_->child_lb_creator_.get()); - - if (legacy_child_lb_creator == nullptr) { - return; - } - - // Each of these expects that an lb_type is set to that type iff the - // returned config for that type exists. - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RingHash, - legacy_child_lb_creator->lbRingHashConfig() != absl::nullopt); - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::Maglev, - legacy_child_lb_creator->lbMaglevConfig() != absl::nullopt); - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RoundRobin, - legacy_child_lb_creator->lbRoundRobinConfig() != absl::nullopt); - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::LeastRequest, - legacy_child_lb_creator->lbLeastRequestConfig() != absl::nullopt); - } - -private: - std::shared_ptr lb_; -}; - -class TestMetadataMatchCriterion : public Router::MetadataMatchCriterion { -public: - TestMetadataMatchCriterion(const std::string& name, const HashedValue& value) - : name_(name), value_(value) {} - - const std::string& name() const override { return name_; } - const HashedValue& value() const override { return value_; } - -private: - std::string name_; - HashedValue value_; -}; - -class TestMetadataMatchCriteria : public Router::MetadataMatchCriteria { -public: - TestMetadataMatchCriteria(const std::map matches) { - for (const auto& it : matches) { - ProtobufWkt::Value v; - v.set_string_value(it.second); - - matches_.emplace_back( - std::make_shared(it.first, HashedValue(v))); - } - } - - TestMetadataMatchCriteria(const std::map matches) { - for (const auto& it : matches) { - matches_.emplace_back( - std::make_shared(it.first, HashedValue(it.second))); - } - } - - const std::vector& - metadataMatchCriteria() const override { - return matches_; - } - - Router::MetadataMatchCriteriaConstPtr - mergeMatchCriteria(const ProtobufWkt::Struct& override) const override { - auto new_criteria = std::make_unique(*this); - - // TODO: this is copied from MetadataMatchCriteriaImpl::extractMetadataMatchCriteria. - // should we start using real impl? - std::vector v; - absl::node_hash_map existing; - - for (const auto& it : matches_) { - existing.emplace(it->name(), v.size()); - v.emplace_back(it); - } - - // Add values from matches, replacing name/values copied from parent. - for (const auto& it : override.fields()) { - const auto index_it = existing.find(it.first); - if (index_it != existing.end()) { - v[index_it->second] = std::make_shared(it.first, it.second); - } else { - v.emplace_back(std::make_shared(it.first, it.second)); - } - } - std::sort(v.begin(), v.end(), - [](const Router::MetadataMatchCriterionConstSharedPtr& a, - const Router::MetadataMatchCriterionConstSharedPtr& b) -> bool { - return a->name() < b->name(); - }); - - new_criteria->matches_ = v; - return new_criteria; - } - - Router::MetadataMatchCriteriaConstPtr - filterMatchCriteria(const std::set& names) const override { - auto new_criteria = std::make_unique(*this); - for (auto it = new_criteria->matches_.begin(); it != new_criteria->matches_.end();) { - if (names.count(it->get()->name()) == 0) { - it = new_criteria->matches_.erase(it); - } else { - it++; - } - } - return new_criteria; - } - -private: - std::vector matches_; -}; - -namespace SubsetLoadBalancerTest { - -class TestLoadBalancerContext : public LoadBalancerContextBase { -public: - TestLoadBalancerContext( - std::initializer_list::value_type> metadata_matches) - : matches_( - new TestMetadataMatchCriteria(std::map(metadata_matches))) {} - - TestLoadBalancerContext( - std::initializer_list::value_type> metadata_matches) - : matches_(new TestMetadataMatchCriteria( - std::map(metadata_matches))) {} - - // Upstream::LoadBalancerContext - absl::optional computeHashKey() override { return {}; } - const Network::Connection* downstreamConnection() const override { return nullptr; } - const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return matches_.get(); } - const Http::RequestHeaderMap* downstreamHeaders() const override { return nullptr; } - - std::shared_ptr matches_; -}; - -enum class UpdateOrder { RemovesFirst, Simultaneous }; - -class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, - public testing::TestWithParam { -public: - SubsetLoadBalancerTest() - : scope_(stats_store_.createScope("testprefix")), stat_names_(stats_store_.symbolTable()), - stats_(stat_names_, *stats_store_.rootScope()) { - least_request_lb_config_.mutable_choice_count()->set_value(2); - } - - using HostMetadata = std::map; - using HostListMetadata = std::map>; - using HostURLMetadataMap = std::map; - - void init() { - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - }); - } - - void configureHostSet(const HostURLMetadataMap& host_metadata, MockHostSet& host_set) { - HostVector hosts; - for (const auto& it : host_metadata) { - hosts.emplace_back(makeHost(it.first, it.second)); - } - - host_set.hosts_ = hosts; - host_set.hosts_per_locality_ = makeHostsPerLocality({hosts}); - host_set.healthy_hosts_ = host_set.hosts_; - host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; - - host_set.runCallbacks({}, {}); - } - - void configureWeightedHostSet(const HostURLMetadataMap& first_locality_host_metadata, - const HostURLMetadataMap& second_locality_host_metadata, - MockHostSet& host_set, LocalityWeights locality_weights) { - HostVector all_hosts; - HostVector first_locality_hosts; - envoy::config::core::v3::Locality first_locality; - first_locality.set_zone("0"); - for (const auto& it : first_locality_host_metadata) { - auto host = makeHost(it.first, it.second, first_locality); - first_locality_hosts.emplace_back(host); - all_hosts.emplace_back(host); - } - - envoy::config::core::v3::Locality second_locality; - second_locality.set_zone("1"); - HostVector second_locality_hosts; - for (const auto& it : second_locality_host_metadata) { - auto host = makeHost(it.first, it.second, second_locality); - second_locality_hosts.emplace_back(host); - all_hosts.emplace_back(host); - } - - host_set.hosts_ = all_hosts; - host_set.hosts_per_locality_ = - makeHostsPerLocality({first_locality_hosts, second_locality_hosts}); - host_set.healthy_hosts_ = host_set.hosts_; - host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; - host_set.locality_weights_ = std::make_shared(locality_weights); - } - - void init(const HostURLMetadataMap& host_metadata) { - HostURLMetadataMap failover; - init(host_metadata, failover); - } - - void init(const HostURLMetadataMap& host_metadata, - const HostURLMetadataMap& failover_host_metadata, bool use_actual_subset_info = false) { - - if (!use_actual_subset_info) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - } - - configureHostSet(host_metadata, host_set_); - if (!failover_host_metadata.empty()) { - configureHostSet(failover_host_metadata, *priority_set_.getMockHostSet(1)); - } - - auto child_lb_creator = std::make_unique( - lb_type_, - lb_type_ == LoadBalancerType::RingHash - ? makeOptRef( - ring_hash_lb_config_) - : absl::nullopt, - lb_type_ == LoadBalancerType::Maglev - ? makeOptRef( - maglev_lb_config_) - : absl::nullopt, - lb_type_ == LoadBalancerType::RoundRobin - ? makeOptRef( - round_robin_lb_config_) - : absl::nullopt, - lb_type_ == LoadBalancerType::LeastRequest - ? makeOptRef( - least_request_lb_config_) - : absl::nullopt, - common_config_); - - lb_ = std::make_shared( - use_actual_subset_info ? static_cast(*actual_subset_info_) - : static_cast(subset_info_), - std::move(child_lb_creator), priority_set_, nullptr, stats_, *scope_, runtime_, random_, - simTime()); - } - - void zoneAwareInit(const std::vector& host_metadata_per_locality, - const std::vector& local_host_metadata_per_locality) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - - std::vector> localities; - for (uint32_t i = 0; i < 10; ++i) { - envoy::config::core::v3::Locality locality; - locality.set_zone(std::to_string(i)); - localities.emplace_back(std::make_shared(locality)); - } - ASSERT(host_metadata_per_locality.size() <= localities.size()); - ASSERT(local_host_metadata_per_locality.size() <= localities.size()); - - HostVector hosts; - std::vector hosts_per_locality; - for (uint32_t i = 0; i < host_metadata_per_locality.size(); ++i) { - const auto& host_metadata = host_metadata_per_locality[i]; - HostVector locality_hosts; - for (const auto& host_entry : host_metadata) { - HostSharedPtr host = makeHost(host_entry.first, host_entry.second, *localities[i]); - hosts.emplace_back(host); - locality_hosts.emplace_back(host); - } - hosts_per_locality.emplace_back(locality_hosts); - } - - host_set_.hosts_ = hosts; - host_set_.hosts_per_locality_ = makeHostsPerLocality(std::move(hosts_per_locality)); - - host_set_.healthy_hosts_ = host_set_.hosts_; - host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; - - local_hosts_ = std::make_shared(); - std::vector local_hosts_per_locality_vector; - for (uint32_t i = 0; i < local_host_metadata_per_locality.size(); ++i) { - const auto& local_host_metadata = local_host_metadata_per_locality[i]; - HostVector local_locality_hosts; - for (const auto& host_entry : local_host_metadata) { - HostSharedPtr host = makeHost(host_entry.first, host_entry.second, *localities[i]); - local_hosts_->emplace_back(host); - local_locality_hosts.emplace_back(host); - } - local_hosts_per_locality_vector.emplace_back(local_locality_hosts); - } - local_hosts_per_locality_ = makeHostsPerLocality(std::move(local_hosts_per_locality_vector)); - - local_priority_set_.updateHosts( - 0, - HostSetImpl::updateHostsParams( - local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), local_hosts_per_locality_, - std::make_shared(), HostsPerLocalityImpl::empty(), - std::make_shared(), HostsPerLocalityImpl::empty()), - {}, {}, {}, absl::nullopt); - - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - - lb_ = std::make_shared(subset_info_, std::move(child_lb_creator), - priority_set_, &local_priority_set_, stats_, *scope_, - runtime_, random_, simTime()); - } - - HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata) { - envoy::config::core::v3::Metadata m; - for (const auto& m_it : metadata) { - Config::Metadata::mutableMetadataValue(m, Config::MetadataFilters::get().ENVOY_LB, m_it.first) - .set_string_value(m_it.second); - } - - return makeTestHost(info_, url, m, simTime()); - } - - HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata, - const envoy::config::core::v3::Locality& locality) { - envoy::config::core::v3::Metadata m; - for (const auto& m_it : metadata) { - Config::Metadata::mutableMetadataValue(m, Config::MetadataFilters::get().ENVOY_LB, m_it.first) - .set_string_value(m_it.second); - } - - return makeTestHost(info_, url, m, locality, simTime()); - } - - HostSharedPtr makeHost(const std::string& url, const HostListMetadata& metadata) { - envoy::config::core::v3::Metadata m; - for (const auto& m_it : metadata) { - auto& metadata = Config::Metadata::mutableMetadataValue( - m, Config::MetadataFilters::get().ENVOY_LB, m_it.first); - for (const auto& value : m_it.second) { - metadata.mutable_list_value()->add_values()->set_string_value(value); - } - } - - return makeTestHost(info_, url, m, simTime()); - } - - ProtobufWkt::Struct makeDefaultSubset(HostMetadata metadata) { - ProtobufWkt::Struct default_subset; - - auto* fields = default_subset.mutable_fields(); - for (const auto& it : metadata) { - ProtobufWkt::Value v; - v.set_string_value(it.second); - fields->insert({it.first, v}); - } - - return default_subset; - } - - SubsetSelectorPtr - makeSelector(const std::set& selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy, - const std::set& fallback_keys_subset, - bool single_host_per_subset = false) { - Protobuf::RepeatedPtrField selector_keys_mapped; - for (const auto& it : selector_keys) { - selector_keys_mapped.Add(std::string(it)); - } - - Protobuf::RepeatedPtrField fallback_keys_subset_mapped; - for (const auto& it : fallback_keys_subset) { - fallback_keys_subset_mapped.Add(std::string(it)); - } - - return std::make_shared( - selector_keys_mapped, fallback_policy, fallback_keys_subset_mapped, single_host_per_subset); - } - - SubsetSelectorPtr makeSelector( - const std::set& selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy = - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED) { - return makeSelector(selector_keys, fallback_policy, {}); - } - - void modifyHosts(HostVector add, HostVector remove, absl::optional add_in_locality = {}, - uint32_t priority = 0) { - MockHostSet& host_set = *priority_set_.getMockHostSet(priority); - for (const auto& host : remove) { - auto it = std::find(host_set.hosts_.begin(), host_set.hosts_.end(), host); - if (it != host_set.hosts_.end()) { - host_set.hosts_.erase(it); - } - host_set.healthy_hosts_ = host_set.hosts_; - - std::vector locality_hosts_copy = host_set.hosts_per_locality_->get(); - for (auto& locality_hosts : locality_hosts_copy) { - auto it = std::find(locality_hosts.begin(), locality_hosts.end(), host); - if (it != locality_hosts.end()) { - locality_hosts.erase(it); - } - } - host_set.hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); - host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; - } - - if (GetParam() == UpdateOrder::RemovesFirst && !remove.empty()) { - host_set.runCallbacks({}, remove); - } - - for (const auto& host : add) { - host_set.hosts_.emplace_back(host); - host_set.healthy_hosts_ = host_set.hosts_; - - if (add_in_locality) { - std::vector locality_hosts_copy = host_set.hosts_per_locality_->get(); - locality_hosts_copy[add_in_locality.value()].emplace_back(host); - host_set.hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); - host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; - } - } - - if (GetParam() == UpdateOrder::RemovesFirst) { - if (!add.empty()) { - host_set.runCallbacks(add, {}); - } - } else if (!add.empty() || !remove.empty()) { - host_set.runCallbacks(add, remove); - } - } - - void modifyLocalHosts(HostVector add, HostVector remove, uint32_t add_in_locality) { - for (const auto& host : remove) { - auto it = std::find(local_hosts_->begin(), local_hosts_->end(), host); - if (it != local_hosts_->end()) { - local_hosts_->erase(it); - } - - std::vector locality_hosts_copy = local_hosts_per_locality_->get(); - for (auto& locality_hosts : locality_hosts_copy) { - auto it = std::find(locality_hosts.begin(), locality_hosts.end(), host); - if (it != locality_hosts.end()) { - locality_hosts.erase(it); - } - } - local_hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); - } - - if (GetParam() == UpdateOrder::RemovesFirst && !remove.empty()) { - local_priority_set_.updateHosts( - 0, - updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), - {}, {}, remove, absl::nullopt); - } - - for (const auto& host : add) { - local_hosts_->emplace_back(host); - std::vector locality_hosts_copy = local_hosts_per_locality_->get(); - locality_hosts_copy[add_in_locality].emplace_back(host); - local_hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); - } - - if (GetParam() == UpdateOrder::RemovesFirst) { - if (!add.empty()) { - local_priority_set_.updateHosts( - 0, - updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), - {}, add, {}, absl::nullopt); - } - } else if (!add.empty() || !remove.empty()) { - local_priority_set_.updateHosts( - 0, - updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), - {}, add, remove, absl::nullopt); - } - } - - void doLbTypeTest(LoadBalancerType type) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - lb_type_ = type; - init({{"tcp://127.0.0.1:80", {{"version", "1.0"}}}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); - - HostSharedPtr added_host = makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}); - modifyHosts({added_host}, {host_set_.hosts_.back()}); - - EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); - } - - MetadataConstSharedPtr buildMetadata(const std::string& version, bool is_default = false) const { - envoy::config::core::v3::Metadata metadata; - - if (!version.empty()) { - Envoy::Config::Metadata::mutableMetadataValue( - metadata, Config::MetadataFilters::get().ENVOY_LB, "version") - .set_string_value(version); - } - - if (is_default) { - Envoy::Config::Metadata::mutableMetadataValue( - metadata, Config::MetadataFilters::get().ENVOY_LB, "default") - .set_string_value("true"); - } - - return std::make_shared(metadata); - } - - MetadataConstSharedPtr buildMetadataWithStage(const std::string& version, - const std::string& stage = "") const { - envoy::config::core::v3::Metadata metadata; - - if (!version.empty()) { - Envoy::Config::Metadata::mutableMetadataValue( - metadata, Config::MetadataFilters::get().ENVOY_LB, "version") - .set_string_value(version); - } - - if (!stage.empty()) { - Envoy::Config::Metadata::mutableMetadataValue( - metadata, Config::MetadataFilters::get().ENVOY_LB, "stage") - .set_string_value(stage); - } - - return std::make_shared(metadata); - } - - ProtobufWkt::Value valueFromJson(std::string json) { - ProtobufWkt::Value v; - TestUtility::loadFromJson(json, v); - return v; - } - - LoadBalancerType lb_type_{LoadBalancerType::RoundRobin}; - NiceMock priority_set_; - MockHostSet& host_set_ = *priority_set_.getMockHostSet(0); - // Mock subset info is used for testing most logic. - NiceMock subset_info_; - // Actual subset info is used for testing actual subset config parsing and behavior. - std::unique_ptr> - actual_subset_info_; - std::shared_ptr info_{new NiceMock()}; - envoy::config::cluster::v3::Cluster::RingHashLbConfig ring_hash_lb_config_; - envoy::config::cluster::v3::Cluster::MaglevLbConfig maglev_lb_config_; - envoy::config::cluster::v3::Cluster::LeastRequestLbConfig least_request_lb_config_; - envoy::config::cluster::v3::Cluster::RoundRobinLbConfig round_robin_lb_config_; - envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_; - NiceMock runtime_; - NiceMock random_; - Stats::IsolatedStoreImpl stats_store_; - Stats::ScopeSharedPtr scope_; - ClusterLbStatNames stat_names_; - ClusterLbStats stats_; - PrioritySetImpl local_priority_set_; - HostVectorSharedPtr local_hosts_; - HostsPerLocalitySharedPtr local_hosts_per_locality_; - std::shared_ptr lb_; -}; - -TEST_F(SubsetLoadBalancerTest, NoFallback) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - init(); - - EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); - EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); - - EXPECT_EQ(nullptr, lb_->peekAnotherHost(nullptr)); - EXPECT_FALSE(lb_->lifetimeCallbacks().has_value()); - std::vector hash_key; - auto mock_host = std::make_shared>(); - EXPECT_FALSE(lb_->selectExistingConnection(nullptr, *mock_host, hash_key).has_value()); -} - -// Validate that SubsetLoadBalancer unregisters its priority set member update -// callback. Regression for heap-use-after-free. -TEST_F(SubsetLoadBalancerTest, DeregisterCallbacks) { - init(); - lb_.reset(); - host_set_.runCallbacks({}, {}); -} - -TEST_P(SubsetLoadBalancerTest, NoFallbackAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - init(); - - EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); - - modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}})}, {host_set_.hosts_.back()}); - - EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); -} - -TEST_F(SubsetLoadBalancerTest, FallbackAnyEndpoint) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - init(); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); - EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); -} - -TEST_P(SubsetLoadBalancerTest, FallbackAnyEndpointAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - init(); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); - - HostSharedPtr added_host = makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}); - modifyHosts({added_host}, {host_set_.hosts_.back()}); - - EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); -} - -TEST_F(SubsetLoadBalancerTest, FallbackDefaultSubset) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "new"}}}, - {"tcp://127.0.0.1:81", {{"version", "default"}}}, - }); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); - EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); -} - -TEST_F(SubsetLoadBalancerTest, FallbackPanicMode) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - EXPECT_CALL(subset_info_, panicModeAny()).WillRepeatedly(Return(true)); - - // The default subset will be empty. - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "none"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "new"}}}, - {"tcp://127.0.0.1:81", {{"version", "default"}}}, - }); - - EXPECT_TRUE(lb_->chooseHost(nullptr) != nullptr); - EXPECT_EQ(1U, stats_.lb_subsets_fallback_panic_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); -} - -TEST_P(SubsetLoadBalancerTest, FallbackPanicModeWithUpdates) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - EXPECT_CALL(subset_info_, panicModeAny()).WillRepeatedly(Return(true)); - - // The default subset will be empty. - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "none"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - init({{"tcp://127.0.0.1:80", {{"version", "default"}}}}); - EXPECT_TRUE(lb_->chooseHost(nullptr) != nullptr); - - // Removing current host, adding a new one. - HostSharedPtr added_host = makeHost("tcp://127.0.0.2:8000", {{"version", "new"}}); - modifyHosts({added_host}, {host_set_.hosts_[0]}); - - EXPECT_EQ(1, host_set_.hosts_.size()); - EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); -} - -TEST_P(SubsetLoadBalancerTest, FallbackDefaultSubsetAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "new"}}}, - {"tcp://127.0.0.1:81", {{"version", "default"}}}, - }); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); - - HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:8000", {{"version", "new"}}); - HostSharedPtr added_host2 = makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}); - - modifyHosts({added_host1, added_host2}, {host_set_.hosts_.back()}); - - EXPECT_EQ(added_host2, lb_->chooseHost(nullptr)); -} - -TEST_F(SubsetLoadBalancerTest, FallbackEmptyDefaultSubsetConvertsToAnyEndpoint) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - EXPECT_CALL(subset_info_, defaultSubset()) - .WillRepeatedly(ReturnRef(ProtobufWkt::Struct::default_instance())); - - init(); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); - EXPECT_EQ(2U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); -} - -TEST_F(SubsetLoadBalancerTest, FallbackOnUnknownMetadata) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - init(); - - TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); - TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_unknown_value)); -} - -TEST_F(SubsetLoadBalancerTest, BalancesSubset) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - }); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_11({{"version", "1.1"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); - EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_selected_.value()); -} - -TEST_P(SubsetLoadBalancerTest, BalancesSubsetAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - }); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_11({{"version", "1.1"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); - EXPECT_EQ(2U, stats_.lb_subsets_created_.value()); - - modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), - makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, - {host_set_.hosts_[1], host_set_.hosts_[2]}); - - TestLoadBalancerContext context_12({{"version", "1.2"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_11)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_12)); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); -} - -TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabled) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); - - init({}); - modifyHosts( - {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), - makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, - {}, {}, 0); - - { - TestLoadBalancerContext context({{"version", "1.0"}}); - EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); - } - { - TestLoadBalancerContext context({{"version", "1.2"}}); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - } - TestLoadBalancerContext context({{"version", "1.2.1"}}); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); -} - -TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabledMultipleLists) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); - - init({}); - modifyHosts( - {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), - makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.2", "1.2"}}}), - makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, - {}, {}, 0); - - { - TestLoadBalancerContext context({{"version", "1.0"}}); - EXPECT_TRUE(host_set_.hosts()[2] == lb_->chooseHost(&context)); - EXPECT_TRUE(host_set_.hosts()[2] == lb_->chooseHost(&context)); - } - { - // This should LB between both hosts marked with version 1.2. - TestLoadBalancerContext context({{"version", "1.2"}}); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); - } - { - // Choose a host multiple times to ensure that hosts()[0] is the *only* - // thing selected for this subset. - TestLoadBalancerContext context({{"version", "1.2.1"}}); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - } - - TestLoadBalancerContext context({{"version", "1.2.2"}}); - EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); -} - -TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabledMultipleListsForSingleHost) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version", "hardware"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); - - init({}); - modifyHosts( - {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}, - {"hardware", std::vector{"a", "b"}}}), - makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.1", "1.1.1"}}, - {"hardware", std::vector{"b", "c"}}})}, - {}, {}, 0); - - { - TestLoadBalancerContext context({{"version", "1.2"}, {"hardware", "a"}}); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - } - - { - TestLoadBalancerContext context({{"version", "1.1"}, {"hardware", "b"}}); - EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); - EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); - } - - { - TestLoadBalancerContext context({{"version", "1.1"}, {"hardware", "a"}}); - EXPECT_TRUE(nullptr == lb_->chooseHost(&context)); - } - - TestLoadBalancerContext context({{"version", "1.2.1"}, {"hardware", "b"}}); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); - EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); -} - -TEST_P(SubsetLoadBalancerTest, ListAsAnyDisable) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({}); - modifyHosts( - {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), - makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, - {}, {}, 0); - - { - TestLoadBalancerContext context({{"version", "1.0"}}); - EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); - } - TestLoadBalancerContext context({{"version", "1.2"}}); - EXPECT_TRUE(nullptr == lb_->chooseHost(&context)); -} - -// Test that adding backends to a failover group causes no problems. -TEST_P(SubsetLoadBalancerTest, UpdateFailover) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - - // Start with an empty lb. Choosing a host should result in failure. - init({}); - EXPECT_TRUE(nullptr == lb_->chooseHost(&context_10).get()); - - // Add hosts to the group at priority 1. - // These hosts should be selected as there are no healthy hosts with priority 0 - modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), - makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, - {}, {}, 1); - EXPECT_FALSE(nullptr == lb_->chooseHost(&context_10).get()); - - // Finally update the priority 0 hosts. The LB should now select hosts. - modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), - makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, - {}, {}, 0); - EXPECT_FALSE(nullptr == lb_->chooseHost(&context_10).get()); -} - -TEST_P(SubsetLoadBalancerTest, OnlyMetadataChanged) { - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_12({{"version", "1.2"}}); - TestLoadBalancerContext context_13({{"version", "1.3"}}); - TestLoadBalancerContext context_default({{"default", "true"}}); - - std::vector subset_selectors = { - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"default"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"default", "true"}}); - - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - // Add hosts initial hosts. - init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, - {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"default", "true"}}}}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); - - // Swap the default version. - host_set_.hosts_[0]->metadata(buildMetadata("1.2", true)); - host_set_.hosts_[1]->metadata(buildMetadata("1.0")); - - host_set_.runCallbacks({}, {}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_13)); - - // Bump 1.0 to 1.3, one subset should be removed. - host_set_.hosts_[1]->metadata(buildMetadata("1.3")); - - // No hosts added nor removed, so we bypass modifyHosts(). - host_set_.runCallbacks({}, {}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - - // Rollback from 1.3 to 1.0. - host_set_.hosts_[1]->metadata(buildMetadata("1.0")); - - host_set_.runCallbacks({}, {}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_13)); - - // Make 1.0 default again. - host_set_.hosts_[1]->metadata(buildMetadata("1.0", true)); - host_set_.hosts_[0]->metadata(buildMetadata("1.2")); - - host_set_.runCallbacks({}, {}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); -} - -TEST_P(SubsetLoadBalancerTest, EmptySubsetsPurged) { - std::vector subset_selectors = {makeSelector({"version"}), - makeSelector({"version", "stage"})}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - // Simple add and remove. - init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, - {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"stage", "prod"}}}}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - - host_set_.hosts_[0]->metadata(buildMetadataWithStage("1.3")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); - - // Move host that was in the version + stage subset into a new version only subset. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.4")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_removed_.value()); - - // Create a new version + stage subset. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.5", "devel")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(7U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_removed_.value()); - - // Now move it back to its original version + stage subset. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.0", "prod")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(9U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(6U, stats_.lb_subsets_removed_.value()); - - // Finally, remove the original version + stage subset again. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.6")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(10U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(8U, stats_.lb_subsets_removed_.value()); -} - -TEST_P(SubsetLoadBalancerTest, EmptySubsetsPurgedCollapsed) { - std::vector subset_selectors = {makeSelector({"version"}), - makeSelector({"version", "stage"})}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - // Init subsets. - init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, - {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"stage", "prod"}}}}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - - // Get rid of 1.0. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2", "prod")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); - - // Get rid of stage prod. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(1U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_removed_.value()); - - // Add stage prod back. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2", "prod")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_removed_.value()); -} - -TEST_P(SubsetLoadBalancerTest, EmptySubsetsPurgedVersionChanged) { - std::vector subset_selectors = {makeSelector({"version"}), - makeSelector({"version", "stage"})}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - // Init subsets. - init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, - {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"stage", "prod"}}}}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - - // Get rid of 1.0. - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2", "prod")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); - - // Change versions. - host_set_.hosts_[0]->metadata(buildMetadataWithStage("1.3")); - host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.4", "prod")); - host_set_.runCallbacks({}, {}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(7U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_removed_.value()); -} - -TEST_P(SubsetLoadBalancerTest, MetadataChangedHostsAddedRemoved) { - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_12({{"version", "1.2"}}); - TestLoadBalancerContext context_13({{"version", "1.3"}}); - TestLoadBalancerContext context_14({{"version", "1.4"}}); - TestLoadBalancerContext context_default({{"default", "true"}}); - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"default", "true"}}); - - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - std::vector subset_selectors = { - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"default"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - // Add hosts initial hosts. - init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, - {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"default", "true"}}}}); - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); - - // Swap the default version. - host_set_.hosts_[0]->metadata(buildMetadata("1.2", true)); - host_set_.hosts_[1]->metadata(buildMetadata("1.0")); - - // Add a new host. - modifyHosts({makeHost("tcp://127.0.0.1:8002", {{"version", "1.3"}})}, {}); - - EXPECT_EQ(4U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_13)); - - // Swap default again and remove the previous one. - host_set_.hosts_[0]->metadata(buildMetadata("1.2")); - host_set_.hosts_[1]->metadata(buildMetadata("1.0", true)); - - modifyHosts({}, {host_set_.hosts_[2]}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); - - // Swap the default version once more, this time adding a new host and removing - // the current default version. - host_set_.hosts_[0]->metadata(buildMetadata("1.2", true)); - host_set_.hosts_[1]->metadata(buildMetadata("1.0")); - - modifyHosts({makeHost("tcp://127.0.0.1:8003", {{"version", "1.4"}})}, {host_set_.hosts_[1]}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_13)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_14)); - - // Make 1.4 default, without hosts being added/removed. - host_set_.hosts_[0]->metadata(buildMetadata("1.2")); - host_set_.hosts_[1]->metadata(buildMetadata("1.4", true)); - - host_set_.runCallbacks({}, {}); - - EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_14)); -} - -TEST_P(SubsetLoadBalancerTest, UpdateRemovingLastSubsetHost) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }); - - HostSharedPtr host_v10 = host_set_.hosts_[0]; - HostSharedPtr host_v11 = host_set_.hosts_[1]; - - TestLoadBalancerContext context({{"version", "1.0"}}); - EXPECT_EQ(host_v10, lb_->chooseHost(&context)); - EXPECT_EQ(1U, stats_.lb_subsets_selected_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_created_.value()); - - modifyHosts({}, {host_v10}); - - // fallback to any endpoint - EXPECT_EQ(host_v11, lb_->chooseHost(&context)); - EXPECT_EQ(1U, stats_.lb_subsets_selected_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(2U, stats_.lb_subsets_created_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); -} - -TEST_P(SubsetLoadBalancerTest, UpdateRemovingUnknownHost) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"stage", "version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"stage", "prod"}, {"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"stage", "prod"}, {"version", "1.1"}}}, - }); - - TestLoadBalancerContext context({{"stage", "prod"}, {"version", "1.0"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context)); - - modifyHosts({}, {makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), - makeHost("tcp://127.0.0.1:8001", {{"stage", "prod"}, {"version", "1.2"}})}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context)); -} - -TEST_F(SubsetLoadBalancerTest, UpdateModifyingOnlyHostHealth) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"hardware"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - }); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_11({{"version", "1.1"}}); - - // All hosts are healthy. - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); - - host_set_.hosts_[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - host_set_.hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_OUTLIER_CHECK); - host_set_.healthy_hosts_ = {host_set_.hosts_[1], host_set_.hosts_[3]}; - host_set_.runCallbacks({}, {}); - - // Unhealthy hosts are excluded. - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); -} - -TEST_F(SubsetLoadBalancerTest, BalancesDisjointSubsets) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"hardware"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "std"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"hardware", "bigmem"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}, {"hardware", "std"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}, {"hardware", "bigmem"}}}, - }); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_bigmem({{"hardware", "bigmem"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_bigmem)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_bigmem)); -} - -TEST_F(SubsetLoadBalancerTest, BalancesOverlappingSubsets) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"stage", "version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.0"}, {"stage", "off"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:84", {{"version", "999"}, {"stage", "dev"}}}, - }); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_10_prod({{"version", "1.0"}, {"stage", "prod"}}); - TestLoadBalancerContext context_dev({{"version", "999"}, {"stage", "dev"}}); - TestLoadBalancerContext context_unknown({{"version", "2.0"}, {"stage", "prod"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_10)); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10_prod)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10_prod)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10_prod)); - - EXPECT_EQ(host_set_.hosts_[4], lb_->chooseHost(&context_dev)); - EXPECT_EQ(host_set_.hosts_[4], lb_->chooseHost(&context_dev)); - - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown)); -} - -TEST_F(SubsetLoadBalancerTest, BalancesNestedSubsets) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"stage", "version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"stage"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.0"}, {"stage", "off"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:84", {{"version", "999"}, {"stage", "dev"}}}, - }); - - TestLoadBalancerContext context_prod({{"stage", "prod"}}); - TestLoadBalancerContext context_prod_10({{"version", "1.0"}, {"stage", "prod"}}); - TestLoadBalancerContext context_unknown_stage({{"stage", "larval"}}); - TestLoadBalancerContext context_unknown_version({{"version", "2.0"}, {"stage", "prod"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_prod)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_prod)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_prod)); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_prod_10)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_prod_10)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_prod_10)); - - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_stage)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_version)); -} - -TEST_F(SubsetLoadBalancerTest, IgnoresUnselectedMetadata) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "ignored"}}}, - {"tcp://127.0.0.1:81", {{"ignore", "value"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - }); - - TestLoadBalancerContext context_ignore({{"ignore", "value"}}); - TestLoadBalancerContext context_version({{"version", "1.0"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_version)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_version)); - - EXPECT_EQ(nullptr, lb_->chooseHost(&context_ignore)); -} - -TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - HostVector hosts; - hosts.emplace_back(makeTestHost(info_, "tcp://127.0.0.1:80", simTime())); - hosts.emplace_back(makeHost("tcp://127.0.0.1:81", {{"version", "1.0"}})); - - host_set_.hosts_ = hosts; - host_set_.hosts_per_locality_ = makeHostsPerLocality({hosts}); - - host_set_.healthy_hosts_ = host_set_.hosts_; - host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; - - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); - - TestLoadBalancerContext context_version({{"version", "1.0"}}); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_version)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_version)); -} - -// TODO(mattklein123): The following 4 tests verify basic functionality with all sub-LB tests. -// Optimally these would also be some type of TEST_P, but that is a little bit complicated as -// modifyHosts() also needs params. Clean this up. -TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRoundRobin) { - doLbTypeTest(LoadBalancerType::RoundRobin); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); -} - -TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesLeastRequest) { - doLbTypeTest(LoadBalancerType::LeastRequest); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); -} - -TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRandom) { - doLbTypeTest(LoadBalancerType::Random); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); -} - -TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRingHash) { - doLbTypeTest(LoadBalancerType::RingHash); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); -} - -TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesMaglev) { - doLbTypeTest(LoadBalancerType::Maglev); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); -} - -TEST_F(SubsetLoadBalancerTest, ZoneAwareFallback) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = {makeSelector( - {"x"}, envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - common_config_.mutable_healthy_panic_threshold()->set_value(40); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 40)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - }, - { - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.1"}}}, - }}, - {{ - {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, - }, - { - {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, - }}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(nullptr)); -} - -TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = {makeSelector( - {"x"}, envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - }, - { - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.1"}}}, - }}, - {{ - {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, - }, - { - {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, - }}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(nullptr)); - - envoy::config::core::v3::Locality local_locality; - local_locality.set_zone("0"); - - modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}, local_locality)}, - {host_set_.hosts_[0]}, absl::optional(0)); - - modifyLocalHosts({makeHost("tcp://127.0.0.1:9000", {{"version", "1.0"}}, local_locality)}, - {local_hosts_->at(0)}, 0); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); -} - -TEST_F(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubset) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "new"}}}, - {"tcp://127.0.0.1:81", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "new"}}}, - {"tcp://127.0.0.1:83", {{"version", "default"}}}, - {"tcp://127.0.0.1:84", {{"version", "new"}}}, - {"tcp://127.0.0.1:85", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:86", {{"version", "new"}}}, - {"tcp://127.0.0.1:87", {{"version", "default"}}}, - {"tcp://127.0.0.1:88", {{"version", "new"}}}, - {"tcp://127.0.0.1:89", {{"version", "default"}}}, - }}, - {{ - {"tcp://127.0.0.1:90", {{"version", "new"}}}, - {"tcp://127.0.0.1:91", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:92", {{"version", "new"}}}, - {"tcp://127.0.0.1:93", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:94", {{"version", "new"}}}, - {"tcp://127.0.0.1:95", {{"version", "default"}}}, - }}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); -} - -TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubsetAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "new"}}}, - {"tcp://127.0.0.1:81", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "new"}}}, - {"tcp://127.0.0.1:83", {{"version", "default"}}}, - {"tcp://127.0.0.1:84", {{"version", "new"}}}, - {"tcp://127.0.0.1:85", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:86", {{"version", "new"}}}, - {"tcp://127.0.0.1:87", {{"version", "default"}}}, - {"tcp://127.0.0.1:88", {{"version", "new"}}}, - {"tcp://127.0.0.1:89", {{"version", "default"}}}, - }}, - {{ - {"tcp://127.0.0.1:90", {{"version", "new"}}}, - {"tcp://127.0.0.1:91", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:92", {{"version", "new"}}}, - {"tcp://127.0.0.1:93", {{"version", "default"}}}, - }, - { - {"tcp://127.0.0.1:94", {{"version", "new"}}}, - {"tcp://127.0.0.1:95", {{"version", "default"}}}, - }}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); - - envoy::config::core::v3::Locality local_locality; - local_locality.set_zone("0"); - - modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}, local_locality)}, - {host_set_.hosts_[1]}, absl::optional(0)); - - modifyLocalHosts({local_hosts_->at(1)}, - {makeHost("tcp://127.0.0.1:9001", {{"version", "default"}}, local_locality)}, 0); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(nullptr)); -} - -TEST_F(SubsetLoadBalancerTest, ZoneAwareBalancesSubsets) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, - }}, - {{ - {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:93", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:94", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:95", {{"version", "1.1"}}}, - }}); - - TestLoadBalancerContext context({{"version", "1.1"}}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); -} - -TEST_P(SubsetLoadBalancerTest, ZoneAwareBalancesSubsetsAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, - }}, - {{ - {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:93", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:94", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:95", {{"version", "1.1"}}}, - }}); - - TestLoadBalancerContext context({{"version", "1.1"}}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); - - envoy::config::core::v3::Locality local_locality; - local_locality.set_zone("0"); - - modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "1.1"}}, local_locality)}, - {host_set_.hosts_[1]}, absl::optional(0)); - - modifyLocalHosts({local_hosts_->at(1)}, - {makeHost("tcp://127.0.0.1:9001", {{"version", "1.1"}}, local_locality)}, 0); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - - // Force request out of small zone. - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); -} - -TEST_F(SubsetLoadBalancerTest, ZoneAwareComplicatedBalancesSubsets) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - // L=local cluster host - // U=upstream host - // - // residuals - // A: 2L 0U 0.00% - // B: 2L 2U 6.67% - // C: 2L 2U 6.67% - // D: 0L 1U 20.00% - // total: 6L 5U 33.33% - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:90", {{"version", "1.1"}}}, - }}, - {{ - {"tcp://127.0.0.1:91", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:92", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:93", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:94", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:95", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:96", {{"version", "1.1"}}}, - }}); - - TestLoadBalancerContext context({{"version", "1.1"}}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(666)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(667)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[2][1], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(1334)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[3][0], lb_->chooseHost(&context)); -} - -TEST_P(SubsetLoadBalancerTest, ZoneAwareComplicatedBalancesSubsetsAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) - .WillRepeatedly(Return(50)); - EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) - .WillRepeatedly(Return(true)); - EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) - .WillRepeatedly(Return(2)); - - // Before update: - // - // L=local cluster host - // U=upstream host - // - // residuals - // A: 2L 0U 0.00% - // B: 2L 2U 6.67% - // C: 2L 2U 6.67% - // D: 0L 1U 20.00% - // total: 6L 5U 33.33% - - zoneAwareInit({{ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:90", {{"version", "1.1"}}}, - }}, - {{ - {"tcp://127.0.0.1:91", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:92", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:93", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:94", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:95", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:96", {{"version", "1.1"}}}, - }}); - - TestLoadBalancerContext context({{"version", "1.1"}}); - - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(666)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(667)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[2][1], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(1334)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[3][0], lb_->chooseHost(&context)); - - envoy::config::core::v3::Locality local_locality; - local_locality.set_zone("0"); - envoy::config::core::v3::Locality locality_2; - locality_2.set_zone("2"); - - modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "1.1"}}, local_locality)}, {}, - absl::optional(0)); - - modifyLocalHosts({makeHost("tcp://127.0.0.1:9001", {{"version", "1.1"}}, locality_2)}, {}, 2); - - // After update: - // - // L=local cluster host - // U=upstream host - // - // residuals - // A: 2L 1U 0.00% - // B: 2L 2U 4.76% - // C: 3L 2U 0.00% - // D: 0L 1U 16.67% - // total: 7L 6U 21.42% - // - // Chance of sampling local host in zone 0: 58.34% - - EXPECT_CALL(random_, random()) - .WillOnce(Return(0)) - .WillOnce(Return(5830)); // 58.31% local routing chance due to rounding error - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(5831)).WillOnce(Return(475)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(476)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[3][0], lb_->chooseHost(&context)); - EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2143)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); -} - -TEST_F(SubsetLoadBalancerTest, DescribeMetadata) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - init(); - - ProtobufWkt::Value str_value; - str_value.set_string_value("abc"); - - ProtobufWkt::Value num_value; - num_value.set_number_value(100); - - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.testDescribeMetadata("version=\"abc\"", {{"version", str_value}}); - tester.testDescribeMetadata("number=100", {{"number", num_value}}); - tester.testDescribeMetadata("x=\"abc\", y=100", {{"x", str_value}, {"y", num_value}}); - tester.testDescribeMetadata("y=100, x=\"abc\"", {{"y", num_value}, {"x", str_value}}); - tester.testDescribeMetadata("", {}); -} - -TEST_F(SubsetLoadBalancerTest, DisabledLocalityWeightAwareness) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - - // We configure a weighted host set that heavily favors the second locality. - configureWeightedHostSet( - { - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - host_set_, {1, 100}); - - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); - - TestLoadBalancerContext context({{"version", "1.1"}}); - - // Since we don't respect locality weights, the first locality is selected. - EXPECT_CALL(random_, random()).WillOnce(Return(0)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(&context)); -} - -// Verifies that we do *not* invoke coarseHealth() on hosts when constructing the load balancer. -// Since health is modified concurrently from multiple threads, it is not safe to call on the worker -// threads. -TEST_F(SubsetLoadBalancerTest, DoesNotCheckHostHealth) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - - auto mock_host = std::make_shared(); - HostVector hosts{mock_host}; - host_set_.hosts_ = hosts; - - EXPECT_CALL(*mock_host, weight()).WillRepeatedly(Return(1)); - - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); -} - -TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); - - // We configure a weighted host set that heavily favors the second locality. - configureWeightedHostSet( - { - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - host_set_, {1, 100}); - - common_config_.mutable_locality_weighted_lb_config(); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); - - TestLoadBalancerContext context({{"version", "1.1"}}); - - // Since we respect locality weights, the second locality is selected. - EXPECT_CALL(random_, random()).WillOnce(Return(0)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(&context)); -} - -TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeights) { - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); - - // We configure a weighted host set is weighted equally between each locality. - configureWeightedHostSet( - { - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - host_set_, {50, 50}); - - common_config_.mutable_locality_weighted_lb_config(); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); - TestLoadBalancerContext context({{"version", "1.1"}}); - - // Since we scale the locality weights by number of hosts removed, we expect to see the second - // locality to be selected less because we've excluded more hosts in that locality than in the - // first. - // The localities are split 50/50, but because of the scaling we expect to see 66/33 instead. - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); -} - -TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeightsRounding) { - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); - - // We configure a weighted host set where the locality weights are very low to test - // that we are rounding computation instead of flooring it. - configureWeightedHostSet( - { - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - { - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, - }, - host_set_, {2, 2}); - - common_config_.mutable_locality_weighted_lb_config(); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); - TestLoadBalancerContext context({{"version", "1.0"}}); - - // We expect to see a 33/66 split because 2 * 1 / 2 = 1 and 2 * 3 / 4 = 1.5 -> 2 - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][2], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); -} - -// Regression for bug where missing locality weights crashed scaling and locality aware subset LBs. -TEST_F(SubsetLoadBalancerTest, ScaleLocalityWeightsWithNoLocalityWeights) { - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); - EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); - - configureHostSet( - { - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, - }, - host_set_); - - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); -} - -TEST_P(SubsetLoadBalancerTest, GaugesUpdatedOnDestroy) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - }); - - EXPECT_EQ(1U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); - - lb_ = nullptr; - - EXPECT_EQ(0U, stats_.lb_subsets_active_.value()); - EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); -} - -TEST_P(SubsetLoadBalancerTest, SubsetSelectorNoFallbackPerSelector) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, - {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, - }); - - TestLoadBalancerContext context_10({{"version", "1.0"}}); - TestLoadBalancerContext context_11({{"version", "1.1"}}); - TestLoadBalancerContext context_12({{"version", "1.2"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_12)); - EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); - EXPECT_EQ(4U, stats_.lb_subsets_selected_.value()); -} - -TEST_P(SubsetLoadBalancerTest, FallbackNotDefinedForIntermediateSelector) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = { - makeSelector( - {"stage"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"stage", "version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"stage", "canary"}}}}); - - TestLoadBalancerContext context_match_host0({{"stage", "dev"}}); - TestLoadBalancerContext context_stage_nx({{"stage", "test"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_stage_nx)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_stage_nx)); -} - -TEST_P(SubsetLoadBalancerTest, SubsetSelectorFallbackOverridesTopLevelOne) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init(); - - TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); - TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); -} - -TEST_P(SubsetLoadBalancerTest, SubsetSelectorNoFallbackMatchesTopLevelOne) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init(); - - TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); - TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); - - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_key)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); -} - -TEST_F(SubsetLoadBalancerTest, AllowRedundantKeysForSubset) { - // Yaml config for subset load balancer. - const std::string yaml = R"EOF( - subset_selectors: - - keys: - - A - fallback_policy: NO_FALLBACK - - keys: - - A - - B - fallback_policy: NO_FALLBACK - - keys: - - A - - B - - C - fallback_policy: NO_FALLBACK - - keys: - - A - - D - fallback_policy: NO_FALLBACK - - keys: - - version - - stage - fallback_policy: NO_FALLBACK - fallback_policy: NO_FALLBACK - allow_redundant_keys: true - )EOF"; - - envoy::extensions::load_balancing_policies::subset::v3::Subset subset_proto_config; - TestUtility::loadFromYaml(yaml, subset_proto_config); - - actual_subset_info_ = std::make_unique>(subset_proto_config); - - // Add hosts initial hosts. - init({{"tcp://127.0.0.1:80", {{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-0"}, {"D", "D-V-0"}}}, - {"tcp://127.0.0.1:81", {{"A", "A-V-1"}, {"B", "B-V-1"}, {"C", "C-V-1"}, {"D", "D-V-1"}}}, - {"tcp://127.0.0.1:82", {{"A", "A-V-2"}, {"B", "B-V-2"}, {"C", "C-V-2"}, {"D", "D-V-2"}}}, - {"tcp://127.0.0.1:83", {{"A", "A-V-3"}, {"B", "B-V-3"}, {"C", "C-V-3"}, {"D", "D-V-3"}}}, - {"tcp://127.0.0.1:84", {{"A", "A-V-4"}, {"B", "B-V-4"}, {"C", "C-V-4"}, {"D", "D-V-4"}}}, - {"tcp://127.0.0.1:85", {{"version", "1.0"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:86", {{"version", "1.0"}, {"stage", "canary"}}}}, - {}, true); - - TestLoadBalancerContext context_empty( - std::initializer_list::value_type>{}); - context_empty.matches_.reset(); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_empty)); - - // Request metadata is same with {version, stage}. - // version, stage will be kept and host 6 will be selected. - TestLoadBalancerContext context_v_s_0({{"version", "1.0"}, {"stage", "canary"}}); - EXPECT_EQ(host_set_.hosts_[6], lb_->chooseHost(&context_v_s_0)); - - // Request metadata is superset of {version, stage}. The redundant key will be ignored. - // version, stage will be kept and host 5 will be selected. - TestLoadBalancerContext context_v_s_1({{"version", "1.0"}, {"stage", "dev"}, {"redundant", "X"}}); - EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&context_v_s_1)); - - // Request metadata is superset of {version, stage}. The redundant key will be ignored. - // But one of value not match, so no host will be selected. - TestLoadBalancerContext context_v_s_2( - {{"version", "1.0"}, {"stage", "prod"}, {"redundant", "X"}}); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_v_s_2)); - - // Request metadata is same with {A, B, C} and is superset of selectors {A}, {A, B}. - // All A, B, C will be kept and host 0 will be selected. - TestLoadBalancerContext context_0({{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-0"}}); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_0)); - - // Request metadata is same with {A, B, C} and is superset of selectors {A}, {A, B}. - // All A, B, C will be kept But one of value not match, so no host will be selected. - TestLoadBalancerContext context_1({{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-X"}}); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_1)); - - // Request metadata is superset of selectors {A}, {A, B} {A, B, C}, {A, D}, the longest win. - // A, B, C will be kept and D will be ignored, so host 1 will be selected. - TestLoadBalancerContext context_2( - {{"A", "A-V-1"}, {"B", "B-V-1"}, {"C", "C-V-1"}, {"D", "D-V-X"}}); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_2)); - - // Request metadata is superset of selectors {A}, {A, B} {A, B, C}, {A, D}, the longest win. - // A, B, C will be kept and D will be ignored, but one of value not match, so no host will be - // selected. - TestLoadBalancerContext context_3( - {{"A", "A-V-1"}, {"B", "B-V-1"}, {"C", "C-V-X"}, {"D", "D-V-X"}}); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_3)); - - // Request metadata is superset of selectors {A}, {A, B}, {A, D}, the longest and first win. - // Only A, B will be kept and D will be ignored, so host 2 will be selected. - TestLoadBalancerContext context_4({{"A", "A-V-2"}, {"B", "B-V-2"}, {"D", "D-V-X"}}); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_4)); - - // Request metadata is superset of selectors {A}, {A, B}, {A, D}, the longest and first win. - // Only A, B will be kept and D will be ignored, but one of value not match, so no host will be - // selected. - TestLoadBalancerContext context_5({{"A", "A-V-3"}, {"B", "B-V-X"}, {"D", "D-V-3"}}); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_5)); - - // Request metadata is superset of selectors {A}, {A, D}, the longest win. - // Only A, D will be kept and C will be ignored, so host 3 will be selected. - TestLoadBalancerContext context_6({{"A", "A-V-3"}, {"C", "C-V-X"}, {"D", "D-V-3"}}); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_6)); -} - -TEST_P(SubsetLoadBalancerTest, SubsetSelectorDefaultAnyFallbackPerSelector) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET), - makeSelector( - {"app"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT), - makeSelector( - {"foo"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"bar", "default"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - // Add hosts initial hosts. - init({{"tcp://127.0.0.1:81", {{"version", "0.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:83", {{"app", "envoy"}}}, - {"tcp://127.0.0.1:84", {{"foo", "abc"}, {"bar", "default"}}}}); - - TestLoadBalancerContext context_ver_10({{"version", "1.0"}}); - TestLoadBalancerContext context_ver_nx({{"version", "x"}}); - TestLoadBalancerContext context_app({{"app", "envoy"}}); - TestLoadBalancerContext context_app_nx({{"app", "ngnix"}}); - TestLoadBalancerContext context_foo({{"foo", "abc"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_app_nx)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_app_nx)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_app)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_ver_nx)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_foo)); -} - -TEST_P(SubsetLoadBalancerTest, SubsetSelectorDefaultAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:80", {{"version", "new"}}}, - {"tcp://127.0.0.1:81", {{"version", "default"}}}, - }); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); - - HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:8000", {{"version", "new"}}); - HostSharedPtr added_host2 = makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}); - - TestLoadBalancerContext context_ver_nx({{"version", "x"}}); - - modifyHosts({added_host1, added_host2}, {host_set_.hosts_.back()}); - - EXPECT_EQ(added_host2, lb_->chooseHost(&context_ver_nx)); -} - -TEST_P(SubsetLoadBalancerTest, SubsetSelectorAnyAfterUpdate) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT)}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({ - {"tcp://127.0.0.1:81", {{"version", "1"}}}, - {"tcp://127.0.0.1:82", {{"version", "2"}}}, - }); - - TestLoadBalancerContext context_ver_nx({{"version", "x"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_ver_nx)); - - HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:83", {{"version", "3"}}); - - modifyHosts({added_host1}, {host_set_.hosts_.back()}); - - EXPECT_EQ(added_host1, lb_->chooseHost(&context_ver_nx)); -} - -TEST_P(SubsetLoadBalancerTest, FallbackForCompoundSelector) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"foo", "bar"}}); - EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - - std::vector subset_selectors = { - makeSelector( - {"version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), - makeSelector( - {"version", "hardware", "stage"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK), - makeSelector( - {"version", "hardware"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET), - makeSelector( - {"version", "stage"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, - {"version"})}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - // Add hosts initial hosts. - init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "c32"}}}, - {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"hardware", "c32"}, {"foo", "bar"}}}, - {"tcp://127.0.0.1:82", {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:83", {{"version", "2.0"}}}}); - - TestLoadBalancerContext context_match_host0({{"version", "1.0"}, {"hardware", "c32"}}); - TestLoadBalancerContext context_ver_nx({{"version", "x"}, {"hardware", "c32"}}); - TestLoadBalancerContext context_stage_nx( - {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "x"}}); - TestLoadBalancerContext context_hardware_nx( - {{"version", "2.0"}, {"hardware", "zzz"}, {"stage", "dev"}}); - TestLoadBalancerContext context_match_host2( - {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "dev"}}); - TestLoadBalancerContext context_ver_20({{"version", "2.0"}}); - TestLoadBalancerContext context_ver_stage_match_host2({{"version", "2.0"}, {"stage", "dev"}}); - TestLoadBalancerContext context_ver_stage_nx({{"version", "2.0"}, {"stage", "canary"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_ver_nx)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_hardware_nx)); - EXPECT_EQ(nullptr, lb_->chooseHost(&context_stage_nx)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_match_host2)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_match_host2)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_20)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_stage_match_host2)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_stage_match_host2)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_ver_stage_nx)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_stage_nx)); -} - -TEST_P(SubsetLoadBalancerTest, KeysSubsetFallbackChained) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - std::vector subset_selectors = { - makeSelector( - {"stage"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK), - makeSelector( - {"stage", "version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, - {"stage"}), - makeSelector( - {"stage", "version", "hardware"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, - {"version", "stage"})}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "c32"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:81", {{"version", "2.0"}, {"hardware", "c64"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:82", {{"version", "1.0"}, {"hardware", "c32"}, {"stage", "test"}}}}); - - TestLoadBalancerContext context_match_host0( - {{"version", "1.0"}, {"hardware", "c32"}, {"stage", "dev"}}); - TestLoadBalancerContext context_hw_nx( - {{"version", "2.0"}, {"hardware", "arm"}, {"stage", "dev"}}); - TestLoadBalancerContext context_ver_hw_nx( - {{"version", "1.2"}, {"hardware", "arm"}, {"stage", "dev"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_hw_nx)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_hw_nx)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_ver_hw_nx)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_ver_hw_nx)); -} - -TEST_P(SubsetLoadBalancerTest, KeysSubsetFallbackToNotExistingSelector) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - - std::vector subset_selectors = {makeSelector( - {"stage", "version"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, - {"stage"})}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "dev"}}}}); - - TestLoadBalancerContext context_nx({{"version", "1.0"}, {"stage", "test"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_nx)); - EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); -} - -TEST_P(SubsetLoadBalancerTest, MetadataFallbackList) { - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - EXPECT_CALL(subset_info_, metadataFallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::FALLBACK_LIST)); - - std::vector subset_selectors = {makeSelector({"version"})}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({{"tcp://127.0.0.1:80", {{"version", "1.0"}}}, - {"tcp://127.0.0.1:81", {{"version", "2.0"}}}, - {"tcp://127.0.0.1:82", {{"version", "3.0"}}}}); - - const auto version1_host = host_set_.hosts_[0]; - const auto version2_host = host_set_.hosts_[1]; - const auto version3_host = host_set_.hosts_[2]; - - // No context. - EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); - - TestLoadBalancerContext context_without_metadata({{"key", "value"}}); - context_without_metadata.matches_ = nullptr; - - // No metadata in context. - EXPECT_EQ(nullptr, lb_->chooseHost(&context_without_metadata)); - - TestLoadBalancerContext context_with_fallback({{"fallback_list", valueFromJson(R""""( - [ - {"version": "2.0"}, - {"version": "1.0"} - ] - )"""")}}); - - // version 2.0 is preferred, should be selected - EXPECT_EQ(version2_host, lb_->chooseHost(&context_with_fallback)); - EXPECT_EQ(version2_host, lb_->chooseHost(&context_with_fallback)); - EXPECT_EQ(version2_host, lb_->chooseHost(&context_with_fallback)); - - modifyHosts({}, {version2_host}); - - // version 1.0 is a fallback, should be used when host with version 2.0 is removed - EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_fallback)); - EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_fallback)); - EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_fallback)); - - // if fallback_list is not a list, it should be ignored - // regular metadata is in effect - ProtobufWkt::Value null_value; - null_value.set_null_value(ProtobufWkt::NullValue::NULL_VALUE); - TestLoadBalancerContext context_with_invalid_fallback_list_null( - {{"version", valueFromJson("\"3.0\"")}, {"fallback_list", null_value}}); - - EXPECT_EQ(version3_host, lb_->chooseHost(&context_with_invalid_fallback_list_null)); - EXPECT_EQ(version3_host, lb_->chooseHost(&context_with_invalid_fallback_list_null)); - - // should ignore fallback list entry which is not a struct - TestLoadBalancerContext context_with_invalid_fallback_list_entry( - {{"fallback_list", valueFromJson(R""""( - [ - "invalid string entry", - {"version": "1.0"} - ] - )"""")}}); - - EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_invalid_fallback_list_entry)); - EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_invalid_fallback_list_entry)); - - // simple metadata with no fallback should work as usual - TestLoadBalancerContext context_no_fallback({{"version", "1.0"}}); - EXPECT_EQ(version1_host, lb_->chooseHost(&context_no_fallback)); - EXPECT_EQ(version1_host, lb_->chooseHost(&context_no_fallback)); - - // fallback metadata overrides regular metadata value - TestLoadBalancerContext context_fallback_overrides_metadata_value( - {{"version", valueFromJson("\"1.0\"")}, {"fallback_list", valueFromJson(R""""( - [ - {"hardware": "arm"}, - {"version": "5.0"}, - {"version": "3.0"} - ] - )"""")}}); - EXPECT_EQ(version3_host, lb_->chooseHost(&context_fallback_overrides_metadata_value)); - EXPECT_EQ(version3_host, lb_->chooseHost(&context_fallback_overrides_metadata_value)); -} - -TEST_P(SubsetLoadBalancerTest, MetadataFallbackDisabled) { - - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - EXPECT_CALL(subset_info_, metadataFallbackPolicy()) - .WillRepeatedly( - Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::METADATA_NO_FALLBACK)); - - std::vector subset_selectors = {makeSelector({"fallback_list"})}; - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({{"tcp://127.0.0.1:80", {{"fallback_list", "lorem"}}}, - {"tcp://127.0.0.1:81", {{"fallback_list", "ipsum"}}}}); - - // should treat 'fallback_list' as a regular metadata key - TestLoadBalancerContext context({{"fallback_list", "ipsum"}}); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context)); -} - -TEST_P(SubsetLoadBalancerTest, MetadataFallbackAndSubsetFallback) { - - EXPECT_CALL(subset_info_, fallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - EXPECT_CALL(subset_info_, metadataFallbackPolicy()) - .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::FALLBACK_LIST)); - - std::vector subset_selectors = { - makeSelector( - {"hardware"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK), - makeSelector( - {"hardware", "stage"}, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, - {"hardware"})}; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - - init({{"tcp://127.0.0.1:80", {{"hardware", "c32"}, {"stage", "production"}}}, - {"tcp://127.0.0.1:81", {{"hardware", "c64"}, {"stage", "canary"}}}, - {"tcp://127.0.0.1:82", {{"hardware", "c64"}, {"stage", "production"}}}}); - - const auto c32_production_host = host_set_.hosts_[0]; - const auto c64_canary_host = host_set_.hosts_[1]; - const auto c64_production_host = host_set_.hosts_[2]; - - TestLoadBalancerContext context_canary_c32_preffered( - {{"stage", valueFromJson("\"canary\"")}, {"fallback_list", valueFromJson(R""""( - [ - {"hardware": "c32"}, - {"hardware": "c64"} - ] - )"""")}}); - - // Should select c32_production_host using first fallback entry, even - // when it doesn't match on requested 'stage' - because of the subset fallback policy. - // There is the c64_canary_host which exactly matches second fallback entry, but - // that entry is not used. - EXPECT_EQ(c32_production_host, lb_->chooseHost(&context_canary_c32_preffered)); - EXPECT_EQ(c32_production_host, lb_->chooseHost(&context_canary_c32_preffered)); - - TestLoadBalancerContext context_canary_c16_preffered( - {{"stage", valueFromJson("\"canary\"")}, {"fallback_list", valueFromJson(R""""( - [ - {"hardware": "c16"}, - {"hardware": "c64"} - ] - )"""")}}); - - // Should select c64_canary_host using second fallback entry. First fallback - // entry doesn't match anything even considering subset fallback policy. - EXPECT_EQ(c64_canary_host, lb_->chooseHost(&context_canary_c16_preffered)); - EXPECT_EQ(c64_canary_host, lb_->chooseHost(&context_canary_c16_preffered)); - - TestLoadBalancerContext context_unknown_or_c64({{"fallback_list", valueFromJson(R""""( - [ - {"unknown": "ipsum"}, - {"hardware": "c64"} - ] - )"""")}}); - - // should select any host using first fallback entry, because of ANY_ENDPOINT - // subset fallback policy - EXPECT_EQ(c32_production_host, lb_->chooseHost(&context_unknown_or_c64)); - EXPECT_EQ(c64_canary_host, lb_->chooseHost(&context_unknown_or_c64)); - EXPECT_EQ(c64_production_host, lb_->chooseHost(&context_unknown_or_c64)); -} - -INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerTest, - testing::ValuesIn({UpdateOrder::RemovesFirst, UpdateOrder::Simultaneous})); - -class SubsetLoadBalancerSingleHostPerSubsetTest : public SubsetLoadBalancerTest { -public: - SubsetLoadBalancerSingleHostPerSubsetTest() - : default_subset_selectors_({ - makeSelector({"key"}, true), - }) { - ON_CALL(subset_info_, subsetSelectors()).WillByDefault(ReturnRef(default_subset_selectors_)); - ON_CALL(subset_info_, fallbackPolicy()) - .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - } - - using SubsetLoadBalancerTest::init; - void init() { - init({ - {"tcp://127.0.0.1:80", {}}, - {"tcp://127.0.0.1:81", {{"key", "a"}}}, - {"tcp://127.0.0.1:82", {{"key", "b"}}}, - - }); - } - - using SubsetLoadBalancerTest::makeSelector; - SubsetSelectorPtr makeSelector(const std::set& selector_keys, - bool single_host_per_subset) { - return makeSelector( - selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED, {}, - single_host_per_subset); - } - - std::vector default_subset_selectors_; -}; - -TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, AcceptMultipleSelectors) { - std::vector subset_selectors = { - makeSelector({"version"}, false), - makeSelector({"stage"}, true), - }; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - ON_CALL(subset_info_, fallbackPolicy()) - .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - init({ - {"tcp://127.0.0.1:80", {}}, - {"tcp://127.0.0.1:81", {{"version", "v1"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:82", {{"version", "v1"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:83", {{"version", "v1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:84", {{"version", "v1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:85", {{"version", "v2"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:86", {{"version", "v2"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:87", {{"version", "v2"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:88", {{"version", "v2"}, {"stage", "prod"}}}, - }); - - TestLoadBalancerContext version_v1({{"version", "v1"}}); - TestLoadBalancerContext version_v2({{"version", "v2"}}); - TestLoadBalancerContext stage_dev({{"stage", "dev"}}); - TestLoadBalancerContext stage_prod({{"stage", "prod"}}); - TestLoadBalancerContext stage_test({{"stage", "test"}}); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&version_v1)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&version_v1)); - - EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&version_v2)); - EXPECT_EQ(host_set_.hosts_[6], lb_->chooseHost(&version_v2)); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&stage_dev)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&stage_dev)); - - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&stage_prod)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&stage_prod)); - - EXPECT_EQ(nullptr, lb_->chooseHost(&stage_test)); -} - -TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, AcceptMultipleKeys) { - std::vector subset_selectors = { - makeSelector({"version", "stage"}, true), - }; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - ON_CALL(subset_info_, fallbackPolicy()) - .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - init({ - {"tcp://127.0.0.1:80", {}}, - {"tcp://127.0.0.1:81", {{"version", "v1"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:82", {{"version", "v1"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:83", {{"version", "v1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:84", {{"version", "v1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:85", {{"version", "v2"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:86", {{"version", "v2"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:87", {{"version", "v2"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:88", {{"version", "v2"}, {"stage", "prod"}}}, - }); - - TestLoadBalancerContext v1_dev({{"version", "v1"}, {"stage", "dev"}}); - TestLoadBalancerContext v1_prod({{"version", "v1"}, {"stage", "prod"}}); - TestLoadBalancerContext v2_dev({{"version", "v2"}, {"stage", "dev"}}); - TestLoadBalancerContext v2_prod({{"version", "v2"}, {"stage", "prod"}}); - TestLoadBalancerContext v2_test({{"version", "v2"}, {"stage", "test"}}); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&v1_dev)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&v1_prod)); - EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&v2_dev)); - EXPECT_EQ(host_set_.hosts_[7], lb_->chooseHost(&v2_prod)); - EXPECT_EQ(nullptr, lb_->chooseHost(&v2_test)); -} - -TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, HybridMultipleSelectorsAndKeys) { - std::vector subset_selectors = { - makeSelector({"version", "stage"}, true), - makeSelector({"stage"}, false), - }; - - EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - ON_CALL(subset_info_, fallbackPolicy()) - .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); - - init({ - {"tcp://127.0.0.1:80", {}}, - {"tcp://127.0.0.1:81", {{"version", "v1"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:82", {{"version", "v1"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:83", {{"version", "v1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:84", {{"version", "v1"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:85", {{"version", "v2"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:86", {{"version", "v2"}, {"stage", "dev"}}}, - {"tcp://127.0.0.1:87", {{"version", "v2"}, {"stage", "prod"}}}, - {"tcp://127.0.0.1:88", {{"version", "v2"}, {"stage", "prod"}}}, - }); - - TestLoadBalancerContext v1_dev({{"version", "v1"}, {"stage", "dev"}}); - TestLoadBalancerContext v1_prod({{"version", "v1"}, {"stage", "prod"}}); - TestLoadBalancerContext v2_dev({{"version", "v2"}, {"stage", "dev"}}); - TestLoadBalancerContext v2_prod({{"version", "v2"}, {"stage", "prod"}}); - TestLoadBalancerContext v2_test({{"version", "v2"}, {"stage", "test"}}); - TestLoadBalancerContext stage_dev({{"stage", "dev"}}); - TestLoadBalancerContext stage_prod({{"stage", "prod"}}); - TestLoadBalancerContext stage_test({{"stage", "test"}}); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&v1_dev)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&v1_prod)); - EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&v2_dev)); - EXPECT_EQ(host_set_.hosts_[7], lb_->chooseHost(&v2_prod)); - EXPECT_EQ(nullptr, lb_->chooseHost(&v2_test)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&stage_dev)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&stage_dev)); - EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&stage_prod)); - EXPECT_EQ(host_set_.hosts_[4], lb_->chooseHost(&stage_prod)); - EXPECT_EQ(nullptr, lb_->chooseHost(&stage_test)); -} - -TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, DuplicateMetadataStat) { - init({ - {"tcp://127.0.0.1:80", {{"key", "a"}}}, - {"tcp://127.0.0.1:81", {{"key", "a"}}}, - {"tcp://127.0.0.1:82", {{"key", "a"}}}, - {"tcp://127.0.0.1:83", {{"key", "b"}}}, - }); - // The first 'a' is the original, the next 2 instances of 'a' are duplicates (counted - // in stat), and 'b' is another non-duplicate. - for (auto& gauge : stats_store_.gauges()) { - ENVOY_LOG_MISC(debug, "name {} value {}", gauge->name(), gauge->value()); - } - EXPECT_EQ(2, TestUtility::findGauge(stats_store_, - "testprefix.lb_subsets_single_host_per_subset_duplicate") - ->value()); -} - -TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, Match) { - init(); - - TestLoadBalancerContext host_1({{"key", "a"}}); - TestLoadBalancerContext host_2({{"key", "b"}}); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_1)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_1)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_2)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_2)); -} - -TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, FallbackOnUnknownMetadata) { - init(); - - TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); - TestLoadBalancerContext context_unknown_value({{"key", "unknown"}}); - - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_unknown_value)); -} - -TEST_P(SubsetLoadBalancerSingleHostPerSubsetTest, Update) { - init(); - - TestLoadBalancerContext host_a({{"key", "a"}}); - TestLoadBalancerContext host_b({{"key", "b"}}); - TestLoadBalancerContext host_c({{"key", "c"}}); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_b)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_b)); - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&host_c)); // fallback - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_c)); // fallback - - HostSharedPtr added_host = makeHost("tcp://127.0.0.1:8000", {{"key", "c"}}); - - // Remove b, add c - modifyHosts({added_host}, {host_set_.hosts_.back()}); - - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_c)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_c)); - EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_b)); // fallback - EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&host_b)); // fallback - EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_b)); // fallback -} - -INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerSingleHostPerSubsetTest, - testing::ValuesIn({UpdateOrder::RemovesFirst, UpdateOrder::Simultaneous})); - -} // namespace SubsetLoadBalancerTest -} // namespace Upstream -} // namespace Envoy diff --git a/test/common/upstream/test_cluster_manager.h b/test/common/upstream/test_cluster_manager.h index 32688a6e9126..71f8629430f5 100644 --- a/test/common/upstream/test_cluster_manager.h +++ b/test/common/upstream/test_cluster_manager.h @@ -21,7 +21,6 @@ #include "source/common/singleton/manager_impl.h" #include "source/common/upstream/cluster_factory_impl.h" #include "source/common/upstream/cluster_manager_impl.h" -#include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "source/extensions/transport_sockets/tls/context_manager_impl.h" #include "test/common/stats/stat_test_utility.h" @@ -160,36 +159,24 @@ class TestClusterManagerFactory : public ClusterManagerFactory { Server::MockOptions& options_ = server_context_.options_; }; -// Helper to intercept calls to postThreadLocalClusterUpdate. -class MockLocalClusterUpdate { -public: - MOCK_METHOD(void, post, - (uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed)); -}; - -class MockLocalHostsRemoved { -public: - MOCK_METHOD(void, post, (const HostVector&)); -}; - // A test version of ClusterManagerImpl that provides a way to get a non-const handle to the // clusters, which is necessary in order to call updateHosts on the priority set. class TestClusterManagerImpl : public ClusterManagerImpl { public: - using ClusterManagerImpl::ClusterManagerImpl; - - TestClusterManagerImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, - ClusterManagerFactory& factory, Stats::Store& stats, - ThreadLocal::Instance& tls, Runtime::Loader& runtime, - const LocalInfo::LocalInfo& local_info, - AccessLog::AccessLogManager& log_manager, - Event::Dispatcher& main_thread_dispatcher, Server::Admin& admin, - ProtobufMessage::ValidationContext& validation_context, Api::Api& api, - Http::Context& http_context, Grpc::Context& grpc_context, - Router::Context& router_context, Server::Instance& server) - : ClusterManagerImpl(bootstrap, factory, stats, tls, runtime, local_info, log_manager, - main_thread_dispatcher, admin, validation_context, api, http_context, - grpc_context, router_context, server) {} + static std::unique_ptr + createAndInit(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + ClusterManagerFactory& factory, Stats::Store& stats, ThreadLocal::Instance& tls, + Runtime::Loader& runtime, const LocalInfo::LocalInfo& local_info, + AccessLog::AccessLogManager& log_manager, Event::Dispatcher& main_thread_dispatcher, + Server::Admin& admin, ProtobufMessage::ValidationContext& validation_context, + Api::Api& api, Http::Context& http_context, Grpc::Context& grpc_context, + Router::Context& router_context, Server::Instance& server) { + auto cluster_manager = std::unique_ptr{new TestClusterManagerImpl( + bootstrap, factory, stats, tls, runtime, local_info, log_manager, main_thread_dispatcher, + admin, validation_context, api, http_context, grpc_context, router_context, server)}; + THROW_IF_NOT_OK(cluster_manager->init(bootstrap)); + return cluster_manager; + } std::map> activeClusters() { std::map> clusters; @@ -199,6 +186,10 @@ class TestClusterManagerImpl : public ClusterManagerImpl { return clusters; } + const ClusterInitializationMap& clusterInitializationMap() const { + return cluster_initialization_map_; + } + OdCdsApiHandlePtr createOdCdsApiHandle(OdCdsApiSharedPtr odcds) { return ClusterManagerImpl::OdCdsApiHandleImpl::create(*this, std::move(odcds)); } @@ -210,43 +201,22 @@ class TestClusterManagerImpl : public ClusterManagerImpl { ClusterDiscoveryManager createAndSwapClusterDiscoveryManager(std::string thread_name) { return ClusterManagerImpl::createAndSwapClusterDiscoveryManager(std::move(thread_name)); } -}; - -// Override postThreadLocalClusterUpdate so we can test that merged updates calls -// it with the right values at the right times. -class MockedUpdatedClusterManagerImpl : public TestClusterManagerImpl { -public: - MockedUpdatedClusterManagerImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, - ClusterManagerFactory& factory, Stats::Store& stats, - ThreadLocal::Instance& tls, Runtime::Loader& runtime, - const LocalInfo::LocalInfo& local_info, - AccessLog::AccessLogManager& log_manager, - Event::Dispatcher& main_thread_dispatcher, Server::Admin& admin, - ProtobufMessage::ValidationContext& validation_context, - Api::Api& api, MockLocalClusterUpdate& local_cluster_update, - MockLocalHostsRemoved& local_hosts_removed, - Http::Context& http_context, Grpc::Context& grpc_context, - Router::Context& router_context, Server::Instance& server) - : TestClusterManagerImpl(bootstrap, factory, stats, tls, runtime, local_info, log_manager, - main_thread_dispatcher, admin, validation_context, api, http_context, - grpc_context, router_context, server), - local_cluster_update_(local_cluster_update), local_hosts_removed_(local_hosts_removed) {} protected: - void postThreadLocalClusterUpdate(ClusterManagerCluster&, - ThreadLocalClusterUpdateParams&& params) override { - for (const auto& per_priority : params.per_priority_update_params_) { - local_cluster_update_.post(per_priority.priority_, per_priority.hosts_added_, - per_priority.hosts_removed_); - } - } - - void postThreadLocalRemoveHosts(const Cluster&, const HostVector& hosts_removed) override { - local_hosts_removed_.post(hosts_removed); - } + using ClusterManagerImpl::ClusterManagerImpl; - MockLocalClusterUpdate& local_cluster_update_; - MockLocalHostsRemoved& local_hosts_removed_; + TestClusterManagerImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap, + ClusterManagerFactory& factory, Stats::Store& stats, + ThreadLocal::Instance& tls, Runtime::Loader& runtime, + const LocalInfo::LocalInfo& local_info, + AccessLog::AccessLogManager& log_manager, + Event::Dispatcher& main_thread_dispatcher, Server::Admin& admin, + ProtobufMessage::ValidationContext& validation_context, Api::Api& api, + Http::Context& http_context, Grpc::Context& grpc_context, + Router::Context& router_context, Server::Instance& server) + : ClusterManagerImpl(bootstrap, factory, stats, tls, runtime, local_info, log_manager, + main_thread_dispatcher, admin, validation_context, api, http_context, + grpc_context, router_context, server) {} }; } // namespace Upstream diff --git a/test/common/upstream/test_local_address_selector.h b/test/common/upstream/test_local_address_selector.h index bc2683b70d31..a4b7407a6a60 100644 --- a/test/common/upstream/test_local_address_selector.h +++ b/test/common/upstream/test_local_address_selector.h @@ -20,7 +20,7 @@ class TestUpstreamLocalAddressSelector : public UpstreamLocalAddressSelector { getUpstreamLocalAddressImpl(const Network::Address::InstanceConstSharedPtr&) const override { ++(*num_calls_); if (return_empty_source_address_) { - return UpstreamLocalAddress(); + return {}; } current_idx_ = (current_idx_ + 1) % upstream_local_addresses_.size(); return upstream_local_addresses_[current_idx_]; @@ -40,7 +40,7 @@ class TestUpstreamLocalAddressSelectorFactory : public UpstreamLocalAddressSelec bool return_empty_source_address = false) : num_calls_(num_calls), return_empty_source_address_{return_empty_source_address} {} - UpstreamLocalAddressSelectorConstSharedPtr createLocalAddressSelector( + absl::StatusOr createLocalAddressSelector( std::vector<::Envoy::Upstream::UpstreamLocalAddress> upstream_local_addresses, absl::optional) const override { return std::make_shared(upstream_local_addresses, num_calls_, diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 4d0bc20708b2..a825d39cadd0 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -26,6 +26,8 @@ #include "source/common/singleton/manager_impl.h" #include "source/extensions/clusters/static/static_cluster.h" #include "source/extensions/clusters/strict_dns/strict_dns_cluster.h" +#include "source/extensions/load_balancing_policies/least_request/config.h" +#include "source/extensions/load_balancing_policies/round_robin/config.h" #include "source/server/transport_socket_config_impl.h" #include "test/common/stats/stat_test_utility.h" @@ -209,6 +211,116 @@ TEST_P(StrictDnsParamTest, ImmediateResolve) { EXPECT_EQ(2UL, cluster.prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); } +TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBasicMillion) { + auto dns_resolver = std::make_shared>(); + ReadyWatcher initialized; + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: strict_dns + )EOF" + std::get<0>(GetParam()) + + R"EOF( + lb_policy: round_robin + load_assignment: + policy: + drop_overloads: + category: test + drop_percentage: + numerator: 35 + denominator: MILLION + )EOF"; + envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + Envoy::Upstream::ClusterFactoryContextImpl factory_context( + server_context_, server_context_.cluster_manager_, nullptr, ssl_context_manager_, nullptr, + false); + StrictDnsClusterImpl cluster(cluster_config, factory_context, dns_resolver); + EXPECT_EQ(0.000035f, cluster.dropOverload().value()); +} + +TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBasicTenThousand) { + auto dns_resolver = std::make_shared>(); + ReadyWatcher initialized; + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: strict_dns + )EOF" + std::get<0>(GetParam()) + + R"EOF( + lb_policy: round_robin + load_assignment: + policy: + drop_overloads: + category: test + drop_percentage: + numerator: 1000 + denominator: TEN_THOUSAND + )EOF"; + envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + Envoy::Upstream::ClusterFactoryContextImpl factory_context( + server_context_, server_context_.cluster_manager_, nullptr, ssl_context_manager_, nullptr, + false); + StrictDnsClusterImpl cluster(cluster_config, factory_context, dns_resolver); + EXPECT_EQ(0.1f, cluster.dropOverload().value()); +} + +TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBadDenominator) { + auto dns_resolver = std::make_shared>(); + ReadyWatcher initialized; + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: strict_dns + )EOF" + std::get<0>(GetParam()) + + R"EOF( + lb_policy: round_robin + load_assignment: + policy: + drop_overloads: + category: test + drop_percentage: + numerator: 35 + denominator: 4 + )EOF"; + + envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + Envoy::Upstream::ClusterFactoryContextImpl factory_context( + server_context_, server_context_.cluster_manager_, nullptr, ssl_context_manager_, nullptr, + false); + EXPECT_THROW_WITH_MESSAGE( + StrictDnsClusterImpl cluster(cluster_config, factory_context, dns_resolver), EnvoyException, + "Cluster drop_overloads config denominator setting is invalid : 4. Valid range 0~2."); +} + +TEST_P(StrictDnsParamTest, DropOverLoadConfigTestMultipleCategory) { + auto dns_resolver = std::make_shared>(); + ReadyWatcher initialized; + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: strict_dns + )EOF" + std::get<0>(GetParam()) + + R"EOF( + lb_policy: round_robin + load_assignment: + policy: + drop_overloads: + - category: foo + drop_percentage: + numerator: 35 + - category: bar + drop_percentage: + numerator: 10 + )EOF"; + + envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + Envoy::Upstream::ClusterFactoryContextImpl factory_context( + server_context_, server_context_.cluster_manager_, nullptr, ssl_context_manager_, nullptr, + false); + EXPECT_THROW_WITH_MESSAGE( + StrictDnsClusterImpl cluster(cluster_config, factory_context, dns_resolver), EnvoyException, + "Cluster drop_overloads config has 2 categories. Envoy only support one."); +} + class StrictDnsClusterImplTest : public testing::Test, public UpstreamImplTestBase { protected: std::shared_ptr dns_resolver_ = @@ -1883,7 +1995,7 @@ TEST_F(StaticClusterImplTest, LoadAssignmentLocality) { EXPECT_EQ("hello", locality.zone()); EXPECT_EQ("world", locality.sub_zone()); } - EXPECT_EQ(nullptr, cluster->prioritySet().hostSetsPerPriority()[0]->localityWeights()); + EXPECT_NE(nullptr, cluster->prioritySet().hostSetsPerPriority()[0]->localityWeights()); EXPECT_FALSE(cluster->info()->addedViaApi()); } @@ -1981,7 +2093,9 @@ TEST_F(StaticClusterImplTest, RingHash) { cluster->initialize([] {}); EXPECT_EQ(1UL, cluster->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); - EXPECT_EQ(LoadBalancerType::RingHash, cluster->info()->lbType()); + EXPECT_EQ(LoadBalancerType::LoadBalancingPolicyConfig, cluster->info()->lbType()); + EXPECT_EQ("envoy.load_balancing_policies.ring_hash", + cluster->info()->loadBalancerFactory()->name()); EXPECT_TRUE(cluster->info()->addedViaApi()); } @@ -2016,8 +2130,14 @@ TEST_F(StaticClusterImplTest, RoundRobinWithSlowStart) { cluster->initialize([] {}); - EXPECT_EQ(LoadBalancerType::RoundRobin, cluster->info()->lbType()); - auto slow_start_config = cluster->info()->lbRoundRobinConfig()->slow_start_config(); + EXPECT_EQ(LoadBalancerType::LoadBalancingPolicyConfig, cluster->info()->lbType()); + EXPECT_EQ("envoy.load_balancing_policies.round_robin", + cluster->info()->loadBalancerFactory()->name()); + auto slow_start_config = + dynamic_cast( + cluster->info()->loadBalancerConfig().ptr()) + ->lbConfig() + ->slow_start_config(); EXPECT_EQ(std::chrono::milliseconds(60000), std::chrono::milliseconds( DurationUtil::durationToMilliseconds(slow_start_config.slow_start_window()))); @@ -2056,8 +2176,15 @@ TEST_F(StaticClusterImplTest, LeastRequestWithSlowStart) { cluster->initialize([] {}); - EXPECT_EQ(LoadBalancerType::LeastRequest, cluster->info()->lbType()); - auto slow_start_config = cluster->info()->lbLeastRequestConfig()->slow_start_config(); + EXPECT_EQ(LoadBalancerType::LoadBalancingPolicyConfig, cluster->info()->lbType()); + EXPECT_EQ("envoy.load_balancing_policies.least_request", + cluster->info()->loadBalancerFactory()->name()); + auto slow_start_config = + dynamic_cast< + const Extensions::LoadBalancingPolices::LeastRequest::LegacyLeastRequestLbConfig*>( + cluster->info()->loadBalancerConfig().ptr()) + ->lbConfig() + ->slow_start_config(); EXPECT_EQ(std::chrono::milliseconds(60000), std::chrono::milliseconds( DurationUtil::durationToMilliseconds(slow_start_config.slow_start_window()))); @@ -2367,7 +2494,8 @@ TEST_F(StaticClusterImplTest, UrlConfig) { EXPECT_EQ(0U, cluster->info()->maxRequestsPerConnection()); EXPECT_EQ(::Envoy::Http2::Utility::OptionsLimits::DEFAULT_HPACK_TABLE_SIZE, cluster->info()->http2Options().hpack_table_size().value()); - EXPECT_EQ(LoadBalancerType::Random, cluster->info()->lbType()); + EXPECT_EQ(LoadBalancerType::LoadBalancingPolicyConfig, cluster->info()->lbType()); + EXPECT_EQ("envoy.load_balancing_policies.random", cluster->info()->loadBalancerFactory()->name()); EXPECT_THAT( std::list({"10.0.0.1:11001", "10.0.0.2:11002"}), ContainerEq(hostListToAddresses(cluster->prioritySet().hostSetsPerPriority()[0]->hosts()))); @@ -2610,6 +2738,36 @@ TEST_F(StaticClusterImplTest, LoadBalancingPolicyWithLbSubsetConfig) { EnvoyException, "cluster: load_balancing_policy cannot be combined with lb_subset_config"); } +// Empty lb_subset_config is set and it should be ignored. +TEST_F(StaticClusterImplTest, EmptyLbSubsetConfig) { + const std::string yaml = R"EOF( + name: staticcluster + connect_timeout: 0.25s + type: static + lb_policy: ROUND_ROBIN + lb_subset_config: {} + load_assignment: + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 10.0.0.1 + port_value: 11001 + )EOF"; + + envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); + + Envoy::Upstream::ClusterFactoryContextImpl factory_context( + server_context_, server_context_.cluster_manager_, nullptr, ssl_context_manager_, nullptr, + true); + + auto cluster = createCluster(cluster_config, factory_context); + + EXPECT_EQ(cluster->info()->loadBalancerFactory()->name(), + "envoy.load_balancing_policies.round_robin"); +} + // Verify that if Envoy does not have a factory for any of the load balancing policies specified in // the load balancing policy config, it is an error. TEST_F(StaticClusterImplTest, LbPolicyConfigThrowsExceptionIfNoLbPoliciesFound) { @@ -3618,7 +3776,8 @@ TEST_F(ClusterInfoImplTest, Metadata) { Config::Metadata::metadataValue(&cluster->info()->metadata(), "com.bar.foo", "baz") .string_value()); EXPECT_EQ(0.3, cluster->info()->lbConfig().healthy_panic_threshold().value()); - EXPECT_EQ(LoadBalancerType::Maglev, cluster->info()->lbType()); + EXPECT_EQ(LoadBalancerType::LoadBalancingPolicyConfig, cluster->info()->lbType()); + EXPECT_EQ("envoy.load_balancing_policies.maglev", cluster->info()->loadBalancerFactory()->name()); } // Verify retry budget default values are honored. @@ -3685,6 +3844,22 @@ TEST_F(ClusterInfoImplTest, RetryBudgetDefaultPopulation) { EXPECT_EQ(min_retry_concurrency, 123UL); } +TEST_F(ClusterInfoImplTest, LoadStatsConflictWithPerEndpointStats) { + std::string yaml = R"EOF( + name: name + type: STRICT_DNS + lb_policy: RANDOM + track_cluster_stats: + per_endpoint_stats: true + )EOF"; + + server_context_.bootstrap_.mutable_cluster_manager()->mutable_load_stats_config(); + + EXPECT_THROW_WITH_MESSAGE(makeCluster(yaml), EnvoyException, + "Only one of cluster per_endpoint_stats and cluster manager " + "load_stats_config can be specified"); +} + TEST_F(ClusterInfoImplTest, UnsupportedPerHostFields) { std::string yaml = R"EOF( name: name @@ -4259,7 +4434,7 @@ class TestHttpFilterConfigFactory : public Server::Configuration::NamedHttpFilte TestHttpFilterConfigFactory(TestFilterConfigFactoryBase& parent) : parent_(parent) {} // NamedNetworkFilterConfigFactory - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, Server::Configuration::FactoryContext&) override { PANIC("not implemented"); diff --git a/test/config/utility.h b/test/config/utility.h index cdc4fe2b2faf..f1914170610d 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -303,8 +303,8 @@ class ConfigHelper { void addVirtualHost(const envoy::config::route::v3::VirtualHost& vhost); // Add an HTTP filter prior to existing filters. - // By default, this prepends a downstream filter, but if downstream is set to - // false it will prepend an upstream filter. + // By default, this prepends a downstream HTTP filter, but if downstream is set to + // false it will prepend an upstream HTTP filter. void prependFilter(const std::string& filter_yaml, bool downstream = true); // Add an HTTP filter prior to existing filters. diff --git a/test/config_test/BUILD b/test/config_test/BUILD index 63ee81132676..a61dc6a0c376 100644 --- a/test/config_test/BUILD +++ b/test/config_test/BUILD @@ -4,7 +4,7 @@ load( "envoy_cc_test_library", "envoy_package", ) -load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS") +load("//bazel:repositories.bzl", "NO_HTTP3_SKIP_TARGETS", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS") load("//source/extensions:all_extensions.bzl", "envoy_all_extensions") licenses(["notice"]) # Apache 2 @@ -64,6 +64,7 @@ envoy_cc_test_library( ] + select({ "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), + "//bazel:disable_http3": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS), "//conditions:default": envoy_all_extensions(), }), ) diff --git a/test/config_test/config_test.cc b/test/config_test/config_test.cc index 009eec46583f..645e6fb74e01 100644 --- a/test/config_test/config_test.cc +++ b/test/config_test/config_test.cc @@ -8,9 +8,9 @@ #include "source/common/api/os_sys_calls_impl.h" #include "source/common/common/fmt.h" +#include "source/common/listener_manager/listener_manager_impl.h" #include "source/common/protobuf/utility.h" #include "source/common/runtime/runtime_features.h" -#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include "source/server/config_validation/server.h" #include "source/server/configuration_impl.h" #include "source/server/options_impl.h" @@ -41,8 +41,8 @@ namespace { // asConfigYaml returns a new config that empties the configPath() and populates configYaml() OptionsImpl asConfigYaml(const OptionsImpl& src, Api::Api& api) { - return Envoy::Server::createTestOptionsImpl("", api.fileSystem().fileReadToEnd(src.configPath()), - src.localAddressIpVersion()); + return Envoy::Server::createTestOptionsImpl( + "", api.fileSystem().fileReadToEnd(src.configPath()).value(), src.localAddressIpVersion()); } static std::vector unsuported_win32_configs = { @@ -65,7 +65,7 @@ class ConfigTest { return api_->threadFactory(); })); ON_CALL(file_system_, fileReadToEnd(_)) - .WillByDefault(Invoke([&](const std::string& file) -> std::string { + .WillByDefault(Invoke([&](const std::string& file) -> absl::StatusOr { return api_->fileSystem().fileReadToEnd(file); })); ON_CALL(os_sys_calls_, close(_)).WillByDefault(Return(Api::SysCallIntResult{0, 0})); diff --git a/test/config_test/example_configs_test.cc b/test/config_test/example_configs_test.cc index 02d0504fbd9a..8802263f5e72 100644 --- a/test/config_test/example_configs_test.cc +++ b/test/config_test/example_configs_test.cc @@ -14,7 +14,8 @@ TEST(ExampleConfigsTest, All) { {TestEnvironment::runfilesPath("test/config_test/example_configs_test_setup.sh")}); Filesystem::InstanceImpl file_system; const auto config_file_count = std::stoi( - file_system.fileReadToEnd(TestEnvironment::temporaryDirectory() + "/config-file-count.txt")); + file_system.fileReadToEnd(TestEnvironment::temporaryDirectory() + "/config-file-count.txt") + .value()); // Change working directory, otherwise we won't be able to read files using relative paths. #ifdef PATH_MAX diff --git a/test/dependencies/curl_test.cc b/test/dependencies/curl_test.cc index 6218a3dea66d..6c402f3d6482 100644 --- a/test/dependencies/curl_test.cc +++ b/test/dependencies/curl_test.cc @@ -15,7 +15,6 @@ TEST(CurlTest, BuiltWithExpectedFeatures) { EXPECT_EQ(0, info->features & CURL_VERSION_KERBEROS4); EXPECT_EQ(0, info->features & CURL_VERSION_SSL); EXPECT_NE(0, info->features & CURL_VERSION_LIBZ); - EXPECT_EQ(0, info->features & CURL_VERSION_NTLM); EXPECT_EQ(0, info->features & CURL_VERSION_GSSNEGOTIATE); EXPECT_NE(0, info->features & CURL_VERSION_ASYNCHDNS); EXPECT_EQ(0, info->features & CURL_VERSION_SPNEGO); diff --git a/test/exe/BUILD b/test/exe/BUILD index e2dc8214d977..7be94770af65 100644 --- a/test/exe/BUILD +++ b/test/exe/BUILD @@ -86,8 +86,8 @@ envoy_cc_test( ] + select({ # gcc RBE build has trouble compiling target with all extensions "//bazel:gcc_build": ["//source/exe:envoy_main_common_with_core_extensions_lib"], - # This dependency MUST be main_common_lib to meet the purpose of this test - "//conditions:default": ["//source/exe:main_common_lib"], + # This dependency MUST be main_common_with_all_extensions_lib to meet the purpose of this test + "//conditions:default": ["//source/exe:main_common_with_all_extensions_lib"], }), ) @@ -114,7 +114,7 @@ envoy_cc_test( # gcc RBE build has trouble compiling target with all extensions "//bazel:gcc_build": ["//source/exe:envoy_main_common_with_core_extensions_lib"], # This dependency MUST be main_common_lib to meet the purpose of this test - "//conditions:default": ["//source/exe:main_common_lib"], + "//conditions:default": ["//source/exe:main_common_with_all_extensions_lib"], }), ) @@ -151,7 +151,7 @@ envoy_cc_test( tags = ["manual"], deps = [ "//source/common/api:api_lib", - "//source/exe:main_common_lib", + "//source/exe:main_common_with_all_extensions_lib", "//test/mocks/runtime:runtime_mocks", "//test/test_common:contention_lib", "//test/test_common:environment_lib", diff --git a/test/extensions/access_loggers/common/access_log_base_test.cc b/test/extensions/access_loggers/common/access_log_base_test.cc index 72d88b87c3eb..a2816d3557b2 100644 --- a/test/extensions/access_loggers/common/access_log_base_test.cc +++ b/test/extensions/access_loggers/common/access_log_base_test.cc @@ -23,9 +23,7 @@ class TestImpl : public ImplBase { int count() { return count_; }; private: - void emitLog(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - AccessLog::AccessLogType) override { + void emitLog(const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&) override { count_++; } @@ -36,7 +34,7 @@ TEST(AccessLogBaseTest, NoFilter) { StreamInfo::MockStreamInfo stream_info; TestImpl logger(nullptr); EXPECT_EQ(logger.count(), 0); - logger.log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + logger.log({}, stream_info); EXPECT_EQ(logger.count(), 1); } @@ -44,10 +42,10 @@ TEST(AccessLogBaseTest, FilterReject) { StreamInfo::MockStreamInfo stream_info; std::unique_ptr filter = std::make_unique(); - EXPECT_CALL(*filter, evaluate(_, _, _, _, _)).WillOnce(Return(false)); + EXPECT_CALL(*filter, evaluate(_, _)).WillOnce(Return(false)); TestImpl logger(std::move(filter)); EXPECT_EQ(logger.count(), 0); - logger.log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + logger.log({}, stream_info); EXPECT_EQ(logger.count(), 0); } diff --git a/test/extensions/access_loggers/file/config_test.cc b/test/extensions/access_loggers/file/config_test.cc index ff2527af5941..7b75312f2b3d 100644 --- a/test/extensions/access_loggers/file/config_test.cc +++ b/test/extensions/access_loggers/file/config_test.cc @@ -22,7 +22,7 @@ namespace File { namespace { TEST(FileAccessLogNegativeTest, ValidateFail) { - NiceMock context; + NiceMock context; EXPECT_THROW(FileAccessLogFactory().createAccessLogInstance( envoy::extensions::access_loggers::file::v3::FileAccessLog(), nullptr, context), @@ -32,7 +32,7 @@ TEST(FileAccessLogNegativeTest, ValidateFail) { TEST(FileAccessLogNegativeTest, InvalidNameFail) { envoy::config::accesslog::v3::AccessLog config; - NiceMock context; + NiceMock context; EXPECT_THROW_WITH_MESSAGE(AccessLog::AccessLogFactory::fromProto(config, context), EnvoyException, "Didn't find a registered implementation for '' with type URL: ''"); @@ -56,7 +56,8 @@ class FileAccessLogTest : public testing::Test { auto file = std::make_shared(); Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, fal_config.path()}; - EXPECT_CALL(context_.access_log_manager_, createAccessLog(file_info)).WillOnce(Return(file)); + EXPECT_CALL(context_.server_factory_context_.access_log_manager_, createAccessLog(file_info)) + .WillOnce(Return(file)); AccessLog::InstanceSharedPtr logger = AccessLog::AccessLogFactory::fromProto(config, context_); @@ -73,8 +74,7 @@ class FileAccessLogTest : public testing::Test { EXPECT_EQ(got, expected); } })); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } Http::TestRequestHeaderMapImpl request_headers_{{":method", "GET"}, {":path", "/bar/foo"}}; @@ -82,7 +82,7 @@ class FileAccessLogTest : public testing::Test { Http::TestResponseTrailerMapImpl response_trailers_; NiceMock stream_info_; - NiceMock context_; + NiceMock context_; }; TEST_F(FileAccessLogTest, DEPRECATED_FEATURE_TEST(LegacyFormatEmpty)) { diff --git a/test/extensions/access_loggers/grpc/BUILD b/test/extensions/access_loggers/grpc/BUILD index 423c8734457f..05d5477338a7 100644 --- a/test/extensions/access_loggers/grpc/BUILD +++ b/test/extensions/access_loggers/grpc/BUILD @@ -35,6 +35,7 @@ envoy_extension_cc_test( extension_names = ["envoy.access_loggers.http_grpc"], deps = [ "//source/extensions/access_loggers/grpc:grpc_access_log_utils", + "//source/extensions/filters/common/expr:cel_state_lib", "//test/mocks/local_info:local_info_mocks", "//test/mocks/ssl:ssl_mocks", "//test/mocks/stream_info:stream_info_mocks", diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc index 782cf5178956..372b7512e631 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc @@ -1,6 +1,9 @@ #include "envoy/data/accesslog/v3/accesslog.pb.h" +#include "source/common/http/header_map_impl.h" +#include "source/common/stream_info/filter_state_impl.h" #include "source/extensions/access_loggers/grpc/grpc_access_log_utils.h" +#include "source/extensions/filters/common/expr/cel_state.h" #include "test/mocks/stream_info/mocks.h" @@ -10,6 +13,8 @@ namespace AccessLoggers { namespace GrpcCommon { namespace { +using Filters::Common::Expr::CelStatePrototype; +using Filters::Common::Expr::CelStateType; using testing::_; using testing::Return; @@ -53,6 +58,105 @@ TEST(UtilityResponseFlagsToAccessLogResponseFlagsTest, All) { EXPECT_EQ(common_access_log_expected.DebugString(), common_access_log.DebugString()); } +// key is present only in downstream streamInfo's filter state +TEST(UtilityExtractCommonAccessLogPropertiesTest, FilterStateFromDownstream) { + NiceMock stream_info; + ON_CALL(stream_info, hasResponseFlag(_)).WillByDefault(Return(true)); + envoy::data::accesslog::v3::AccessLogCommon common_access_log; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config; + config.mutable_filter_state_objects_to_log()->Add("downstream_peer"); + CelStatePrototype prototype(true, CelStateType::Bytes, "", + StreamInfo::FilterState::LifeSpan::FilterChain); + auto state = std::make_unique<::Envoy::Extensions::Filters::Common::Expr::CelState>(prototype); + state->setValue("value_from_downstream_peer"); + stream_info.filter_state_->setData("downstream_peer", std::move(state), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + + Utility::extractCommonAccessLogProperties( + common_access_log, *Http::StaticEmptyHeaders::get().request_headers.get(), stream_info, + config, envoy::data::accesslog::v3::AccessLogType::TcpConnectionEnd); + + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->contains("downstream_peer"), true); + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->count("downstream_peer"), 1); + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->size(), 1); + auto any = (*(common_access_log.mutable_filter_state_objects()))["downstream_peer"]; + ProtobufWkt::BytesValue gotState; + any.UnpackTo(&gotState); + EXPECT_EQ(gotState.value(), "value_from_downstream_peer"); +} + +// key is present only in the upstream streamInfo's filter state +TEST(UtilityExtractCommonAccessLogPropertiesTest, FilterStateFromUpstream) { + NiceMock stream_info; + ON_CALL(stream_info, hasResponseFlag(_)).WillByDefault(Return(true)); + envoy::data::accesslog::v3::AccessLogCommon common_access_log; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config; + config.mutable_filter_state_objects_to_log()->Add("upstream_peer"); + CelStatePrototype prototype(true, CelStateType::Bytes, "", + StreamInfo::FilterState::LifeSpan::FilterChain); + auto state = std::make_unique<::Envoy::Extensions::Filters::Common::Expr::CelState>(prototype); + auto filter_state = + std::make_shared(StreamInfo::FilterState::LifeSpan::FilterChain); + state->setValue("value_from_upstream_peer"); + filter_state->setData("upstream_peer", std::move(state), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + stream_info.upstreamInfo()->setUpstreamFilterState(filter_state); + + Utility::extractCommonAccessLogProperties( + common_access_log, *Http::StaticEmptyHeaders::get().request_headers.get(), stream_info, + config, envoy::data::accesslog::v3::AccessLogType::TcpConnectionEnd); + + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->contains("upstream_peer"), true); + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->count("upstream_peer"), 1); + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->size(), 1); + auto any = (*(common_access_log.mutable_filter_state_objects()))["upstream_peer"]; + ProtobufWkt::BytesValue gotState; + any.UnpackTo(&gotState); + EXPECT_EQ(gotState.value(), "value_from_upstream_peer"); +} + +// key is present in both the streamInfo's filter state +TEST(UtilityExtractCommonAccessLogPropertiesTest, + FilterStateFromDownstreamIfSameKeyInBothStreamInfo) { + NiceMock stream_info; + ON_CALL(stream_info, hasResponseFlag(_)).WillByDefault(Return(true)); + envoy::data::accesslog::v3::AccessLogCommon common_access_log; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config; + config.mutable_filter_state_objects_to_log()->Add("same_key"); + CelStatePrototype prototype(true, CelStateType::Bytes, "", + StreamInfo::FilterState::LifeSpan::FilterChain); + auto downstream_state = + std::make_unique<::Envoy::Extensions::Filters::Common::Expr::CelState>(prototype); + downstream_state->setValue("value_from_downstream_peer"); + stream_info.filter_state_->setData("same_key", std::move(downstream_state), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + + auto upstream_state = + std::make_unique<::Envoy::Extensions::Filters::Common::Expr::CelState>(prototype); + auto filter_state = + std::make_shared(StreamInfo::FilterState::LifeSpan::FilterChain); + upstream_state->setValue("value_from_upstream_peer"); + filter_state->setData("same_key", std::move(upstream_state), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + stream_info.upstreamInfo()->setUpstreamFilterState(filter_state); + + Utility::extractCommonAccessLogProperties( + common_access_log, *Http::StaticEmptyHeaders::get().request_headers.get(), stream_info, + config, envoy::data::accesslog::v3::AccessLogType::TcpConnectionEnd); + + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->contains("same_key"), true); + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->count("same_key"), 1); + ASSERT_EQ(common_access_log.mutable_filter_state_objects()->size(), 1); + auto any = (*(common_access_log.mutable_filter_state_objects()))["same_key"]; + ProtobufWkt::BytesValue gotState; + any.UnpackTo(&gotState); + EXPECT_EQ(gotState.value(), "value_from_downstream_peer"); +} + } // namespace } // namespace GrpcCommon } // namespace AccessLoggers diff --git a/test/extensions/access_loggers/grpc/http_config_test.cc b/test/extensions/access_loggers/grpc/http_config_test.cc index 209763cf6f05..c43ca17579b8 100644 --- a/test/extensions/access_loggers/grpc/http_config_test.cc +++ b/test/extensions/access_loggers/grpc/http_config_test.cc @@ -1,7 +1,7 @@ +#include "envoy/access_log/access_log_config.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/registry/registry.h" -#include "envoy/server/access_log_config.h" #include "envoy/stats/scope.h" #include "source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h" @@ -23,9 +23,8 @@ namespace { class HttpGrpcAccessLogConfigTest : public testing::Test { public: void SetUp() override { - factory_ = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.http_grpc"); + factory_ = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.http_grpc"); ASSERT_NE(nullptr, factory_); message_ = factory_->createEmptyConfigProto(); @@ -42,7 +41,8 @@ class HttpGrpcAccessLogConfigTest : public testing::Test { TestUtility::jsonConvert(http_grpc_access_log_, *message_); if (cluster_name == good_cluster) { - EXPECT_CALL(context_.cluster_manager_.async_client_manager_, factoryForGrpcService(_, _, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.async_client_manager_, + factoryForGrpcService(_, _, _)) .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { return std::make_unique>(); })); @@ -58,10 +58,10 @@ class HttpGrpcAccessLogConfigTest : public testing::Test { } AccessLog::FilterPtr filter_; - NiceMock context_; + NiceMock context_; envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig http_grpc_access_log_; ProtobufTypes::MessagePtr message_; - Server::Configuration::AccessLogInstanceFactory* factory_{}; + AccessLog::AccessLogInstanceFactory* factory_{}; }; // Normal OK configuration. diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 2becfc4819d8..a8e8a17e757d 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -83,7 +83,7 @@ TEST(HttpGrpcAccessLog, TlsLifetimeCheck) { class HttpGrpcAccessLogTest : public testing::Test { public: void init() { - ON_CALL(*filter_, evaluate(_, _, _, _, _)).WillByDefault(Return(true)); + ON_CALL(*filter_, evaluate(_, _)).WillByDefault(Return(true)); config_.mutable_common_config()->set_log_name("hello_log"); config_.mutable_common_config()->add_filter_state_objects_to_log("string_accessor"); config_.mutable_common_config()->add_filter_state_objects_to_log("uint32_accessor"); @@ -155,8 +155,7 @@ class HttpGrpcAccessLogTest : public testing::Test { response: {{}} )EOF", request_method, request_method.length() + 7)); - access_log_->log(&request_headers, nullptr, nullptr, stream_info, - AccessLog::AccessLogType::NotSet); + access_log_->log({&request_headers}, stream_info); } Stats::IsolatedStoreImpl scope_; @@ -248,7 +247,7 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } { @@ -286,7 +285,7 @@ response: {} request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } { @@ -406,8 +405,7 @@ protocol_version: HTTP10 response_body_bytes: 20 response_code_details: "via_upstream" )EOF"); - access_log_->log(&request_headers, &response_headers, nullptr, stream_info, - AccessLog::AccessLogType::NotSet); + access_log_->log({&request_headers, &response_headers}, stream_info); } { @@ -448,8 +446,7 @@ protocol_version: HTTP10 request_headers_bytes: 16 response: {} )EOF"); - access_log_->log(&request_headers, nullptr, nullptr, stream_info, - AccessLog::AccessLogType::NotSet); + access_log_->log({&request_headers}, stream_info); } { @@ -521,8 +518,7 @@ response: {} request_headers_bytes: 16 response: {} )EOF"); - access_log_->log(&request_headers, nullptr, nullptr, stream_info, - AccessLog::AccessLogType::NotSet); + access_log_->log({&request_headers}, stream_info); } // TLSv1.2 @@ -578,7 +574,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } // TLSv1.1 @@ -634,7 +630,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } // TLSv1 @@ -690,7 +686,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } // Unknown TLS version (TLSv1.4) @@ -746,7 +742,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } // Intermediate log entry. @@ -814,7 +810,7 @@ response: {} request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } } @@ -910,8 +906,7 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { "x-logged-trailer": "value,response_trailer_value" "x-empty-trailer": "" )EOF"); - access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info, - AccessLog::AccessLogType::NotSet); + access_log_->log({&request_headers, &response_headers, &response_trailers}, stream_info); } } @@ -994,8 +989,7 @@ TEST_F(HttpGrpcAccessLogTest, SanitizeUTF8) { "x-trailer": "{0},{0}" )EOF", "prefix!!suffix")); - access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info, - AccessLog::AccessLogType::NotSet); + access_log_->log({&request_headers, &response_headers, &response_trailers}, stream_info); } } @@ -1057,7 +1051,7 @@ tag: ltag request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } TEST_F(HttpGrpcAccessLogTest, CustomTagTestMetadata) { @@ -1116,7 +1110,7 @@ tag: mtag request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } TEST_F(HttpGrpcAccessLogTest, CustomTagTestMetadataDefaultValue) { @@ -1172,7 +1166,7 @@ tag: mtag request: {} response: {} )EOF"); - access_log_->log(nullptr, nullptr, nullptr, stream_info, AccessLog::AccessLogType::NotSet); + access_log_->log({}, stream_info); } } // namespace diff --git a/test/extensions/access_loggers/grpc/tcp_config_test.cc b/test/extensions/access_loggers/grpc/tcp_config_test.cc index bb18e7b81b42..4f35c4d65d29 100644 --- a/test/extensions/access_loggers/grpc/tcp_config_test.cc +++ b/test/extensions/access_loggers/grpc/tcp_config_test.cc @@ -1,7 +1,7 @@ +#include "envoy/access_log/access_log_config.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/registry/registry.h" -#include "envoy/server/access_log_config.h" #include "envoy/stats/scope.h" #include "source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h" @@ -23,9 +23,8 @@ namespace { class TcpGrpcAccessLogConfigTest : public testing::Test { public: void SetUp() override { - factory_ = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.tcp_grpc"); + factory_ = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.tcp_grpc"); ASSERT_NE(nullptr, factory_); message_ = factory_->createEmptyConfigProto(); @@ -42,7 +41,8 @@ class TcpGrpcAccessLogConfigTest : public testing::Test { TestUtility::jsonConvert(tcp_grpc_access_log_, *message_); if (cluster_name == good_cluster) { - EXPECT_CALL(context_.cluster_manager_.async_client_manager_, factoryForGrpcService(_, _, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.async_client_manager_, + factoryForGrpcService(_, _, _)) .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { return std::make_unique>(); })); @@ -58,10 +58,10 @@ class TcpGrpcAccessLogConfigTest : public testing::Test { } AccessLog::FilterPtr filter_; - NiceMock context_; + NiceMock context_; envoy::extensions::access_loggers::grpc::v3::TcpGrpcAccessLogConfig tcp_grpc_access_log_; ProtobufTypes::MessagePtr message_; - Server::Configuration::AccessLogInstanceFactory* factory_{}; + AccessLog::AccessLogInstanceFactory* factory_{}; }; // Normal OK configuration. diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index 2fe620e16cea..069462aa9665 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -27,7 +27,6 @@ using namespace std::chrono_literals; using ::Envoy::AccessLog::FilterPtr; using ::Envoy::AccessLog::MockFilter; -using envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig; using opentelemetry::proto::common::v1::AnyValue; using opentelemetry::proto::common::v1::KeyValueList; using opentelemetry::proto::logs::v1::LogRecord; @@ -64,7 +63,7 @@ class MockGrpcAccessLoggerCache : public GrpcAccessLoggerCache { class AccessLogTest : public testing::Test { public: AccessLogPtr makeAccessLog(const AnyValue& body_config, const KeyValueList& attributes_config) { - ON_CALL(*filter_, evaluate(_, _, _, _, _)).WillByDefault(Return(true)); + ON_CALL(*filter_, evaluate(_, _)).WillByDefault(Return(true)); *config_.mutable_body() = body_config; *config_.mutable_attributes() = attributes_config; config_.mutable_common_config()->set_log_name("test_log"); @@ -145,8 +144,7 @@ TEST_F(AccessLogTest, Marshalling) { value: string_value: "10" )EOF"); - access_log->log(&request_headers, &response_headers, nullptr, stream_info, - Envoy::AccessLog::AccessLogType::NotSet); + access_log->log({&request_headers, &response_headers}, stream_info); } // Test log with empty config. @@ -160,8 +158,7 @@ TEST_F(AccessLogTest, EmptyConfig) { expectLog(R"EOF( time_unix_nano: 3600000000000 )EOF"); - access_log->log(&request_headers, &response_headers, nullptr, stream_info, - Envoy::AccessLog::AccessLogType::NotSet); + access_log->log({&request_headers, &response_headers}, stream_info); } } // namespace diff --git a/test/extensions/access_loggers/open_telemetry/config_test.cc b/test/extensions/access_loggers/open_telemetry/config_test.cc index 30db7a23d947..36ad90e731c5 100644 --- a/test/extensions/access_loggers/open_telemetry/config_test.cc +++ b/test/extensions/access_loggers/open_telemetry/config_test.cc @@ -1,7 +1,7 @@ +#include "envoy/access_log/access_log_config.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/extensions/access_loggers/open_telemetry/v3/logs_service.pb.h" #include "envoy/registry/registry.h" -#include "envoy/server/access_log_config.h" #include "envoy/stats/scope.h" #include "source/extensions/access_loggers/open_telemetry/access_log_impl.h" @@ -24,15 +24,15 @@ namespace { class OpenTelemetryAccessLogConfigTest : public testing::Test { public: void SetUp() override { - factory_ = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.open_telemetry"); + factory_ = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.open_telemetry"); ASSERT_NE(nullptr, factory_); message_ = factory_->createEmptyConfigProto(); ASSERT_NE(nullptr, message_); - EXPECT_CALL(context_.cluster_manager_.async_client_manager_, factoryForGrpcService(_, _, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.async_client_manager_, + factoryForGrpcService(_, _, _)) .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { return std::make_unique>(); })); @@ -49,7 +49,7 @@ class OpenTelemetryAccessLogConfigTest : public testing::Test { envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig access_log_config_; ProtobufTypes::MessagePtr message_; - Server::Configuration::AccessLogInstanceFactory* factory_{}; + Envoy::AccessLog::AccessLogInstanceFactory* factory_{}; }; // Normal OK configuration. diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc index 5cf53412cda3..11557702df2b 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc @@ -72,15 +72,9 @@ static void BM_OpenTelemetryAccessLogFormatter(benchmark::State& state) { std::unique_ptr otel_formatter = makeOpenTelemetryFormatter(); size_t output_bytes = 0; - Http::TestRequestHeaderMapImpl request_headers; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; - std::string body; + for (auto _ : state) { // NOLINT: Silences warning about dead store - output_bytes += otel_formatter - ->format(request_headers, response_headers, response_trailers, *stream_info, - body, AccessLog::AccessLogType::NotSet) - .ByteSize(); + output_bytes += otel_formatter->format({}, *stream_info).ByteSize(); } benchmark::DoNotOptimize(output_bytes); } diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc index 1322b282ebf1..2adedb958afc 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc @@ -113,10 +113,6 @@ void verifyOpenTelemetryOutput(KeyValueList output, OpenTelemetryFormatMap expec TEST(SubstitutionFormatterTest, OpenTelemetryFormatterPlainStringTest) { StreamInfo::MockStreamInfo stream_info; - Http::TestRequestHeaderMapImpl request_header; - Http::TestResponseHeaderMapImpl response_header; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -133,17 +129,11 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterPlainStringTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterTypesTest) { StreamInfo::MockStreamInfo stream_info; - Http::TestRequestHeaderMapImpl request_header; - Http::TestResponseHeaderMapImpl response_header; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -198,18 +188,13 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterTypesTest) { - string_value: "HTTP/1.1" )EOF", expected); - const KeyValueList output = formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet); + const KeyValueList output = formatter.format({}, stream_info); EXPECT_TRUE(TestUtility::protoEqual(output, expected)); } // Test that nested values are formatted properly, including inter-type nesting. TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNestedObjectsTest) { StreamInfo::MockStreamInfo stream_info; - Http::TestRequestHeaderMapImpl request_header; - Http::TestResponseHeaderMapImpl response_header; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -411,17 +396,12 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNestedObjectsTest) { - string_value: "HTTP/1.1" )EOF", expected); - const KeyValueList output = formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet); + const KeyValueList output = formatter.format({}, stream_info); EXPECT_TRUE(TestUtility::protoEqual(output, expected)); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterSingleOperatorTest) { StreamInfo::MockStreamInfo stream_info; - Http::TestRequestHeaderMapImpl request_header; - Http::TestResponseHeaderMapImpl response_header; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -438,17 +418,11 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterSingleOperatorTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } TEST(SubstitutionFormatterTest, EmptyOpenTelemetryFormatterTest) { StreamInfo::MockStreamInfo stream_info; - Http::TestRequestHeaderMapImpl request_header; - Http::TestResponseHeaderMapImpl response_header; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -465,17 +439,13 @@ TEST(SubstitutionFormatterTest, EmptyOpenTelemetryFormatterTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNonExistentHeaderTest) { StreamInfo::MockStreamInfo stream_info; Http::TestRequestHeaderMapImpl request_header{{"some_request_header", "SOME_REQUEST_HEADER"}}; Http::TestResponseHeaderMapImpl response_header{{"some_response_header", "SOME_RESPONSE_HEADER"}}; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; OpenTelemetryFormatMap expected = {{"protocol", "HTTP/1.1"}, {"some_request_header", "SOME_REQUEST_HEADER"}, @@ -504,8 +474,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNonExistentHeaderTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), + verifyOpenTelemetryOutput(formatter.format({&request_header, &response_header}, stream_info), expected); } @@ -515,8 +484,6 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterAlternateHeaderTest) { {"request_present_header", "REQUEST_PRESENT_HEADER"}}; Http::TestResponseHeaderMapImpl response_header{ {"response_present_header", "RESPONSE_PRESENT_HEADER"}}; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; OpenTelemetryFormatMap expected = { {"request_present_header_or_request_absent_header", "REQUEST_PRESENT_HEADER"}, @@ -546,8 +513,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterAlternateHeaderTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), + verifyOpenTelemetryOutput(formatter.format({&request_header, &response_header}, stream_info), expected); } @@ -556,7 +522,6 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterDynamicMetadataTest) { Http::TestRequestHeaderMapImpl request_header{{"first", "GET"}, {":path", "/"}}; Http::TestResponseHeaderMapImpl response_header{{"second", "PUT"}, {"test", "test"}}; Http::TestResponseTrailerMapImpl response_trailer{{"third", "POST"}, {"test-2", "test-2"}}; - std::string body; envoy::config::core::v3::Metadata metadata; populateMetadataTestData(metadata); @@ -583,9 +548,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterDynamicMetadataTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput( + formatter.format({&request_header, &response_header, &response_trailer}, stream_info), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataTest) { @@ -593,7 +558,6 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataTest) { Http::TestRequestHeaderMapImpl request_header{{"first", "GET"}, {":path", "/"}}; Http::TestResponseHeaderMapImpl response_header{{"second", "PUT"}, {"test", "test"}}; Http::TestResponseTrailerMapImpl response_trailer{{"third", "POST"}, {"test-2", "test-2"}}; - std::string body; envoy::config::core::v3::Metadata metadata; populateMetadataTestData(metadata); @@ -629,9 +593,9 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput( + formatter.format({&request_header, &response_header, &response_trailer}, stream_info), + expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataNoClusterInfoTest) { @@ -639,7 +603,6 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataNoClusterIn Http::TestRequestHeaderMapImpl request_header{{"first", "GET"}, {":path", "/"}}; Http::TestResponseHeaderMapImpl response_header{{"second", "PUT"}, {"test", "test"}}; Http::TestResponseTrailerMapImpl response_trailer{{"third", "POST"}, {"test-2", "test-2"}}; - std::string body; OpenTelemetryFormatMap expected = {{"test_key", "-"}}; @@ -656,25 +619,21 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataNoClusterIn // Empty optional (absl::nullopt) { EXPECT_CALL(Const(stream_info), upstreamClusterInfo()).WillOnce(Return(absl::nullopt)); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput( + formatter.format({&request_header, &response_header, &response_trailer}, stream_info), + expected); } // Empty cluster info (nullptr) { EXPECT_CALL(Const(stream_info), upstreamClusterInfo()).WillOnce(Return(nullptr)); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput( + formatter.format({&request_header, &response_header, &response_trailer}, stream_info), + expected); } } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateTest) { - Http::TestRequestHeaderMapImpl request_headers; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; StreamInfo::MockStreamInfo stream_info; - std::string body; stream_info.filter_state_->setData("test_key", std::make_unique("test_value"), StreamInfo::FilterState::StateType::ReadOnly); @@ -699,17 +658,12 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateTest) { - Http::TestRequestHeaderMapImpl request_headers; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; + StreamInfo::MockStreamInfo stream_info; - std::string body; const StreamInfo::FilterStateSharedPtr upstream_filter_state = std::make_shared(StreamInfo::FilterState::LifeSpan::Request); @@ -744,19 +698,13 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } // Test new specifier (PLAIN/TYPED) of FilterState. Ensure that after adding additional specifier, // the FilterState can call the serializeAsProto or serializeAsString methods correctly. TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateSpeciferTest) { - Http::TestRequestHeaderMapImpl request_headers; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; StreamInfo::MockStreamInfo stream_info; - std::string body; stream_info.filter_state_->setData( "test_key", std::make_unique("test_value"), StreamInfo::FilterState::StateType::ReadOnly); @@ -780,19 +728,13 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateSpeciferTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } // Test new specifier (PLAIN/TYPED) of FilterState. Ensure that after adding additional specifier, // the FilterState can call the serializeAsProto or serializeAsString methods correctly. TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateSpeciferTest) { - Http::TestRequestHeaderMapImpl request_headers; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; StreamInfo::MockStreamInfo stream_info; - std::string body; stream_info.upstream_info_ = std::make_shared(); stream_info.upstream_info_->setUpstreamFilterState( @@ -822,16 +764,11 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateSpecife key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_headers, response_headers, response_trailers, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } // Error specifier will cause an exception to be thrown. TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateErrorSpeciferTest) { - Http::TestRequestHeaderMapImpl request_headers; - Http::TestResponseHeaderMapImpl response_headers; - Http::TestResponseTrailerMapImpl response_trailers; StreamInfo::MockStreamInfo stream_info; std::string body; stream_info.filter_state_->setData( @@ -887,10 +824,6 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateErrorSp TEST(SubstitutionFormatterTest, OpenTelemetryFormatterStartTimeTest) { StreamInfo::MockStreamInfo stream_info; - Http::TestRequestHeaderMapImpl request_header; - Http::TestResponseHeaderMapImpl response_header; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; time_t expected_time_in_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); @@ -924,9 +857,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterStartTimeTest) { key_mapping); OpenTelemetryFormatter formatter(key_mapping); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), - expected); + verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { @@ -935,8 +866,6 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { Http::TestRequestHeaderMapImpl request_header{{"some_request_header", "SOME_REQUEST_HEADER"}}; Http::TestResponseHeaderMapImpl response_header{ {"some_response_header", "SOME_RESPONSE_HEADER"}}; - Http::TestResponseTrailerMapImpl response_trailer; - std::string body; OpenTelemetryFormatMap expected = { {"multi_token_field", "HTTP/1.1 plainstring SOME_REQUEST_HEADER SOME_RESPONSE_HEADER"}}; @@ -954,8 +883,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - verifyOpenTelemetryOutput(formatter.format(request_header, response_header, response_trailer, - stream_info, body, AccessLog::AccessLogType::NotSet), + verifyOpenTelemetryOutput(formatter.format({&request_header, &response_header}, stream_info), expected); } } diff --git a/test/extensions/access_loggers/stream/stream_test_base.h b/test/extensions/access_loggers/stream/stream_test_base.h index 790ff373d79c..be3f20da117b 100644 --- a/test/extensions/access_loggers/stream/stream_test_base.h +++ b/test/extensions/access_loggers/stream/stream_test_base.h @@ -37,7 +37,8 @@ class StreamAccessLogTest : public testing::Test { auto file = std::make_shared(); Filesystem::FilePathAndType file_info{destination_type, ""}; - EXPECT_CALL(context_.access_log_manager_, createAccessLog(file_info)).WillOnce(Return(file)); + EXPECT_CALL(context_.server_factory_context_.access_log_manager_, createAccessLog(file_info)) + .WillOnce(Return(file)); AccessLog::InstanceSharedPtr logger = AccessLog::AccessLogFactory::fromProto(config, context_); @@ -54,8 +55,7 @@ class StreamAccessLogTest : public testing::Test { EXPECT_EQ(got, expected); } })); - logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_, - AccessLog::AccessLogType::NotSet); + logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info_); } Http::TestRequestHeaderMapImpl request_headers_{{":method", "GET"}, {":path", "/bar/foo"}}; diff --git a/test/extensions/access_loggers/wasm/BUILD b/test/extensions/access_loggers/wasm/BUILD index d17d06f86952..47d6aa71e1c9 100644 --- a/test/extensions/access_loggers/wasm/BUILD +++ b/test/extensions/access_loggers/wasm/BUILD @@ -26,6 +26,7 @@ envoy_extension_cc_test( "//source/extensions/access_loggers/wasm:config", "//test/extensions/access_loggers/wasm/test_data:test_cpp_plugin", "//test/extensions/common/wasm:wasm_runtime", + "//test/mocks/network:network_mocks", "//test/mocks/server:server_mocks", "//test/test_common:environment_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/access_loggers/wasm/config_test.cc b/test/extensions/access_loggers/wasm/config_test.cc index 1ba26b57e53d..4e14e1b47590 100644 --- a/test/extensions/access_loggers/wasm/config_test.cc +++ b/test/extensions/access_loggers/wasm/config_test.cc @@ -10,6 +10,7 @@ #include "source/extensions/common/wasm/wasm.h" #include "test/extensions/common/wasm/wasm_runtime.h" +#include "test/mocks/network/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/printers.h" @@ -31,12 +32,15 @@ class WasmAccessLogConfigTest : public testing::TestWithParam> { protected: WasmAccessLogConfigTest() : api_(Api::createApiForTest(stats_store_)) { - ON_CALL(context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(context_.server_factory_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(context_, scope()).WillByDefault(ReturnRef(scope_)); - ON_CALL(context_, listenerMetadata()).WillByDefault(ReturnRef(listener_metadata_)); + ON_CALL(context_, listenerInfo()).WillByDefault(ReturnRef(listener_info_)); + ON_CALL(listener_info_, metadata()).WillByDefault(ReturnRef(listener_metadata_)); ON_CALL(context_, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(context_, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(context_.server_factory_context_, clusterManager()) + .WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(context_.server_factory_context_, mainThreadDispatcher()) + .WillByDefault(ReturnRef(dispatcher_)); ON_CALL(log_stream_info_, requestComplete()) .WillByDefault(Return(std::chrono::milliseconds(30))); } @@ -52,6 +56,7 @@ class WasmAccessLogConfigTest })); } + NiceMock listener_info_; NiceMock context_; Stats::IsolatedStoreImpl stats_store_; Stats::Scope& scope_{*stats_store_.rootScope()}; @@ -71,9 +76,8 @@ INSTANTIATE_TEST_SUITE_P(Runtimes, WasmAccessLogConfigTest, Envoy::Extensions::Common::Wasm::wasmTestParamsToString); TEST_P(WasmAccessLogConfigTest, CreateWasmFromEmpty) { - auto factory = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.wasm"); + auto factory = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.wasm"); ASSERT_NE(factory, nullptr); ProtobufTypes::MessagePtr message = factory->createEmptyConfigProto(); @@ -88,9 +92,8 @@ TEST_P(WasmAccessLogConfigTest, CreateWasmFromEmpty) { } TEST_P(WasmAccessLogConfigTest, CreateWasmFromWASM) { - auto factory = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.wasm"); + auto factory = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.wasm"); ASSERT_NE(factory, nullptr); envoy::extensions::access_loggers::wasm::v3::WasmAccessLog config; @@ -121,23 +124,20 @@ TEST_P(WasmAccessLogConfigTest, CreateWasmFromWASM) { Http::TestRequestHeaderMapImpl request_header; Http::TestResponseHeaderMapImpl response_header; Http::TestResponseTrailerMapImpl response_trailer; - instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, - AccessLog::AccessLogType::NotSet); + instance->log({&request_header, &response_header, &response_trailer}, log_stream_info_); filter = std::make_unique>(); AccessLog::InstanceSharedPtr filter_instance = factory->createAccessLogInstance(config, std::move(filter), context_); - filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, - AccessLog::AccessLogType::NotSet); + filter_instance->log({&request_header, &response_header, &response_trailer}, log_stream_info_); } TEST_P(WasmAccessLogConfigTest, YamlLoadFromFileWasmInvalidConfig) { if (std::get<0>(GetParam()) == "null") { return; } - auto factory = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.wasm"); + auto factory = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.wasm"); ASSERT_NE(factory, nullptr); const std::string invalid_yaml = @@ -182,8 +182,7 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromFileWasmInvalidConfig) { AccessLog::InstanceSharedPtr filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_, - AccessLog::AccessLogType::NotSet); + filter_instance->log({}, log_stream_info_); } TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { @@ -228,8 +227,7 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { })); AccessLog::InstanceSharedPtr filter_instance = factory.createAccessLogInstance(proto_config, nullptr, context_); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_, - AccessLog::AccessLogType::NotSet); + filter_instance->log({}, log_stream_info_); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); auto response = Http::ResponseMessagePtr{new Http::ResponseMessageImpl( @@ -237,17 +235,15 @@ TEST_P(WasmAccessLogConfigTest, YamlLoadFromRemoteWasmCreateFilter) { response->body().add(code); async_callbacks->onSuccess(request, std::move(response)); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); - filter_instance->log(nullptr, nullptr, nullptr, log_stream_info_, - AccessLog::AccessLogType::NotSet); + filter_instance->log({}, log_stream_info_); } TEST_P(WasmAccessLogConfigTest, FailedToGetThreadLocalPlugin) { if (std::get<0>(GetParam()) == "null") { return; } - auto factory = - Registry::FactoryRegistry::getFactory( - "envoy.access_loggers.wasm"); + auto factory = Registry::FactoryRegistry::getFactory( + "envoy.access_loggers.wasm"); ASSERT_NE(factory, nullptr); NiceMock threadlocal; @@ -269,7 +265,7 @@ TEST_P(WasmAccessLogConfigTest, FailedToGetThreadLocalPlugin) { envoy::extensions::access_loggers::wasm::v3::WasmAccessLog proto_config; TestUtility::loadFromYaml(yaml, proto_config); - EXPECT_CALL(context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); + EXPECT_CALL(context_.server_factory_context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); threadlocal.registered_ = true; AccessLog::InstanceSharedPtr filter_instance = factory->createAccessLogInstance(proto_config, nullptr, context_); @@ -279,13 +275,11 @@ TEST_P(WasmAccessLogConfigTest, FailedToGetThreadLocalPlugin) { Http::TestResponseHeaderMapImpl response_header; Http::TestResponseTrailerMapImpl response_trailer; - filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, - AccessLog::AccessLogType::NotSet); + filter_instance->log({&request_header, &response_header, &response_trailer}, log_stream_info_); // Even if the thread local plugin handle returns nullptr, `log` should not raise error or // exception. threadlocal.data_[0] = std::make_shared(nullptr); - filter_instance->log(&request_header, &response_header, &response_trailer, log_stream_info_, - AccessLog::AccessLogType::NotSet); + filter_instance->log({&request_header, &response_header, &response_trailer}, log_stream_info_); } } // namespace Wasm diff --git a/test/extensions/bootstrap/internal_listener/BUILD b/test/extensions/bootstrap/internal_listener/BUILD index 339f63bee567..0ddc7fec65ac 100644 --- a/test/extensions/bootstrap/internal_listener/BUILD +++ b/test/extensions/bootstrap/internal_listener/BUILD @@ -72,6 +72,7 @@ envoy_extension_cc_test( srcs = ["active_internal_listener_test.cc"], extension_names = ["envoy.bootstrap.internal_listener"], deps = [ + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:listen_socket_lib", @@ -79,7 +80,6 @@ envoy_extension_cc_test( "//source/common/stats:stats_lib", "//source/extensions/bootstrap/internal_listener:active_internal_listener", "//source/extensions/bootstrap/internal_listener:thread_local_registry", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//test/mocks/access_log:access_log_mocks", "//test/mocks/network:network_mocks", diff --git a/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc b/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc index b31209dba88d..d739f795d5a8 100644 --- a/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc +++ b/test/extensions/bootstrap/internal_listener/active_internal_listener_test.cc @@ -5,12 +5,12 @@ #include "envoy/network/listener.h" #include "envoy/stats/scope.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/raw_buffer_socket.h" #include "source/extensions/bootstrap/internal_listener/active_internal_listener.h" #include "source/extensions/bootstrap/internal_listener/thread_local_registry.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/common.h" @@ -271,7 +271,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable() : connection_balancer), access_logs_({access_log}), inline_filter_chain_manager_(filter_chain_manager), - init_manager_(nullptr), ignore_global_conn_limit_(ignore_global_conn_limit) { + init_manager_(nullptr), ignore_global_conn_limit_(ignore_global_conn_limit), + listener_info_(std::make_shared>()) { socket_factories_.emplace_back(std::make_unique()); ON_CALL(*socket_, socketType()).WillByDefault(Return(socket_type)); } @@ -323,9 +324,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> inline_filter_chain_manager_; std::unique_ptr init_manager_; const bool ignore_global_conn_limit_; - envoy::config::core::v3::TrafficDirection direction_; + const Network::ListenerInfoConstSharedPtr listener_info_; }; using TestListenerPtr = std::unique_ptr; @@ -397,6 +397,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable access_log_; TestScopedRuntime scoped_runtime_; Runtime::Loader& runtime_{scoped_runtime_.loader()}; + testing::NiceMock random_; }; TEST_F(ConnectionHandlerTest, DisableInternalListener) { @@ -410,7 +411,7 @@ TEST_F(ConnectionHandlerTest, DisableInternalListener) { internal_listener->socket_factories_[0].get()), localAddress()) .WillRepeatedly(ReturnRef(local_address)); - handler_->addListener(absl::nullopt, *internal_listener, runtime_); + handler_->addListener(absl::nullopt, *internal_listener, runtime_, random_); auto internal_listener_cb = handler_->findByAddress(local_address); ASSERT_TRUE(internal_listener_cb.has_value()); @@ -438,7 +439,7 @@ TEST_F(ConnectionHandlerTest, InternalListenerInplaceUpdate) { internal_listener->socket_factories_[0].get()), localAddress()) .WillRepeatedly(ReturnRef(local_address)); - handler_->addListener(absl::nullopt, *internal_listener, runtime_); + handler_->addListener(absl::nullopt, *internal_listener, runtime_, random_); ASSERT_NE(internal_listener, nullptr); @@ -448,7 +449,7 @@ TEST_F(ConnectionHandlerTest, InternalListenerInplaceUpdate) { addInternalListener(new_listener_tag, "test_internal_listener", std::chrono::milliseconds(), false, overridden_filter_chain_manager); - handler_->addListener(old_listener_tag, *new_test_listener, runtime_); + handler_->addListener(old_listener_tag, *new_test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); @@ -456,7 +457,7 @@ TEST_F(ConnectionHandlerTest, InternalListenerInplaceUpdate) { EXPECT_CALL(manager_, findFilterChain(_, _)).Times(0); EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); internal_listener_cb.value().get().onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); diff --git a/test/extensions/bootstrap/wasm/test_data/speed_cpp.cc b/test/extensions/bootstrap/wasm/test_data/speed_cpp.cc index d0d9e9906674..3447622b78b2 100644 --- a/test/extensions/bootstrap/wasm/test_data/speed_cpp.cc +++ b/test/extensions/bootstrap/wasm/test_data/speed_cpp.cc @@ -138,28 +138,28 @@ std::string check_compiler; void (*test_fn)() = nullptr; -void empty_test() {} +void emptyTest() {} -void get_current_time_test() { +void getCurrentTimeTest() { uint64_t t; if (WasmResult::Ok != proxy_get_current_time_nanoseconds(&t)) { logError("bad result from getCurrentTimeNanoseconds"); } } -void small_string_check_compiler_test() { +void smallStringCheckCompilerTest() { check_compiler = "foo"; check_compiler += "bar"; check_compiler = ""; } -void small_string_test() { +void smallStringTest() { std::string s = "foo"; s += "bar"; xDoNotRemove = s.size(); } -void small_string_check_compiler1000_test() { +void smallStringCheckCompiler1000Test() { for (int x = 0; x < 1000; x++) { check_compiler = "foo"; check_compiler += "bar"; @@ -167,7 +167,7 @@ void small_string_check_compiler1000_test() { check_compiler = ""; } -void small_string1000_test() { +void smallString1000Test() { for (int x = 0; x < 1000; x++) { std::string s = "foo"; s += "bar"; @@ -175,14 +175,14 @@ void small_string1000_test() { } } -void large_string_test() { +void largeStringTest() { std::string s(1024, 'f'); std::string d(1024, 'o'); s += d; xDoNotRemove += s.size(); } -void large_string1000_test() { +void largeString1000Test() { for (int x = 0; x < 1000; x++) { std::string s(1024, 'f'); std::string d(1024, 'o'); @@ -191,7 +191,7 @@ void large_string1000_test() { } } -void get_property_test() { +void getPropertyTest() { std::string property = "plugin_root_id"; const char* value_ptr = nullptr; size_t value_size = 0; @@ -202,7 +202,7 @@ void get_property_test() { ::free(reinterpret_cast(const_cast(value_ptr))); } -void grpc_service_test() { +void grpcServiceTest() { std::string value = "foo"; GrpcService grpc_service; grpc_service.mutable_envoy_grpc()->set_cluster_name(value); @@ -210,7 +210,7 @@ void grpc_service_test() { grpc_service.SerializeToString(&grpc_service_string); } -void grpc_service1000_test() { +void grpcService1000Test() { std::string value = "foo"; for (int x = 0; x < 1000; x++) { GrpcService grpc_service; @@ -220,7 +220,7 @@ void grpc_service1000_test() { } } -void modify_metadata_test() { +void modifyMetadataTest() { auto path = getRequestHeader(":path"); addRequestHeader("newheader", "newheadervalue"); auto server = getRequestHeader("server"); @@ -229,7 +229,7 @@ void modify_metadata_test() { removeRequestHeader("newheader"); } -void modify_metadata1000_test() { +void modifyMetadata1000Test() { for (int x = 0; x < 1000; x++) { auto path = getRequestHeader(":path"); addRequestHeader("newheader", "newheadervalue"); @@ -240,25 +240,25 @@ void modify_metadata1000_test() { } } -void json_serialize_test() { +void jsonSerializeTest() { google::protobuf::Struct proto; google::protobuf::util::JsonStringToMessage(test_json, &proto).IgnoreError(); } -void json_deserialize_test() { +void jsonDeserializeTest() { std::string json; google::protobuf::util::MessageToJsonString(test_proto, &json).IgnoreError(); xDoNotRemove += json.size(); } -void json_deserialize_empty_test() { +void jsonDeserializeEmptyTest() { std::string json; google::protobuf::Struct empty; google::protobuf::util::MessageToJsonString(empty, &json).IgnoreError(); xDoNotRemove = json.size(); } -void convert_to_filter_state_test() { +void convertToFilterStateTest() { auto start = reinterpret_cast(&*test_json.begin()); auto end = start + test_json.size(); std::string encoded_json = base64Encode(start, end); @@ -278,40 +278,40 @@ WASM_EXPORT(uint32_t, proxy_on_vm_start, (uint32_t, uint32_t configuration_size) &size); std::string configuration(configuration_ptr, size); if (configuration == "empty") { - test_fn = &empty_test; + test_fn = &emptyTest; } else if (configuration == "get_current_time") { - test_fn = &get_current_time_test; + test_fn = &getCurrentTimeTest; } else if (configuration == "small_string") { - test_fn = &small_string_test; + test_fn = &smallStringTest; } else if (configuration == "small_string1000") { - test_fn = &small_string1000_test; + test_fn = &smallString1000Test; } else if (configuration == "small_string_check_compiler") { - test_fn = &small_string_check_compiler_test; + test_fn = &smallStringCheckCompilerTest; } else if (configuration == "small_string_check_compiler1000") { - test_fn = &small_string_check_compiler1000_test; + test_fn = &smallStringCheckCompiler1000Test; } else if (configuration == "large_string") { - test_fn = &large_string_test; + test_fn = &largeStringTest; } else if (configuration == "large_string1000") { - test_fn = &large_string1000_test; + test_fn = &largeString1000Test; } else if (configuration == "get_property") { - test_fn = &get_property_test; + test_fn = &getPropertyTest; } else if (configuration == "grpc_service") { - test_fn = &grpc_service_test; + test_fn = &grpcServiceTest; } else if (configuration == "grpc_service1000") { - test_fn = &grpc_service1000_test; + test_fn = &grpcService1000Test; } else if (configuration == "modify_metadata") { - test_fn = &modify_metadata_test; + test_fn = &modifyMetadataTest; } else if (configuration == "modify_metadata1000") { - test_fn = &modify_metadata1000_test; + test_fn = &modifyMetadata1000Test; } else if (configuration == "json_serialize") { - test_fn = &json_serialize_test; + test_fn = &jsonSerializeTest; } else if (configuration == "json_deserialize") { google::protobuf::util::JsonStringToMessage(test_json, &test_proto).IgnoreError(); - test_fn = &json_deserialize_test; + test_fn = &jsonDeserializeTest; } else if (configuration == "json_deserialize_empty") { - test_fn = &json_deserialize_empty_test; + test_fn = &jsonDeserializeEmptyTest; } else if (configuration == "convert_to_filter_state") { - test_fn = &convert_to_filter_state_test; + test_fn = &convertToFilterStateTest; } else { std::string message = "on_start " + configuration; proxy_log(LogLevel::info, message.c_str(), message.size()); diff --git a/test/extensions/bootstrap/wasm/test_data/stats_cpp.cc b/test/extensions/bootstrap/wasm/test_data/stats_cpp.cc index f36f85c685af..51b21a4c28bc 100644 --- a/test/extensions/bootstrap/wasm/test_data/stats_cpp.cc +++ b/test/extensions/bootstrap/wasm/test_data/stats_cpp.cc @@ -7,7 +7,7 @@ #include "include/proxy-wasm/null_plugin.h" #endif -template std::unique_ptr wrap_unique(T* ptr) { return std::unique_ptr(ptr); } +template std::unique_ptr wrapUnique(T* ptr) { return std::unique_ptr(ptr); } START_WASM_PLUGIN(WasmStatsCpp) @@ -71,11 +71,11 @@ WASM_EXPORT(void, proxy_on_tick, (uint32_t)) { // Test the high level interface. WASM_EXPORT(void, proxy_on_log, (uint32_t /* context_zero */)) { - auto c = wrap_unique( + auto c = wrapUnique( Counter::New("test_counter", "string_tag", "int_tag", "bool_tag")); auto g = - wrap_unique(Gauge::New("test_gauge", "string_tag1", "string_tag2")); - auto h = wrap_unique(Histogram::New("test_histogram", "int_tag", + wrapUnique(Gauge::New("test_gauge", "string_tag1", "string_tag2")); + auto h = wrapUnique(Histogram::New("test_histogram", "int_tag", "string_tag", "bool_tag")); c->increment(1, "test_tag", 7, true); @@ -90,9 +90,9 @@ WASM_EXPORT(void, proxy_on_log, (uint32_t /* context_zero */)) { logWarn(std::string("get gauge = ") + std::to_string(g->get("test_tag1", "test_tag2"))); h->record(3, 7, "test_tag", true); - auto base_h = wrap_unique(Counter::New("test_histogram", "int_tag")); + auto base_h = wrapUnique(Counter::New("test_histogram", "int_tag")); auto complete_h = - wrap_unique(base_h->extendAndResolve(7, "string_tag", "bool_tag")); + wrapUnique(base_h->extendAndResolve(7, "string_tag", "bool_tag")); auto simple_h = complete_h->resolve("test_tag", true); logError(std::string("h_id = ") + complete_h->nameFromIdSlow(simple_h.metric_id)); diff --git a/test/extensions/clusters/aggregate/BUILD b/test/extensions/clusters/aggregate/BUILD index f0a927f83651..3dd5655a73df 100644 --- a/test/extensions/clusters/aggregate/BUILD +++ b/test/extensions/clusters/aggregate/BUILD @@ -17,6 +17,7 @@ envoy_extension_cc_test( extension_names = ["envoy.clusters.aggregate"], deps = [ "//source/extensions/clusters/aggregate:cluster", + "//source/extensions/load_balancing_policies/cluster_provided:config", "//source/extensions/transport_sockets/raw_buffer:config", "//test/common/upstream:utility_lib", "//test/mocks/protobuf:protobuf_mocks", @@ -43,6 +44,8 @@ envoy_extension_cc_test( "//source/common/upstream:cluster_manager_lib", "//source/extensions/clusters/aggregate:cluster", "//source/extensions/clusters/static:static_cluster_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//test/common/upstream:test_cluster_manager", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/aggregate/cluster_update_test.cc b/test/extensions/clusters/aggregate/cluster_update_test.cc index 08a1d0d47af9..67c029b52799 100644 --- a/test/extensions/clusters/aggregate/cluster_update_test.cc +++ b/test/extensions/clusters/aggregate/cluster_update_test.cc @@ -42,11 +42,11 @@ class AggregateClusterUpdateTest : public Event::TestUsingSimulatedTime, auto bootstrap = parseBootstrapFromV2Yaml(yaml_config); const bool use_deferred_cluster = GetParam(); bootstrap.mutable_cluster_manager()->set_enable_deferred_cluster_creation(use_deferred_cluster); - cluster_manager_ = std::make_unique( + cluster_manager_ = Upstream::TestClusterManagerImpl::createAndInit( bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, *factory_.api_, http_context_, grpc_context_, router_context_, server_); - cluster_manager_->initializeSecondaryClusters(bootstrap); + ASSERT_TRUE(cluster_manager_->initializeSecondaryClusters(bootstrap).ok()); EXPECT_EQ(cluster_manager_->activeClusters().size(), 1); cluster_ = cluster_manager_->getThreadLocalCluster("aggregate_cluster"); } @@ -277,11 +277,11 @@ TEST_P(AggregateClusterUpdateTest, InitializeAggregateClusterAfterOtherClusters) )EOF"; auto bootstrap = parseBootstrapFromV2Yaml(config); - cluster_manager_ = std::make_unique( + cluster_manager_ = Upstream::TestClusterManagerImpl::createAndInit( bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, *factory_.api_, http_context_, grpc_context_, router_context_, server_); - cluster_manager_->initializeSecondaryClusters(bootstrap); + ASSERT_TRUE(cluster_manager_->initializeSecondaryClusters(bootstrap).ok()); EXPECT_EQ(cluster_manager_->activeClusters().size(), 2); cluster_ = cluster_manager_->getThreadLocalCluster("aggregate_cluster"); auto primary = cluster_manager_->getThreadLocalCluster("primary"); diff --git a/test/extensions/clusters/dynamic_forward_proxy/BUILD b/test/extensions/clusters/dynamic_forward_proxy/BUILD index d1093317bb7a..1527806a0f0f 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/BUILD +++ b/test/extensions/clusters/dynamic_forward_proxy/BUILD @@ -19,6 +19,8 @@ envoy_extension_cc_test( deps = [ "//source/common/router:string_accessor_lib", "//source/extensions/clusters/dynamic_forward_proxy:cluster", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/network/dns_resolver/cares:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:config", diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index bbe4ab7ed27e..75f48aadfdf7 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -57,7 +57,9 @@ class ClusterTest : public testing::Test, // actually correct. It's possible this will have to change in the future. EXPECT_CALL(*dns_cache_manager_->dns_cache_, addUpdateCallbacks_(_)) .WillOnce(DoAll(SaveArgAddress(&update_callbacks_), Return(nullptr))); - cluster_.reset(new Cluster(cluster_config, config, factory_context, *this)); + auto cache = dns_cache_manager_->getCache(config.dns_cache_config()).value(); + cluster_.reset( + new Cluster(cluster_config, std::move(cache), config, factory_context, this->get())); thread_aware_lb_ = std::make_unique(*cluster_); lb_factory_ = thread_aware_lb_->factory(); refreshLb(); diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD index d9a4b138500a..09e0f5909326 100644 --- a/test/extensions/clusters/eds/BUILD +++ b/test/extensions/clusters/eds/BUILD @@ -16,6 +16,7 @@ envoy_cc_test( deps = [ "//source/common/config:utility_lib", "//source/extensions/clusters/eds:eds_lib", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:config", "//source/server:transport_socket_config_lib", @@ -54,6 +55,7 @@ envoy_cc_benchmark_binary( "//source/extensions/clusters/eds:eds_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", "//source/extensions/config_subscription/grpc/xds_mux:grpc_mux_lib", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", @@ -85,6 +87,7 @@ envoy_cc_test( deps = [ "//source/common/config:utility_lib", "//source/extensions/clusters/eds:leds_lib", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/stats:stat_test_utility_lib", diff --git a/test/extensions/clusters/eds/eds_speed_test.cc b/test/extensions/clusters/eds/eds_speed_test.cc index c5b10480216f..d87d22a5bfaf 100644 --- a/test/extensions/clusters/eds/eds_speed_test.cc +++ b/test/extensions/clusters/eds/eds_speed_test.cc @@ -68,9 +68,9 @@ class EdsSpeedTest { /*target_xds_authority_=*/"", /*eds_resources_cache_=*/nullptr}; if (use_unified_mux_) { - grpc_mux_.reset(new Config::XdsMux::GrpcMuxSotw(grpc_mux_context, true)); + grpc_mux_ = std::make_shared(grpc_mux_context, true); } else { - grpc_mux_.reset(new Config::GrpcMuxImpl(grpc_mux_context, true)); + grpc_mux_ = std::make_shared(grpc_mux_context, true); } resetCluster(R"EOF( name: name diff --git a/test/extensions/clusters/eds/eds_test.cc b/test/extensions/clusters/eds/eds_test.cc index 02495fb926c8..95e123a421b1 100644 --- a/test/extensions/clusters/eds/eds_test.cc +++ b/test/extensions/clusters/eds/eds_test.cc @@ -258,7 +258,7 @@ TEST_F(EdsTest, OnConfigUpdateWrongName) { TestUtility::decodeResources({cluster_load_assignment}, "cluster_name"); initialize(); try { - EXPECT_TRUE(eds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()); + THROW_IF_NOT_OK(eds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "")); } catch (const EnvoyException& e) { eds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); @@ -284,7 +284,7 @@ TEST_F(EdsTest, OnConfigUpdateWrongSize) { const auto decoded_resources = TestUtility::decodeResources( {cluster_load_assignment, cluster_load_assignment}, "cluster_name"); try { - EXPECT_TRUE(eds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()); + THROW_IF_NOT_OK(eds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "")); } catch (const EnvoyException& e) { eds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); @@ -1820,7 +1820,6 @@ TEST_F(EdsTest, EndpointLocality) { EXPECT_EQ("hello", locality.zone()); EXPECT_EQ("world", locality.sub_zone()); } - EXPECT_EQ(nullptr, cluster_->prioritySet().hostSetsPerPriority()[0]->localityWeights()); } TEST_F(EdsTest, EndpointCombineDuplicateLocalities) { @@ -1869,7 +1868,6 @@ TEST_F(EdsTest, EndpointCombineDuplicateLocalities) { EXPECT_EQ("hello", locality.zone()); EXPECT_EQ("world", locality.sub_zone()); } - EXPECT_EQ(nullptr, cluster_->prioritySet().hostSetsPerPriority()[0]->localityWeights()); } // Validate that onConfigUpdate() updates the endpoint locality of an existing endpoint. @@ -1912,7 +1910,6 @@ TEST_F(EdsTest, EndpointLocalityUpdated) { EXPECT_EQ("hello", locality.zone()); EXPECT_EQ("world", locality.sub_zone()); } - EXPECT_EQ(nullptr, cluster_->prioritySet().hostSetsPerPriority()[0]->localityWeights()); // Update locality now locality->set_region("space"); @@ -1934,6 +1931,12 @@ TEST_F(EdsTest, EndpointLocalityUpdated) { // Validate that onConfigUpdate() does not propagate locality weights to the host set when // locality weighted balancing isn't configured and the cluster does not use LB policy extensions. TEST_F(EdsTest, EndpointLocalityWeightsIgnored) { + TestScopedRuntime runtime; + runtime.mergeValues({{"envoy.reloadable_features.convert_legacy_lb_config", "false"}}); + + // Reset the cluster after the runtime change. + resetCluster(); + envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment; cluster_load_assignment.set_cluster_name("fare"); @@ -2830,11 +2833,9 @@ TEST_F(EdsTest, OnConfigUpdateLedsAndEndpoints) { const auto decoded_resources = TestUtility::decodeResources({cluster_load_assignment}, "cluster_name"); - EXPECT_THROW_WITH_MESSAGE( - EXPECT_TRUE(eds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").ok()), - EnvoyException, - "A ClusterLoadAssignment for cluster fare cannot include both LEDS " - "(resource: xdstp://foo/leds/collection) and a list of endpoints."); + EXPECT_EQ(eds_callbacks_->onConfigUpdate(decoded_resources.refvec_, "").message(), + "A ClusterLoadAssignment for cluster fare cannot include both LEDS " + "(resource: xdstp://foo/leds/collection) and a list of endpoints."); } class EdsCachedAssignmentTest : public testing::Test { diff --git a/test/extensions/clusters/logical_dns/BUILD b/test/extensions/clusters/logical_dns/BUILD index 2c1b6a44d443..c5d3e64052c6 100644 --- a/test/extensions/clusters/logical_dns/BUILD +++ b/test/extensions/clusters/logical_dns/BUILD @@ -16,6 +16,7 @@ envoy_cc_test( "//source/common/network:utility_lib", "//source/common/upstream:upstream_lib", "//source/extensions/clusters/logical_dns:logical_dns_cluster_lib", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/original_dst/BUILD b/test/extensions/clusters/original_dst/BUILD index bfb33cea1edc..1b8d43c8ff76 100644 --- a/test/extensions/clusters/original_dst/BUILD +++ b/test/extensions/clusters/original_dst/BUILD @@ -17,6 +17,7 @@ envoy_cc_test( "//source/common/network:utility_lib", "//source/common/upstream:upstream_lib", "//source/extensions/clusters/original_dst:original_dst_cluster_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", "//source/extensions/transport_sockets/raw_buffer:config", "//test/common/upstream:utility_lib", "//test/mocks:common_lib", diff --git a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc index 9914f56a096a..ae5d1a65de9a 100644 --- a/test/extensions/clusters/original_dst/original_dst_cluster_test.cc +++ b/test/extensions/clusters/original_dst/original_dst_cluster_test.cc @@ -68,7 +68,7 @@ class TestLoadBalancerContext : public LoadBalancerContextBase { class OriginalDstClusterTest : public Event::TestUsingSimulatedTime, public testing::Test { public: - OriginalDstClusterTest() {} + OriginalDstClusterTest() = default; void setupFromYaml(const std::string& yaml, bool expect_success = true) { if (expect_success) { @@ -1065,8 +1065,8 @@ TEST_F(OriginalDstClusterTest, UseFilterState) { // Filter state takes priority over header override. NiceMock connection; connection.stream_info_.filterState()->setData( - Network::DestinationAddress::key(), - std::make_shared( + Upstream::OriginalDstClusterFilterStateKey, + std::make_shared( std::make_shared("10.10.11.11", 6666)), StreamInfo::FilterState::StateType::ReadOnly); TestLoadBalancerContext lb_context1(&connection, Http::Headers::get().EnvoyOriginalDstHost.get(), @@ -1133,8 +1133,8 @@ TEST_F(OriginalDstClusterTest, UseFilterStateWithPortOverride) { // Filter state takes priority over header override. NiceMock connection; connection.stream_info_.filterState()->setData( - Network::DestinationAddress::key(), - std::make_shared( + Upstream::OriginalDstClusterFilterStateKey, + std::make_shared( std::make_shared("10.10.11.11", 6666)), StreamInfo::FilterState::StateType::ReadOnly); TestLoadBalancerContext lb_context1(&connection, Http::Headers::get().EnvoyOriginalDstHost.get(), diff --git a/test/extensions/clusters/redis/BUILD b/test/extensions/clusters/redis/BUILD index e989050baa9a..89d7d26fbd3d 100644 --- a/test/extensions/clusters/redis/BUILD +++ b/test/extensions/clusters/redis/BUILD @@ -24,6 +24,8 @@ envoy_extension_cc_test( "//source/extensions/clusters/redis:redis_cluster", "//source/extensions/clusters/redis:redis_cluster_lb", "//source/extensions/filters/network/redis_proxy:config", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", @@ -68,6 +70,7 @@ envoy_extension_cc_test( "//source/extensions/filters/network/common/redis:client_interface", "//source/extensions/filters/network/common/redis:codec_lib", "//source/extensions/filters/network/common/redis:supported_commands_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", @@ -98,6 +101,8 @@ envoy_extension_cc_test( "//source/extensions/clusters/redis:redis_cluster", "//source/extensions/clusters/redis:redis_cluster_lb", "//source/extensions/filters/network/redis_proxy:config", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//test/integration:ads_integration_lib", "//test/integration:integration_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index f861cdc6404f..b6767d9fb6a6 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -31,7 +31,6 @@ using testing::_; using testing::ContainerEq; -using testing::Eq; using testing::NiceMock; using testing::Ref; using testing::Return; diff --git a/test/extensions/common/aws/BUILD b/test/extensions/common/aws/BUILD index 55ebdf79f19f..63440356fae7 100644 --- a/test/extensions/common/aws/BUILD +++ b/test/extensions/common/aws/BUILD @@ -3,6 +3,7 @@ load( "envoy_cc_mock", "envoy_cc_test", "envoy_package", + "envoy_select_boringssl", ) licenses(["notice"]) # Apache 2 @@ -14,8 +15,11 @@ envoy_cc_mock( srcs = ["mocks.cc"], hdrs = ["mocks.h"], deps = [ + "//source/common/http:message_lib", "//source/extensions/common/aws:credentials_provider_interface", + "//source/extensions/common/aws:metadata_fetcher_lib", "//source/extensions/common/aws:signer_interface", + "//test/mocks/upstream:cluster_manager_mocks", ], ) @@ -35,8 +39,14 @@ envoy_cc_test( envoy_cc_test( name = "utility_test", srcs = ["utility_test.cc"], + copts = envoy_select_boringssl( + [ + "-DENVOY_SSL_FIPS", + ], + ), deps = [ "//source/extensions/common/aws:utility_lib", + "//test/extensions/common/aws:aws_mocks", "//test/test_common:utility_lib", ], ) @@ -50,15 +60,33 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "metadata_fetcher_test", + srcs = ["metadata_fetcher_test.cc"], + deps = [ + "//source/extensions/common/aws:metadata_fetcher_lib", + "//test/extensions/common/aws:aws_mocks", + "//test/extensions/filters/http/common:mock_lib", + "//test/mocks/api:api_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:environment_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "credentials_provider_impl_test", srcs = ["credentials_provider_impl_test.cc"], deps = [ "//source/extensions/common/aws:credentials_provider_impl_lib", + "//source/extensions/common/aws:metadata_fetcher_lib", "//test/extensions/common/aws:aws_mocks", "//test/mocks/api:api_mocks", "//test/mocks/event:event_mocks", "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:factory_context_mocks", "//test/test_common:environment_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:test_runtime_lib", diff --git a/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc b/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc index 6e71984ee8a7..89946ef92e06 100644 --- a/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc +++ b/test/extensions/common/aws/aws_metadata_fetcher_integration_test.cc @@ -99,7 +99,7 @@ TEST_F(AwsMetadataIntegrationTestSuccess, Success) { const auto authority = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), lookupPort("listener_0")); auto headers = Http::RequestHeaderMapPtr{new Http::TestRequestHeaderMapImpl{ - {":path", "/"}, {":authority", authority}, {":method", "GET"}}}; + {":path", "/"}, {":authority", authority}, {":scheme", "http"}, {":method", "GET"}}}; Http::RequestMessageImpl message(std::move(headers)); const auto response = Utility::fetchMetadata(message); @@ -116,6 +116,7 @@ TEST_F(AwsMetadataIntegrationTestSuccess, AuthToken) { auto headers = Http::RequestHeaderMapPtr{ new Http::TestRequestHeaderMapImpl{{":path", "/"}, {":authority", authority}, + {":scheme", "http"}, {":method", "GET"}, {"authorization", "AUTH_TOKEN"}}}; Http::RequestMessageImpl message(std::move(headers)); @@ -155,6 +156,7 @@ TEST_F(AwsMetadataIntegrationTestSuccess, Redirect) { auto headers = Http::RequestHeaderMapPtr{ new Http::TestRequestHeaderMapImpl{{":path", "/redirect"}, {":authority", authority}, + {":scheme", "http"}, {":method", "GET"}, {"authorization", "AUTH_TOKEN"}}}; Http::RequestMessageImpl message(std::move(headers)); @@ -182,6 +184,7 @@ TEST_F(AwsMetadataIntegrationTestFailure, Failure) { auto headers = Http::RequestHeaderMapPtr{ new Http::TestRequestHeaderMapImpl{{":path", "/"}, {":authority", authority}, + {":scheme", "http"}, {":method", "GET"}, {"authorization", "AUTH_TOKEN"}}}; @@ -210,7 +213,7 @@ TEST_F(AwsMetadataIntegrationTestTimeout, Timeout) { const auto authority = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), lookupPort("listener_0")); auto headers = Http::RequestHeaderMapPtr{new Http::TestRequestHeaderMapImpl{ - {":path", "/"}, {":authority", authority}, {":method", "GET"}}}; + {":path", "/"}, {":authority", authority}, {":scheme", "http"}, {":method", "GET"}}}; Http::RequestMessageImpl message(std::move(headers)); const auto start_time = timeSystem().monotonicTime(); diff --git a/test/extensions/common/aws/credentials_provider_impl_test.cc b/test/extensions/common/aws/credentials_provider_impl_test.cc index a93c9ccf770b..a48a16bd0c54 100644 --- a/test/extensions/common/aws/credentials_provider_impl_test.cc +++ b/test/extensions/common/aws/credentials_provider_impl_test.cc @@ -1,18 +1,28 @@ +#include +#include + #include "source/extensions/common/aws/credentials_provider_impl.h" +#include "source/extensions/common/aws/metadata_fetcher.h" #include "test/extensions/common/aws/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/event/mocks.h" #include "test/mocks/runtime/mocks.h" +#include "test/mocks/server/factory_context.h" #include "test/test_common/environment.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/test_runtime.h" +using Envoy::Extensions::Common::Aws::MetadataFetcher; +using Envoy::Extensions::Common::Aws::MetadataFetcherPtr; +using Envoy::Extensions::Common::Aws::MockMetadataFetcher; using testing::_; +using testing::Eq; using testing::InSequence; using testing::NiceMock; using testing::Ref; using testing::Return; +using testing::Throw; namespace Envoy { namespace Extensions { @@ -48,6 +58,40 @@ aws_secret_access_key = profile4_secret aws_session_token = profile4_token )"; +MATCHER_P(WithName, expectedName, "") { + *result_listener << "\nexpected { name: \"" << expectedName << "\"} but got {name: \"" + << arg.name() << "\"}\n"; + return ExplainMatchResult(expectedName, arg.name(), result_listener); +} + +MATCHER_P(WithAttribute, expectedCluster, "") { + const auto argSocketAddress = + arg.load_assignment().endpoints()[0].lb_endpoints()[0].endpoint().address().socket_address(); + const auto expectedSocketAddress = expectedCluster.load_assignment() + .endpoints()[0] + .lb_endpoints()[0] + .endpoint() + .address() + .socket_address(); + + *result_listener << "\nexpected {cluster name: \"" << expectedCluster.name() << "\", type: \"" + << expectedCluster.type() << "\", socket address: \"" + << expectedSocketAddress.address() << "\", port: \"" + << expectedSocketAddress.port_value() << "\", transport socket enabled: \"" + << expectedCluster.has_transport_socket() << "\"},\n but got {cluster name: \"" + << arg.name() << "\", type: \"" << arg.type() << "\", socket address: \"" + << argSocketAddress.address() << "\", port: \"" << argSocketAddress.port_value() + << "\", transport socket enabled: \"" << arg.has_transport_socket() << "\"}\n"; + return ExplainMatchResult(expectedCluster.name(), arg.name(), result_listener) && + ExplainMatchResult(expectedCluster.type(), arg.type(), result_listener) && + ExplainMatchResult(expectedSocketAddress.address(), argSocketAddress.address(), + result_listener) && + ExplainMatchResult(expectedSocketAddress.port_value(), argSocketAddress.port_value(), + result_listener) && + ExplainMatchResult(expectedCluster.has_transport_socket(), arg.has_transport_socket(), + result_listener); +} + class EvironmentCredentialsProviderTest : public testing::Test { public: ~EvironmentCredentialsProviderTest() override { @@ -250,17 +294,856 @@ messageMatches(const Http::TestRequestHeaderMapImpl& expected_headers) { return testing::MakeMatcher(new MessageMatcher(expected_headers)); } +// Begin unit test for new option via Http Async client. class InstanceProfileCredentialsProviderTest : public testing::Test { public: InstanceProfileCredentialsProviderTest() - : api_(Api::createApiForTest(time_system_)), - provider_(*api_, [this](Http::RequestMessage& message) -> absl::optional { + : api_(Api::createApiForTest(time_system_)), raw_metadata_fetcher_(new MockMetadataFetcher) {} + + void setupProvider() { + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "true"}}); + ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + provider_ = std::make_shared( + *api_, context_, + [this](Http::RequestMessage& message) -> absl::optional { + return this->fetch_metadata_.fetch(message); + }, + [this](Upstream::ClusterManager&, absl::string_view) { + metadata_fetcher_.reset(raw_metadata_fetcher_); + return std::move(metadata_fetcher_); + }, + "credentials_provider_cluster"); + } + + void setupProviderWithContext() { + EXPECT_CALL(context_.init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + setupProvider(); + expected_duration_ = provider_->getCacheDuration(); + init_target_handle_->initialize(init_watcher_); + } + + void expectSessionToken(const uint64_t status_code, const std::string&& token) { + Http::TestRequestHeaderMapImpl headers{{":path", "/latest/api/token"}, + {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, + {":method", "PUT"}, + {"X-aws-ec2-metadata-token-ttl-seconds", "21600"}}; + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, token = std::move(token)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!token.empty()) { + receiver.onMetadataSuccess(std::move(token)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); + } + + void expectCredentialListing(const uint64_t status_code, const std::string&& instance_role) { + Http::TestRequestHeaderMapImpl headers{{":path", "/latest/meta-data/iam/security-credentials"}, + {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, + {":method", "GET"}}; + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, instance_role = std::move(instance_role)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!instance_role.empty()) { + receiver.onMetadataSuccess(std::move(instance_role)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); + } + + void expectCredentialListingSecure(const uint64_t status_code, + const std::string&& instance_role) { + Http::TestRequestHeaderMapImpl headers{{":path", "/latest/meta-data/iam/security-credentials"}, + {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, + {":method", "GET"}, + {"X-aws-ec2-metadata-token", "TOKEN"}}; + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, instance_role = std::move(instance_role)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!instance_role.empty()) { + receiver.onMetadataSuccess(std::move(instance_role)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); + } + + void expectDocument(const uint64_t status_code, const std::string&& credential_document_value) { + Http::TestRequestHeaderMapImpl headers{ + {":path", "/latest/meta-data/iam/security-credentials/doc1"}, + {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, + {":method", "GET"}}; + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, + credential_document_value = std::move(credential_document_value)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!credential_document_value.empty()) { + receiver.onMetadataSuccess(std::move(credential_document_value)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); + } + + void expectDocumentSecure(const uint64_t status_code, + const std::string&& credential_document_value) { + Http::TestRequestHeaderMapImpl headers{ + {":path", "/latest/meta-data/iam/security-credentials/doc1"}, + {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, + {":method", "GET"}, + {"X-aws-ec2-metadata-token", "TOKEN"}}; + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, + credential_document_value = std::move(credential_document_value)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!credential_document_value.empty()) { + receiver.onMetadataSuccess(std::move(credential_document_value)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); + } + + TestScopedRuntime scoped_runtime_; + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + NiceMock fetch_metadata_; + MockMetadataFetcher* raw_metadata_fetcher_; + MetadataFetcherPtr metadata_fetcher_; + NiceMock cluster_manager_; + NiceMock context_; + InstanceProfileCredentialsProviderPtr provider_; + Init::TargetHandlePtr init_target_handle_; + NiceMock init_watcher_; + Event::MockTimer* timer_{}; + std::chrono::milliseconds expected_duration_; +}; + +TEST_F(InstanceProfileCredentialsProviderTest, TestAddMissingCluster) { + // Setup without thread local cluster yet + envoy::config::cluster::v3::Cluster expected_cluster; + constexpr static const char* kStaticCluster = R"EOF( +name: credentials_provider_cluster +type: static +connectTimeout: 2s +lb_policy: ROUND_ROBIN +loadAssignment: + clusterName: credentials_provider_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: "169.254.169.254" + portValue: 80 +typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: + accept_http_10: true + )EOF"; + MessageUtil::loadFromYaml(kStaticCluster, expected_cluster, + ProtobufMessage::getNullValidationVisitor()); + + EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cluster_manager_, addOrUpdateCluster(WithAttribute(expected_cluster), _)) + .WillOnce(Return(true)); + + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1"))); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + expectDocumentSecure(200, std::move(R"EOF( + { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" + } + )EOF")); + + setupProviderWithContext(); +} + +TEST_F(InstanceProfileCredentialsProviderTest, TestClusterMissing) { + // Setup without thread local cluster + Http::RequestMessageImpl message; + + EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cluster_manager_, addOrUpdateCluster(WithName("credentials_provider_cluster"), _)) + .WillOnce(Throw(EnvoyException("exeption message"))); + + // init_watcher ready is not called. + init_watcher_.expectReady().Times(0); + setupProvider(); + // Below line is not testing anything, will just avoid asan failure with memory leak. + metadata_fetcher_.reset(raw_metadata_fetcher_); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FailedCredentialListingUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(403 /*Forbidden*/, std::move(std::string())); + expectCredentialListing(403 /*Forbidden*/, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FailedCredentialListingSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(401 /*Unauthorized*/, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyCredentialListingUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string(""))); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called once for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyCredentialListingSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string(""))); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyListCredentialListingUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("\n"))); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called once for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyListCredentialListingSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("\n"))); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FailedDocumentUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1\ndoc2\ndoc3"))); + expectDocument(401 /*Unauthorized*/, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called thrice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FailedDocumentSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1\ndoc2\ndoc3"))); + expectDocumentSecure(401 /*Unauthorized*/, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called thrice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, MissingDocumentUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1\ndoc2\ndoc3"))); + expectDocument(200, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called thrice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, MissingDocumentSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1\ndoc2\ndoc3"))); + expectDocumentSecure(200, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called thrice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, MalformedDocumentUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1"))); + expectDocument(200, std::move(R"EOF( + not json + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called thrice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, MalformedDocumentSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1"))); + expectDocumentSecure(200, std::move(R"EOF( + not json + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is called thrice + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyValuesUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1"))); + expectDocument(200, std::move(R"EOF( + { + "AccessKeyId": "", + "SecretAccessKey": "", + "Token": "" + } + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is not called again as we don't expect any more call to fetch until timeout. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyValuesSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1"))); + expectDocumentSecure(200, std::move(R"EOF( + { + "AccessKeyId": "", + "SecretAccessKey": "", + "Token": "" + } + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + // Cancel is not called again as we don't expect any more call to fetch until timeout. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentialsUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1"))); + expectDocument(200, std::move(R"EOF( + { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" + } + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Cancel is not called again as we don't expect any more call to fetch until timeout. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("token", cached_credentials.sessionToken().value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentialsSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1"))); + expectDocumentSecure(200, std::move(R"EOF( + { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" + } + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started after fetch done from init. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Cancel is not called again as we don't expect any more call to fetch until timeout. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("token", cached_credentials.sessionToken().value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, RefreshOnCredentialExpirationUnsecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1"))); + expectDocument(200, std::move(R"EOF( + { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" + } + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Cancel is not called again as we don't expect any more call to fetch until timeout. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + expectSessionToken(200, std::move(std::string())); + expectCredentialListing(200, std::move(std::string("doc1"))); + expectDocument(200, std::move(R"EOF( + { + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token1" + } + )EOF")); + + // Expect timer to have expired but we would re-start the timer eventually after refresh. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + // Cancel will be called thrice back to back to back. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + time_system_.advanceTimeWait(std::chrono::minutes(61)); + timer_->invokeCallback(); + + // We don't expect timer to be reset again for new fetch. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Similarly we won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto new_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", new_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", new_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token1", new_credentials.sessionToken().value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, RefreshOnCredentialExpirationSecure) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1"))); + expectDocumentSecure(200, std::move(R"EOF( + { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" + } + )EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Cancel is called twice. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(2); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Cancel is not called again as we don't expect any more call to fetch until timeout. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + expectSessionToken(200, std::move("TOKEN")); + expectCredentialListingSecure(200, std::move(std::string("doc1"))); + expectDocumentSecure(200, std::move(R"EOF( + { + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token1" + } + )EOF")); + + // Expect timer to have expired but we would re-start the timer eventually after refresh. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + // Cancel will be called thrice back to back to back. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(3); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + time_system_.advanceTimeWait(std::chrono::minutes(61)); + timer_->invokeCallback(); + + // We don't expect timer to be reset again for new fetch. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Similarly we won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto new_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", new_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", new_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token1", new_credentials.sessionToken().value()); +} +// End unit test for new option via Http Async client. + +// Begin unit test for deprecated option using Libcurl client. +// TODO(suniltheta): Remove this test class once libcurl is removed from Envoy. +class InstanceProfileCredentialsProviderUsingLibcurlTest : public testing::Test { +public: + InstanceProfileCredentialsProviderUsingLibcurlTest() + : api_(Api::createApiForTest(time_system_)) {} + + void setupProvider() { + provider_ = std::make_shared( + *api_, absl::nullopt, + [this](Http::RequestMessage& message) -> absl::optional { return this->fetch_metadata_.fetch(message); - }) {} + }, + nullptr, "credentials_provider_cluster"); + } void expectSessionToken(const absl::optional& token) { Http::TestRequestHeaderMapImpl headers{{":path", "/latest/api/token"}, {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, {":method", "PUT"}, {"X-aws-ec2-metadata-token-ttl-seconds", "21600"}}; EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(token)); @@ -269,6 +1152,7 @@ class InstanceProfileCredentialsProviderTest : public testing::Test { void expectCredentialListing(const absl::optional& listing) { Http::TestRequestHeaderMapImpl headers{{":path", "/latest/meta-data/iam/security-credentials"}, {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, {":method", "GET"}}; EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(listing)); } @@ -276,6 +1160,7 @@ class InstanceProfileCredentialsProviderTest : public testing::Test { void expectCredentialListingSecure(const absl::optional& listing) { Http::TestRequestHeaderMapImpl headers{{":path", "/latest/meta-data/iam/security-credentials"}, {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, {":method", "GET"}, {"X-aws-ec2-metadata-token", "TOKEN"}}; EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(listing)); @@ -285,6 +1170,7 @@ class InstanceProfileCredentialsProviderTest : public testing::Test { Http::TestRequestHeaderMapImpl headers{ {":path", "/latest/meta-data/iam/security-credentials/doc1"}, {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, {":method", "GET"}}; EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(document)); } @@ -293,6 +1179,7 @@ class InstanceProfileCredentialsProviderTest : public testing::Test { Http::TestRequestHeaderMapImpl headers{ {":path", "/latest/meta-data/iam/security-credentials/doc1"}, {":authority", "169.254.169.254:80"}, + {":scheme", "http"}, {":method", "GET"}, {"X-aws-ec2-metadata-token", "TOKEN"}}; EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(document)); @@ -301,90 +1188,119 @@ class InstanceProfileCredentialsProviderTest : public testing::Test { Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; NiceMock fetch_metadata_; - InstanceProfileCredentialsProvider provider_; + InstanceProfileCredentialsProviderPtr provider_; }; -TEST_F(InstanceProfileCredentialsProviderTest, FailedCredentialListing) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, FailedCredentialListingUnsecure) { + setupProvider(); expectSessionToken(absl::optional()); expectCredentialListing(absl::optional()); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, FailedCredentialListingSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, FailedCredentialListingSecure) { + setupProvider(); expectSessionToken("TOKEN"); expectCredentialListingSecure(absl::optional()); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, EmptyCredentialListing) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, EmptyCredentialListingUnsecure) { + setupProvider(); expectSessionToken(absl::optional()); expectCredentialListing(""); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, EmptyCredentialListingSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, EmptyCredentialListingSecure) { + setupProvider(); + expectSessionToken("TOKEN"); + expectCredentialListingSecure("\n"); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, EmptyListCredentialListingUnsecure) { + setupProvider(); + expectSessionToken(absl::optional()); + expectCredentialListing("\n"); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, EmptyListCredentialListingSecure) { + setupProvider(); expectSessionToken("TOKEN"); expectCredentialListingSecure(""); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, MissingDocument) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, MissingDocumentUnsecure) { + setupProvider(); expectSessionToken(absl::optional()); expectCredentialListing("doc1\ndoc2\ndoc3"); expectDocument(absl::optional()); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, MissingDocumentSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, MissingDocumentSecure) { + setupProvider(); expectSessionToken("TOKEN"); expectCredentialListingSecure("doc1\ndoc2\ndoc3"); expectDocumentSecure(absl::optional()); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, MalformedDocumenet) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, MalformedDocumentUnsecure) { + setupProvider(); expectSessionToken(absl::optional()); expectCredentialListing("doc1"); expectDocument(R"EOF( not json -)EOF"); - const auto credentials = provider_.getCredentials(); + )EOF"); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, MalformedDocumentSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, MalformedDocumentSecure) { + setupProvider(); expectSessionToken("TOKEN"); expectCredentialListingSecure("doc1"); expectDocumentSecure(R"EOF( not json )EOF"); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, EmptyValues) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, EmptyValuesUnsecure) { + setupProvider(); expectSessionToken(absl::optional()); expectCredentialListing("doc1"); expectDocument(R"EOF( @@ -393,14 +1309,15 @@ TEST_F(InstanceProfileCredentialsProviderTest, EmptyValues) { "SecretAccessKey": "", "Token": "" } -)EOF"); - const auto credentials = provider_.getCredentials(); + )EOF"); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, EmptyValuesSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, EmptyValuesSecure) { + setupProvider(); expectSessionToken("TOKEN"); expectCredentialListingSecure("doc1"); expectDocumentSecure(R"EOF( @@ -410,13 +1327,14 @@ TEST_F(InstanceProfileCredentialsProviderTest, EmptyValuesSecure) { "Token": "" } )EOF"); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentials) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, FullCachedCredentialsUnsecure) { + setupProvider(); expectSessionToken(absl::optional()); expectCredentialListing("doc1"); expectDocument(R"EOF( @@ -425,18 +1343,19 @@ TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentials) { "SecretAccessKey": "secret", "Token": "token" } -)EOF"); - const auto credentials = provider_.getCredentials(); + )EOF"); + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); - const auto cached_credentials = provider_.getCredentials(); + const auto cached_credentials = provider_->getCredentials(); EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); EXPECT_EQ("token", cached_credentials.sessionToken().value()); } -TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentialsSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, FullCachedCredentialsSecure) { + setupProvider(); expectSessionToken("TOKEN"); expectCredentialListingSecure("doc1"); expectDocumentSecure(R"EOF( @@ -446,17 +1365,18 @@ TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentialsSecure) { "Token": "token" } )EOF"); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); - const auto cached_credentials = provider_.getCredentials(); + const auto cached_credentials = provider_->getCredentials(); EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); EXPECT_EQ("token", cached_credentials.sessionToken().value()); } -TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpiration) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, CredentialExpirationUnsecure) { + setupProvider(); InSequence sequence; expectSessionToken(absl::optional()); expectCredentialListing("doc1"); @@ -466,8 +1386,8 @@ TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpiration) { "SecretAccessKey": "secret", "Token": "token" } -)EOF"); - const auto credentials = provider_.getCredentials(); + )EOF"); + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); @@ -480,14 +1400,15 @@ TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpiration) { "SecretAccessKey": "new_secret", "Token": "new_token" } -)EOF"); - const auto new_credentials = provider_.getCredentials(); + )EOF"); + const auto new_credentials = provider_->getCredentials(); EXPECT_EQ("new_akid", new_credentials.accessKeyId().value()); EXPECT_EQ("new_secret", new_credentials.secretAccessKey().value()); EXPECT_EQ("new_token", new_credentials.sessionToken().value()); } -TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpirationSecure) { +TEST_F(InstanceProfileCredentialsProviderUsingLibcurlTest, CredentialExpirationSecure) { + setupProvider(); InSequence sequence; expectSessionToken("TOKEN"); expectCredentialListingSecure("doc1"); @@ -498,7 +1419,7 @@ TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpirationSecure) { "Token": "token" } )EOF"); - const auto credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); @@ -512,152 +1433,1287 @@ TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpirationSecure) { "Token": "new_token" } )EOF"); - const auto new_credentials = provider_.getCredentials(); + const auto new_credentials = provider_->getCredentials(); EXPECT_EQ("new_akid", new_credentials.accessKeyId().value()); EXPECT_EQ("new_secret", new_credentials.secretAccessKey().value()); EXPECT_EQ("new_token", new_credentials.sessionToken().value()); } +// End unit test for deprecated option using Libcurl client. +// Begin unit test for new option via Http Async client. class TaskRoleCredentialsProviderTest : public testing::Test { public: TaskRoleCredentialsProviderTest() - : api_(Api::createApiForTest(time_system_)), - provider_( - *api_, - [this](Http::RequestMessage& message) -> absl::optional { - return this->fetch_metadata_.fetch(message); - }, - "169.254.170.2:80/path/to/doc", "auth_token") { + : api_(Api::createApiForTest(time_system_)), raw_metadata_fetcher_(new MockMetadataFetcher) { // Tue Jan 2 03:04:05 UTC 2018 time_system_.setSystemTime(std::chrono::milliseconds(1514862245000)); } - void expectDocument(const absl::optional& document) { + void setupProvider() { + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "true"}}); + ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + provider_ = std::make_shared( + *api_, context_, + [this](Http::RequestMessage& message) -> absl::optional { + return this->fetch_metadata_.fetch(message); + }, + [this](Upstream::ClusterManager&, absl::string_view) { + metadata_fetcher_.reset(raw_metadata_fetcher_); + return std::move(metadata_fetcher_); + }, + "169.254.170.2:80/path/to/doc", "auth_token", "credentials_provider_cluster"); + } + + void setupProviderWithContext() { + EXPECT_CALL(context_.init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + setupProvider(); + expected_duration_ = provider_->getCacheDuration(); + init_target_handle_->initialize(init_watcher_); + } + + void expectDocument(const uint64_t status_code, const std::string&& document) { Http::TestRequestHeaderMapImpl headers{{":path", "/path/to/doc"}, {":authority", "169.254.170.2:80"}, + {":scheme", "http"}, {":method", "GET"}, {"authorization", "auth_token"}}; - EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(document)); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, document = std::move(document)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!document.empty()) { + receiver.onMetadataSuccess(std::move(document)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); } + TestScopedRuntime scoped_runtime_; Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; NiceMock fetch_metadata_; - TaskRoleCredentialsProvider provider_; + MockMetadataFetcher* raw_metadata_fetcher_; + MetadataFetcherPtr metadata_fetcher_; + NiceMock cluster_manager_; + NiceMock context_; + TaskRoleCredentialsProviderPtr provider_; + Init::TargetHandlePtr init_target_handle_; + NiceMock init_watcher_; + Event::MockTimer* timer_{}; + std::chrono::milliseconds expected_duration_; }; +TEST_F(TaskRoleCredentialsProviderTest, TestAddMissingCluster) { + // Setup without thread local cluster yet + envoy::config::cluster::v3::Cluster expected_cluster; + constexpr static const char* kStaticCluster = R"EOF( +name: credentials_provider_cluster +type: static +connectTimeout: 2s +lb_policy: ROUND_ROBIN +loadAssignment: + clusterName: credentials_provider_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: "169.254.170.2" + portValue: 80 +typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: + accept_http_10: true + )EOF"; + MessageUtil::loadFromYaml(kStaticCluster, expected_cluster, + ProtobufMessage::getNullValidationVisitor()); + + EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cluster_manager_, addOrUpdateCluster(WithAttribute(expected_cluster), _)) + .WillOnce(Return(true)); + + expectDocument(200, std::move(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "2018-01-02T03:05:00Z" +} +)EOF")); + + setupProviderWithContext(); +} + +TEST_F(TaskRoleCredentialsProviderTest, TestClusterMissing) { + // Setup without thread local cluster + Http::RequestMessageImpl message; + + EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cluster_manager_, addOrUpdateCluster(WithName("credentials_provider_cluster"), _)) + .WillOnce(Throw(EnvoyException("exeption message"))); + // init_watcher ready is not called. + init_watcher_.expectReady().Times(0); + setupProvider(); + // Below line is not testing anything, will just avoid asan failure with memory leak. + metadata_fetcher_.reset(raw_metadata_fetcher_); +} + TEST_F(TaskRoleCredentialsProviderTest, FailedFetchingDocument) { - expectDocument(absl::optional()); - const auto credentials = provider_.getCredentials(); + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(403 /*Forbidden*/, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } -TEST_F(TaskRoleCredentialsProviderTest, MalformedDocumenet) { - expectDocument(R"EOF( +TEST_F(TaskRoleCredentialsProviderTest, EmptyDocument) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderTest, MalformedDocument) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + + expectDocument(200, std::move(R"EOF( not json -)EOF"); - const auto credentials = provider_.getCredentials(); +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } TEST_F(TaskRoleCredentialsProviderTest, EmptyValues) { - expectDocument(R"EOF( + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + + expectDocument(200, std::move(R"EOF( { "AccessKeyId": "", "SecretAccessKey": "", "Token": "", "Expiration": "" } -)EOF"); - const auto credentials = provider_.getCredentials(); +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success with updating + // expiration time. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); EXPECT_FALSE(credentials.accessKeyId().has_value()); EXPECT_FALSE(credentials.secretAccessKey().has_value()); EXPECT_FALSE(credentials.sessionToken().has_value()); } TEST_F(TaskRoleCredentialsProviderTest, FullCachedCredentials) { - expectDocument(R"EOF( + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(R"EOF( { "AccessKeyId": "akid", "SecretAccessKey": "secret", "Token": "token", "Expiration": "2018-01-02T03:05:00Z" } -)EOF"); - const auto credentials = provider_.getCredentials(); +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); - const auto cached_credentials = provider_.getCredentials(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto cached_credentials = provider_->getCredentials(); EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); EXPECT_EQ("token", cached_credentials.sessionToken().value()); } -TEST_F(TaskRoleCredentialsProviderTest, NormalCredentialExpiration) { - InSequence sequence; - expectDocument(R"EOF( +TEST_F(TaskRoleCredentialsProviderTest, RefreshOnNormalCredentialExpiration) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + + expectDocument(200, std::move(R"EOF( { "AccessKeyId": "akid", "SecretAccessKey": "secret", "Token": "token", "Expiration": "2019-01-02T03:04:05Z" } -)EOF"); - const auto credentials = provider_.getCredentials(); +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); - time_system_.advanceTimeWait(std::chrono::hours(2)); - expectDocument(R"EOF( + + expectDocument(200, std::move(R"EOF( { "AccessKeyId": "new_akid", "SecretAccessKey": "new_secret", "Token": "new_token", "Expiration": "2019-01-02T03:04:05Z" } -)EOF"); - const auto cached_credentials = provider_.getCredentials(); +)EOF")); + // Expect timer to have expired but we would re-start the timer eventually after refresh. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + // Cancel will be called once more. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + time_system_.advanceTimeWait(std::chrono::minutes(61)); + timer_->invokeCallback(); + + // We don't expect timer to be reset again for new fetch. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Similarly we won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto cached_credentials = provider_->getCredentials(); EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); } TEST_F(TaskRoleCredentialsProviderTest, TimestampCredentialExpiration) { - InSequence sequence; - expectDocument(R"EOF( + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(R"EOF( { "AccessKeyId": "akid", "SecretAccessKey": "secret", "Token": "token", "Expiration": "2018-01-02T03:04:05Z" } -)EOF"); - const auto credentials = provider_.getCredentials(); +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // Need to disable and restart timer since credentials are expired and fetched again + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + // We call cancel once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + + const auto credentials = provider_->getCredentials(); EXPECT_EQ("akid", credentials.accessKeyId().value()); EXPECT_EQ("secret", credentials.secretAccessKey().value()); EXPECT_EQ("token", credentials.sessionToken().value()); - expectDocument(R"EOF( + + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + expectDocument(200, std::move(R"EOF( { "AccessKeyId": "new_akid", "SecretAccessKey": "new_secret", "Token": "new_token", "Expiration": "2019-01-02T03:04:05Z" } +)EOF")); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} +// End unit test for new option via Http Async client. + +// Begin unit test for deprecated option using Libcurl client. +// TODO(suniltheta): Remove this test class once libcurl is removed from Envoy. +class TaskRoleCredentialsProviderUsingLibcurlTest : public testing::Test { +public: + TaskRoleCredentialsProviderUsingLibcurlTest() : api_(Api::createApiForTest(time_system_)) { + // Tue Jan 2 03:04:05 UTC 2018 + time_system_.setSystemTime(std::chrono::milliseconds(1514862245000)); + } + + void setupProvider() { + provider_ = std::make_shared( + *api_, absl::nullopt, + [this](Http::RequestMessage& message) -> absl::optional { + return this->fetch_metadata_.fetch(message); + }, + nullptr, "169.254.170.2:80/path/to/doc", "auth_token", "credentials_provider_cluster"); + } + + void expectDocument(const absl::optional& document) { + Http::TestRequestHeaderMapImpl headers{{":path", "/path/to/doc"}, + {":authority", "169.254.170.2:80"}, + {":scheme", "http"}, + {":method", "GET"}, + {"authorization", "auth_token"}}; + EXPECT_CALL(fetch_metadata_, fetch(messageMatches(headers))).WillOnce(Return(document)); + } + + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + NiceMock fetch_metadata_; + TaskRoleCredentialsProviderPtr provider_; +}; + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, FailedFetchingDocument) { + setupProvider(); + expectDocument(absl::optional()); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, EmptyDocument) { + setupProvider(); + expectDocument(""); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, MalformedDocument) { + setupProvider(); + expectDocument(R"EOF( +not json +)EOF"); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, EmptyValues) { + setupProvider(); + expectDocument(R"EOF( +{ + "AccessKeyId": "", + "SecretAccessKey": "", + "Token": "", + "Expiration": "" +} +)EOF"); + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, FullCachedCredentials) { + setupProvider(); + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "2018-01-02T03:05:00Z" +} )EOF"); - const auto cached_credentials = provider_.getCredentials(); + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("token", cached_credentials.sessionToken().value()); +} + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, NormalCredentialExpiration) { + setupProvider(); + InSequence sequence; + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "2019-01-02T03:04:05Z" +} +)EOF"); + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + time_system_.advanceTimeWait(std::chrono::hours(2)); + expectDocument(R"EOF( +{ + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token", + "Expiration": "2019-01-02T03:04:05Z" +} +)EOF"); + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} + +TEST_F(TaskRoleCredentialsProviderUsingLibcurlTest, TimestampCredentialExpiration) { + setupProvider(); + InSequence sequence; + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "2018-01-02T03:04:05Z" +} +)EOF"); + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + expectDocument(R"EOF( +{ + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token", + "Expiration": "2019-01-02T03:04:05Z" +} +)EOF"); + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} +// End unit test for deprecated option using Libcurl client. + +class WebIdentityCredentialsProviderTest : public testing::Test { +public: + WebIdentityCredentialsProviderTest() + : api_(Api::createApiForTest(time_system_)), raw_metadata_fetcher_(new MockMetadataFetcher) { + // Tue Jan 2 03:04:05 UTC 2018 + time_system_.setSystemTime(std::chrono::milliseconds(1514862245000)); + } + + void setupProvider() { + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "true"}}); + ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + provider_ = std::make_shared( + *api_, context_, + [this](Http::RequestMessage& message) -> absl::optional { + return this->fetch_metadata_.fetch(message); + }, + [this](Upstream::ClusterManager&, absl::string_view) { + metadata_fetcher_.reset(raw_metadata_fetcher_); + return std::move(metadata_fetcher_); + }, + TestEnvironment::writeStringToFileForTest("web_token_file", "web_token"), + "sts.region.amazonaws.com:443", "aws:iam::123456789012:role/arn", "role-session-name", + "credentials_provider_cluster"); + } + + void setupProviderWithContext() { + EXPECT_CALL(context_.init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + setupProvider(); + expected_duration_ = provider_->getCacheDuration(); + init_target_handle_->initialize(init_watcher_); + } + + void setupProviderWithLibcurl() { + ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + provider_ = std::make_shared( + *api_, context_, + [this](Http::RequestMessage& message) -> absl::optional { + return this->fetch_metadata_.fetch(message); + }, + [this](Upstream::ClusterManager&, absl::string_view) { + metadata_fetcher_.reset(raw_metadata_fetcher_); + return std::move(metadata_fetcher_); + }, + TestEnvironment::writeStringToFileForTest("web_token_file", "web_token"), + "sts.region.amazonaws.com:443", "aws:iam::123456789012:role/arn", "role-session-name", + "credentials_provider_cluster"); + } + + void expectDocument(const uint64_t status_code, const std::string&& document) { + Http::TestRequestHeaderMapImpl headers{{":path", + "/?Action=AssumeRoleWithWebIdentity" + "&Version=2011-06-15&RoleSessionName=role-session-name" + "&RoleArn=aws:iam::123456789012:role/arn" + "&WebIdentityToken=web_token"}, + {":authority", "sts.region.amazonaws.com"}, + {":scheme", "https"}, + {":method", "GET"}, + {"Accept", "application/json"}}; + EXPECT_CALL(*raw_metadata_fetcher_, fetch(messageMatches(headers), _, _)) + .WillRepeatedly(Invoke([this, status_code, document = std::move(document)]( + Http::RequestMessage&, Tracing::Span&, + MetadataFetcher::MetadataReceiver& receiver) { + if (status_code == enumToInt(Http::Code::OK)) { + if (!document.empty()) { + receiver.onMetadataSuccess(std::move(document)); + } else { + EXPECT_CALL( + *raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata))) + .WillRepeatedly(testing::Return("InvalidMetadata")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata); + } + } else { + EXPECT_CALL(*raw_metadata_fetcher_, + failureToString(Eq(MetadataFetcher::MetadataReceiver::Failure::Network))) + .WillRepeatedly(testing::Return("Network")); + receiver.onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network); + } + })); + } + + TestScopedRuntime scoped_runtime_; + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + NiceMock fetch_metadata_; + MockMetadataFetcher* raw_metadata_fetcher_; + MetadataFetcherPtr metadata_fetcher_; + NiceMock cluster_manager_; + NiceMock context_; + WebIdentityCredentialsProviderPtr provider_; + Init::TargetHandlePtr init_target_handle_; + NiceMock init_watcher_; + Event::MockTimer* timer_{}; + std::chrono::milliseconds expected_duration_; +}; + +TEST_F(WebIdentityCredentialsProviderTest, TestAddMissingCluster) { + // Setup without thread local cluster yet + envoy::config::cluster::v3::Cluster expected_cluster; + constexpr static const char* kStaticCluster = R"EOF( +name: credentials_provider_cluster +type: logical_dns +connectTimeout: 2s +lb_policy: ROUND_ROBIN +loadAssignment: + clusterName: credentials_provider_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: "sts.region.amazonaws.com" + portValue: 443 +typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: + accept_http_10: true +transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + )EOF"; + MessageUtil::loadFromYaml(kStaticCluster, expected_cluster, + ProtobufMessage::getNullValidationVisitor()); + + EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cluster_manager_, addOrUpdateCluster(WithAttribute(expected_cluster), _)) + .WillOnce(Return(true)); + + // Time 2018-01-02T03:05:00Z in unix_timestamp is 1514862300 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "SessionToken": "token", + "Expiration": 1514862300 + } + } + } +} +)EOF")); + setupProviderWithContext(); +} + +TEST_F(WebIdentityCredentialsProviderTest, TestClusterMissing) { + // Setup without thread local cluster + Http::RequestMessageImpl message; + + EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cluster_manager_, addOrUpdateCluster(WithName("credentials_provider_cluster"), _)) + .WillOnce(Throw(EnvoyException("exeption message"))); + // init_watcher ready is not called. + init_watcher_.expectReady().Times(0); + setupProvider(); + // Below line is not testing anything, will just avoid asan failure with memory leak. + metadata_fetcher_.reset(raw_metadata_fetcher_); +} + +TEST_F(WebIdentityCredentialsProviderTest, FailedFetchingDocument) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(403 /*Forbidden*/, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, EmptyDocument) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(std::string())); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, MalformedDocument) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + + expectDocument(200, std::move(R"EOF( +not json +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, UnexpectedResponse) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "UnexpectedResponse": "" + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success with updating + // expiration time. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, NoCredentials) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": "" + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success with updating + // expiration time. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, EmptyCredentials) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": "" + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success with updating + // expiration time. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, FullCachedCredentials) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + // Time 2018-01-02T03:05:00Z in unix_timestamp is 1514862300 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "SessionToken": "token", + "Expiration": 1514862300 + } + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("token", cached_credentials.sessionToken().value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, CredentialsWithWrongFormat) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": 1, + "SecretAccessKey": 2, + "SessionToken": 3 + } + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // Cancel is called for fetching once again as previous attempt wasn't a success with updating + // expiration time. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, FullCachedCredentialsBadExpirationFormat) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + // Time 2018-01-02T03:04:05Z in unix_timestamp is 1514862245 + // STS API call with "Accept: application/json" is expected to return Exception in `Integer` unix + // timestamp format. However, if non integer is returned for Expiration field, then the value will + // be ignored and instead the expiration is set to 1 hour in future. + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "SessionToken": "token", + "Expiration": "2018-01-02T03:04:05Z" + } + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + // Even though as per `Expiration` field (in wrong format) the credentials are expired + // the credentials won't be refreshed until the next refresh period (1hr) or new expiration + // value implicitly set to a value same as refresh interval. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + // Time 2019-01-02T03:04:05Z in unix_timestamp is 1546398245 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "SessionToken": "new_token", + "Expiration": "2019-01-02T03:04:05Z" + } + } + } +} +)EOF")); + // Expect timer to have expired but we would re-start the timer eventually after refresh. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + // Cancel will be called once more. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + time_system_.advanceTimeWait(std::chrono::minutes(61)); + timer_->invokeCallback(); + + // We don't expect timer to be reset again for new fetch. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Similarly we won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, FullCachedCredentialsWithMissingExpiration) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + // STS API call with "Accept: application/json" is expected to return Exception in `Integer` unix + // timestamp format. However, if Expiration field is empty, then the expiration will set to 1 hour + // in future. + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "SessionToken": "token" + } + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + // The credentials won't be refreshed until the next refresh period (1hr) or new expiration + // value implicitly set to a value same as refresh interval. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + // Time 2019-01-02T03:04:05Z in unix_timestamp is 1546398245 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "SessionToken": "new_token" + } + } + } +} +)EOF")); + // Expect timer to have expired but we would re-start the timer eventually after refresh. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + // Cancel will be called once more. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + time_system_.advanceTimeWait(std::chrono::minutes(61)); + timer_->invokeCallback(); + + // We don't expect timer to be reset again for new fetch. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Similarly we won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, RefreshOnNormalCredentialExpiration) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + // Time 2018-01-02T04:04:05Z in unix_timestamp is 1514865845 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "SessionToken": "token", + "Expiration": 1514865845 + } + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // No need to restart timer since credentials are fetched from cache. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // We don't expect any more call to cancel or fetch again. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + // Time 2019-01-02T03:05:00Z in unix_timestamp is 1546398300 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "SessionToken": "new_token", + "Expiration": 1546398300 + } + } + } +} +)EOF")); + // Expect timer to have expired but we would re-start the timer eventually after refresh. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + // Cancel will be called once more. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + time_system_.advanceTimeWait(std::chrono::minutes(61)); + timer_->invokeCallback(); + + // We don't expect timer to be reset again for new fetch. + EXPECT_CALL(*timer_, disableTimer()).Times(0); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)).Times(0); + // Similarly we won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto cached_credentials = provider_->getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} + +TEST_F(WebIdentityCredentialsProviderTest, TimestampCredentialExpiration) { + // Setup timer. + timer_ = new NiceMock(&context_.dispatcher_); + // Time 2018-01-02T03:04:05Z in unix_timestamp is 1514862245 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "SessionToken": "token", + "Expiration": 1514862245 + } + } + } +} +)EOF")); + // init_watcher ready is called. + init_watcher_.expectReady(); + // Expect refresh timer to be started. + EXPECT_CALL(*timer_, enableTimer(_, nullptr)); + setupProviderWithContext(); + + // init_watcher ready is not called again. + init_watcher_.expectReady().Times(0); + // Need to disable and restart timer since credentials are expired and fetched again + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + // We call cancel once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + + const auto credentials = provider_->getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + + // Cancel is called once. + EXPECT_CALL(*raw_metadata_fetcher_, cancel()); + // Time 2019-01-02T03:04:05Z in unix_timestamp is 1546398245 + expectDocument(200, std::move(R"EOF( +{ + "AssumeRoleWithWebIdentityResponse": { + "AssumeRoleWithWebIdentityResult": { + "Credentials": { + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "SessionToken": "new_token", + "Expiration": 1546398245 + } + } + } +} +)EOF")); + // Expect refresh timer to be stopped and started. + EXPECT_CALL(*timer_, disableTimer()); + EXPECT_CALL(*timer_, enableTimer(expected_duration_, nullptr)); + const auto cached_credentials = provider_->getCredentials(); EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); } +TEST_F(WebIdentityCredentialsProviderTest, LibcurlEnabled) { + setupProviderWithLibcurl(); + // Won't call fetch or cancel on metadata fetcher. + EXPECT_CALL(*raw_metadata_fetcher_, fetch(_, _, _)).Times(0); + EXPECT_CALL(*raw_metadata_fetcher_, cancel()).Times(0); + + const auto credentials = provider_->getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); + + // Below line is not testing anything, will just avoid asan failure with memory leak. + metadata_fetcher_.reset(raw_metadata_fetcher_); +} + class DefaultCredentialsProviderChainTest : public testing::Test { public: DefaultCredentialsProviderChainTest() : api_(Api::createApiForTest(time_system_)) { + ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); + cluster_manager_.initializeThreadLocalClusters({"credentials_provider_cluster"}); EXPECT_CALL(factories_, createEnvironmentCredentialsProvider()); } @@ -666,6 +2722,9 @@ class DefaultCredentialsProviderChainTest : public testing::Test { TestEnvironment::unsetEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI"); TestEnvironment::unsetEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN"); TestEnvironment::unsetEnvVar("AWS_EC2_METADATA_DISABLED"); + TestEnvironment::unsetEnvVar("AWS_WEB_IDENTITY_TOKEN_FILE"); + TestEnvironment::unsetEnvVar("AWS_ROLE_ARN"); + TestEnvironment::unsetEnvVar("AWS_ROLE_SESSION_NAME"); } class MockCredentialsProviderChainFactories : public CredentialsProviderChainFactories { @@ -673,63 +2732,81 @@ class DefaultCredentialsProviderChainTest : public testing::Test { MOCK_METHOD(CredentialsProviderSharedPtr, createEnvironmentCredentialsProvider, (), (const)); MOCK_METHOD(CredentialsProviderSharedPtr, createCredentialsFileCredentialsProvider, (Api::Api&), (const)); - MOCK_METHOD(CredentialsProviderSharedPtr, createTaskRoleCredentialsProvider, - (Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher&, + MOCK_METHOD(CredentialsProviderSharedPtr, createWebIdentityCredentialsProvider, + (Api::Api&, ServerFactoryContextOptRef, + const MetadataCredentialsProviderBase::CurlMetadataFetcher&, + CreateMetadataFetcherCb, absl::string_view, absl::string_view, absl::string_view, absl::string_view, absl::string_view), (const)); + MOCK_METHOD(CredentialsProviderSharedPtr, createTaskRoleCredentialsProvider, + (Api::Api&, ServerFactoryContextOptRef, + const MetadataCredentialsProviderBase::CurlMetadataFetcher&, + CreateMetadataFetcherCb, absl::string_view, absl::string_view, absl::string_view), + (const)); MOCK_METHOD(CredentialsProviderSharedPtr, createInstanceProfileCredentialsProvider, - (Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher& fetcher), + (Api::Api&, ServerFactoryContextOptRef, + const MetadataCredentialsProviderBase::CurlMetadataFetcher&, + CreateMetadataFetcherCb, absl::string_view), (const)); }; + TestScopedRuntime scoped_runtime_; Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; + NiceMock cluster_manager_; + NiceMock context_; NiceMock factories_; }; TEST_F(DefaultCredentialsProviderChainTest, NoEnvironmentVars) { EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); - EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)); + + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST_F(DefaultCredentialsProviderChainTest, CredentialsFileDisabled) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.enable_aws_credentials_file", "false"}}); - + scoped_runtime_.mergeValues({{"envoy.reloadable_features.enable_aws_credentials_file", "false"}}); EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))).Times(0); - EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST_F(DefaultCredentialsProviderChainTest, MetadataDisabled) { TestEnvironment::setEnvVar("AWS_EC2_METADATA_DISABLED", "true", 1); EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); - EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)).Times(0); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)) + .Times(0); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST_F(DefaultCredentialsProviderChainTest, MetadataNotDisabled) { TestEnvironment::setEnvVar("AWS_EC2_METADATA_DISABLED", "false", 1); EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); - EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST_F(DefaultCredentialsProviderChainTest, RelativeUri) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/path/to/creds", 1); EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); - EXPECT_CALL(factories_, createTaskRoleCredentialsProvider(Ref(*api_), _, + EXPECT_CALL(factories_, createTaskRoleCredentialsProvider(Ref(*api_), _, _, _, _, "169.254.170.2:80/path/to/creds", "")); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST_F(DefaultCredentialsProviderChainTest, FullUriNoAuthorizationToken) { TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); - EXPECT_CALL(factories_, - createTaskRoleCredentialsProvider(Ref(*api_), _, "http://host/path/to/creds", "")); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + EXPECT_CALL(factories_, createTaskRoleCredentialsProvider(Ref(*api_), _, _, _, _, + "http://host/path/to/creds", "")); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST_F(DefaultCredentialsProviderChainTest, FullUriWithAuthorizationToken) { @@ -737,8 +2814,46 @@ TEST_F(DefaultCredentialsProviderChainTest, FullUriWithAuthorizationToken) { TestEnvironment::setEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN", "auth_token", 1); EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); EXPECT_CALL(factories_, createTaskRoleCredentialsProvider( - Ref(*api_), _, "http://host/path/to/creds", "auth_token")); - DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); + Ref(*api_), _, _, _, _, "http://host/path/to/creds", "auth_token")); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, NoWebIdentityRoleArn) { + TestEnvironment::setEnvVar("AWS_WEB_IDENTITY_TOKEN_FILE", "/path/to/web_token", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, NoWebIdentitySessionName) { + TestEnvironment::setEnvVar("AWS_WEB_IDENTITY_TOKEN_FILE", "/path/to/web_token", 1); + TestEnvironment::setEnvVar("AWS_ROLE_ARN", "aws:iam::123456789012:role/arn", 1); + time_system_.setSystemTime(std::chrono::milliseconds(1234567890)); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); + EXPECT_CALL(factories_, + createWebIdentityCredentialsProvider( + Ref(*api_), _, _, _, _, "/path/to/web_token", "sts.region.amazonaws.com:443", + "aws:iam::123456789012:role/arn", "1234567890000000")); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)); + + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, WebIdentityWithSessionName) { + TestEnvironment::setEnvVar("AWS_WEB_IDENTITY_TOKEN_FILE", "/path/to/web_token", 1); + TestEnvironment::setEnvVar("AWS_ROLE_ARN", "aws:iam::123456789012:role/arn", 1); + TestEnvironment::setEnvVar("AWS_ROLE_SESSION_NAME", "role-session-name", 1); + EXPECT_CALL(factories_, createCredentialsFileCredentialsProvider(Ref(*api_))); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _, _, _, _)); + EXPECT_CALL(factories_, + createWebIdentityCredentialsProvider( + Ref(*api_), _, _, _, _, "/path/to/web_token", "sts.region.amazonaws.com:443", + "aws:iam::123456789012:role/arn", "role-session-name")); + DefaultCredentialsProviderChain chain(*api_, context_, "region", DummyMetadataFetcher(), + factories_); } TEST(CredentialsProviderChainTest, getCredentials_noCredentials) { diff --git a/test/extensions/common/aws/metadata_fetcher_test.cc b/test/extensions/common/aws/metadata_fetcher_test.cc new file mode 100644 index 000000000000..da4702211d00 --- /dev/null +++ b/test/extensions/common/aws/metadata_fetcher_test.cc @@ -0,0 +1,289 @@ +#include +#include +#include + +#include "source/common/http/headers.h" +#include "source/common/http/message_impl.h" +#include "source/common/http/utility.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/common/aws/metadata_fetcher.h" + +#include "test/extensions/common/aws/mocks.h" +#include "test/extensions/filters/http/common/mock.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/test_common/environment.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +using Envoy::Extensions::HttpFilters::Common::MockUpstream; +using testing::_; +using testing::AllOf; +using testing::Mock; +using testing::NiceMock; +using testing::Return; +using testing::UnorderedElementsAre; + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Aws { + +MATCHER_P(OptionsHasBufferBodyForRetry, expectedValue, "") { + *result_listener << "\nexpected { buffer_body_for_retry: \"" << expectedValue + << "\"} but got {buffer_body_for_retry: \"" << arg.buffer_body_for_retry + << "\"}\n"; + return ExplainMatchResult(expectedValue, arg.buffer_body_for_retry, result_listener); +} + +MATCHER_P(NumRetries, expectedRetries, "") { + *result_listener << "\nexpected { num_retries: \"" << expectedRetries + << "\"} but got {num_retries: \"" << arg.num_retries().value() << "\"}\n"; + return ExplainMatchResult(expectedRetries, arg.num_retries().value(), result_listener); +} + +MATCHER_P(PerTryTimeout, expectedTimeout, "") { + *result_listener << "\nexpected { per_try_timeout: \"" << expectedTimeout + << "\"} but got { per_try_timeout: \"" << arg.per_try_timeout().seconds() + << "\"}\n"; + return ExplainMatchResult(expectedTimeout, arg.per_try_timeout().seconds(), result_listener); +} + +MATCHER_P(PerTryIdleTimeout, expectedIdleTimeout, "") { + *result_listener << "\nexpected { per_try_idle_timeout: \"" << expectedIdleTimeout + << "\"} but got { per_try_idle_timeout: \"" + << arg.per_try_idle_timeout().seconds() << "\"}\n"; + return ExplainMatchResult(expectedIdleTimeout, arg.per_try_idle_timeout().seconds(), + result_listener); +} + +MATCHER_P(RetryOnModes, expectedModes, "") { + const std::string& retry_on = arg.retry_on(); + std::set retry_on_modes = absl::StrSplit(retry_on, ','); + *result_listener << "\nexpected retry_on modes doesn't match " + << "received { retry_on modes: \"" << retry_on << "\"}\n"; + return ExplainMatchResult(expectedModes, retry_on_modes, result_listener); +} + +MATCHER_P(OptionsHasRetryPolicy, policyMatcher, "") { + if (!arg.retry_policy.has_value()) { + *result_listener << "Expected options to have retry policy, but it was unset"; + return false; + } + return ExplainMatchResult(policyMatcher, arg.retry_policy.value(), result_listener); +} + +class MetadataFetcherTest : public testing::Test { +public: + void setupFetcher() { + mock_factory_ctx_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"cluster_name"}); + fetcher_ = MetadataFetcher::create(mock_factory_ctx_.server_factory_context_.cluster_manager_, + "cluster_name"); + EXPECT_TRUE(fetcher_ != nullptr); + } + + testing::NiceMock mock_factory_ctx_; + std::unique_ptr fetcher_; + NiceMock parent_span_; +}; + +TEST_F(MetadataFetcherTest, TestGetSuccess) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + std::string body = "not_empty"; + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", body); + MockMetadataReceiver receiver; + EXPECT_CALL(receiver, onMetadataSuccess(std::move(body))); + EXPECT_CALL(receiver, onMetadataError(_)).Times(0); + + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestRequestMatchAndSpanPassedDown) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + + message.headers().setScheme(Http::Headers::get().SchemeValues.Http); + message.headers().setMethod(Http::Headers::get().MethodValues.Get); + message.headers().setHost("169.254.170.2:80"); + message.headers().setPath("/v2/credentials/c68caeb5-ef71-4914-8170-111111111111"); + message.headers().setCopy(Http::LowerCaseString(":pseudo-header"), "peudo-header-value"); + message.headers().setCopy(Http::LowerCaseString("X-aws-ec2-metadata-token"), "Token"); + + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", + "not_empty"); + MockMetadataReceiver receiver; + Http::MockAsyncClientRequest httpClientRequest( + &mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_); + + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_, + send_(_, _, _)) + .WillOnce(Invoke( + [this, &httpClientRequest]( + Http::RequestMessagePtr& request, Http::AsyncClient::Callbacks& cb, + const Http::AsyncClient::RequestOptions& options) -> Http::AsyncClient::Request* { + Http::TestRequestHeaderMapImpl injected_headers = { + {":method", "GET"}, + {":scheme", "http"}, + {":authority", "169.254.170.2"}, + {":path", "/v2/credentials/c68caeb5-ef71-4914-8170-111111111111"}, + {"X-aws-ec2-metadata-token", "Token"}}; + EXPECT_THAT(request->headers(), IsSupersetOfHeaders(injected_headers)); + EXPECT_TRUE(request->headers().get(Http::LowerCaseString(":pseudo-header")).empty()); + + // Verify expectations for span + EXPECT_TRUE(options.parent_span_ == &this->parent_span_); + EXPECT_TRUE(options.child_span_name_ == "AWS Metadata Fetch"); + + // Let's say this ends up with a failure then verify it is handled properly by calling + // onMetadataError. + cb.onFailure(httpClientRequest, Http::AsyncClient::FailureReason::Reset); + return &httpClientRequest; + })); + EXPECT_CALL(receiver, onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network)); + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestGet400) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, "400", + "not_empty"); + MockMetadataReceiver receiver; + EXPECT_CALL(receiver, onMetadataSuccess(_)).Times(0); + EXPECT_CALL(receiver, onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network)); + + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestGet400NoBody) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, "400", ""); + MockMetadataReceiver receiver; + EXPECT_CALL(receiver, onMetadataSuccess(_)).Times(0); + EXPECT_CALL(receiver, onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network)); + + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestGetNoBody) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", ""); + MockMetadataReceiver receiver; + EXPECT_CALL(receiver, onMetadataSuccess(_)).Times(0); + EXPECT_CALL(receiver, + onMetadataError(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata)); + + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestHttpFailure) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, + Http::AsyncClient::FailureReason::Reset); + MockMetadataReceiver receiver; + EXPECT_CALL(receiver, onMetadataSuccess(_)).Times(0); + EXPECT_CALL(receiver, onMetadataError(MetadataFetcher::MetadataReceiver::Failure::Network)); + + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestClusterNotFound) { + // Setup without thread local cluster + fetcher_ = MetadataFetcher::create(mock_factory_ctx_.server_factory_context_.cluster_manager_, + "cluster_name"); + Http::RequestMessageImpl message; + MockMetadataReceiver receiver; + + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(receiver, onMetadataSuccess(_)).Times(0); + EXPECT_CALL(receiver, onMetadataError(MetadataFetcher::MetadataReceiver::Failure::MissingConfig)); + + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestCancel) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + Http::MockAsyncClientRequest request(&(mock_factory_ctx_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.async_client_)); + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, &request); + MockMetadataReceiver receiver; + EXPECT_CALL(request, cancel()); + EXPECT_CALL(receiver, onMetadataSuccess(_)).Times(0); + EXPECT_CALL(receiver, onMetadataError(_)).Times(0); + + // Act + fetcher_->fetch(message, parent_span_, receiver); + // Proper cancel + fetcher_->cancel(); + Mock::VerifyAndClearExpectations(&request); + Mock::VerifyAndClearExpectations(&receiver); + // Re-entrant cancel should do nothing. + EXPECT_CALL(request, cancel()).Times(0); + fetcher_->cancel(); +} + +TEST_F(MetadataFetcherTest, TestDefaultRetryPolicy) { + // Setup + setupFetcher(); + Http::RequestMessageImpl message; + MockUpstream mock_result(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", + "not_empty"); + MockMetadataReceiver receiver; + + EXPECT_CALL( + mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_, + send_(_, _, + AllOf(OptionsHasBufferBodyForRetry(true), + OptionsHasRetryPolicy(AllOf( + NumRetries(3), PerTryTimeout(5), PerTryIdleTimeout(1), + RetryOnModes(UnorderedElementsAre("5xx", "gateway-error", "connect-failure", + "refused-stream", "reset"))))))) + .WillOnce(Return(nullptr)); + // Act + fetcher_->fetch(message, parent_span_, receiver); +} + +TEST_F(MetadataFetcherTest, TestFailureToStringConversion) { + // Setup + setupFetcher(); + EXPECT_EQ(fetcher_->failureToString(MetadataFetcher::MetadataReceiver::Failure::Network), + "Network"); + EXPECT_EQ(fetcher_->failureToString(MetadataFetcher::MetadataReceiver::Failure::InvalidMetadata), + "InvalidMetadata"); + EXPECT_EQ(fetcher_->failureToString(MetadataFetcher::MetadataReceiver::Failure::MissingConfig), + "MissingConfig"); +} + +} // namespace Aws +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/aws/mocks.h b/test/extensions/common/aws/mocks.h index 5c3b0c7041af..6db726a8f936 100644 --- a/test/extensions/common/aws/mocks.h +++ b/test/extensions/common/aws/mocks.h @@ -1,8 +1,14 @@ #pragma once +#include "envoy/http/message.h" + +#include "source/common/http/message_impl.h" #include "source/extensions/common/aws/credentials_provider.h" +#include "source/extensions/common/aws/metadata_fetcher.h" #include "source/extensions/common/aws/signer.h" +#include "test/mocks/upstream/cluster_manager.h" + #include "gmock/gmock.h" namespace Envoy { @@ -10,6 +16,21 @@ namespace Extensions { namespace Common { namespace Aws { +class MockMetadataFetcher : public MetadataFetcher { +public: + MOCK_METHOD(void, cancel, ()); + MOCK_METHOD(absl::string_view, failureToString, (MetadataFetcher::MetadataReceiver::Failure)); + MOCK_METHOD(void, fetch, + (Http::RequestMessage & message, Tracing::Span& parent_span, + MetadataFetcher::MetadataReceiver& receiver)); +}; + +class MockMetadataReceiver : public MetadataFetcher::MetadataReceiver { +public: + MOCK_METHOD(void, onMetadataSuccess, (const std::string&& body)); + MOCK_METHOD(void, onMetadataError, (MetadataFetcher::MetadataReceiver::Failure reason)); +}; + class MockCredentialsProvider : public CredentialsProvider { public: MockCredentialsProvider(); diff --git a/test/extensions/common/aws/utility_test.cc b/test/extensions/common/aws/utility_test.cc index 629f28c2c957..fa5d2c41deae 100644 --- a/test/extensions/common/aws/utility_test.cc +++ b/test/extensions/common/aws/utility_test.cc @@ -1,11 +1,18 @@ #include "source/extensions/common/aws/utility.h" +#include "test/extensions/common/aws/mocks.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" +using testing::_; using testing::ElementsAre; +using testing::InSequence; +using testing::NiceMock; using testing::Pair; +using testing::Ref; +using testing::Return; +using testing::Throw; namespace Envoy { namespace Extensions { @@ -13,6 +20,12 @@ namespace Common { namespace Aws { namespace { +MATCHER_P(WithName, expectedName, "") { + *result_listener << "\nexpected { name: \"" << expectedName << "\"} but got {name: \"" + << arg.name() << "\"}\n"; + return ExplainMatchResult(expectedName, arg.name(), result_listener); +} + // Headers must be in alphabetical order by virtue of std::map TEST(UtilityTest, CanonicalizeHeadersInAlphabeticalOrder) { Http::TestRequestHeaderMapImpl headers{ @@ -346,6 +359,80 @@ TEST(UtilityTest, JoinCanonicalHeaderNamesWithEmptyMap) { EXPECT_EQ("", names); } +// Verify that we don't add a thread local cluster if it already exists. +TEST(UtilityTest, ThreadLocalClusterExistsAlready) { + NiceMock cluster_; + NiceMock cm_; + EXPECT_CALL(cm_, getThreadLocalCluster(_)).WillOnce(Return(&cluster_)); + EXPECT_CALL(cm_, addOrUpdateCluster(_, _)).Times(0); + EXPECT_TRUE(Utility::addInternalClusterStatic(cm_, "cluster_name", + envoy::config::cluster::v3::Cluster::STATIC, "")); +} + +// Verify that if thread local cluster doesn't exist we can create a new one. +TEST(UtilityTest, AddStaticClusterSuccess) { + NiceMock cm_; + EXPECT_CALL(cm_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cm_, addOrUpdateCluster(WithName("cluster_name"), _)).WillOnce(Return(true)); + EXPECT_TRUE(Utility::addInternalClusterStatic( + cm_, "cluster_name", envoy::config::cluster::v3::Cluster::STATIC, "127.0.0.1:80")); +} + +// Handle exception when adding thread local cluster fails. +TEST(UtilityTest, AddStaticClusterFailure) { + NiceMock cm_; + EXPECT_CALL(cm_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cm_, addOrUpdateCluster(WithName("cluster_name"), _)) + .WillOnce(Throw(EnvoyException("exeption message"))); + EXPECT_FALSE(Utility::addInternalClusterStatic( + cm_, "cluster_name", envoy::config::cluster::v3::Cluster::STATIC, "127.0.0.1:80")); +} + +// Verify that for uri argument in addInternalClusterStatic port value is optional +// and can contain request path which will be ignored. +TEST(UtilityTest, AddStaticClusterSuccessEvenWithMissingPort) { + NiceMock cm_; + EXPECT_CALL(cm_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(cm_, addOrUpdateCluster(WithName("cluster_name"), _)).WillOnce(Return(true)); + EXPECT_TRUE(Utility::addInternalClusterStatic( + cm_, "cluster_name", envoy::config::cluster::v3::Cluster::STATIC, "127.0.0.1/something")); +} + +// The region is simply interpolated into sts.{}.amazonaws.com for most regions +// https://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region. +TEST(UtilityTest, GetNormalAndFipsSTSEndpoints) { + EXPECT_EQ("sts.ap-south-1.amazonaws.com", Utility::getSTSEndpoint("ap-south-1")); + EXPECT_EQ("sts.some-new-region.amazonaws.com", Utility::getSTSEndpoint("some-new-region")); +#ifdef ENVOY_SSL_FIPS + // Under FIPS mode the Envoy should fetch the credentials from FIPS the dedicated endpoints. + EXPECT_EQ("sts-fips.us-east-1.amazonaws.com", Utility::getSTSEndpoint("us-east-1")); + EXPECT_EQ("sts-fips.us-east-2.amazonaws.com", Utility::getSTSEndpoint("us-east-2")); + EXPECT_EQ("sts-fips.us-west-1.amazonaws.com", Utility::getSTSEndpoint("us-west-1")); + EXPECT_EQ("sts-fips.us-west-2.amazonaws.com", Utility::getSTSEndpoint("us-west-2")); + // Even if FIPS mode is enabled ca-central-1 doesn't have a dedicated fips endpoint yet. + EXPECT_EQ("sts.ca-central-1.amazonaws.com", Utility::getSTSEndpoint("ca-central-1")); +#else + EXPECT_EQ("sts.us-east-1.amazonaws.com", Utility::getSTSEndpoint("us-east-1")); + EXPECT_EQ("sts.us-east-2.amazonaws.com", Utility::getSTSEndpoint("us-east-2")); + EXPECT_EQ("sts.us-west-1.amazonaws.com", Utility::getSTSEndpoint("us-west-1")); + EXPECT_EQ("sts.us-west-2.amazonaws.com", Utility::getSTSEndpoint("us-west-2")); + EXPECT_EQ("sts.ca-central-1.amazonaws.com", Utility::getSTSEndpoint("ca-central-1")); +#endif +} + +// China regions: https://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region. +TEST(UtilityTest, GetChinaSTSEndpoints) { + EXPECT_EQ("sts.cn-north-1.amazonaws.com.cn", Utility::getSTSEndpoint("cn-north-1")); + EXPECT_EQ("sts.cn-northwest-1.amazonaws.com.cn", Utility::getSTSEndpoint("cn-northwest-1")); +} + +// GovCloud regions: https://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region. +TEST(UtilityTest, GetGovCloudSTSEndpoints) { + // No difference between fips vs non-fips endpoints in GovCloud. + EXPECT_EQ("sts.us-gov-east-1.amazonaws.com", Utility::getSTSEndpoint("us-gov-east-1")); + EXPECT_EQ("sts.us-gov-west-1.amazonaws.com", Utility::getSTSEndpoint("us-gov-west-1")); +} + } // namespace } // namespace Aws } // namespace Common diff --git a/test/extensions/common/dynamic_forward_proxy/BUILD b/test/extensions/common/dynamic_forward_proxy/BUILD index ea7ed5952f45..8625b7c3bb7c 100644 --- a/test/extensions/common/dynamic_forward_proxy/BUILD +++ b/test/extensions/common/dynamic_forward_proxy/BUILD @@ -24,6 +24,7 @@ envoy_cc_test( "//test/mocks/thread_local:thread_local_mocks", "//test/test_common:registry_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto", diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index 212e38eceeb5..e16e0953d33c 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -7,7 +7,6 @@ #include "source/common/network/resolver_impl.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h" #include "source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" -#include "source/server/factory_context_base_impl.h" #include "test/extensions/common/dynamic_forward_proxy/mocks.h" #include "test/mocks/filesystem/mocks.h" @@ -18,8 +17,11 @@ #include "test/mocks/thread_local/mocks.h" #include "test/test_common/registry.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" +#include "absl/strings/str_cat.h" + using testing::AtLeast; using testing::DoAll; using testing::InSequence; @@ -37,29 +39,36 @@ static const absl::optional kNoTtl = absl::nullopt; class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedTime { public: DnsCacheImplTest() : registered_dns_factory_(dns_resolver_factory_) {} - void initialize(std::vector preresolve_hostnames = {}, uint32_t max_hosts = 1024) { + void initialize( + std::vector> preresolve_hostnames = {}, + uint32_t max_hosts = 1024) { config_.set_name("foo"); config_.set_dns_lookup_family(envoy::config::cluster::v3::Cluster::V4_ONLY); config_.mutable_max_hosts()->set_value(max_hosts); if (!preresolve_hostnames.empty()) { - for (const auto& hostname : preresolve_hostnames) { + for (const auto& [host, port] : preresolve_hostnames) { envoy::config::core::v3::SocketAddress* address = config_.add_preresolve_hostnames(); - address->set_address(hostname); - address->set_port_value(443); + address->set_address(host); + address->set_port_value(port); } } - EXPECT_CALL(context_.dispatcher_, isThreadSafe).WillRepeatedly(Return(true)); + EXPECT_CALL(context_.server_factory_context_.dispatcher_, isThreadSafe) + .WillRepeatedly(Return(true)); EXPECT_CALL(dns_resolver_factory_, createDnsResolver(_, _, _)) .WillRepeatedly(Return(resolver_)); - dns_cache_ = std::make_unique(context_, config_); + auto status_or_cache = DnsCacheImpl::createDnsCacheImpl(context_, config_); + THROW_IF_STATUS_NOT_OK(status_or_cache, throw); + dns_cache_ = status_or_cache.value(); update_callbacks_handle_ = dns_cache_->addUpdateCallbacks(update_callbacks_); } ~DnsCacheImplTest() override { - dns_cache_.reset(); - EXPECT_EQ(0, TestUtility::findGauge(context_.store_, "dns_cache.foo.num_hosts")->value()); + if (dns_cache_.get()) { + dns_cache_.reset(); + EXPECT_EQ(0, TestUtility::findGauge(context_.store_, "dns_cache.foo.num_hosts")->value()); + } } void checkStats(uint64_t query_attempt, uint64_t query_success, uint64_t query_failure, @@ -78,10 +87,10 @@ class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedT TestUtility::findGauge(context_.store_, "dns_cache.foo.num_hosts")->value()); } - NiceMock context_; + NiceMock context_; envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config_; std::shared_ptr resolver_{std::make_shared()}; - std::unique_ptr dns_cache_; + std::shared_ptr dns_cache_; MockUpdateCallbacks update_callbacks_; DnsCache::AddUpdateCallbacksHandlePtr update_callbacks_handle_; NiceMock dns_resolver_factory_; @@ -111,6 +120,7 @@ MATCHER_P3(DnsHostInfoEquals, address, resolved_host, is_ip_address, "") { } MATCHER(DnsHostInfoAddressIsNull, "") { return arg->address() == nullptr; } +MATCHER(DnsHostInfoFirstResolveCompleteTrue, "") { return arg->firstResolveComplete() == false; } void verifyCaresDnsConfigAndUnpack( const envoy::config::core::v3::TypedExtensionConfig& typed_dns_resolver_config, @@ -123,20 +133,35 @@ void verifyCaresDnsConfigAndUnpack( typed_dns_resolver_config.typed_config().UnpackTo(&cares); } -TEST_F(DnsCacheImplTest, PreresolveSuccess) { +class DnsCacheImplPreresolveTest : public DnsCacheImplTest, + public testing::WithParamInterface { +public: + bool normalizeDfpHost() { return GetParam(); } +}; + +INSTANTIATE_TEST_SUITE_P(DnsCachePreresolveNormalizedDfpHost, DnsCacheImplPreresolveTest, + testing::Bool()); + +TEST_P(DnsCacheImplPreresolveTest, PreresolveSuccess) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.normalize_host_for_preresolve_dfp_dns", + absl::StrCat(normalizeDfpHost())}}); + Network::DnsResolver::ResolveCb resolve_cb; - std::string hostname = "bar.baz.com:443"; - EXPECT_CALL(*resolver_, resolve("bar.baz.com", _, _)) - .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("bar.baz.com:443", - DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false))); + std::string host = "bar.baz.com"; + uint32_t port = 443; + std::string authority = absl::StrCat(host, ":", port); + EXPECT_CALL(*resolver_, resolve(host, _, _)) + .WillRepeatedly(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate(authority, DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false))); EXPECT_CALL(update_callbacks_, - onDnsResolutionComplete("bar.baz.com:443", + onDnsResolutionComplete(authority, DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false), Network::DnsResolver::ResolutionStatus::Success)); - initialize({"bar.baz.com:443"} /* preresolve_hostnames */); + initialize({{normalizeDfpHost() ? host : authority, port}} /* preresolve_hostnames */); resolve_cb(Network::DnsResolver::ResolutionStatus::Success, TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -144,18 +169,55 @@ TEST_F(DnsCacheImplTest, PreresolveSuccess) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); MockLoadDnsCacheEntryCallbacks callbacks; - auto result = dns_cache_->loadDnsCacheEntry("bar.baz.com", 443, false, callbacks); + if (normalizeDfpHost()) { + // Retrieve with the hostname and port in the "host". + auto result = dns_cache_->loadDnsCacheEntry(authority, port, false, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); + EXPECT_EQ(result.handle_, nullptr); + EXPECT_NE(absl::nullopt, result.host_info_); + } + // Retrieve with the hostname only in the "host". + auto result = dns_cache_->loadDnsCacheEntry(host, port, false, callbacks); EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); EXPECT_EQ(result.handle_, nullptr); EXPECT_NE(absl::nullopt, result.host_info_); } -TEST_F(DnsCacheImplTest, PreresolveFailure) { +TEST_P(DnsCacheImplPreresolveTest, PreresolveFailure) { EXPECT_THROW_WITH_MESSAGE( - initialize({"bar.baz.com"} /* preresolve_hostnames */, 0 /* max_hosts */), EnvoyException, + initialize({{"bar.baz.com", 443}} /* preresolve_hostnames */, 0 /* max_hosts */), + EnvoyException, "DNS Cache [foo] configured with preresolve_hostnames=1 larger than max_hosts=0"); } +TEST_F(DnsCacheImplTest, DnsFirstResolveComplete) { + // This test relies on below runtime flag to be true. + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.dns_cache_set_first_resolve_complete", "true"}}); + Network::DnsResolver::ResolveCb resolve_cb; + std::string hostname = "bar.baz.com:443"; + EXPECT_CALL(*resolver_, resolve("bar.baz.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, DnsHostInfoFirstResolveCompleteTrue())); + EXPECT_CALL(update_callbacks_, + onDnsResolutionComplete("bar.baz.com:443", + DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false), + Network::DnsResolver::ResolutionStatus::Success)); + + initialize({{"bar.baz.com", 443}}); + resolve_cb(Network::DnsResolver::ResolutionStatus::Success, + TestUtility::makeDnsResponse({"10.0.0.1"})); + checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + + MockLoadDnsCacheEntryCallbacks callbacks; + auto result = dns_cache_->loadDnsCacheEntry("bar.baz.com", 443, false, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); + EXPECT_EQ(result.handle_, nullptr); + EXPECT_NE(absl::nullopt, result.host_info_); +} + // Basic successful resolution and then re-resolution. TEST_F(DnsCacheImplTest, ResolveSuccess) { initialize(); @@ -163,8 +225,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -254,8 +318,10 @@ TEST_F(DnsCacheImplTest, ForceRefresh) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -286,8 +352,10 @@ TEST_F(DnsCacheImplTest, Ipv4Address) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("127.0.0.1", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -318,8 +386,10 @@ TEST_F(DnsCacheImplTest, Ipv4AddressWithPort) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("127.0.0.1", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -350,8 +420,10 @@ TEST_F(DnsCacheImplTest, Ipv6Address) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("::1", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -379,8 +451,10 @@ TEST_F(DnsCacheImplTest, Ipv6AddressWithPort) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("::1", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -408,8 +482,10 @@ TEST_F(DnsCacheImplTest, TTL) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -467,8 +543,8 @@ TEST_F(DnsCacheImplTest, TTL) { 1 /* added */, 1 /* removed */, 0 /* num hosts */); // Make sure we don't get a cache hit the next time the host is requested. - new Event::MockTimer(&context_.dispatcher_); // resolve_timer - timeout_timer = new Event::MockTimer(&context_.dispatcher_); + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); // resolve_timer + timeout_timer = new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -488,8 +564,10 @@ TEST_F(DnsCacheImplTest, TTLWithMinRefreshRate) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -523,8 +601,10 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(1000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -575,16 +655,17 @@ TEST_F(DnsCacheImplTest, InlineResolve) { MockLoadDnsCacheEntryCallbacks callbacks; Event::PostCb post_cb; - EXPECT_CALL(context_.dispatcher_, post(_)).WillOnce([&post_cb](Event::PostCb cb) { - post_cb = std::move(cb); - }); + EXPECT_CALL(context_.server_factory_context_.dispatcher_, post(_)) + .WillOnce([&post_cb](Event::PostCb cb) { post_cb = std::move(cb); }); auto result = dns_cache_->loadDnsCacheEntry("localhost", 80, false, callbacks); EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); EXPECT_NE(result.handle_, nullptr); EXPECT_EQ(absl::nullopt, result.host_info_); - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("localhost", _, _)) .WillOnce(Invoke([](const std::string&, Network::DnsLookupFamily, @@ -614,8 +695,10 @@ TEST_F(DnsCacheImplTest, ResolveTimeout) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -650,8 +733,10 @@ TEST_F(DnsCacheImplTest, ResolveFailure) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -702,8 +787,10 @@ TEST_F(DnsCacheImplTest, ResolveFailureWithFailureRefreshRate) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -720,7 +807,7 @@ TEST_F(DnsCacheImplTest, ResolveFailureWithFailureRefreshRate) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), Network::DnsResolver::ResolutionStatus::Failure)); - ON_CALL(context_.api_.random_, random()).WillByDefault(Return(8000)); + ON_CALL(context_.server_factory_context_.api_.random_, random()).WillByDefault(Return(8000)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(1000), _)); resolve_cb(Network::DnsResolver::ResolutionStatus::Failure, TestUtility::makeDnsResponse({})); checkStats(1 /* attempt */, 0 /* success */, 1 /* failure */, 0 /* address changed */, @@ -751,8 +838,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithEmptyResult) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -1066,7 +1155,8 @@ TEST_F(DnsCacheImplTest, UseTcpForDnsLookupsOptionSetDeprecatedField) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; EXPECT_CALL(dns_resolver_factory_, createDnsResolver(_, _, _)) .WillOnce(DoAll(SaveArg<2>(&typed_dns_resolver_config), Return(resolver_))); - DnsCacheImpl dns_cache_(context_, config_); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context_, config_).value(); envoy::extensions::network::dns_resolver::cares::v3::CaresDnsResolverConfig cares; verifyCaresDnsConfigAndUnpack(typed_dns_resolver_config, cares); // `true` here means dns_resolver_options.use_tcp_for_dns_lookups is set to true. @@ -1081,7 +1171,8 @@ TEST_F(DnsCacheImplTest, UseTcpForDnsLookupsOptionSet) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; EXPECT_CALL(dns_resolver_factory_, createDnsResolver(_, _, _)) .WillOnce(DoAll(SaveArg<2>(&typed_dns_resolver_config), Return(resolver_))); - DnsCacheImpl dns_cache_(context_, config_); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context_, config_).value(); envoy::extensions::network::dns_resolver::cares::v3::CaresDnsResolverConfig cares; verifyCaresDnsConfigAndUnpack(typed_dns_resolver_config, cares); // `true` here means dns_resolver_options.use_tcp_for_dns_lookups is set to true. @@ -1096,7 +1187,8 @@ TEST_F(DnsCacheImplTest, NoDefaultSearchDomainOptionSet) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; EXPECT_CALL(dns_resolver_factory_, createDnsResolver(_, _, _)) .WillOnce(DoAll(SaveArg<2>(&typed_dns_resolver_config), Return(resolver_))); - DnsCacheImpl dns_cache_(context_, config_); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context_, config_).value(); envoy::extensions::network::dns_resolver::cares::v3::CaresDnsResolverConfig cares; verifyCaresDnsConfigAndUnpack(typed_dns_resolver_config, cares); // `true` here means dns_resolver_options.no_default_search_domain is set to true. @@ -1108,7 +1200,8 @@ TEST_F(DnsCacheImplTest, UseTcpForDnsLookupsOptionUnSet) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; EXPECT_CALL(dns_resolver_factory_, createDnsResolver(_, _, _)) .WillOnce(DoAll(SaveArg<2>(&typed_dns_resolver_config), Return(resolver_))); - DnsCacheImpl dns_cache_(context_, config_); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context_, config_).value(); envoy::extensions::network::dns_resolver::cares::v3::CaresDnsResolverConfig cares; verifyCaresDnsConfigAndUnpack(typed_dns_resolver_config, cares); // `false` here means dns_resolver_options.use_tcp_for_dns_lookups is set to false. @@ -1120,7 +1213,8 @@ TEST_F(DnsCacheImplTest, NoDefaultSearchDomainOptionUnSet) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; EXPECT_CALL(dns_resolver_factory_, createDnsResolver(_, _, _)) .WillOnce(DoAll(SaveArg<2>(&typed_dns_resolver_config), Return(resolver_))); - DnsCacheImpl dns_cache_(context_, config_); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context_, config_).value(); envoy::extensions::network::dns_resolver::cares::v3::CaresDnsResolverConfig cares; verifyCaresDnsConfigAndUnpack(typed_dns_resolver_config, cares); // `false` here means dns_resolver_options.no_default_search_domain is set to false. @@ -1129,34 +1223,34 @@ TEST_F(DnsCacheImplTest, NoDefaultSearchDomainOptionUnSet) { // DNS cache manager config tests. TEST(DnsCacheManagerImplTest, LoadViaConfig) { - NiceMock context; + NiceMock context; DnsCacheManagerImpl cache_manager(context); envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config1; config1.set_name("foo"); - auto cache1 = cache_manager.getCache(config1); + auto cache1 = cache_manager.getCache(config1).value(); EXPECT_NE(cache1, nullptr); envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config2; config2.set_name("foo"); - EXPECT_EQ(cache1, cache_manager.getCache(config2)); + EXPECT_EQ(cache1, cache_manager.getCache(config2).value()); envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config3; config3.set_name("bar"); - auto cache2 = cache_manager.getCache(config3); + auto cache2 = cache_manager.getCache(config3).value(); EXPECT_NE(cache2, nullptr); EXPECT_NE(cache1, cache2); envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config4; config4.set_name("foo"); config4.set_dns_lookup_family(envoy::config::cluster::v3::Cluster::V6_ONLY); - EXPECT_THROW_WITH_MESSAGE(cache_manager.getCache(config4), EnvoyException, - "config specified DNS cache 'foo' with different settings"); + EXPECT_EQ(cache_manager.getCache(config4).status().message(), + "config specified DNS cache 'foo' with different settings"); } TEST(DnsCacheManagerImplTest, LookupByName) { - NiceMock context; + NiceMock context; DnsCacheManagerImpl cache_manager(context); EXPECT_EQ(cache_manager.lookUpCacheByName("foo"), nullptr); @@ -1164,7 +1258,7 @@ TEST(DnsCacheManagerImplTest, LookupByName) { envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config1; config1.set_name("foo"); - auto cache1 = cache_manager.getCache(config1); + auto cache1 = cache_manager.getCache(config1).value(); EXPECT_NE(cache1, nullptr); auto cache2 = cache_manager.lookUpCacheByName("foo"); @@ -1173,7 +1267,7 @@ TEST(DnsCacheManagerImplTest, LookupByName) { } TEST(DnsCacheConfigOptionsTest, EmtpyDnsResolutionConfig) { - NiceMock context; + NiceMock context; envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config; std::shared_ptr resolver{std::make_shared()}; envoy::config::core::v3::TypedExtensionConfig empty_typed_dns_resolver_config; @@ -1185,12 +1279,13 @@ TEST(DnsCacheConfigOptionsTest, EmtpyDnsResolutionConfig) { EXPECT_CALL(dns_resolver_factory, createDnsResolver(_, _, ProtoEq(empty_typed_dns_resolver_config))) .WillOnce(Return(resolver)); - DnsCacheImpl dns_cache_(context, config); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context, config).value(); } // Test dns_resolution_config is in place, use it. TEST(DnsCacheConfigOptionsTest, NonEmptyDnsResolutionConfig) { - NiceMock context; + NiceMock context; envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config; std::shared_ptr resolver{std::make_shared()}; envoy::config::core::v3::Address resolvers; @@ -1207,12 +1302,13 @@ TEST(DnsCacheConfigOptionsTest, NonEmptyDnsResolutionConfig) { Registry::InjectFactory registered_dns_factory(dns_resolver_factory); EXPECT_CALL(dns_resolver_factory, createDnsResolver(_, _, ProtoEq(typed_dns_resolver_config))) .WillOnce(Return(resolver)); - DnsCacheImpl dns_cache_(context, config); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context, config).value(); } // Test dns_resolution_config is in place, use it and overriding use_tcp_for_dns_lookups. TEST(DnsCacheConfigOptionsTest, NonEmptyDnsResolutionConfigOverridingUseTcp) { - NiceMock context; + NiceMock context; std::shared_ptr resolver{std::make_shared()}; envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config; @@ -1244,13 +1340,14 @@ TEST(DnsCacheConfigOptionsTest, NonEmptyDnsResolutionConfigOverridingUseTcp) { Registry::InjectFactory registered_dns_factory(dns_resolver_factory); EXPECT_CALL(dns_resolver_factory, createDnsResolver(_, _, ProtoEq(typed_dns_resolver_config))) .WillOnce(Return(resolver)); - DnsCacheImpl dns_cache_(context, config); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context, config).value(); } // Test the case that the typed_dns_resolver_config is specified, and it overrides all // other configuration, like config.dns_resolution_config, and config.use_tcp_for_dns_lookups. TEST(DnsCacheConfigOptionsTest, NonEmptyTypedDnsResolverConfig) { - NiceMock context; + NiceMock context; std::shared_ptr resolver{std::make_shared()}; envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config; @@ -1288,7 +1385,8 @@ TEST(DnsCacheConfigOptionsTest, NonEmptyTypedDnsResolverConfig) { EXPECT_CALL(dns_resolver_factory, createDnsResolver(_, _, ProtoEq(expected_typed_dns_resolver_config))) .WillOnce(Return(resolver)); - DnsCacheImpl dns_cache_(context, config); + std::shared_ptr dns_cache = + DnsCacheImpl::createDnsCacheImpl(context, config).value(); } // Note: this test is done here, rather than a TYPED_TEST_SUITE in @@ -1337,7 +1435,7 @@ TEST(UtilityTest, PrepareDnsRefreshStrategy) { TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { auto* time_source = new NiceMock(); - context_.dispatcher_.time_system_.reset(time_source); + context_.server_factory_context_.dispatcher_.time_system_.reset(time_source); // Configure the cache. MockKeyValueStoreFactory factory; @@ -1352,7 +1450,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { // Make sure there's an attempt to load from the key value store. EXPECT_CALL(*store, iterate(_)); // Make sure the result is sent to the worker threads. - EXPECT_CALL(context_.thread_local_, runOnAllThreads(_)).Times(2); + EXPECT_CALL(context_.server_factory_context_.thread_local_, runOnAllThreads(_)).Times(2); return ret; })); @@ -1368,8 +1466,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; - Event::MockTimer* resolve_timer = new Event::MockTimer(&context_.dispatcher_); - Event::MockTimer* timeout_timer = new Event::MockTimer(&context_.dispatcher_); + Event::MockTimer* resolve_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); + Event::MockTimer* timeout_timer = + new Event::MockTimer(&context_.server_factory_context_.dispatcher_); EXPECT_CALL(*timeout_timer, enableTimer(std::chrono::milliseconds(5000), nullptr)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); @@ -1471,7 +1571,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { TEST_F(DnsCacheImplTest, CacheLoad) { auto* time_source = new NiceMock(); - context_.dispatcher_.time_system_.reset(time_source); + context_.server_factory_context_.dispatcher_.time_system_.reset(time_source); // Configure the cache. MockKeyValueStoreFactory factory; @@ -1527,17 +1627,17 @@ TEST_F(DnsCacheImplTest, CacheLoad) { // Make sure the cache manager can handle the context going out of scope. TEST(DnsCacheManagerImplTest, TestLifetime) { - NiceMock context; + NiceMock context; std::unique_ptr cache_manager; { - Server::FactoryContextBaseImpl scoped_context(context); + Server::GenericFactoryContextImpl scoped_context(context); cache_manager = std::make_unique(scoped_context); } envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig config1; config1.set_name("foo"); - EXPECT_TRUE(cache_manager->getCache(config1) != nullptr); + EXPECT_TRUE(cache_manager->getCache(config1).value() != nullptr); } TEST(NoramlizeHost, NormalizeHost) { diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h index 1a613271efdf..9baaab626b0a 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.h +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -78,7 +78,7 @@ class MockDnsCacheManager : public DnsCacheManager { MockDnsCacheManager(); ~MockDnsCacheManager() override; - MOCK_METHOD(DnsCacheSharedPtr, getCache, + MOCK_METHOD(absl::StatusOr, getCache, (const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config)); MOCK_METHOD(DnsCacheSharedPtr, lookUpCacheByName, (absl::string_view cache_name)); @@ -95,6 +95,7 @@ class MockDnsHostInfo : public DnsHostInfo { MOCK_METHOD(const std::string&, resolvedHost, (), (const)); MOCK_METHOD(bool, isIpAddress, (), (const)); MOCK_METHOD(void, touch, ()); + MOCK_METHOD(bool, firstResolveComplete, (), (const)); Network::Address::InstanceConstSharedPtr address_; std::vector address_list_; diff --git a/test/extensions/common/proxy_protocol/BUILD b/test/extensions/common/proxy_protocol/BUILD index 33b8ac48892b..42a3b20db339 100644 --- a/test/extensions/common/proxy_protocol/BUILD +++ b/test/extensions/common/proxy_protocol/BUILD @@ -26,11 +26,11 @@ envoy_cc_test( "//source/common/buffer:buffer_lib", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:listener_lib", "//source/extensions/common/proxy_protocol:proxy_protocol_header_lib", "//source/extensions/filters/listener/proxy_protocol:proxy_protocol_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/buffer:buffer_mocks", "//test/mocks/network:network_mocks", "//test/test_common:environment_lib", diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index cd1cb650507c..ee5732e61735 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -3,13 +3,14 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/common/basic_resource_impl.h" #include "source/common/event/dispatcher_impl.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/extensions/common/proxy_protocol/proxy_protocol_header.h" #include "source/extensions/filters/listener/proxy_protocol/proxy_protocol.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" #include "test/mocks/buffer/mocks.h" +#include "test/mocks/common.h" #include "test/mocks/network/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" @@ -45,7 +46,8 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam>()) { socket_factories_.emplace_back(std::make_unique()); EXPECT_CALL(*static_cast(socket_factories_[0].get()), socketType()) @@ -56,7 +58,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam(socket_factories_[0].get()), getListenSocket(_)) .WillOnce(Return(socket_)); - connection_handler_->addListener(absl::nullopt, *this, runtime_); + connection_handler_->addListener(absl::nullopt, *this, runtime_, random_); conn_ = dispatcher_->createClientConnection(socket_->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr, @@ -81,15 +83,11 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam empty_access_logs_; std::unique_ptr init_manager_; NiceMock runtime_; + testing::NiceMock random_; + const Network::ListenerInfoConstSharedPtr listener_info_; }; // Parameterize the listener socket address version. diff --git a/test/extensions/common/tap/admin_test.cc b/test/extensions/common/tap/admin_test.cc index f25c56a4340b..df3aa2783c15 100644 --- a/test/extensions/common/tap/admin_test.cc +++ b/test/extensions/common/tap/admin_test.cc @@ -153,8 +153,8 @@ config_id: test_config_id using Extensions::Common::Tap::TapSinkFactory; class MockTapSinkFactory : public TapSinkFactory { public: - MockTapSinkFactory() {} - ~MockTapSinkFactory() override{}; + MockTapSinkFactory() = default; + ~MockTapSinkFactory() override = default; MOCK_METHOD(SinkPtr, createSinkPtr, (const Protobuf::Message& config, SinkContext), (override)); @@ -200,7 +200,7 @@ TEST(TypedExtensionConfigTest, AddTestConfigHttpContext) { Registry::InjectFactory factory(factory_impl); NiceMock factory_context; - TestConfigImpl(tap_config, NULL, factory_context); + TestConfigImpl(tap_config, nullptr, factory_context); } TEST(TypedExtensionConfigTest, AddTestConfigTransportSocketContext) { @@ -234,7 +234,7 @@ TEST(TypedExtensionConfigTest, AddTestConfigTransportSocketContext) { Registry::InjectFactory factory(factory_impl); NiceMock factory_context; - TestConfigImpl(tap_config, NULL, factory_context); + TestConfigImpl(tap_config, nullptr, factory_context); } // Make sure warn if using a pipe address for the admin handler. diff --git a/test/extensions/common/wasm/BUILD b/test/extensions/common/wasm/BUILD index bfbfb093be58..eaf3b573c9c8 100644 --- a/test/extensions/common/wasm/BUILD +++ b/test/extensions/common/wasm/BUILD @@ -133,6 +133,7 @@ envoy_cc_test( deps = [ "//source/common/network:filter_state_dst_address_lib", "//source/common/tcp_proxy", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/common/wasm:wasm_hdr", "//source/extensions/common/wasm:wasm_lib", "//test/mocks/http:http_mocks", diff --git a/test/extensions/common/wasm/foreign_test.cc b/test/extensions/common/wasm/foreign_test.cc index 2d5969976418..a29c20eed483 100644 --- a/test/extensions/common/wasm/foreign_test.cc +++ b/test/extensions/common/wasm/foreign_test.cc @@ -2,6 +2,7 @@ #include "source/common/network/filter_state_dst_address.h" #include "source/common/stats/isolated_store_impl.h" #include "source/common/tcp_proxy/tcp_proxy.h" +#include "source/extensions/clusters/original_dst/original_dst_cluster.h" #include "source/extensions/common/wasm/ext/set_envoy_filter_state.pb.h" #include "source/extensions/common/wasm/wasm.h" @@ -21,7 +22,7 @@ class TestContext : public Context {}; class ForeignTest : public testing::Test { public: - ForeignTest() {} + ForeignTest() = default; void initializeFilterCallbacks() { ctx_.initializeReadFilterCallbacks(read_filter_callbacks_); } @@ -102,14 +103,14 @@ TEST_F(ForeignTest, ForeignFunctionSetEnvoyFilterTest) { EXPECT_TRUE(stream_info->filterState()->hasData( TcpProxy::PerConnectionCluster::key())); - args.set_path(Network::DestinationAddress::key()); + args.set_path(Upstream::OriginalDstClusterFilterStateKey); args.set_value("1.2.3.4:80"); args.set_span(envoy::source::extensions::common::wasm::LifeSpan::DownstreamRequest); args.SerializeToString(&in); result = function(wasm, in, [](size_t size) { return malloc(size); }); EXPECT_EQ(result, WasmResult::Ok); - EXPECT_TRUE(stream_info->filterState()->hasData( - Network::DestinationAddress::key())); + EXPECT_TRUE(stream_info->filterState()->hasData( + Upstream::OriginalDstClusterFilterStateKey)); } } // namespace Wasm diff --git a/test/extensions/common/wasm/test_data/test_context_cpp.cc b/test/extensions/common/wasm/test_data/test_context_cpp.cc index 225fda61b8c5..02d9264cedc5 100644 --- a/test/extensions/common/wasm/test_data/test_context_cpp.cc +++ b/test/extensions/common/wasm/test_data/test_context_cpp.cc @@ -106,7 +106,7 @@ class PanicReplyContext : public Context { FilterDataStatus PanicReplyContext::onRequestBody(size_t, bool) { sendLocalResponse(200, "not send", "body", {}); int* badptr = nullptr; - *badptr = 0; + *badptr = 0; // NOLINT(clang-analyzer-core.NullDereference) return FilterDataStatus::Continue; } diff --git a/test/extensions/common/wasm/test_data/test_restriction_cpp.cc b/test/extensions/common/wasm/test_data/test_restriction_cpp.cc index 75f425ca1e71..c9125de93ba2 100644 --- a/test/extensions/common/wasm/test_data/test_restriction_cpp.cc +++ b/test/extensions/common/wasm/test_data/test_restriction_cpp.cc @@ -27,7 +27,6 @@ WASM_EXPORT(void, proxy_on_context_create, (uint32_t context_id, uint32_t parent (void)(parent_context_id); std::string log_message = "after proxy_on_context_create: written by proxy_log"; proxy_log(LogLevel::info, log_message.c_str(), log_message.size()); - return; } END_WASM_PLUGIN diff --git a/test/extensions/compression/zstd/zstd_compression_test.cc b/test/extensions/compression/zstd/zstd_compression_test.cc index 17e749eaeb65..25a5b3efcaac 100644 --- a/test/extensions/compression/zstd/zstd_compression_test.cc +++ b/test/extensions/compression/zstd/zstd_compression_test.cc @@ -28,8 +28,9 @@ class ZstdCompressionTest { protected: ZstdCompressionTest() : api_(Api::createApiForTest()), dispatcher_(setupDispatcher()) { TestEnvironment::createPath(TestEnvironment::temporaryPath("envoy_test")); - EXPECT_CALL(mock_context_, api()).WillRepeatedly(ReturnRef(*api_)); - EXPECT_CALL(mock_context_, mainThreadDispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + EXPECT_CALL(mock_context_.server_factory_context_, api()).WillRepeatedly(ReturnRef(*api_)); + EXPECT_CALL(mock_context_.server_factory_context_, mainThreadDispatcher()) + .WillRepeatedly(ReturnRef(*dispatcher_)); } void drainBuffer(Buffer::OwnedImpl& buffer) { diff --git a/test/extensions/config_subscription/filesystem/BUILD b/test/extensions/config_subscription/filesystem/BUILD index 9aec42cb7597..0b2d4f2cee0f 100644 --- a/test/extensions/config_subscription/filesystem/BUILD +++ b/test/extensions/config_subscription/filesystem/BUILD @@ -25,7 +25,7 @@ envoy_cc_test( envoy_cc_test_library( name = "filesystem_subscription_test_harness", - srcs = ["filesystem_subscription_test_harness.h"], + hdrs = ["filesystem_subscription_test_harness.h"], deps = [ "//source/common/config:utility_lib", "//source/common/event:dispatcher_lib", diff --git a/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc index 5dfb9a7afa57..0dec36a101b0 100644 --- a/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc @@ -41,6 +41,7 @@ using testing::IsSubstring; using testing::NiceMock; using testing::Return; using testing::ReturnRef; +using testing::SaveArg; namespace Envoy { namespace Config { @@ -276,6 +277,62 @@ TEST_F(GrpcMuxImplTest, ResetStream) { expectSendMessage("foo", {}, ""); } +// Validate cached nonces are cleared on reconnection. +TEST_F(GrpcMuxImplTest, ReconnectionResetsNonceAndAcks) { + OpaqueResourceDecoderSharedPtr resource_decoder( + std::make_shared>("cluster_name")); + // Create the retry timer that will invoke the callback that will trigger + // reconnection when the gRPC connection is closed. + Event::MockTimer* grpc_stream_retry_timer{new Event::MockTimer()}; + Event::MockTimer* ttl_mgr_timer{new NiceMock()}; + Event::TimerCb grpc_stream_retry_timer_cb; + EXPECT_CALL(dispatcher_, createTimer_(_)) + .WillOnce( + testing::DoAll(SaveArg<0>(&grpc_stream_retry_timer_cb), Return(grpc_stream_retry_timer))) + // Happens when adding a type url watch. + .WillRepeatedly(Return(ttl_mgr_timer)); + setup(); + InSequence s; + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + auto foo_sub = grpc_mux_->addWatch(type_url, {"x", "y"}, callbacks_, resource_decoder, {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + // Send on connection. + expectSendMessage(type_url, {"x", "y"}, {}, true); + grpc_mux_->start(); + + // Create a reply with some nonce. + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("3000"); + response->set_nonce("111"); + auto add_response_resource = [](const std::string& name, + envoy::service::discovery::v3::DiscoveryResponse& response) { + envoy::config::endpoint::v3::ClusterLoadAssignment cla; + cla.set_cluster_name(name); + auto res = response.add_resources(); + res->PackFrom(cla); + }; + add_response_resource("x", *response); + add_response_resource("y", *response); + { + // Pause EDS to allow the ACK to be cached. + auto resume_eds = grpc_mux_->pause(type_url); + // Send the reply. + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + // Now disconnect, gRPC stream retry timer will kick in and reconnection will happen. + EXPECT_CALL(*grpc_stream_retry_timer, enableTimer(_, _)) + .WillOnce(Invoke(grpc_stream_retry_timer_cb)); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Canceled, ""); + + // Unpausing will initiate a new request, with the same resources, version, + // but empty nonce. + expectSendMessage(type_url, {"x", "y"}, "3000", true, ""); + } + expectSendMessage(type_url, {}, "3000", false); +} + // Validate pause-resume behavior. TEST_F(GrpcMuxImplTest, PauseResume) { setup(); @@ -1281,7 +1338,7 @@ TEST_F(GrpcMuxImplTest, RemoveCachedResourceOnLastSubscription) { */ class NullGrpcMuxImplTest : public testing::Test { public: - NullGrpcMuxImplTest() {} + NullGrpcMuxImplTest() = default; NullGrpcMuxImpl null_mux_; NiceMock callbacks_; }; diff --git a/test/extensions/config_subscription/grpc/grpc_stream_test.cc b/test/extensions/config_subscription/grpc/grpc_stream_test.cc index 6d594c2ff2f6..a0aaf9549ec8 100644 --- a/test/extensions/config_subscription/grpc/grpc_stream_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_stream_test.cc @@ -110,21 +110,21 @@ TEST_F(GrpcStreamTest, LogClose) { // Failures with statuses that do not need special handling. They are always logged in the same // way and so never saved. { - EXPECT_FALSE(grpc_stream_->getCloseStatus().has_value()); + EXPECT_FALSE(grpc_stream_->getCloseStatusForTest().has_value()); // Benign status: debug. EXPECT_CALL(callbacks_, onEstablishmentFailure()); EXPECT_LOG_CONTAINS("debug", "gRPC config stream to test_destination closed", { grpc_stream_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Ok, "Ok"); }); - EXPECT_FALSE(grpc_stream_->getCloseStatus().has_value()); + EXPECT_FALSE(grpc_stream_->getCloseStatusForTest().has_value()); // Non-retriable failure: warn. EXPECT_CALL(callbacks_, onEstablishmentFailure()); EXPECT_LOG_CONTAINS("warn", "gRPC config stream to test_destination closed", { grpc_stream_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::NotFound, "Not Found"); }); - EXPECT_FALSE(grpc_stream_->getCloseStatus().has_value()); + EXPECT_FALSE(grpc_stream_->getCloseStatusForTest().has_value()); } // Repeated failures that warn after enough time. { @@ -133,7 +133,7 @@ TEST_F(GrpcStreamTest, LogClose) { EXPECT_LOG_CONTAINS("debug", "gRPC config stream to test_destination closed", { grpc_stream_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Unavailable, "Unavailable"); }); - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::Unavailable); // Different retriable failure: warn. @@ -147,7 +147,7 @@ TEST_F(GrpcStreamTest, LogClose) { Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded, "Deadline Exceeded"); }); - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded); // Same retriable failure after a short amount of time: debug. @@ -157,7 +157,7 @@ TEST_F(GrpcStreamTest, LogClose) { grpc_stream_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded, "Deadline Exceeded"); }); - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded); // Same retriable failure after a long time: warn. @@ -169,7 +169,7 @@ TEST_F(GrpcStreamTest, LogClose) { grpc_stream_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded, "Deadline Exceeded"); }); - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded); // Warn again, using the newest message. @@ -180,7 +180,7 @@ TEST_F(GrpcStreamTest, LogClose) { grpc_stream_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded, "new message"); }); - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::DeadlineExceeded); // Different retriable failure, using the most recent error message from the previous one. @@ -193,7 +193,7 @@ TEST_F(GrpcStreamTest, LogClose) { grpc_stream_->onRemoteClose( Grpc::Status::WellKnownGrpcStatus::Unavailable, "Unavailable"); }); - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::Unavailable); } @@ -204,12 +204,12 @@ TEST_F(GrpcStreamTest, LogClose) { grpc_stream_->establishNewStream(); EXPECT_TRUE(grpc_stream_->grpcStreamAvailable()); // Status isn't cleared yet. - EXPECT_EQ(grpc_stream_->getCloseStatus().value(), + EXPECT_EQ(grpc_stream_->getCloseStatusForTest().value(), Grpc::Status::WellKnownGrpcStatus::Unavailable); auto response = std::make_unique(); grpc_stream_->onReceiveMessage(std::move(response)); - EXPECT_FALSE(grpc_stream_->getCloseStatus().has_value()); + EXPECT_FALSE(grpc_stream_->getCloseStatusForTest().has_value()); } } diff --git a/test/extensions/config_subscription/rest/BUILD b/test/extensions/config_subscription/rest/BUILD index ac81ea0f96e0..b8ebd9f86b0b 100644 --- a/test/extensions/config_subscription/rest/BUILD +++ b/test/extensions/config_subscription/rest/BUILD @@ -19,7 +19,7 @@ envoy_cc_test( envoy_cc_test_library( name = "http_subscription_test_harness", - srcs = ["http_subscription_test_harness.h"], + hdrs = ["http_subscription_test_harness.h"], deps = [ "//envoy/http:async_client_interface", "//source/common/common:utility_lib", diff --git a/test/extensions/filters/common/expr/BUILD b/test/extensions/filters/common/expr/BUILD index 2a0cfbae87be..ddda2980b9b8 100644 --- a/test/extensions/filters/common/expr/BUILD +++ b/test/extensions/filters/common/expr/BUILD @@ -21,8 +21,10 @@ envoy_extension_cc_test( "//source/common/network:filter_state_dst_address_lib", "//source/common/router:string_accessor_lib", "//source/common/stream_info:stream_info_lib", + "//source/extensions/clusters/original_dst:original_dst_cluster_lib", "//source/extensions/filters/common/expr:cel_state_lib", "//source/extensions/filters/common/expr:context_lib", + "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", "//test/mocks/router:router_mocks", "//test/mocks/ssl:ssl_mocks", diff --git a/test/extensions/filters/common/expr/context_test.cc b/test/extensions/filters/common/expr/context_test.cc index ddeb6e9f987c..556424e8c3cc 100644 --- a/test/extensions/filters/common/expr/context_test.cc +++ b/test/extensions/filters/common/expr/context_test.cc @@ -7,6 +7,7 @@ #include "source/extensions/filters/common/expr/cel_state.h" #include "source/extensions/filters/common/expr/context.h" +#include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/router/mocks.h" #include "test/mocks/ssl/mocks.h" @@ -490,6 +491,8 @@ TEST(Context, ConnectionAttributes) { const absl::optional connection_termination_details = "unauthorized"; EXPECT_CALL(info, connectionTerminationDetails()) .WillRepeatedly(ReturnRef(connection_termination_details)); + const std::string downstream_transport_failure_reason = "TlsError"; + info.setDownstreamTransportFailureReason(downstream_transport_failure_reason); EXPECT_CALL(*downstream_ssl_info, peerCertificatePresented()).WillRepeatedly(Return(true)); EXPECT_CALL(*upstream_host, address()).WillRepeatedly(Return(upstream_address)); @@ -672,6 +675,13 @@ TEST(Context, ConnectionAttributes) { EXPECT_EQ(connection_termination_details.value(), value.value().StringOrDie().value()); } + { + auto value = connection[CelValue::CreateStringView(DownstreamTransportFailureReason)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ(downstream_transport_failure_reason, value.value().StringOrDie().value()); + } + { auto value = upstream[CelValue::CreateStringView(TLSVersion)]; EXPECT_TRUE(value.has_value()); @@ -794,7 +804,7 @@ TEST(Context, FilterStateAttributes) { const std::string ip_string = "ip"; const std::string port_string = "port"; filter_state.setData(address_key, - std::make_unique( + std::make_unique( std::make_shared("10.10.11.11", 6666)), StreamInfo::FilterState::StateType::ReadOnly); { @@ -820,6 +830,7 @@ TEST(Context, FilterStateAttributes) { } TEST(Context, XDSAttributes) { + NiceMock local_info; NiceMock info; std::shared_ptr> cluster_info( new NiceMock()); @@ -837,8 +848,15 @@ TEST(Context, XDSAttributes) { filter_chain_info->filter_chain_name_ = "fake_filter_chain_name"; info.downstream_connection_info_provider_->setFilterChainInfo(filter_chain_info); + auto listener_info = std::make_shared>(); + envoy::config::core::v3::Metadata listener_metadata; + EXPECT_CALL(*listener_info, metadata()).WillRepeatedly(ReturnRef(listener_metadata)); + EXPECT_CALL(*listener_info, direction()) + .WillRepeatedly(Return(envoy::config::core::v3::TrafficDirection::OUTBOUND)); + info.downstream_connection_info_provider_->setListenerInfo(listener_info); + Protobuf::Arena arena; - XDSWrapper wrapper(arena, info); + XDSWrapper wrapper(arena, info, &local_info); { const auto value = wrapper[CelValue::CreateStringView(ClusterName)]; @@ -876,6 +894,18 @@ TEST(Context, XDSAttributes) { ASSERT_TRUE(value.value().IsString()); EXPECT_EQ(chain_name, value.value().StringOrDie().value()); } + { + const auto value = wrapper[CelValue::CreateStringView(ListenerMetadata)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsMessage()); + EXPECT_EQ(&listener_metadata, value.value().MessageOrDie()); + } + { + const auto value = wrapper[CelValue::CreateStringView(ListenerDirection)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(2, value.value().Int64OrDie()); + } { const auto value = wrapper[CelValue::CreateStringView(XDS)]; EXPECT_FALSE(value.has_value()); @@ -884,6 +914,11 @@ TEST(Context, XDSAttributes) { const auto value = wrapper[CelValue::CreateInt64(5)]; EXPECT_FALSE(value.has_value()); } + { + const auto value = wrapper[CelValue::CreateStringView(Node)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsMessage()); + } } } // namespace diff --git a/test/extensions/filters/common/expr/evaluator_fuzz_test.cc b/test/extensions/filters/common/expr/evaluator_fuzz_test.cc index 2446fb4a8ac3..a09c51bb4a2a 100644 --- a/test/extensions/filters/common/expr/evaluator_fuzz_test.cc +++ b/test/extensions/filters/common/expr/evaluator_fuzz_test.cc @@ -44,7 +44,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::common::expr::EvaluatorTest // Evaluate the CEL expression. Protobuf::Arena arena; - Expr::evaluate(*expr, arena, *stream_info, &request_headers, &response_headers, + Expr::evaluate(*expr, arena, nullptr, *stream_info, &request_headers, &response_headers, &response_trailers); } catch (const CelException& e) { ENVOY_LOG_MISC(debug, "CelException: {}", e.what()); diff --git a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc index 83674fc8f177..9486088ba523 100644 --- a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc +++ b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc @@ -66,11 +66,11 @@ class CheckRequestUtilsTest : public testing::Test { auto metadata_val = MessageUtil::keyValueStruct("foo", "bar"); (*metadata_context.mutable_filter_metadata())["meta.key"] = metadata_val; - CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, std::move(context_extensions), - std::move(metadata_context), request, - /*max_request_bytes=*/0, /*pack_as_bytes=*/false, - include_peer_certificate, want_tls_session != nullptr, - labels, nullptr); + CheckRequestUtils::createHttpCheck( + &callbacks_, request_headers, std::move(context_extensions), std::move(metadata_context), + envoy::config::core::v3::Metadata(), request, /*max_request_bytes=*/0, + /*pack_as_bytes=*/false, include_peer_certificate, want_tls_session != nullptr, labels, + nullptr); EXPECT_EQ("source", request.attributes().source().principal()); EXPECT_EQ("destination", request.attributes().destination().principal()); @@ -78,7 +78,6 @@ class CheckRequestUtilsTest : public testing::Test { EXPECT_EQ("value", request.attributes().context_extensions().at("key")); EXPECT_EQ("value_1", request.attributes().destination().labels().at("label_1")); EXPECT_EQ("value_2", request.attributes().destination().labels().at("label_2")); - EXPECT_EQ("bar", request.attributes() .metadata_context() .filter_metadata() @@ -86,6 +85,7 @@ class CheckRequestUtilsTest : public testing::Test { .fields() .at("foo") .string_value()); + EXPECT_TRUE(request.attributes().has_route_metadata_context()); if (include_peer_certificate) { EXPECT_EQ(cert_data_, request.attributes().source().certificate()); @@ -190,7 +190,7 @@ TEST_F(CheckRequestUtilsTest, BasicHttp) { expectBasicHttp(); CheckRequestUtils::createHttpCheck( &callbacks_, request_headers, Protobuf::Map(), - envoy::config::core::v3::Metadata(), request_, size, + envoy::config::core::v3::Metadata(), envoy::config::core::v3::Metadata(), request_, size, /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, /*include_tls_session=*/false, Protobuf::Map(), nullptr); ASSERT_EQ(size, request_.attributes().request().http().body().size()); @@ -218,9 +218,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithDuplicateHeaders) { expectBasicHttp(); CheckRequestUtils::createHttpCheck( &callbacks_, request_headers, Protobuf::Map(), - envoy::config::core::v3::Metadata(), request_, size, - /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, - /*include_tls_session=*/false, Protobuf::Map(), nullptr); + envoy::config::core::v3::Metadata(), envoy::config::core::v3::Metadata(), request_, size, + /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, /*include_tls_session=*/false, + Protobuf::Map(), nullptr); ASSERT_EQ(size, request_.attributes().request().http().body().size()); EXPECT_EQ(buffer_->toString().substr(0, size), request_.attributes().request().http().body()); EXPECT_EQ(",foo,bar", request_.attributes().request().http().headers().at("x-duplicate-header")); @@ -247,7 +247,7 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithRequestHeaderMatchers) { CheckRequestUtils::createHttpCheck( &callbacks_, request_headers, Protobuf::Map(), - envoy::config::core::v3::Metadata(), request_, size, + envoy::config::core::v3::Metadata(), envoy::config::core::v3::Metadata(), request_, size, /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, /*include_tls_session=*/false, Protobuf::Map(), createRequestHeaderMatchers()); ASSERT_EQ(size, request_.attributes().request().http().body().size()); @@ -270,9 +270,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithPartialBody) { expectBasicHttp(); CheckRequestUtils::createHttpCheck( &callbacks_, headers_, Protobuf::Map(), - envoy::config::core::v3::Metadata(), request_, size, - /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, - /*include_tls_session=*/false, Protobuf::Map(), nullptr); + envoy::config::core::v3::Metadata(), envoy::config::core::v3::Metadata(), request_, size, + /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, /*include_tls_session=*/false, + Protobuf::Map(), nullptr); ASSERT_EQ(size, request_.attributes().request().http().body().size()); EXPECT_EQ(buffer_->toString().substr(0, size), request_.attributes().request().http().body()); EXPECT_EQ("true", request_.attributes().request().http().headers().at( @@ -290,9 +290,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBody) { expectBasicHttp(); CheckRequestUtils::createHttpCheck( &callbacks_, headers_, Protobuf::Map(), - envoy::config::core::v3::Metadata(), request_, buffer_->length(), /*pack_as_bytes=*/false, - /*include_peer_certificate=*/false, /*include_tls_session=*/false, - Protobuf::Map(), nullptr); + envoy::config::core::v3::Metadata(), envoy::config::core::v3::Metadata(), request_, + buffer_->length(), /*pack_as_bytes=*/false, /*include_peer_certificate=*/false, + /*include_tls_session=*/false, Protobuf::Map(), nullptr); ASSERT_EQ(buffer_->length(), request_.attributes().request().http().body().size()); EXPECT_EQ(buffer_->toString().substr(0, buffer_->length()), request_.attributes().request().http().body()); @@ -323,9 +323,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBodyPackAsBytes) { // request_.SerializeToString() still returns "true" when it is failed to serialize the data. CheckRequestUtils::createHttpCheck( &callbacks_, headers_, Protobuf::Map(), - envoy::config::core::v3::Metadata(), request_, buffer_->length(), /*pack_as_bytes=*/true, - /*include_peer_certificate=*/false, /*include_tls_session=*/false, - Protobuf::Map(), nullptr); + envoy::config::core::v3::Metadata(), envoy::config::core::v3::Metadata(), request_, + buffer_->length(), /*pack_as_bytes=*/true, /*include_peer_certificate=*/false, + /*include_tls_session=*/false, Protobuf::Map(), nullptr); // TODO(dio): Find a way to test this without using function from testing::internal namespace. testing::internal::CaptureStderr(); diff --git a/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc index 57a9fd690675..b8772aa3b7e5 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc @@ -20,7 +20,6 @@ using testing::Eq; using testing::Invoke; using testing::Ref; using testing::Return; -using testing::Values; using testing::WhenDynamicCastTo; namespace Envoy { diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index 3ab57375af02..e7ca63a9a97b 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -374,7 +374,7 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithAddedAuthzHeadersFromStreamInf expected_header.second); StreamInfo::MockStreamInfo stream_info; - EXPECT_CALL(stream_info, getRequestHeaders()).Times(2).WillRepeatedly(Return(&request_headers)); + EXPECT_CALL(stream_info, getRequestHeaders()).WillOnce(Return(&request_headers)); envoy::service::auth::v3::CheckRequest request; client_->check(request_callbacks_, request, parent_span_, stream_info); diff --git a/test/extensions/filters/common/original_src/original_src_socket_option_test.cc b/test/extensions/filters/common/original_src/original_src_socket_option_test.cc index c123e0253f68..06877612512e 100644 --- a/test/extensions/filters/common/original_src/original_src_socket_option_test.cc +++ b/test/extensions/filters/common/original_src/original_src_socket_option_test.cc @@ -11,8 +11,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; - namespace Envoy { namespace Extensions { namespace Filters { diff --git a/test/extensions/filters/common/rbac/matchers_test.cc b/test/extensions/filters/common/rbac/matchers_test.cc index 6d9a582df765..c1a1dd37fb18 100644 --- a/test/extensions/filters/common/rbac/matchers_test.cc +++ b/test/extensions/filters/common/rbac/matchers_test.cc @@ -35,9 +35,7 @@ void checkMatcher( EXPECT_EQ(expected, matcher.matches(connection, headers, info)); } -PortRangeMatcher createPortRangeMatcher(envoy::type::v3::Int32Range range) { - return PortRangeMatcher(range); -} +PortRangeMatcher createPortRangeMatcher(envoy::type::v3::Int32Range range) { return {range}; } TEST(AlwaysMatcher, AlwaysMatches) { checkMatcher(RBAC::AlwaysMatcher(), true); } diff --git a/test/extensions/filters/common/set_filter_state/BUILD b/test/extensions/filters/common/set_filter_state/BUILD new file mode 100644 index 000000000000..9f5df8463e76 --- /dev/null +++ b/test/extensions/filters/common/set_filter_state/BUILD @@ -0,0 +1,22 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "filter_config_test", + srcs = ["filter_config_test.cc"], + deps = [ + "//source/common/router:string_accessor_lib", + "//source/extensions/filters/common/set_filter_state:filter_config_lib", + "//source/server:generic_factory_context_lib", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/common/set_filter_state/filter_config_test.cc b/test/extensions/filters/common/set_filter_state/filter_config_test.cc new file mode 100644 index 000000000000..bfbece8b40cd --- /dev/null +++ b/test/extensions/filters/common/set_filter_state/filter_config_test.cc @@ -0,0 +1,257 @@ +#include "source/common/router/string_accessor_impl.h" +#include "source/extensions/filters/common/set_filter_state/filter_config.h" +#include "source/server/generic_factory_context.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace SetFilterState { +namespace { + +class ObjectBarFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "bar"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +class ObjectFooFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "foo"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + if (data == "BAD_VALUE") { + return nullptr; + } + return std::make_unique(data); + } +}; + +REGISTER_FACTORY(ObjectBarFactory, StreamInfo::FilterState::ObjectFactory); +REGISTER_FACTORY(ObjectFooFactory, StreamInfo::FilterState::ObjectFactory); + +class ConfigTest : public testing::Test { +public: + void initialize(const std::vector& values, + LifeSpan life_span = LifeSpan::FilterChain) { + std::vector proto_values; + proto_values.reserve(values.size()); + for (const auto& value : values) { + FilterStateValueProto proto_value; + TestUtility::loadFromYaml(value, proto_value); + proto_values.push_back(proto_value); + } + config_ = std::make_shared( + Protobuf::RepeatedPtrField(proto_values.begin(), proto_values.end()), + life_span, context_); + } + void update() { config_->updateFilterState({&header_map_}, info_); } + NiceMock context_; + Http::TestRequestHeaderMapImpl header_map_{{"test-header", "test-value"}}; + NiceMock info_; + ConfigSharedPtr config_; +}; + +TEST_F(ConfigTest, SetValue) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "XXX" + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "XXX"); + EXPECT_EQ(0, info_.filterState()->objectsSharedWithUpstreamConnection()->size()); +} + +TEST_F(ConfigTest, SetValueConnection) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "XXX" + )YAML"}, + LifeSpan::Connection); + update(); + EXPECT_TRUE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "XXX"); + EXPECT_EQ(0, info_.filterState()->objectsSharedWithUpstreamConnection()->size()); +} + +TEST_F(ConfigTest, UpdateValue) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "XXX" + )YAML"}); + info_.filterState()->setData("foo", std::make_unique("OLD"), + StateType::Mutable); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "XXX"); + EXPECT_EQ(0, info_.filterState()->objectsSharedWithUpstreamConnection()->size()); +} + +TEST_F(ConfigTest, SetValueFromHeader) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "%REQ(test-header)%" + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "test-value"); + EXPECT_EQ(0, info_.filterState()->objectsSharedWithUpstreamConnection()->size()); +} + +TEST_F(ConfigTest, MultipleValues) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "XXX" + )YAML", + R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "YYY" + )YAML", + R"YAML( + object_key: bar + format_string: + text_format_source: + inline_string: "ZZZ" + )YAML"}); + update(); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + const auto* bar = info_.filterState()->getDataReadOnly("bar"); + ASSERT_NE(nullptr, bar); + EXPECT_EQ(foo->serializeAsString(), "YYY"); + EXPECT_EQ(bar->serializeAsString(), "ZZZ"); + EXPECT_EQ(0, info_.filterState()->objectsSharedWithUpstreamConnection()->size()); +} + +TEST_F(ConfigTest, BadValue) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "BAD_VALUE" + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + EXPECT_EQ(nullptr, foo); +} + +TEST_F(ConfigTest, MissingKey) { + EXPECT_THROW_WITH_MESSAGE(initialize({R"YAML( + object_key: unknown_key + format_string: + text_format_source: + inline_string: "XXX" + )YAML"}), + EnvoyException, "'unknown_key' does not have an object factory"); +} + +TEST_F(ConfigTest, EmptyValue) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "" + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), ""); + EXPECT_EQ(0, info_.filterState()->objectsSharedWithUpstreamConnection()->size()); +} + +TEST_F(ConfigTest, EmptyValueSkip) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "" + skip_if_empty: true + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + EXPECT_EQ(nullptr, foo); +} + +TEST_F(ConfigTest, SetValueUpstreamSharedOnce) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "XXX" + shared_with_upstream: ONCE + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "XXX"); + const auto objects = info_.filterState()->objectsSharedWithUpstreamConnection(); + EXPECT_EQ(1, objects->size()); + EXPECT_EQ(StreamSharing::None, objects->at(0).stream_sharing_); + EXPECT_EQ(StateType::Mutable, objects->at(0).state_type_); + EXPECT_EQ("foo", objects->at(0).name_); + EXPECT_EQ(foo, objects->at(0).data_.get()); +} + +TEST_F(ConfigTest, SetValueUpstreamSharedTransitive) { + initialize({R"YAML( + object_key: foo + format_string: + text_format_source: + inline_string: "XXX" + shared_with_upstream: TRANSITIVE + read_only: true + )YAML"}); + update(); + EXPECT_FALSE(info_.filterState()->hasDataAtOrAboveLifeSpan(LifeSpan::Request)); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "XXX"); + const auto objects = info_.filterState()->objectsSharedWithUpstreamConnection(); + EXPECT_EQ(1, objects->size()); + EXPECT_EQ(StreamSharing::SharedWithUpstreamConnection, objects->at(0).stream_sharing_); + EXPECT_EQ(StateType::ReadOnly, objects->at(0).state_type_); + EXPECT_EQ("foo", objects->at(0).name_); + EXPECT_EQ(foo, objects->at(0).data_.get()); +} + +} // namespace +} // namespace SetFilterState +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/adaptive_concurrency/BUILD b/test/extensions/filters/http/adaptive_concurrency/BUILD index a068e3d4d61c..287454b822c8 100644 --- a/test/extensions/filters/http/adaptive_concurrency/BUILD +++ b/test/extensions/filters/http/adaptive_concurrency/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", "envoy_package", ) load( @@ -27,15 +28,20 @@ envoy_extension_cc_test( ], ) +envoy_cc_test_library( + name = "adaptive_concurrency_filter_integration_test_lib", + hdrs = ["adaptive_concurrency_filter_integration_test.h"], +) + envoy_extension_cc_test( name = "adaptive_concurrency_integration_test", size = "large", srcs = [ "adaptive_concurrency_filter_integration_test.cc", - "adaptive_concurrency_filter_integration_test.h", ], extension_names = ["envoy.filters.http.adaptive_concurrency"], deps = [ + ":adaptive_concurrency_filter_integration_test_lib", "//source/extensions/filters/http/adaptive_concurrency:config", "//source/extensions/filters/http/fault:config", "//test/integration:http_integration_lib", diff --git a/test/extensions/filters/http/admission_control/admission_control_filter_test.cc b/test/extensions/filters/http/admission_control/admission_control_filter_test.cc index ef76731bd79f..cd13fe27a98e 100644 --- a/test/extensions/filters/http/admission_control/admission_control_filter_test.cc +++ b/test/extensions/filters/http/admission_control/admission_control_filter_test.cc @@ -67,8 +67,8 @@ class AdmissionControlTest : public testing::Test { std::shared_ptr makeConfig(const std::string& yaml) { AdmissionControlProto proto; TestUtility::loadFromYamlAndValidate(yaml, proto); - auto tls = - ThreadLocal::TypedSlot::makeUnique(context_.threadLocal()); + auto tls = ThreadLocal::TypedSlot::makeUnique( + context_.server_factory_context_.threadLocal()); evaluator_ = std::make_shared(); return std::make_shared(proto, runtime_, random_, scope_, std::move(tls), diff --git a/test/extensions/filters/http/admission_control/config_test.cc b/test/extensions/filters/http/admission_control/config_test.cc index 556b134a12ea..8e6990e8c344 100644 --- a/test/extensions/filters/http/admission_control/config_test.cc +++ b/test/extensions/filters/http/admission_control/config_test.cc @@ -33,8 +33,8 @@ class AdmissionControlConfigTest : public testing::Test { std::shared_ptr makeConfig(const std::string& yaml) { AdmissionControlProto proto; TestUtility::loadFromYamlAndValidate(yaml, proto); - auto tls = - ThreadLocal::TypedSlot::makeUnique(context_.threadLocal()); + auto tls = ThreadLocal::TypedSlot::makeUnique( + context_.server_factory_context_.threadLocal()); auto evaluator = std::make_unique(proto.success_criteria()); return std::make_shared(proto, runtime_, random_, scope_, std::move(tls), std::move(evaluator)); @@ -75,8 +75,11 @@ sampling_window: 1337s TestUtility::loadFromYamlAndValidate(yaml, proto); NiceMock factory_context; EXPECT_THROW_WITH_MESSAGE( - admission_control_filter_factory.createFilterFactoryFromProtoTyped( - proto, "whatever", dual_info_, factory_context.getServerFactoryContext()), + admission_control_filter_factory + .createFilterFactoryFromProtoTyped(proto, "whatever", dual_info_, + factory_context.serverFactoryContext()) + .status() + .IgnoreError(), EnvoyException, "Success rate threshold cannot be less than 1.0%."); } @@ -103,8 +106,11 @@ sampling_window: 1337s TestUtility::loadFromYamlAndValidate(yaml, proto); NiceMock factory_context; EXPECT_THROW_WITH_MESSAGE( - admission_control_filter_factory.createFilterFactoryFromProtoTyped( - proto, "whatever", dual_info_, factory_context.getServerFactoryContext()), + admission_control_filter_factory + .createFilterFactoryFromProtoTyped(proto, "whatever", dual_info_, + factory_context.serverFactoryContext()) + .status() + .IgnoreError(), EnvoyException, "Success rate threshold cannot be less than 1.0%."); } diff --git a/test/extensions/filters/http/alternate_protocols_cache/filter_test.cc b/test/extensions/filters/http/alternate_protocols_cache/filter_test.cc index 920a6f296f4c..8026efc25152 100644 --- a/test/extensions/filters/http/alternate_protocols_cache/filter_test.cc +++ b/test/extensions/filters/http/alternate_protocols_cache/filter_test.cc @@ -162,6 +162,62 @@ TEST_F(FilterTest, ValidAltSvc) { filter_->onDestroy(); } +TEST_F(FilterTest, ValidAltSvcMissingPort) { + Http::TestResponseHeaderMapImpl headers{ + {":status", "200"}, {"alt-svc", "h3-29=\":443\"; ma=86400, h3=\":443\"; ma=60"}}; + Http::HttpServerPropertiesCache::Origin expected_origin("https", "host1", 443); + MonotonicTime now = simTime().monotonicTime(); + const std::vector expected_protocols = { + Http::HttpServerPropertiesCache::AlternateProtocol("h3-29", "", 443, + now + std::chrono::seconds(86400)), + Http::HttpServerPropertiesCache::AlternateProtocol("h3", "", 443, + now + std::chrono::seconds(60)), + }; + + std::string hostname = "host1"; + + // Set up the cluster info correctly to have a cache configuration. + envoy::extensions::filters::http::alternate_protocols_cache::v3::FilterConfig proto_config; + proto_config.mutable_alternate_protocols_cache_options()->set_name("foo"); + auto info = std::make_shared>(); + callbacks_.stream_info_.upstream_cluster_info_ = info; + absl::optional options = + proto_config.alternate_protocols_cache_options(); + ON_CALL(*info, alternateProtocolsCacheOptions()).WillByDefault(ReturnRef(options)); + EXPECT_CALL(*alternate_protocols_cache_manager_, getCache(_, _)) + .Times(testing::AnyNumber()) + .WillOnce(Return(alternate_protocols_cache_)); + + EXPECT_CALL(callbacks_, streamInfo()) + .Times(testing::AtLeast(1)) + .WillRepeatedly(ReturnRef(callbacks_.stream_info_)); + EXPECT_CALL(callbacks_.stream_info_, upstreamClusterInfo()) + .Times(testing::AtLeast(1)) + .WillRepeatedly(Return(info)); + EXPECT_CALL(callbacks_.stream_info_, upstreamInfo()).Times(testing::AtLeast(1)); + // Get the pointer to MockHostDescription. + std::shared_ptr hd = + std::dynamic_pointer_cast( + callbacks_.stream_info_.upstreamInfo()->upstreamHost()); + EXPECT_CALL(*hd, hostname()).WillOnce(ReturnRef(hostname)); + // The address() call returns nullptr, so we won't get a port, but the filter should use the + // default port. + EXPECT_CALL(*hd, address()).WillOnce(Return(nullptr)); + EXPECT_CALL(*alternate_protocols_cache_, setAlternatives(_, _)) + .WillOnce(testing::DoAll( + testing::WithArg<0>(Invoke([expected_origin](auto& actual_origin) { + EXPECT_EQ(expected_origin, actual_origin) + << dumpOrigin(expected_origin) << dumpOrigin(actual_origin); + })), + testing::WithArg<1>(Invoke([expected_protocols](auto& actual_protocols) { + EXPECT_EQ(expected_protocols, actual_protocols) << dumpAlternative(actual_protocols[0]); + ; + })))); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); + filter_->onDestroy(); +} + TEST_F(FilterTest, ValidAltSvcLegacy) { TestScopedRuntime scoped_runtime; scoped_runtime.mergeValues( diff --git a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc index b42b5977b839..bee8f3ad2a2e 100644 --- a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc +++ b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_integration_test.cc @@ -23,6 +23,7 @@ class AwsLambdaFilterIntegrationTest : public testing::TestWithParam context; AwsLambdaFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; auto has_expected_settings = [](std::shared_ptr stream_filter) { auto filter = std::static_pointer_cast(stream_filter); @@ -63,7 +64,8 @@ arn: "arn:aws:lambda:region:424242:function:fun" testing::NiceMock context; AwsLambdaFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; auto has_expected_settings = [](std::shared_ptr stream_filter) { auto filter = std::static_pointer_cast(stream_filter); @@ -110,8 +112,9 @@ arn: "arn:aws:lambda:region:424242:fun" testing::NiceMock context; AwsLambdaFilterFactory factory; - EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "stats", context), - EnvoyException); + EXPECT_THROW( + factory.createFilterFactoryFromProto(proto_config, "stats", context).status().IgnoreError(), + EnvoyException); } TEST(AwsLambdaFilterConfigTest, PerRouteConfigWithInvalidARNThrows) { diff --git a/test/extensions/filters/http/aws_request_signing/config_test.cc b/test/extensions/filters/http/aws_request_signing/config_test.cc index 95e8773f5f96..a3e581ffe73e 100644 --- a/test/extensions/filters/http/aws_request_signing/config_test.cc +++ b/test/extensions/filters/http/aws_request_signing/config_test.cc @@ -45,7 +45,8 @@ host_rewrite: new-host testing::NiceMock context; AwsRequestSigningFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamDecoderFilter(_)); cb(filter_callbacks); diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index c7b66374d8f9..aa237f13aab5 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -24,8 +24,8 @@ TEST(Factory, GlobalEmptyConfig) { NiceMock context; - EXPECT_CALL(context.dispatcher_, createTimer_(_)).Times(0); - auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + EXPECT_CALL(context.server_factory_context_.dispatcher_, createTimer_(_)).Times(0); + auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); callback(filter_callback); diff --git a/test/extensions/filters/http/basic_auth/BUILD b/test/extensions/filters/http/basic_auth/BUILD new file mode 100644 index 000000000000..39e573d580cd --- /dev/null +++ b/test/extensions/filters/http/basic_auth/BUILD @@ -0,0 +1,45 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "filter_test", + srcs = ["filter_test.cc"], + extension_names = ["envoy.filters.http.basic_auth"], + deps = [ + "//source/extensions/filters/http/basic_auth:basic_auth_lib", + "//test/mocks/server:server_mocks", + "@envoy_api//envoy/extensions/filters/http/basic_auth/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.filters.http.basic_auth"], + deps = [ + "//source/extensions/filters/http/basic_auth:config", + "//test/mocks/server:server_mocks", + ], +) + +envoy_extension_cc_test( + name = "basic_auth_integration_test", + size = "large", + srcs = ["basic_auth_integration_test.cc"], + extension_names = ["envoy.filters.http.basic_auth"], + deps = [ + "//source/extensions/filters/http/basic_auth:config", + "//test/integration:http_protocol_integration_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/basic_auth/basic_auth_integration_test.cc b/test/extensions/filters/http/basic_auth/basic_auth_integration_test.cc new file mode 100644 index 000000000000..c66d98084b1f --- /dev/null +++ b/test/extensions/filters/http/basic_auth/basic_auth_integration_test.cc @@ -0,0 +1,119 @@ +#include "test/integration/http_protocol_integration.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { +namespace { + +class BasicAuthIntegrationTest : public HttpProtocolIntegrationTest { +public: + void initializeFilter() { + // user1, test1 + // user2, test2 + const std::string filter_config = + R"EOF( +name: envoy.filters.http.basic_auth +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth + users: + inline_string: |- + user1:{SHA}tESsBmE/yNY3lb6a0L6vVQEZNqw= + user2:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= +)EOF"; + config_helper_.prependFilter(filter_config); + initialize(); + } +}; + +// BasicAuth integration tests that should run with all protocols +class BasicAuthIntegrationTestAllProtocols : public BasicAuthIntegrationTest {}; + +INSTANTIATE_TEST_SUITE_P( + Protocols, BasicAuthIntegrationTestAllProtocols, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParamsWithoutHTTP3()), + HttpProtocolIntegrationTest::protocolTestParamsToString); + +// Request with valid credential +TEST_P(BasicAuthIntegrationTestAllProtocols, ValidCredential) { + initializeFilter(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Basic dXNlcjE6dGVzdDE="}, // user1, test1 + }); + + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + +// Request without credential +TEST_P(BasicAuthIntegrationTestAllProtocols, NoCredential) { + initializeFilter(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + }); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("401", response->headers().getStatusValue()); + EXPECT_EQ("User authentication failed. Missing username and password.", response->body()); +} + +// Request without wrong password +TEST_P(BasicAuthIntegrationTestAllProtocols, WrongPasswrod) { + initializeFilter(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Basic dXNlcjE6dGVzdDI="}, // user1, test2 + }); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("401", response->headers().getStatusValue()); + EXPECT_EQ("User authentication failed. Invalid username/password combination.", response->body()); +} + +// Request with none-existed user +TEST_P(BasicAuthIntegrationTestAllProtocols, NoneExistedUser) { + initializeFilter(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"Authorization", "Basic dXNlcjM6dGVzdDI="}, // user3, test2 + }); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("401", response->headers().getStatusValue()); + EXPECT_EQ("User authentication failed. Invalid username/password combination.", response->body()); +} +} // namespace +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/basic_auth/config_test.cc b/test/extensions/filters/http/basic_auth/config_test.cc new file mode 100644 index 000000000000..5ed811995e35 --- /dev/null +++ b/test/extensions/filters/http/basic_auth/config_test.cc @@ -0,0 +1,152 @@ +#include "source/extensions/filters/http/basic_auth/basic_auth_filter.h" +#include "source/extensions/filters/http/basic_auth/config.h" + +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { + +TEST(Factory, ValidConfig) { + const std::string yaml = R"( + users: + inline_string: |- + # comment line + user1:{SHA}tESsBmE/yNY3lb6a0L6vVQEZNqw= + user2:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + Http::MockFilterChainFactoryCallbacks filter_callback; + EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); + callback.value()(filter_callback); +} + +TEST(Factory, InvalidConfigNoColon) { + const std::string yaml = R"( + users: + inline_string: |- + user1{SHA}tESsBmE/yNY3lb6a0L6vVQEZNqw= + user2:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_THROW( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException); +} + +TEST(Factory, InvalidConfigDuplicateUsers) { + const std::string yaml = R"( + users: + inline_string: |- + user1:{SHA}tESsBmE/yNY3lb6a0L6vVQEZNqw= + user1:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "basic auth: duplicate users"); +} + +TEST(Factory, InvalidConfigNoUser) { + const std::string yaml = R"( + users: + inline_string: |- + :{SHA}tESsBmE/yNY3lb6a0L6vVQEZNqw= + user2:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "basic auth: empty user name or password"); +} + +TEST(Factory, InvalidConfigNoPassword) { + const std::string yaml = R"( + users: + inline_string: |- + user1: + user2:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "basic auth: empty user name or password"); +} + +TEST(Factory, InvalidConfigNoHash) { + const std::string yaml = R"( + users: + inline_string: |- + user1:{SHA} + user2:{SHA}EJ9LPFDXsN9ynSmbxvjp75Bmlx8= + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "basic auth: invalid htpasswd format, invalid SHA hash length"); +} + +TEST(Factory, InvalidConfigNotSHA) { + const std::string yaml = R"( + users: + inline_string: |- + user1:{SHA}tESsBmE/yNY3lb6a0L6vVQEZNqw= + user2:$apr1$0vAnUTEB$4EJJr0GR3y48WF2AiieWs. + )"; + + BasicAuthFilterFactory factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + NiceMock context; + + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "basic auth: unsupported htpasswd format: please use {SHA}"); +} + +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/basic_auth/filter_test.cc b/test/extensions/filters/http/basic_auth/filter_test.cc new file mode 100644 index 000000000000..375c005705bc --- /dev/null +++ b/test/extensions/filters/http/basic_auth/filter_test.cc @@ -0,0 +1,136 @@ +#include "envoy/extensions/filters/http/basic_auth/v3/basic_auth.pb.h" + +#include "source/extensions/filters/http/basic_auth/basic_auth_filter.h" + +#include "test/mocks/http/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace BasicAuth { + +class FilterTest : public testing::Test { +public: + FilterTest() { + UserMap users; + users.insert({"user1", {"user1", "tESsBmE/yNY3lb6a0L6vVQEZNqw="}}); // user1:test1 + users.insert({"user2", {"user2", "EJ9LPFDXsN9ynSmbxvjp75Bmlx8="}}); // user2:test2 + config_ = std::make_unique(std::move(users), "stats", *stats_.rootScope()); + filter_ = std::make_shared(config_); + filter_->setDecoderFilterCallbacks(decoder_filter_callbacks_); + } + + NiceMock stats_; + NiceMock decoder_filter_callbacks_; + FilterConfigConstSharedPtr config_; + std::shared_ptr filter_; +}; + +TEST_F(FilterTest, BasicAuth) { + // user1:test1 + Http::TestRequestHeaderMapImpl request_headers_user1{{"Authorization", "Basic dXNlcjE6dGVzdDE="}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(request_headers_user1, true)); + + // user2:test2 + Http::TestRequestHeaderMapImpl request_headers_user2{{"Authorization", "Basic dXNlcjI6dGVzdDI="}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(request_headers_user2, true)); +} + +TEST_F(FilterTest, UserNotExist) { + // user3:test2 + Http::TestRequestHeaderMapImpl request_headers_user1{{"Authorization", "Basic dXNlcjM6dGVzdDI="}}; + + EXPECT_CALL(decoder_filter_callbacks_, sendLocalReply(_, _, _, _, _)) + .WillOnce(Invoke([&](Http::Code code, absl::string_view body, + std::function, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::Unauthorized, code); + EXPECT_EQ("User authentication failed. Invalid username/password combination.", body); + EXPECT_EQ(grpc_status, absl::nullopt); + EXPECT_EQ(details, "invalid_credential_for_basic_auth"); + })); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_user1, true)); +} + +TEST_F(FilterTest, InvalidPassword) { + // user1:test2 + Http::TestRequestHeaderMapImpl request_headers_user1{{"Authorization", "Basic dXNlcjE6dGVzdDI="}}; + + EXPECT_CALL(decoder_filter_callbacks_, sendLocalReply(_, _, _, _, _)) + .WillOnce(Invoke([&](Http::Code code, absl::string_view body, + std::function, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::Unauthorized, code); + EXPECT_EQ("User authentication failed. Invalid username/password combination.", body); + EXPECT_EQ(grpc_status, absl::nullopt); + EXPECT_EQ(details, "invalid_credential_for_basic_auth"); + })); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_user1, true)); +} + +TEST_F(FilterTest, NoAuthHeader) { + Http::TestRequestHeaderMapImpl request_headers_user1; + + EXPECT_CALL(decoder_filter_callbacks_, sendLocalReply(_, _, _, _, _)) + .WillOnce(Invoke([&](Http::Code code, absl::string_view body, + std::function, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::Unauthorized, code); + EXPECT_EQ("User authentication failed. Missing username and password.", body); + EXPECT_EQ(grpc_status, absl::nullopt); + EXPECT_EQ(details, "no_credential_for_basic_auth"); + })); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_user1, true)); +} + +TEST_F(FilterTest, HasAuthHeaderButNotForBasic) { + Http::TestRequestHeaderMapImpl request_headers_user1{{"Authorization", "Bearer xxxxxxx"}}; + + EXPECT_CALL(decoder_filter_callbacks_, sendLocalReply(_, _, _, _, _)) + .WillOnce(Invoke([&](Http::Code code, absl::string_view body, + std::function, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::Unauthorized, code); + EXPECT_EQ("User authentication failed. Expected 'Basic' authentication scheme.", body); + EXPECT_EQ(grpc_status, absl::nullopt); + EXPECT_EQ(details, "invalid_scheme_for_basic_auth"); + })); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_user1, true)); +} + +TEST_F(FilterTest, HasAuthHeaderButNoColon) { + Http::TestRequestHeaderMapImpl request_headers_user1{{"Authorization", "Basic dXNlcjE="}}; + + EXPECT_CALL(decoder_filter_callbacks_, sendLocalReply(_, _, _, _, _)) + .WillOnce(Invoke([&](Http::Code code, absl::string_view body, + std::function, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::Unauthorized, code); + EXPECT_EQ("User authentication failed. Invalid basic credential format.", body); + EXPECT_EQ(grpc_status, absl::nullopt); + EXPECT_EQ(details, "invalid_format_for_basic_auth"); + })); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_user1, true)); +} + +} // namespace BasicAuth +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc index 4bf081a13f7a..ee3d5607dc0f 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc @@ -147,7 +147,7 @@ ConfigHelper::HttpModifierFunction overrideConfig(const std::string& json_config ->Mutable(0) ->mutable_typed_per_filter_config(); - (*config)["envoy.filters.http.buffer"].PackFrom(buffer_per_route); + (*config)["buffer"].PackFrom(buffer_per_route); }; } diff --git a/test/extensions/filters/http/buffer/config_test.cc b/test/extensions/filters/http/buffer/config_test.cc index ae58549733d0..7166a27bd219 100644 --- a/test/extensions/filters/http/buffer/config_test.cc +++ b/test/extensions/filters/http/buffer/config_test.cc @@ -28,7 +28,8 @@ TEST(BufferFilterFactoryTest, BufferFilterCorrectYaml) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; BufferFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -40,7 +41,7 @@ TEST(BufferFilterFactoryTest, BufferFilterCorrectProto) { NiceMock context; BufferFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -52,7 +53,7 @@ TEST(BufferFilterFactoryTest, BufferFilterCorrectProtoUpstreamFactory) { NiceMock context; BufferFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -67,7 +68,7 @@ TEST(BufferFilterFactoryTest, BufferFilterEmptyProto) { config.mutable_max_request_bytes()->set_value(1028); NiceMock context; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -80,7 +81,7 @@ TEST(BufferFilterFactoryTest, BufferFilterNoMaxRequestBytes) { *dynamic_cast(empty_proto.get()); NiceMock context; - EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(config, "stats", context).value(), EnvoyException, "Proto constraint validation failed"); } diff --git a/test/extensions/filters/http/cache/BUILD b/test/extensions/filters/http/cache/BUILD index 65930a8771f3..f45131e56f5d 100644 --- a/test/extensions/filters/http/cache/BUILD +++ b/test/extensions/filters/http/cache/BUILD @@ -9,16 +9,6 @@ licenses(["notice"]) # Apache 2 envoy_package() -envoy_cc_test_library( - name = "common", - srcs = ["common.cc"], - hdrs = ["common.h"], - deps = [ - "//source/extensions/filters/http/cache:cache_headers_utils_lib", - "//source/extensions/filters/http/cache:http_cache_lib", - ], -) - envoy_cc_test_library( name = "mocks", hdrs = ["mocks.h"], @@ -32,7 +22,6 @@ envoy_extension_cc_test( srcs = ["cache_headers_utils_test.cc"], extension_names = ["envoy.filters.http.cache"], deps = [ - ":common", "//envoy/http:header_map_interface", "//source/common/http:header_map_lib", "//source/extensions/filters/http/cache:cache_headers_utils_lib", @@ -64,11 +53,11 @@ envoy_extension_cc_test( srcs = ["http_cache_test.cc"], extension_names = ["envoy.filters.http.cache"], deps = [ - ":common", "//source/extensions/filters/http/cache:http_cache_lib", "//source/extensions/http/cache/simple_http_cache:config", "//test/mocks/http:http_mocks", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], ) @@ -78,7 +67,6 @@ envoy_extension_cc_test( srcs = ["range_utils_test.cc"], extension_names = ["envoy.filters.http.cache"], deps = [ - ":common", "//source/extensions/filters/http/cache:range_utils_lib", "//test/test_common:utility_lib", ], @@ -89,7 +77,6 @@ envoy_extension_cc_test( srcs = ["cache_filter_test.cc"], extension_names = ["envoy.filters.http.cache"], deps = [ - ":common", ":mocks", "//source/extensions/filters/http/cache:cache_filter_lib", "//source/extensions/filters/http/cache:cache_filter_logging_info_lib", @@ -162,7 +149,6 @@ envoy_extension_cc_test_library( deps = [ "//source/extensions/filters/http/cache:cache_headers_utils_lib", "//source/extensions/filters/http/cache:http_cache_lib", - "//test/extensions/filters/http/cache:common", "//test/mocks/server:factory_context_mocks", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/filters/http/cache/cache_filter_test.cc b/test/extensions/filters/http/cache/cache_filter_test.cc index fd86617efb86..d5a91f1a719b 100644 --- a/test/extensions/filters/http/cache/cache_filter_test.cc +++ b/test/extensions/filters/http/cache/cache_filter_test.cc @@ -5,7 +5,6 @@ #include "source/extensions/filters/http/cache/cache_filter_logging_info.h" #include "source/extensions/http/cache/simple_http_cache/simple_http_cache.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/extensions/filters/http/cache/mocks.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/simulated_time_system.h" @@ -31,15 +30,15 @@ class CacheFilterTest : public ::testing::Test { // The filter has to be created as a shared_ptr to enable shared_from_this() which is used in the // cache callbacks. CacheFilterSharedPtr makeFilter(std::shared_ptr cache, bool auto_destroy = true) { - std::shared_ptr filter(new CacheFilter(config_, /*stats_prefix=*/"", - context_.scope(), context_.timeSource(), - cache), - [auto_destroy](CacheFilter* f) { - if (auto_destroy) { - f->onDestroy(); - } - delete f; - }); + std::shared_ptr filter( + new CacheFilter(config_, /*stats_prefix=*/"", context_.scope(), + context_.server_factory_context_.timeSource(), cache), + [auto_destroy](CacheFilter* f) { + if (auto_destroy) { + f->onDestroy(); + } + delete f; + }); filter_state_ = std::make_shared( StreamInfo::FilterState::LifeSpan::FilterChain); filter->setDecoderFilterCallbacks(decoder_callbacks_); @@ -1150,21 +1149,18 @@ TEST_F(CacheFilterDeathTest, StreamTimeoutDuringLookup) { } TEST(LookupStatusDeathTest, ResolveLookupStatusRequireValidationAndInitialIsBug) { - GTEST_SKIP(); // TODO(issue #29217): Remove skip. EXPECT_ENVOY_BUG( CacheFilter::resolveLookupStatus(CacheEntryStatus::RequiresValidation, FilterState::Initial), "Unexpected filter state in requestCacheStatus"); } TEST(LookupStatusDeathTest, ResolveLookupStatusRequireValidationAndDecodeServingFromCacheIsBug) { - GTEST_SKIP(); // TODO(issue #29217): Remove skip. EXPECT_ENVOY_BUG(CacheFilter::resolveLookupStatus(CacheEntryStatus::RequiresValidation, FilterState::DecodeServingFromCache), "Unexpected filter state in requestCacheStatus"); } TEST(LookupStatusDeathTest, ResolveLookupStatusRequireValidationAndDestroyedIsBug) { - GTEST_SKIP(); // TODO(issue #29217): Remove skip. EXPECT_ENVOY_BUG(CacheFilter::resolveLookupStatus(CacheEntryStatus::RequiresValidation, FilterState::Destroyed), "Unexpected filter state in requestCacheStatus"); diff --git a/test/extensions/filters/http/cache/cache_headers_utils_test.cc b/test/extensions/filters/http/cache/cache_headers_utils_test.cc index 35a9a7dd60d2..2b5a0de3304e 100644 --- a/test/extensions/filters/http/cache/cache_headers_utils_test.cc +++ b/test/extensions/filters/http/cache/cache_headers_utils_test.cc @@ -10,7 +10,6 @@ #include "source/common/http/header_utility.h" #include "source/extensions/filters/http/cache/cache_headers_utils.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -155,6 +154,27 @@ TEST_P(RequestCacheControlTest, RequestCacheControlTest) { EXPECT_EQ(expected_request_cache_control, RequestCacheControl(cache_control_header)); } +// operator<<(ostream&, const RequestCacheControl&) is only used in tests, but lives in //source, +// and so needs test coverage. This test provides that coverage, to keep the coverage test happy. +TEST(RequestCacheControl, StreamingTest) { + std::ostringstream os; + RequestCacheControl request_cache_control( + "no-cache, no-store, no-transform, only-if-cached, max-age=0, min-fresh=0, max-stale=0"); + os << request_cache_control; + EXPECT_EQ(os.str(), "{must_validate, no_store, no_transform, only_if_cached, max-age=0, " + "min-fresh=0, max-stale=0}"); +} + +// operator<<(ostream&, const ResponseCacheControl&) is only used in tests, but lives in //source, +// and so needs test coverage. This test provides that coverage, to keep the coverage test happy. +TEST(ResponseCacheControl, StreamingTest) { + std::ostringstream os; + ResponseCacheControl response_cache_control( + "no-cache, must-revalidate, no-store, no-transform, max-age=0"); + os << response_cache_control; + EXPECT_EQ(os.str(), "{must_validate, no_store, no_transform, no_stale, max-age=0}"); +} + struct TestResponseCacheControl : public ResponseCacheControl { TestResponseCacheControl(bool must_validate, bool no_store, bool no_transform, bool no_stale, bool is_public, OptionalDuration max_age) { diff --git a/test/extensions/filters/http/cache/common.cc b/test/extensions/filters/http/cache/common.cc deleted file mode 100644 index 93c4c995c95e..000000000000 --- a/test/extensions/filters/http/cache/common.cc +++ /dev/null @@ -1,72 +0,0 @@ -#include "test/extensions/filters/http/cache/common.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Cache { - -std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control) { - std::vector fields; - - if (request_cache_control.must_validate_) { - fields.push_back("must_validate"); - } - if (request_cache_control.no_store_) { - fields.push_back("no_store"); - } - if (request_cache_control.no_transform_) { - fields.push_back("no_transform"); - } - if (request_cache_control.only_if_cached_) { - fields.push_back("only_if_cached"); - } - if (request_cache_control.max_age_.has_value()) { - fields.push_back( - absl::StrCat("max-age=", std::to_string(request_cache_control.max_age_->count()))); - } - if (request_cache_control.min_fresh_.has_value()) { - fields.push_back( - absl::StrCat("min-fresh=", std::to_string(request_cache_control.min_fresh_->count()))); - } - if (request_cache_control.max_stale_.has_value()) { - fields.push_back( - absl::StrCat("max-stale=", std::to_string(request_cache_control.max_stale_->count()))); - } - - return os << "{" << absl::StrJoin(fields, ", ") << "}"; -} - -std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control) { - std::vector fields; - - if (response_cache_control.must_validate_) { - fields.push_back("must_validate"); - } - if (response_cache_control.no_store_) { - fields.push_back("no_store"); - } - if (response_cache_control.no_transform_) { - fields.push_back("no_transform"); - } - if (response_cache_control.no_stale_) { - fields.push_back("no_stale"); - } - if (response_cache_control.is_public_) { - fields.push_back("public"); - } - if (response_cache_control.max_age_.has_value()) { - fields.push_back( - absl::StrCat("max-age=", std::to_string(response_cache_control.max_age_->count()))); - } - - return os << "{" << absl::StrJoin(fields, ", ") << "}"; -} - -std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range) { - return os << "[" << range.begin() << "," << range.end() << ")"; -} - -} // namespace Cache -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/filters/http/cache/common.h b/test/extensions/filters/http/cache/common.h deleted file mode 100644 index 6b6145c18eba..000000000000 --- a/test/extensions/filters/http/cache/common.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "source/extensions/filters/http/cache/cache_headers_utils.h" -#include "source/extensions/filters/http/cache/http_cache.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Cache { - -std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control); - -std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control); - -std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range); - -} // namespace Cache -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/filters/http/cache/config_test.cc b/test/extensions/filters/http/cache/config_test.cc index 109d1e000fd7..e892ccb662ee 100644 --- a/test/extensions/filters/http/cache/config_test.cc +++ b/test/extensions/filters/http/cache/config_test.cc @@ -25,7 +25,8 @@ class CacheFilterFactoryTest : public ::testing::Test { TEST_F(CacheFilterFactoryTest, Basic) { config_.mutable_typed_config()->PackFrom( envoy::extensions::http::cache::simple_http_cache::v3::SimpleHttpCacheConfig()); - Http::FilterFactoryCb cb = factory_.createFilterFactoryFromProto(config_, "stats", context_); + Http::FilterFactoryCb cb = + factory_.createFilterFactoryFromProto(config_, "stats", context_).value(); Http::StreamFilterSharedPtr filter; EXPECT_CALL(filter_callback_, addStreamFilter(_)).WillOnce(::testing::SaveArg<0>(&filter)); cb(filter_callback_); @@ -35,7 +36,8 @@ TEST_F(CacheFilterFactoryTest, Basic) { TEST_F(CacheFilterFactoryTest, Disabled) { config_.mutable_disabled()->set_value(true); - Http::FilterFactoryCb cb = factory_.createFilterFactoryFromProto(config_, "stats", context_); + Http::FilterFactoryCb cb = + factory_.createFilterFactoryFromProto(config_, "stats", context_).value(); Http::StreamFilterSharedPtr filter; EXPECT_CALL(filter_callback_, addStreamFilter(_)).WillOnce(::testing::SaveArg<0>(&filter)); cb(filter_callback_); @@ -44,13 +46,17 @@ TEST_F(CacheFilterFactoryTest, Disabled) { } TEST_F(CacheFilterFactoryTest, NoTypedConfig) { - EXPECT_THROW(factory_.createFilterFactoryFromProto(config_, "stats", context_), EnvoyException); + EXPECT_THROW( + factory_.createFilterFactoryFromProto(config_, "stats", context_).status().IgnoreError(), + EnvoyException); } TEST_F(CacheFilterFactoryTest, UnregisteredTypedConfig) { config_.mutable_typed_config()->PackFrom( envoy::extensions::filters::http::cache::v3::CacheConfig()); - EXPECT_THROW(factory_.createFilterFactoryFromProto(config_, "stats", context_), EnvoyException); + EXPECT_THROW( + factory_.createFilterFactoryFromProto(config_, "stats", context_).status().IgnoreError(), + EnvoyException); } } // namespace diff --git a/test/extensions/filters/http/cache/http_cache_implementation_test_common.h b/test/extensions/filters/http/cache/http_cache_implementation_test_common.h index 362c7c22ea66..85c4ba8981fd 100644 --- a/test/extensions/filters/http/cache/http_cache_implementation_test_common.h +++ b/test/extensions/filters/http/cache/http_cache_implementation_test_common.h @@ -7,7 +7,6 @@ #include "source/extensions/filters/http/cache/cache_headers_utils.h" #include "source/extensions/filters/http/cache/http_cache.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/mocks/event/mocks.h" #include "test/mocks/http/mocks.h" #include "test/test_common/simulated_time_system.h" diff --git a/test/extensions/filters/http/cache/http_cache_test.cc b/test/extensions/filters/http/cache/http_cache_test.cc index fa13cf30b4a1..54c53fb5f2cf 100644 --- a/test/extensions/filters/http/cache/http_cache_test.cc +++ b/test/extensions/filters/http/cache/http_cache_test.cc @@ -4,9 +4,9 @@ #include "source/extensions/filters/http/cache/cache_headers_utils.h" #include "source/extensions/filters/http/cache/http_cache.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/mocks/http/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -294,6 +294,17 @@ TEST_F(LookupRequestTest, PragmaNoFallback) { } TEST(HttpCacheTest, StableHashKey) { + TestScopedRuntime runtime; + runtime.mergeValues({{"envoy.restart_features.use_fast_protobuf_hash", "true"}}); + Key key; + key.set_host("example.com"); + ASSERT_EQ(stableHashKey(key), 6153940628716543519u); +} + +TEST(HttpCacheTest, StableHashKeyWithSlowHash) { + // TODO(ravenblack): This test should be removed when the runtime guard is removed. + TestScopedRuntime runtime; + runtime.mergeValues({{"envoy.restart_features.use_fast_protobuf_hash", "false"}}); Key key; key.set_host("example.com"); ASSERT_EQ(stableHashKey(key), 9582653837550152292u); diff --git a/test/extensions/filters/http/cache/range_utils_test.cc b/test/extensions/filters/http/cache/range_utils_test.cc index 84bc3c0028b1..980a4dc52fec 100644 --- a/test/extensions/filters/http/cache/range_utils_test.cc +++ b/test/extensions/filters/http/cache/range_utils_test.cc @@ -5,7 +5,6 @@ #include "source/extensions/filters/http/cache/range_utils.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -327,6 +326,14 @@ TEST(GetRangeDetailsTest, NotSatisfiableRange) { ASSERT_TRUE(result->ranges_.empty()); } +// operator<<(ostream&, const AdjustedByteRange&) is only used in tests, but lives in //source, +// and so needs test coverage. This test provides that coverage, to keep the coverage test happy. +TEST(AdjustedByteRange, StreamingTest) { + std::ostringstream os; + os << AdjustedByteRange(0, 1); + EXPECT_EQ(os.str(), "[0,1)"); +} + } // namespace } // namespace Cache } // namespace HttpFilters diff --git a/test/extensions/filters/http/cdn_loop/config_test.cc b/test/extensions/filters/http/cdn_loop/config_test.cc index 5e9b89d3c636..fe9db1b43d20 100644 --- a/test/extensions/filters/http/cdn_loop/config_test.cc +++ b/test/extensions/filters/http/cdn_loop/config_test.cc @@ -27,7 +27,7 @@ TEST(CdnLoopFilterFactoryTest, ValidValuesWork) { config.set_cdn_id("cdn"); CdnLoopFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); cb(filter_callbacks); EXPECT_NE(filter.get(), nullptr); EXPECT_NE(dynamic_cast(filter.get()), nullptr); @@ -39,7 +39,7 @@ TEST(CdnLoopFilterFactoryTest, BlankCdnIdThrows) { envoy::extensions::filters::http::cdn_loop::v3::CdnLoopConfig config; CdnLoopFilterFactory factory; - EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context).value(), ProtoValidationException, HasSubstr("value length must be at least")); } @@ -50,7 +50,7 @@ TEST(CdnLoopFilterFactoryTest, InvalidCdnId) { config.set_cdn_id("[not-token-or-ip"); CdnLoopFilterFactory factory; - EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context).value(), EnvoyException, HasSubstr("is not a valid CDN identifier")); } @@ -61,7 +61,7 @@ TEST(CdnLoopFilterFactoryTest, InvalidCdnIdNonHeaderWhitespace) { config.set_cdn_id("\r\n"); CdnLoopFilterFactory factory; - EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context).value(), EnvoyException, HasSubstr("is not a valid CDN identifier")); } @@ -72,7 +72,7 @@ TEST(CdnLoopFilterFactoryTest, InvalidParsedCdnIdNotInput) { config.set_cdn_id("cdn,cdn"); CdnLoopFilterFactory factory; - EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THAT_THROWS_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context).value(), EnvoyException, HasSubstr("is not a valid CDN identifier")); } diff --git a/test/extensions/filters/http/common/empty_http_filter_config.h b/test/extensions/filters/http/common/empty_http_filter_config.h index d28ac5fe737e..d78f9ec268ac 100644 --- a/test/extensions/filters/http/common/empty_http_filter_config.h +++ b/test/extensions/filters/http/common/empty_http_filter_config.h @@ -19,10 +19,10 @@ namespace Common { */ class EmptyHttpFilterConfig : public Server::Configuration::NamedHttpFilterConfigFactory { public: - virtual Http::FilterFactoryCb createFilter(const std::string& stat_prefix, - Server::Configuration::FactoryContext& context) PURE; + virtual absl::StatusOr + createFilter(const std::string& stat_prefix, Server::Configuration::FactoryContext& context) PURE; - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) override { return createFilter(stat_prefix, context); @@ -46,14 +46,14 @@ class EmptyHttpFilterConfig : public Server::Configuration::NamedHttpFilterConfi class UpstreamFilterConfig : public Server::Configuration::UpstreamHttpFilterConfigFactory { public: - virtual Http::FilterFactoryCb + virtual absl::StatusOr createDualFilter(const std::string& stat_prefix, Server::Configuration::ServerFactoryContext& context) PURE; - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string& stat_prefix, Server::Configuration::UpstreamFactoryContext& context) override { - return createDualFilter(stat_prefix, context.getServerFactoryContext()); + return createDualFilter(stat_prefix, context.serverFactoryContext()); } }; @@ -61,9 +61,10 @@ class EmptyHttpDualFilterConfig : public EmptyHttpFilterConfig, public UpstreamF public: EmptyHttpDualFilterConfig(const std::string& name) : EmptyHttpFilterConfig(name) {} - Http::FilterFactoryCb createFilter(const std::string& stat_prefix, - Server::Configuration::FactoryContext& context) override { - return createDualFilter(stat_prefix, context.getServerFactoryContext()); + absl::StatusOr + createFilter(const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) override { + return createDualFilter(stat_prefix, context.serverFactoryContext()); } }; diff --git a/test/extensions/filters/http/common/fuzz/BUILD b/test/extensions/filters/http/common/fuzz/BUILD index 121a5be03d8c..d0b460b12cd5 100644 --- a/test/extensions/filters/http/common/fuzz/BUILD +++ b/test/extensions/filters/http/common/fuzz/BUILD @@ -51,6 +51,7 @@ envoy_cc_test_library( "//source/common/tracing:http_tracer_lib", "//test/mocks/buffer:buffer_mocks", "//test/mocks/http:http_mocks", + "//test/mocks/network:network_mocks", "//test/mocks/server:factory_context_mocks", "//test/proto:bookstore_proto_cc_proto", "//test/test_common:registry_lib", diff --git a/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h b/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h index 7e415e679992..81877f8e4932 100644 --- a/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h +++ b/test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h @@ -38,8 +38,7 @@ class HttpFilterFuzzer { // This executes the access logger with the fuzzed headers/trailers. void accessLog(AccessLog::Instance* access_logger, const StreamInfo::StreamInfo& stream_info) { ENVOY_LOG_MISC(debug, "Access logging"); - access_logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info, - AccessLog::AccessLogType::NotSet); + access_logger->log({&request_headers_, &response_headers_, &response_trailers_}, stream_info); } // Fuzzed headers and trailers are needed for access logging, reset the data and destroy filters. diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.cc b/test/extensions/filters/http/common/fuzz/uber_filter.cc index e6af5ebab809..cfabe6ffa186 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_filter.cc @@ -73,7 +73,7 @@ void UberFilterFuzzer::fuzz( proto_config, factory_context_.messageValidationVisitor(), factory); // Clean-up config with filter-specific logic before it runs through validations. cleanFuzzedConfig(proto_config.name(), message.get()); - cb_ = factory.createFilterFactoryFromProto(*message, "stats", factory_context_); + cb_ = factory.createFilterFactoryFromProto(*message, "stats", factory_context_).value(); cb_(filter_callback_); } catch (const EnvoyException& e) { ENVOY_LOG_MISC(debug, "Controlled exception {}", e.what()); diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.h b/test/extensions/filters/http/common/fuzz/uber_filter.h index 3a8ca9757b4d..76c5f27b79a7 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.h +++ b/test/extensions/filters/http/common/fuzz/uber_filter.h @@ -7,6 +7,7 @@ #include "test/mocks/api/mocks.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/network/mocks.h" #include "test/mocks/server/factory_context.h" #include "test/mocks/stream_info/mocks.h" #include "test/test_common/test_runtime.h" @@ -39,6 +40,7 @@ class UberFilterFuzzer : public HttpFilterFuzzer { private: NiceMock cluster_manager_; NiceMock factory_context_; + NiceMock listener_info_; NiceMock filter_callback_; std::shared_ptr resolver_{std::make_shared()}; Http::FilterFactoryCb cb_; diff --git a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc index 46afefa2d0dd..911596e1de35 100644 --- a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc @@ -141,7 +141,8 @@ void UberFilterFuzzer::perFilterSetup() { addr_ = std::make_shared("1.2.3.4", 1111); connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(addr_); connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress(addr_); - ON_CALL(factory_context_, clusterManager()).WillByDefault(testing::ReturnRef(cluster_manager_)); + ON_CALL(factory_context_.server_factory_context_, clusterManager()) + .WillByDefault(testing::ReturnRef(cluster_manager_)); ON_CALL(cluster_manager_.thread_local_cluster_.async_client_, send_(_, _, _)) .WillByDefault(Return(&async_request_)); @@ -164,16 +165,18 @@ void UberFilterFuzzer::perFilterSetup() { .WillByDefault(testing::Return(resolver_)); // Prepare expectations for TAP config. - ON_CALL(factory_context_, admin()) - .WillByDefault(testing::Return(OptRef{factory_context_.admin_})); - ON_CALL(factory_context_.admin_, addHandler(_, _, _, _, _, _)) + ON_CALL(factory_context_.server_factory_context_, admin()) + .WillByDefault( + testing::Return(OptRef{factory_context_.server_factory_context_.admin_})); + ON_CALL(factory_context_.server_factory_context_.admin_, addHandler(_, _, _, _, _, _)) + .WillByDefault(testing::Return(true)); + ON_CALL(factory_context_.server_factory_context_.admin_, removeHandler(_)) .WillByDefault(testing::Return(true)); - ON_CALL(factory_context_.admin_, removeHandler(_)).WillByDefault(testing::Return(true)); // Prepare expectations for WASM filter. - ON_CALL(factory_context_, listenerMetadata()) - .WillByDefault(testing::ReturnRef(listener_metadata_)); - ON_CALL(factory_context_.api_, customStatNamespaces()) + ON_CALL(factory_context_, listenerInfo()).WillByDefault(testing::ReturnRef(listener_info_)); + ON_CALL(listener_info_, metadata()).WillByDefault(testing::ReturnRef(listener_metadata_)); + ON_CALL(factory_context_.server_factory_context_.api_, customStatNamespaces()) .WillByDefault(testing::ReturnRef(custom_stat_namespaces_)); // Prepare expectations for AWSRequestSigning filter diff --git a/test/extensions/filters/http/common/jwks_fetcher_test.cc b/test/extensions/filters/http/common/jwks_fetcher_test.cc index 09a90ddbd00f..59184dda5b0e 100644 --- a/test/extensions/filters/http/common/jwks_fetcher_test.cc +++ b/test/extensions/filters/http/common/jwks_fetcher_test.cc @@ -54,8 +54,10 @@ class JwksFetcherTest : public testing::Test { public: void setupFetcher(const std::string& config_str) { TestUtility::loadFromYaml(config_str, remote_jwks_); - mock_factory_ctx_.cluster_manager_.initializeThreadLocalClusters({"pubkey_cluster"}); - fetcher_ = JwksFetcher::create(mock_factory_ctx_.cluster_manager_, remote_jwks_); + mock_factory_ctx_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"pubkey_cluster"}); + fetcher_ = JwksFetcher::create(mock_factory_ctx_.server_factory_context_.cluster_manager_, + remote_jwks_); EXPECT_TRUE(fetcher_ != nullptr); } @@ -69,7 +71,8 @@ class JwksFetcherTest : public testing::Test { TEST_F(JwksFetcherTest, TestGetSuccess) { // Setup setupFetcher(config); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, "200", publicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", + publicKey); MockJwksReceiver receiver; EXPECT_CALL(receiver, onJwksSuccessImpl(testing::_)); EXPECT_CALL(receiver, onJwksError(testing::_)).Times(0); @@ -81,7 +84,8 @@ TEST_F(JwksFetcherTest, TestGetSuccess) { TEST_F(JwksFetcherTest, TestGet400) { // Setup setupFetcher(config); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, "400", "invalid"); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, "400", + "invalid"); MockJwksReceiver receiver; EXPECT_CALL(receiver, onJwksSuccessImpl(testing::_)).Times(0); EXPECT_CALL(receiver, onJwksError(JwksFetcher::JwksReceiver::Failure::Network)); @@ -93,7 +97,7 @@ TEST_F(JwksFetcherTest, TestGet400) { TEST_F(JwksFetcherTest, TestGetNoBody) { // Setup setupFetcher(config); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, "200", ""); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", ""); MockJwksReceiver receiver; EXPECT_CALL(receiver, onJwksSuccessImpl(testing::_)).Times(0); EXPECT_CALL(receiver, onJwksError(JwksFetcher::JwksReceiver::Failure::Network)); @@ -105,7 +109,8 @@ TEST_F(JwksFetcherTest, TestGetNoBody) { TEST_F(JwksFetcherTest, TestGetInvalidJwks) { // Setup setupFetcher(config); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, "200", "invalid"); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", + "invalid"); MockJwksReceiver receiver; EXPECT_CALL(receiver, onJwksSuccessImpl(testing::_)).Times(0); EXPECT_CALL(receiver, onJwksError(JwksFetcher::JwksReceiver::Failure::InvalidJwks)); @@ -117,7 +122,7 @@ TEST_F(JwksFetcherTest, TestGetInvalidJwks) { TEST_F(JwksFetcherTest, TestHttpFailure) { // Setup setupFetcher(config); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, Http::AsyncClient::FailureReason::Reset); MockJwksReceiver receiver; EXPECT_CALL(receiver, onJwksSuccessImpl(testing::_)).Times(0); @@ -130,9 +135,9 @@ TEST_F(JwksFetcherTest, TestHttpFailure) { TEST_F(JwksFetcherTest, TestCancel) { // Setup setupFetcher(config); - Http::MockAsyncClientRequest request( - &(mock_factory_ctx_.cluster_manager_.thread_local_cluster_.async_client_)); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, &request); + Http::MockAsyncClientRequest request(&(mock_factory_ctx_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.async_client_)); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, &request); MockJwksReceiver receiver; EXPECT_CALL(request, cancel()); EXPECT_CALL(receiver, onJwksSuccessImpl(testing::_)).Times(0); @@ -149,11 +154,13 @@ TEST_F(JwksFetcherTest, TestCancel) { TEST_F(JwksFetcherTest, TestSpanPassedDown) { // Setup setupFetcher(config); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, "200", publicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", + publicKey); NiceMock receiver; // Expectations for span - EXPECT_CALL(mock_factory_ctx_.cluster_manager_.thread_local_cluster_.async_client_, + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_, send_(_, _, _)) .WillOnce(Invoke( [this](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks&, @@ -182,8 +189,10 @@ class JwksFetcherRetryingTest : public testing::TestWithParam receiver; // Expectations for envoy.config.core.v3.RetryPolicy to envoy.config.route.v3.RetryPolicy // used by async client. // execution deep down in async_client_'s route entry implementation // is not exercised here, just the configuration adaptation. - EXPECT_CALL(mock_factory_ctx_.cluster_manager_.thread_local_cluster_.async_client_, + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_, send_(_, _, _)) .WillOnce(Invoke( [](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks&, diff --git a/test/extensions/filters/http/composite/BUILD b/test/extensions/filters/http/composite/BUILD index f846deb7b778..fd0777e78c5a 100644 --- a/test/extensions/filters/http/composite/BUILD +++ b/test/extensions/filters/http/composite/BUILD @@ -19,8 +19,12 @@ envoy_extension_cc_test( "//source/common/http:header_map_lib", "//source/extensions/filters/http/composite:config", "//source/extensions/filters/http/composite:filter_lib", + "//source/extensions/filters/http/fault:config", + "//source/extensions/filters/http/fault:fault_filter_lib", "//test/mocks/access_log:access_log_mocks", "//test/mocks/http:http_mocks", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/server:instance_mocks", ], ) @@ -45,7 +49,7 @@ envoy_extension_cc_test( "//test/integration/filters:set_response_code_filter_config_proto_cc_proto", "//test/integration/filters:set_response_code_filter_lib", "//test/proto:helloworld_proto_cc_proto", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/composite/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", diff --git a/test/extensions/filters/http/composite/composite_filter_integration_test.cc b/test/extensions/filters/http/composite/composite_filter_integration_test.cc index fe75b1a45213..8521c3666de8 100644 --- a/test/extensions/filters/http/composite/composite_filter_integration_test.cc +++ b/test/extensions/filters/http/composite/composite_filter_integration_test.cc @@ -33,6 +33,7 @@ using envoy::extensions::common::matching::v3::ExtensionWithMatcherPerRoute; using envoy::extensions::filters::http::composite::v3::ExecuteFilterAction; using envoy::type::matcher::v3::HttpRequestHeaderMatchInput; using test::integration::filters::SetResponseCodeFilterConfig; +using test::integration::filters::SetResponseCodePerRouteFilterConfig; using xds::type::matcher::v3::Matcher_OnMatch; class CompositeFilterIntegrationTest : public testing::TestWithParam, @@ -87,6 +88,25 @@ class CompositeFilterIntegrationTest : public testing::TestWithParammutable_virtual_hosts(0); + auto* route = vh->mutable_routes()->Mutable(0); + route->mutable_match()->set_prefix(route_prefix); + route->mutable_route()->set_cluster("cluster_0"); + (*route->mutable_typed_per_filter_config())[filter_name].PackFrom( + set_response_code_per_route_config); + }); + } + void prependCompositeFilter(const std::string& name = "composite") { config_helper_.prependFilter(absl::StrFormat(R"EOF( name: %s @@ -119,11 +139,111 @@ class CompositeFilterIntegrationTest : public testing::TestWithParamadd_clusters(); + ecds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + ecds_cluster->set_name("ecds_cluster"); + ecds_cluster->mutable_load_assignment()->set_cluster_name("ecds_cluster"); + }); + HttpIntegrationTest::initialize(); + } + + void createUpstreams() override { + BaseIntegrationTest::createUpstreams(); + addFakeUpstream(Http::CodecType::HTTP2); + } }; INSTANTIATE_TEST_SUITE_P(IpVersions, CompositeFilterIntegrationTest, @@ -153,6 +273,51 @@ TEST_P(CompositeFilterIntegrationTest, TestBasic) { } } +// Verifies that if we don't match the match action the request is proxied as normal, while if the +// match action is hit we apply the specified dynamic filter to the stream. +TEST_P(CompositeFilterIntegrationTest, TestBasicDynamicFilter) { + prependCompositeDynamicFilter("composite-dynamic"); + initialize(); + test_server_->waitForCounterGe( + "extension_config_discovery.http_filter.set-response-code.config_reload", 1); + test_server_->waitUntilListenersReady(); + test_server_->waitForGaugeGe("listener_manager.workers_started", 1); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + { + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 1024); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_THAT(response->headers(), Http::HttpStatusIs("200")); + } + + { + auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_THAT(response->headers(), Http::HttpStatusIs("403")); + } +} + +// Verifies that if ECDS response is not sent, the missing filter config is applied that returns +// 500. +TEST_P(CompositeFilterIntegrationTest, TestMissingDynamicFilter) { + prependMissingCompositeDynamicFilter("composite-dynamic-missing"); + + initialize(); + test_server_->waitForCounterGe( + "extension_config_discovery.http_filter.missing-config.config_fail", 1); + test_server_->waitUntilListenersReady(); + test_server_->waitForGaugeGe("listener_manager.workers_started", 1); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_THAT(response->headers(), Http::HttpStatusIs("500")); +} + // Verifies function of the per-route config in the ExtensionWithMatcher class. TEST_P(CompositeFilterIntegrationTest, TestPerRoute) { prependCompositeFilter(); @@ -166,6 +331,23 @@ TEST_P(CompositeFilterIntegrationTest, TestPerRoute) { EXPECT_THAT(response->headers(), Http::HttpStatusIs("401")); } +// Verifies set_response_code filter's per-route config overrides the filter config. +TEST_P(CompositeFilterIntegrationTest, TestPerRouteResponseCodeConfig) { + std::string top_level_filter_name = "match_delegate_filter"; + prependCompositeFilter(top_level_filter_name); + + addResponseCodeFilterPerRouteConfig(/*filter_name=*/top_level_filter_name, + /*route_prefix=*/"/somepath", + /*code=*/406); + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeRequestWithBody(match_request_headers_, 1024); + ASSERT_TRUE(response->waitForEndStream()); + // Verifies that 406 from per route config is used, rather than 403 from filter config. + EXPECT_THAT(response->headers(), Http::HttpStatusIs("406")); +} + // Test an empty match tree resolving with a per route config. TEST_P(CompositeFilterIntegrationTest, TestPerRouteEmptyMatcher) { config_helper_.prependFilter(R"EOF( diff --git a/test/extensions/filters/http/composite/filter_test.cc b/test/extensions/filters/http/composite/filter_test.cc index 4085800097ac..bbfc1407e94f 100644 --- a/test/extensions/filters/http/composite/filter_test.cc +++ b/test/extensions/filters/http/composite/filter_test.cc @@ -2,10 +2,13 @@ #include "envoy/http/metadata_interface.h" +#include "source/extensions/filters/http/composite/action.h" #include "source/extensions/filters/http/composite/filter.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/mocks/server/instance.h" #include "gtest/gtest.h" @@ -228,10 +231,46 @@ TEST_F(FilterTest, StreamFilterDelegationMultipleAccessLoggers) { EXPECT_CALL(*encode_filter, onDestroy()); filter_.onDestroy(); - EXPECT_CALL(*access_log_1, log(_, _, _, _, _)); - EXPECT_CALL(*access_log_2, log(_, _, _, _, _)); - filter_.log(nullptr, nullptr, nullptr, StreamInfo::MockStreamInfo(), - AccessLog::AccessLogType::NotSet); + EXPECT_CALL(*access_log_1, log(_, _)); + EXPECT_CALL(*access_log_2, log(_, _)); + filter_.log({}, StreamInfo::MockStreamInfo()); +} + +// Validate that when dynamic_config and typed_config are both set, an exception is thrown. +TEST(ConfigTest, TestConfig) { + const std::string yaml_string = R"EOF( + typed_config: + name: set-response-code + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault + abort: + http_status: 503 + percentage: + numerator: 0 + denominator: HUNDRED + dynamic_config: + name: set-response-code + config_discovery: + config_source: + resource_api_version: V3 + path_config_source: + path: "{{ test_tmpdir }}/set_response_code.yaml" + type_urls: + - type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig +)EOF"; + + envoy::extensions::filters::http::composite::v3::ExecuteFilterAction config; + TestUtility::loadFromYaml(yaml_string, config); + + testing::NiceMock server_factory_context; + testing::NiceMock factory_context; + Envoy::Http::Matching::HttpFilterActionContext action_context{"test", factory_context, + server_factory_context}; + ExecuteFilterActionFactory factory; + EXPECT_THROW_WITH_MESSAGE( + factory.createActionFactoryCb(config, action_context, + ProtobufMessage::getStrictValidationVisitor()), + EnvoyException, "Error: Only one of `dynamic_config` or `typed_config` can be set."); } } // namespace diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index ac1b443fc27f..58530912582d 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -3,6 +3,7 @@ load( "envoy_benchmark_test", "envoy_cc_benchmark_binary", "envoy_cc_test", + "envoy_cc_test_library", "envoy_package", "envoy_proto_library", ) @@ -101,14 +102,19 @@ envoy_benchmark_test( tags = ["fails_on_windows"], ) +envoy_cc_test_library( + name = "compressor_integration_tests_lib", + hdrs = ["compressor_integration_tests.h"], +) + envoy_cc_test( name = "compressor_integration_tests", size = "large", srcs = [ "compressor_integration_tests.cc", - "compressor_integration_tests.h", ], deps = [ + ":compressor_integration_tests_lib", "//source/common/http:header_map_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/compression/gzip/compressor:config", diff --git a/test/extensions/filters/http/compressor/compressor_filter_test.cc b/test/extensions/filters/http/compressor/compressor_filter_test.cc index 3cb207fc7db1..cf4a2885c5a6 100644 --- a/test/extensions/filters/http/compressor/compressor_filter_test.cc +++ b/test/extensions/filters/http/compressor/compressor_filter_test.cc @@ -430,6 +430,7 @@ TEST_F(CompressorFilterTest, EmptyResponse) { // Verify removeAcceptEncoding header. TEST_F(CompressorFilterTest, RemoveAcceptEncodingHeader) { + // Filter true, no response direction overrides. Header is removed. { Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; setUpFilter(R"EOF( @@ -442,9 +443,12 @@ TEST_F(CompressorFilterTest, RemoveAcceptEncodingHeader) { } } )EOF"); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); EXPECT_FALSE(headers.has("accept-encoding")); } + + // Filter false, no response direction overrides. Header is present. { Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; setUpFilter(R"EOF( @@ -456,6 +460,169 @@ TEST_F(CompressorFilterTest, RemoveAcceptEncodingHeader) { } } )EOF"); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_TRUE(headers.has("accept-encoding")); + EXPECT_EQ("deflate, test, gzip, br", headers.get_("accept-encoding")); + } + + // Filter true, response direction overrides present but no override. Header is removed. + { + Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; + setUpFilter(R"EOF( +{ + "remove_accept_encoding_header": true, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" + } + } +} +)EOF"); + CompressorPerRoute per_route_proto; + per_route_proto.mutable_overrides()->mutable_response_direction_config(); + + std::unique_ptr per_route_config = + std::make_unique(per_route_proto); + ON_CALL(decoder_callbacks_, mostSpecificPerFilterConfig()) + .WillByDefault(Return(per_route_config.get())); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_FALSE(headers.has("accept-encoding")); + } + + // Filter false, response direction overrides present but no override. Header is present. + { + Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; + setUpFilter(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" + } + } +} +)EOF"); + CompressorPerRoute per_route_proto; + per_route_proto.mutable_overrides()->mutable_response_direction_config(); + + std::unique_ptr per_route_config = + std::make_unique(per_route_proto); + ON_CALL(decoder_callbacks_, mostSpecificPerFilterConfig()) + .WillByDefault(Return(per_route_config.get())); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_TRUE(headers.has("accept-encoding")); + EXPECT_EQ("deflate, test, gzip, br", headers.get_("accept-encoding")); + } + + // Filter true, per-route override true. Header is removed. + { + Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; + setUpFilter(R"EOF( +{ + "remove_accept_encoding_header": true, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" + } + } +} +)EOF"); + CompressorPerRoute per_route_proto; + per_route_proto.mutable_overrides() + ->mutable_response_direction_config() + ->mutable_remove_accept_encoding_header() + ->set_value(true); + + std::unique_ptr per_route_config = + std::make_unique(per_route_proto); + ON_CALL(decoder_callbacks_, mostSpecificPerFilterConfig()) + .WillByDefault(Return(per_route_config.get())); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_FALSE(headers.has("accept-encoding")); + } + + // Filter true, per-route override false. Header is present. + { + Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; + setUpFilter(R"EOF( +{ + "remove_accept_encoding_header": true, + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" + } + } +} +)EOF"); + CompressorPerRoute per_route_proto; + per_route_proto.mutable_overrides() + ->mutable_response_direction_config() + ->mutable_remove_accept_encoding_header() + ->set_value(false); + + std::unique_ptr per_route_config = + std::make_unique(per_route_proto); + ON_CALL(decoder_callbacks_, mostSpecificPerFilterConfig()) + .WillByDefault(Return(per_route_config.get())); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_TRUE(headers.has("accept-encoding")); + EXPECT_EQ("deflate, test, gzip, br", headers.get_("accept-encoding")); + } + + // Filter false, per-route override true. Header is removed. + { + Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; + setUpFilter(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" + } + } +} +)EOF"); + CompressorPerRoute per_route_proto; + per_route_proto.mutable_overrides() + ->mutable_response_direction_config() + ->mutable_remove_accept_encoding_header() + ->set_value(true); + + std::unique_ptr per_route_config = + std::make_unique(per_route_proto); + ON_CALL(decoder_callbacks_, mostSpecificPerFilterConfig()) + .WillByDefault(Return(per_route_config.get())); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); + EXPECT_FALSE(headers.has("accept-encoding")); + } + + // Filter false, per-route override false. Header is present. + { + Http::TestRequestHeaderMapImpl headers = {{"accept-encoding", "deflate, test, gzip, br"}}; + setUpFilter(R"EOF( +{ + "compressor_library": { + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.compression.gzip.compressor.v3.Gzip" + } + } +} +)EOF"); + CompressorPerRoute per_route_proto; + per_route_proto.mutable_overrides() + ->mutable_response_direction_config() + ->mutable_remove_accept_encoding_header() + ->set_value(false); + + std::unique_ptr per_route_config = + std::make_unique(per_route_proto); + ON_CALL(decoder_callbacks_, mostSpecificPerFilterConfig()) + .WillByDefault(Return(per_route_config.get())); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); EXPECT_TRUE(headers.has("accept-encoding")); EXPECT_EQ("deflate, test, gzip, br", headers.get_("accept-encoding")); diff --git a/test/extensions/filters/http/compressor/config_test.cc b/test/extensions/filters/http/compressor/config_test.cc index f253ee7e9d5a..1d609712f851 100644 --- a/test/extensions/filters/http/compressor/config_test.cc +++ b/test/extensions/filters/http/compressor/config_test.cc @@ -27,10 +27,10 @@ TEST(CompressorFilterFactoryTests, UnregisteredCompressorLibraryConfig) { TestUtility::loadFromYaml(yaml_string, proto_config); CompressorFilterFactory factory; NiceMock context; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context), - EnvoyException, - "Didn't find a registered implementation for type: " - "'test.mock_compressor_library.Unregistered'"); + EXPECT_THAT( + factory.createFilterFactoryFromProto(proto_config, "stats", context).status().message(), + testing::HasSubstr("Didn't find a registered implementation for type: " + "'test.mock_compressor_library.Unregistered'")); } TEST(CompressorFilterFactoryTests, EmptyPerRouteConfig) { diff --git a/test/extensions/filters/http/connect_grpc_bridge/config_test.cc b/test/extensions/filters/http/connect_grpc_bridge/config_test.cc index 61ae0cd36096..5d337e778131 100644 --- a/test/extensions/filters/http/connect_grpc_bridge/config_test.cc +++ b/test/extensions/filters/http/connect_grpc_bridge/config_test.cc @@ -21,7 +21,7 @@ TEST(ConnectGrpcBridgeFilterConfigTest, ConnectGrpcBridgeFilter) { NiceMock context; ConnectGrpcFilterConfigFactory factory; envoy::extensions::filters::http::connect_grpc_bridge::v3::FilterConfig config; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)).Times(AtLeast(1)); cb(filter_callback); diff --git a/test/extensions/filters/http/connect_grpc_bridge/connect_grpc_bridge_filter_test.cc b/test/extensions/filters/http/connect_grpc_bridge/connect_grpc_bridge_filter_test.cc index 7589e62fe26b..2c99607397ca 100644 --- a/test/extensions/filters/http/connect_grpc_bridge/connect_grpc_bridge_filter_test.cc +++ b/test/extensions/filters/http/connect_grpc_bridge/connect_grpc_bridge_filter_test.cc @@ -437,6 +437,27 @@ TEST_F(ConnectGrpcBridgeFilterTest, UnaryGetRequestTimeout) { EXPECT_EQ(false, request_headers_.has(Http::CustomHeaders::get().ConnectTimeoutMs)); } +TEST_F(ConnectGrpcBridgeFilterTest, UnaryRequestWithNoBodyNorTrailers) { + request_headers_.setCopy(Http::CustomHeaders::get().ConnectProtocolVersion, "1"); + request_headers_.setContentType("application/proto"); + + Buffer::OwnedImpl data{}; + + EXPECT_CALL(decoder_callbacks_, addDecodedData(_, true)) + .WillOnce( + Invoke(([&](Buffer::Instance& d, bool) { EXPECT_EQ('\0', d.drainInt()); }))); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers_, true)); +} + +TEST_F(ConnectGrpcBridgeFilterTest, UnaryRequestRemovesContentLength) { + request_headers_.setCopy(Http::CustomHeaders::get().ConnectProtocolVersion, "1"); + request_headers_.setContentType(Http::Headers::get().ContentTypeValues.Grpc); + request_headers_.setContentLength(1337); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers_, false)); + EXPECT_EQ("", request_headers_.get_("content-length")); +} + TEST_F(ConnectGrpcBridgeFilterTest, StreamingSupportedContentType) { request_headers_.setContentType("application/connect+proto"); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers_, false)); diff --git a/test/extensions/filters/http/cors/cors_filter_test.cc b/test/extensions/filters/http/cors/cors_filter_test.cc index 1d9e87e2cd93..cf40ac441fb3 100644 --- a/test/extensions/filters/http/cors/cors_filter_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_test.cc @@ -63,7 +63,7 @@ class CorsFilterTest : public testing::Test { filter_.setEncoderFilterCallbacks(encoder_callbacks_); } - bool IsCorsRequest() { return filter_.is_cors_request_; } + bool isCorsRequest() { return filter_.is_cors_request_; } NiceMock decoder_callbacks_; NiceMock encoder_callbacks_; @@ -84,7 +84,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { // Cors policies in the 'typed_per_filter_config'. { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(2, filter_.policiesForTest().size()); EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(0)); EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(1)); @@ -103,7 +103,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { })); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(1, filter_.policiesForTest().size()); EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(0)); } @@ -123,7 +123,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { Invoke([](std::function) {})); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(2, filter_.policiesForTest().size()); EXPECT_EQ(nullptr, filter_.policiesForTest().at(0)); EXPECT_EQ(nullptr, filter_.policiesForTest().at(1)); @@ -143,7 +143,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { Invoke([](std::function) {})); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(2, filter_.policiesForTest().size()); EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(0)); EXPECT_EQ(nullptr, filter_.policiesForTest().at(1)); @@ -162,7 +162,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { Invoke([](std::function) {})); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(2, filter_.policiesForTest().size()); EXPECT_EQ(nullptr, filter_.policiesForTest().at(0)); EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(1)); @@ -174,7 +174,7 @@ TEST_F(CorsFilterTest, RequestWithoutOrigin) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -192,7 +192,7 @@ TEST_F(CorsFilterTest, RequestWithOrigin) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -208,7 +208,7 @@ TEST_F(CorsFilterTest, OptionsRequestWithoutOrigin) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -224,7 +224,7 @@ TEST_F(CorsFilterTest, OptionsRequestWithOrigin) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -295,7 +295,7 @@ TEST_F(CorsFilterTest, OptionsRequestWithOriginCorsEnabled) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -311,7 +311,7 @@ TEST_F(CorsFilterTest, OptionsRequestWithoutAccessRequestMethod) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -337,7 +337,7 @@ TEST_F(CorsFilterTest, OptionsRequestMatchingOriginByWildcard) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -367,7 +367,7 @@ TEST_F(CorsFilterTest, OptionsRequestWithOriginCorsEnabledShadowDisabled) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -395,7 +395,7 @@ TEST_F(CorsFilterTest, OptionsRequestWithOriginCorsEnabledShadowEnabled) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -415,7 +415,7 @@ TEST_F(CorsFilterTest, OptionsRequestNotMatchingOrigin) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(1, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -434,7 +434,7 @@ TEST_F(CorsFilterTest, OptionsRequestEmptyOriginList) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(1, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -465,7 +465,7 @@ TEST_F(CorsFilterTest, ValidOptionsRequestWithAllowCredentialsTrue) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -491,7 +491,7 @@ TEST_F(CorsFilterTest, ValidOptionsRequestWithAllowCredentialsFalse) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -615,7 +615,7 @@ TEST_F(CorsFilterTest, RedirectRoute) { .WillByDefault(Return(&direct_response_entry_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers_, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_trailers_)); @@ -673,7 +673,7 @@ TEST_F(CorsFilterTest, NoCorsEntry) { .WillByDefault(Return(nullptr)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -709,7 +709,7 @@ TEST_F(CorsFilterTest, NoRouteCorsEntry) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -742,7 +742,7 @@ TEST_F(CorsFilterTest, NoVHostCorsEntry) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -773,7 +773,7 @@ TEST_F(CorsFilterTest, OptionsRequestMatchingOriginByRegex) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -794,7 +794,7 @@ TEST_F(CorsFilterTest, OptionsRequestNotMatchingOriginByRegex) { EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(false, IsCorsRequest()); + EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(1, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -879,7 +879,7 @@ TEST_F(CorsFilterTest, OptionsRequestMatchingOriginByWildcardWithPNA) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -913,7 +913,7 @@ TEST_F(CorsFilterTest, OptionsRequestMatchingOriginByWildcardWithoutPNA) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -945,7 +945,7 @@ TEST_F(CorsFilterTest, OptionsRequestMatchingOriginByWildcardWithInvalidPNA) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers, false)); - EXPECT_EQ(true, IsCorsRequest()); + EXPECT_EQ(true, isCorsRequest()); EXPECT_EQ(0, stats_.counter("test.cors.origin_invalid").value()); EXPECT_EQ(1, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); diff --git a/test/extensions/filters/http/custom_response/config_test.cc b/test/extensions/filters/http/custom_response/config_test.cc index f72c938a35c4..92670ca9a728 100644 --- a/test/extensions/filters/http/custom_response/config_test.cc +++ b/test/extensions/filters/http/custom_response/config_test.cc @@ -25,7 +25,7 @@ TEST(CustomResponseFilterConfigTest, CustomResponseFilter) { EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; ::Envoy::Http::FilterFactoryCb cb = - factory.createFilterFactoryFromProto(filter_config, "stats", context); + factory.createFilterFactoryFromProto(filter_config, "stats", context).value(); ::Envoy::Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -40,10 +40,11 @@ TEST(CustomResponseFilterConfigTest, InvalidURI) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(filter_config, "stats", context), - EnvoyException, - "Invalid uri specified for redirection for custom response: " - "%#?*s://foo.example/gateway_error"); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(filter_config, "stats", context).IgnoreError(), + EnvoyException, + "Invalid uri specified for redirection for custom response: " + "%#?*s://foo.example/gateway_error"); } // Specifying neither host nor path redirect is valid when the routing decision @@ -66,7 +67,7 @@ TEST(CustomResponseFilterConfigTest, NoHostAndPathRedirect) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; - EXPECT_NO_THROW(factory.createFilterFactoryFromProto(filter_config, "stats", context)); + EXPECT_TRUE(factory.createFilterFactoryFromProto(filter_config, "stats", context).ok()); } TEST(CustomResponseFilterConfigTest, PrefixRewrite) { @@ -88,8 +89,9 @@ TEST(CustomResponseFilterConfigTest, PrefixRewrite) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(filter_config, "stats", context), - EnvoyException, "prefix_rewrite is not supported for Custom Response"); + EXPECT_THROW_WITH_MESSAGE( + EXPECT_TRUE(factory.createFilterFactoryFromProto(filter_config, "stats", context).ok()), + EnvoyException, "prefix_rewrite is not supported for Custom Response"); } TEST(CustomResponseFilterConfigTest, RegexRewrite) { @@ -114,8 +116,9 @@ TEST(CustomResponseFilterConfigTest, RegexRewrite) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(filter_config, "stats", context), - EnvoyException, "regex_rewrite is not supported for Custom Response"); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(filter_config, "stats", context).IgnoreError(), + EnvoyException, "regex_rewrite is not supported for Custom Response"); } TEST(CustomResponseFilterConfigTest, HashFragmentUri) { @@ -135,10 +138,11 @@ TEST(CustomResponseFilterConfigTest, HashFragmentUri) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(filter_config, "stats", context), - EnvoyException, - "Invalid uri specified for redirection for custom response: " - "example.foo/abc/cde#ab"); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(filter_config, "stats", context).IgnoreError(), + EnvoyException, + "Invalid uri specified for redirection for custom response: " + "example.foo/abc/cde#ab"); } TEST(CustomResponseFilterConfigTest, HashFragmentPathRedirect) { @@ -159,10 +163,11 @@ TEST(CustomResponseFilterConfigTest, HashFragmentPathRedirect) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); CustomResponseFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(filter_config, "stats", context), - EnvoyException, - "#fragment is not supported for custom response. Specified " - "path_redirect is /abc/cde#ab"); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(filter_config, "stats", context).IgnoreError(), + EnvoyException, + "#fragment is not supported for custom response. Specified " + "path_redirect is /abc/cde#ab"); } } // namespace diff --git a/test/extensions/filters/http/custom_response/custom_response_integration_test.cc b/test/extensions/filters/http/custom_response/custom_response_integration_test.cc index e1a54885fa68..1991be48d2dc 100644 --- a/test/extensions/filters/http/custom_response/custom_response_integration_test.cc +++ b/test/extensions/filters/http/custom_response/custom_response_integration_test.cc @@ -240,7 +240,7 @@ TEST_P(CustomResponseIntegrationTest, LocalReplyWithFormatter) { sendRequestAndWaitForResponse(default_request_headers_, 0, unauthorized_response_, 0); // Verify that we get the modified status value. EXPECT_EQ("499", response->headers().getStatusValue()); - EXPECT_EQ("not allowed", response->body()); + EXPECT_EQ("{\"message\":\"not allowed\"}\n", response->body()); EXPECT_EQ( "x-bar", response->headers().get(::Envoy::Http::LowerCaseString("foo"))[0]->value().getStringView()); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc index 512f660e6d8a..b37113e27ff2 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc @@ -41,17 +41,18 @@ class ProxyFilterTest : public testing::Test, } void setupSocketMatcher() { - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); transport_socket_match_ = new NiceMock( Network::UpstreamTransportSocketFactoryPtr(transport_socket_factory_)); - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ ->transport_socket_matcher_.reset(transport_socket_match_); dfp_cluster_ = std::make_shared>(); auto cluster = std::dynamic_pointer_cast( dfp_cluster_); Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( - factory_context_); + *factory_context_.server_factory_context_.singleton_manager_); cluster_store_factory.get()->save("fake_cluster", cluster); } @@ -59,10 +60,11 @@ class ProxyFilterTest : public testing::Test, EXPECT_CALL(*dns_cache_manager_, getCache(_)); Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( - factory_context_); + *factory_context_.server_factory_context_.singleton_manager_); envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig proto_config; - filter_config_ = std::make_shared(proto_config, *this, cluster_store_factory, - factory_context_); + filter_config_ = std::make_shared( + proto_config, dns_cache_manager_->getCache(proto_config.dns_cache_config()).value(), + this->get(), cluster_store_factory, factory_context_); filter_ = std::make_unique(filter_config_); filter_->setDecoderFilterCallbacks(callbacks_); @@ -77,20 +79,20 @@ class ProxyFilterTest : public testing::Test, // kind we need to do DNS entries for. CustomClusterType cluster_type; cluster_type.set_name("envoy.clusters.dynamic_forward_proxy"); - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->cluster_type_ = + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->cluster_type_ = std::make_unique( cluster_type); // Configure max pending to 1 so we can test circuit breaking. - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( - 0, 1, 0, 0, 0, 100); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->resetResourceManager(0, 1, 0, 0, 0, 100); } ~ProxyFilterTest() override { - EXPECT_TRUE( - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resource_manager_ - ->pendingRequests() - .canCreate()); + EXPECT_TRUE(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->resource_manager_->pendingRequests() + .canCreate()); } Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { @@ -118,7 +120,7 @@ TEST_F(ProxyFilterTest, HttpDefaultPort) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) @@ -146,7 +148,7 @@ TEST_F(ProxyFilterTest, HttpsDefaultPort) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) @@ -174,7 +176,7 @@ TEST_F(ProxyFilterTest, CacheOverflow) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) @@ -203,7 +205,7 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflow) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) @@ -223,7 +225,7 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflow) { auto filter2 = std::make_unique(filter_config_); filter2->setDecoderFilterCallbacks(callbacks_); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()); @@ -247,7 +249,7 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); @@ -267,7 +269,7 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) { auto filter2 = std::make_unique(filter_config_); filter2->setDecoderFilterCallbacks(callbacks_); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()); @@ -280,9 +282,9 @@ TEST_F(ProxyFilterTest, CircuitBreakerOverflowWithDnsCacheResourceManager) { filter2->decodeHeaders(request_headers_, false)); // Cluster circuit breaker overflow counter won't be incremented. - EXPECT_EQ(0, - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->trafficStats() - ->upstream_rq_pending_overflow_.value()); + EXPECT_EQ(0, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->trafficStats() + ->upstream_rq_pending_overflow_.value()); filter2->onDestroy(); EXPECT_CALL(*handle, onDestroy()); filter_->onDestroy(); @@ -301,19 +303,20 @@ TEST_F(ProxyFilterTest, NoCluster) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) .WillOnce(Return(nullptr)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); } // No cluster type leads to skipping DNS lookups. TEST_F(ProxyFilterTest, NoClusterType) { - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->cluster_type_ = nullptr; + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->cluster_type_ = nullptr; InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); } @@ -321,13 +324,14 @@ TEST_F(ProxyFilterTest, NoClusterType) { TEST_F(ProxyFilterTest, NonDynamicForwardProxy) { CustomClusterType cluster_type; cluster_type.set_name("envoy.cluster.static"); - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->cluster_type_ = + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->cluster_type_ = std::make_unique(cluster_type); InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); } @@ -341,7 +345,7 @@ TEST_F(ProxyFilterTest, HostRewrite) { ProxyPerRouteConfig config(proto_config); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); @@ -372,7 +376,7 @@ TEST_F(ProxyFilterTest, HostRewriteViaHeader) { ProxyPerRouteConfig config(proto_config); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); @@ -400,12 +404,13 @@ TEST_F(ProxyFilterTest, SubClusterNotExists) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); // get DFPCluster, not exists. - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq("DFPCluster:foo:80"))); // "true" means another thread already created it. EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)) .WillOnce(Return(std::make_pair(true, absl::nullopt))); @@ -418,16 +423,18 @@ TEST_F(ProxyFilterTest, SubClusterNotExists) { // Thread local cluster exists. TEST_F(ProxyFilterTest, SubClusterExists) { - factory_context_.cluster_manager_.initializeThreadLocalClusters({"DFPCluster:foo:80"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"DFPCluster:foo:80"}); InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); // get DFPCluster, exists. - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq("DFPCluster:foo:80"))); EXPECT_CALL(*(dfp_cluster_.get()), touch(_)).WillOnce(Return(true)); // should not create. EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)).Times(0); @@ -442,12 +449,13 @@ TEST_F(ProxyFilterTest, SubClusterOverflow) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); // get DFPCluster - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq("DFPCluster:foo:80"))); // reach the max_sub_clusters limitation. EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)) .WillOnce(Return(std::make_pair(false, absl::nullopt))); @@ -465,12 +473,12 @@ TEST_F(ProxyFilterTest, SubClusterOverflow) { // DFP cluster is removed early. TEST_F(ProxyFilterTest, DFPClusterIsGone) { Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( - factory_context_); + *factory_context_.server_factory_context_.singleton_manager_); cluster_store_factory.get()->remove("fake_cluster"); InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).Times(0); @@ -490,12 +498,13 @@ TEST_F(ProxyFilterTest, SubClusterInitTimeout) { InSequence s; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*(dfp_cluster_.get()), enableSubCluster()).WillOnce(Return(true)); // get DFPCluster, not exists. - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(Eq("DFPCluster:foo:80"))); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq("DFPCluster:foo:80"))); // "true" means another thread already created it. EXPECT_CALL(*(dfp_cluster_.get()), createSubClusterConfig(_, _, _)) .WillOnce(Return(std::make_pair(true, absl::nullopt))); @@ -521,9 +530,11 @@ class UpstreamResolvedHostFilterStateHelper : public ProxyFilterTest { proto_config.set_save_upstream_address(true); Extensions::Common::DynamicForwardProxy::DFPClusterStoreFactory cluster_store_factory( - factory_context_); - filter_config_ = std::make_shared(proto_config, *this, cluster_store_factory, - factory_context_); + factory_context_.serverFactoryContext().singletonManager()); + + filter_config_ = std::make_shared( + proto_config, dns_cache_manager_->getCache(proto_config.dns_cache_config()).value(), + this->get(), cluster_store_factory, factory_context_); filter_ = std::make_unique(filter_config_); filter_->setDecoderFilterCallbacks(callbacks_); @@ -547,7 +558,7 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, AddResolvedHostFilterStateMetadata host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 80); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) @@ -598,7 +609,7 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, UpdateResolvedHostFilterStateMetad host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 80); EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) @@ -649,7 +660,7 @@ TEST_F(UpstreamResolvedHostFilterStateHelper, IgnoreFilterStateMetadataNullAddre host_info->address_ = nullptr; EXPECT_CALL(callbacks_, route()); - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); EXPECT_CALL(callbacks_, route()); EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) diff --git a/test/extensions/filters/http/ext_authz/BUILD b/test/extensions/filters/http/ext_authz/BUILD index db532996b13b..6aaad4beb600 100644 --- a/test/extensions/filters/http/ext_authz/BUILD +++ b/test/extensions/filters/http/ext_authz/BUILD @@ -56,6 +56,7 @@ envoy_extension_cc_test( "//test/mocks/server:factory_context_mocks", "//test/test_common:real_threads_test_helper_lib", "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/ext_authz/v3:pkg_cc_proto", ], diff --git a/test/extensions/filters/http/ext_authz/config_test.cc b/test/extensions/filters/http/ext_authz/config_test.cc index 2eddc7f01cec..c9d716c2dab2 100644 --- a/test/extensions/filters/http/ext_authz/config_test.cc +++ b/test/extensions/filters/http/ext_authz/config_test.cc @@ -1,3 +1,4 @@ +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/core/v3/grpc_service.pb.h" #include "envoy/extensions/filters/http/ext_authz/v3/ext_authz.pb.h" #include "envoy/extensions/filters/http/ext_authz/v3/ext_authz.pb.validate.h" @@ -15,6 +16,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +using envoy::config::bootstrap::v3::Bootstrap; using testing::_; using testing::Invoke; using testing::NiceMock; @@ -30,8 +32,9 @@ class TestAsyncClientManagerImpl : public Grpc::AsyncClientManagerImpl { public: TestAsyncClientManagerImpl(Upstream::ClusterManager& cm, ThreadLocal::Instance& tls, TimeSource& time_source, Api::Api& api, - const Grpc::StatNames& stat_names) - : Grpc::AsyncClientManagerImpl(cm, tls, time_source, api, stat_names) {} + const Grpc::StatNames& stat_names, + const Bootstrap::GrpcAsyncClientManagerConfig& config) + : Grpc::AsyncClientManagerImpl(cm, tls, time_source, api, stat_names, config) {} Grpc::AsyncClientFactoryPtr factoryForGrpcService(const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) override { return std::make_unique>(); @@ -45,7 +48,8 @@ class ExtAuthzFilterTest : public Event::TestUsingSimulatedTime, ExtAuthzFilterTest() : RealThreadsTestHelper(5), stat_names_(symbol_table_) { runOnMainBlocking([&]() { async_client_manager_ = std::make_unique( - context_.cluster_manager_, tls(), api().timeSource(), api(), stat_names_); + context_.server_factory_context_.cluster_manager_, tls(), api().timeSource(), api(), + stat_names_, Bootstrap::GrpcAsyncClientManagerConfig()); }); } @@ -62,8 +66,7 @@ class ExtAuthzFilterTest : public Event::TestUsingSimulatedTime, Http::FilterFactoryCb createFilterFactory( const envoy::extensions::filters::http::ext_authz::v3::ExtAuthz& ext_authz_config) { // Delegate call to mock async client manager to real async client manager. - ON_CALL(context_, getServerFactoryContext()).WillByDefault(testing::ReturnRef(server_context_)); - ON_CALL(context_.cluster_manager_.async_client_manager_, + ON_CALL(context_.server_factory_context_.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClientWithHashKey(_, _, _)) .WillByDefault( Invoke([&](const Envoy::Grpc::GrpcServiceConfigWithHashKey& config_with_hash_key, @@ -72,7 +75,7 @@ class ExtAuthzFilterTest : public Event::TestUsingSimulatedTime, config_with_hash_key, scope, skip_cluster_check); })); ExtAuthzFilterConfig factory; - return factory.createFilterFactoryFromProto(ext_authz_config, "stats", context_); + return factory.createFilterFactoryFromProto(ext_authz_config, "stats", context_).value(); } Http::StreamFilterSharedPtr createFilterFromFilterFactory(Http::FilterFactoryCb filter_factory) { @@ -85,7 +88,6 @@ class ExtAuthzFilterTest : public Event::TestUsingSimulatedTime, } private: - NiceMock server_context_; Stats::SymbolTableImpl symbol_table_; Grpc::StatNames stat_names_; diff --git a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc index e5f9a74b1a35..a7b48eb4f768 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc @@ -27,7 +27,8 @@ using Headers = std::vector>; class ExtAuthzGrpcIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { public: - ExtAuthzGrpcIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, ipVersion()) {} + ExtAuthzGrpcIntegrationTest() + : HttpIntegrationTest(Http::CodecType::HTTP1, ExtAuthzGrpcIntegrationTest::ipVersion()) {} void createUpstreams() override { HttpIntegrationTest::createUpstreams(); @@ -183,6 +184,7 @@ class ExtAuthzGrpcIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, attributes->clear_source(); attributes->clear_destination(); attributes->clear_metadata_context(); + attributes->clear_route_metadata_context(); attributes->mutable_request()->clear_time(); http_request->clear_id(); http_request->clear_headers(); diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index 2b32da46875e..96f74729dabe 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -36,9 +36,12 @@ using Envoy::Http::LowerCaseString; using testing::_; +using testing::Contains; using testing::InSequence; using testing::Invoke; +using testing::Key; using testing::NiceMock; +using testing::Not; using testing::Return; using testing::ReturnRef; using testing::Values; @@ -58,8 +61,8 @@ template class HttpFilterTestBase : public T { if (!yaml.empty()) { TestUtility::loadFromYaml(yaml, proto_config); } - config_.reset(new FilterConfig(proto_config, *stats_store_.rootScope(), runtime_, http_context_, - "ext_authz_prefix", bootstrap_)); + config_ = std::make_shared(proto_config, *stats_store_.rootScope(), runtime_, + http_context_, "ext_authz_prefix", bootstrap_); client_ = new Filters::Common::ExtAuthz::MockClient(); filter_ = std::make_unique(config_, Filters::Common::ExtAuthz::ClientPtr{client_}); filter_->setDecoderFilterCallbacks(decoder_filter_callbacks_); @@ -363,6 +366,7 @@ TEST_F(HttpFilterTest, ErrorOpen) { envoy_grpc: cluster_name: "ext_authz_server" failure_mode_allow: true + failure_mode_allow_header_add: true )EOF"); ON_CALL(decoder_filter_callbacks_, connection()) @@ -400,6 +404,7 @@ TEST_F(HttpFilterTest, ImmediateErrorOpen) { envoy_grpc: cluster_name: "ext_authz_server" failure_mode_allow: true + failure_mode_allow_header_add: true )EOF"); ON_CALL(decoder_filter_callbacks_, connection()) @@ -433,13 +438,9 @@ TEST_F(HttpFilterTest, ImmediateErrorOpen) { EXPECT_EQ(request_headers_.get_("x-envoy-auth-failure-mode-allowed"), "true"); } -// Test when failure_mode_allow is set with runtime flag closed and the response from the +// Test when failure_mode_allow is set with header add closed and the response from the // authorization service is Error that the request is allowed to continue. -TEST_F(HttpFilterTest, ErrorOpenWithRuntimeFlagClose) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.http_ext_auth_failure_mode_allow_header_add", "false"}}); - +TEST_F(HttpFilterTest, ErrorOpenWithHeaderAddClose) { InSequence s; initialize(R"EOF( @@ -448,6 +449,7 @@ TEST_F(HttpFilterTest, ErrorOpenWithRuntimeFlagClose) { envoy_grpc: cluster_name: "ext_authz_server" failure_mode_allow: true + failure_mode_allow_header_add: false )EOF"); ON_CALL(decoder_filter_callbacks_, connection()) @@ -474,13 +476,9 @@ TEST_F(HttpFilterTest, ErrorOpenWithRuntimeFlagClose) { EXPECT_EQ(request_headers_.get_("x-envoy-auth-failure-mode-allowed"), EMPTY_STRING); } -// Test when failure_mode_allow is set with runtime flag closed and the response from the +// Test when failure_mode_allow is set with header add closed and the response from the // authorization service is an immediate Error that the request is allowed to continue. -TEST_F(HttpFilterTest, ImmediateErrorOpenWithRuntimeFlagClose) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues( - {{"envoy.reloadable_features.http_ext_auth_failure_mode_allow_header_add", "false"}}); - +TEST_F(HttpFilterTest, ImmediateErrorOpenWithHeaderAddClose) { InSequence s; initialize(R"EOF( @@ -489,6 +487,7 @@ TEST_F(HttpFilterTest, ImmediateErrorOpenWithRuntimeFlagClose) { envoy_grpc: cluster_name: "ext_authz_server" failure_mode_allow: true + failure_mode_allow_header_add: false )EOF"); ON_CALL(decoder_filter_callbacks_, connection()) @@ -1512,6 +1511,174 @@ TEST_F(HttpFilterTest, ConnectionMetadataContext) { "not.selected.data")); } +// Verifies that specified route metadata is passed along in the check request +TEST_F(HttpFilterTest, RouteMetadataContext) { + initialize(R"EOF( + transport_api_version: V3 + grpc_service: + envoy_grpc: + cluster_name: "ext_authz_server" + route_metadata_context_namespaces: + - request.connection.route.have.data + - request.route.have.data + - connection.route.have.data + - route.has.data + - request.has.data + - untyped.and.typed.route.data + - typed.route.data + - untyped.route.data + route_typed_metadata_context_namespaces: + - untyped.and.typed.route.data + - typed.route.data + - untyped.route.data + metadata_context_namespaces: + - request.connection.route.have.data + - request.route.have.data + - connection.route.have.data + - connection.has.data + - route.has.data + )EOF"); + + const std::string route_yaml = R"EOF( + filter_metadata: + request.connection.route.have.data: + data: route + request.route.have.data: + data: route + connection.route.have.data: + data: route + route.has.data: + data: route + untyped.and.typed.route.data: + data: route_untyped + untyped.route.data: + data: route_untyped + typed_filter_metadata: + untyped.and.typed.route.data: + '@type': type.googleapis.com/helloworld.HelloRequest + name: route_typed + typed.route.data: + '@type': type.googleapis.com/helloworld.HelloRequest + name: route_typed + )EOF"; + + const std::string request_yaml = R"EOF( + filter_metadata: + request.connection.route.have.data: + data: request + request.route.have.data: + data: request + )EOF"; + + const std::string connection_yaml = R"EOF( + filter_metadata: + request.connection.route.have.data: + data: connection + connection.route.have.data: + data: connection + connection.has.data: + data: connection + )EOF"; + + prepareCheck(); + + envoy::config::core::v3::Metadata request_metadata, connection_metadata, route_metadata; + TestUtility::loadFromYaml(request_yaml, request_metadata); + TestUtility::loadFromYaml(connection_yaml, connection_metadata); + TestUtility::loadFromYaml(route_yaml, route_metadata); + ON_CALL(decoder_filter_callbacks_.stream_info_, dynamicMetadata()) + .WillByDefault(ReturnRef(request_metadata)); + connection_.stream_info_.metadata_ = connection_metadata; + ON_CALL(*decoder_filter_callbacks_.route_, metadata()).WillByDefault(ReturnRef(route_metadata)); + + envoy::service::auth::v3::CheckRequest check_request; + EXPECT_CALL(*client_, check(_, _, _, _)) + .WillOnce( + Invoke([&](Filters::Common::ExtAuthz::RequestCallbacks&, + const envoy::service::auth::v3::CheckRequest& check_param, Tracing::Span&, + const StreamInfo::StreamInfo&) -> void { check_request = check_param; })); + + filter_->decodeHeaders(request_headers_, false); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + + for (const auto& namespace_from_route : std::vector{ + "request.connection.route.have.data", + "request.route.have.data", + "connection.route.have.data", + "route.has.data", + }) { + ASSERT_THAT(check_request.attributes().route_metadata_context().filter_metadata(), + Contains(Key(namespace_from_route))); + EXPECT_EQ("route", check_request.attributes() + .route_metadata_context() + .filter_metadata() + .at(namespace_from_route) + .fields() + .at("data") + .string_value()); + } + EXPECT_THAT(check_request.attributes().route_metadata_context().filter_metadata(), + Not(Contains(Key("request.has.data")))); + + for (const auto& namespace_from_request : + std::vector{"request.connection.route.have.data", "request.route.have.data"}) { + ASSERT_THAT(check_request.attributes().metadata_context().filter_metadata(), + Contains(Key(namespace_from_request))); + EXPECT_EQ("request", check_request.attributes() + .metadata_context() + .filter_metadata() + .at(namespace_from_request) + .fields() + .at("data") + .string_value()); + } + for (const auto& namespace_from_connection : + std::vector{"connection.route.have.data", "connection.has.data"}) { + ASSERT_THAT(check_request.attributes().metadata_context().filter_metadata(), + Contains(Key(namespace_from_connection))); + EXPECT_EQ("connection", check_request.attributes() + .metadata_context() + .filter_metadata() + .at(namespace_from_connection) + .fields() + .at("data") + .string_value()); + } + EXPECT_THAT(check_request.attributes().metadata_context().filter_metadata(), + Not(Contains(Key("route.has.data")))); + + for (const auto& namespace_from_route_untyped : + std::vector{"untyped.and.typed.route.data", "untyped.route.data"}) { + ASSERT_THAT(check_request.attributes().route_metadata_context().filter_metadata(), + Contains(Key(namespace_from_route_untyped))); + EXPECT_EQ("route_untyped", check_request.attributes() + .route_metadata_context() + .filter_metadata() + .at(namespace_from_route_untyped) + .fields() + .at("data") + .string_value()); + } + EXPECT_THAT(check_request.attributes().route_metadata_context().filter_metadata(), + Not(Contains(Key("typed.route.data")))); + + for (const auto& namespace_from_route_typed : + std::vector{"untyped.and.typed.route.data", "typed.route.data"}) { + ASSERT_THAT(check_request.attributes().route_metadata_context().typed_filter_metadata(), + Contains(Key(namespace_from_route_typed))); + helloworld::HelloRequest hello; + EXPECT_TRUE(check_request.attributes() + .route_metadata_context() + .typed_filter_metadata() + .at(namespace_from_route_typed) + .UnpackTo(&hello)); + EXPECT_EQ("route_typed", hello.name()); + } + EXPECT_THAT(check_request.attributes().route_metadata_context().typed_filter_metadata(), + Not(Contains(Key("untyped.route.data")))); +} + // Test that filter can be disabled via the filter_enabled field. TEST_F(HttpFilterTest, FilterDisabled) { initialize(R"EOF( @@ -2382,6 +2549,51 @@ TEST_P(HttpFilterTestParam, DeniedResponseWith401) { .value()); } +// Test that a denied response results in the connection closing with a 401 response to the client. +TEST_P(HttpFilterTestParam, DeniedResponseWith401NoClusterResponseCodeStats) { + initialize(R"EOF( + transport_api_version: V3 + grpc_service: + envoy_grpc: + cluster_name: "ext_authz_server" + charge_cluster_response_stats: + value: false + )EOF"); + + InSequence s; + + prepareCheck(); + EXPECT_CALL(*client_, check(_, _, _, _)) + .WillOnce( + Invoke([&](Filters::Common::ExtAuthz::RequestCallbacks& callbacks, + const envoy::service::auth::v3::CheckRequest&, Tracing::Span&, + const StreamInfo::StreamInfo&) -> void { request_callbacks_ = &callbacks; })); + + EXPECT_CALL(decoder_filter_callbacks_.stream_info_, + setResponseFlag(Envoy::StreamInfo::ResponseFlag::UnauthorizedExternalService)); + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "401"}}; + EXPECT_CALL(decoder_filter_callbacks_, + encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); + EXPECT_CALL(decoder_filter_callbacks_, continueDecoding()).Times(0); + + Filters::Common::ExtAuthz::Response response{}; + response.status = Filters::Common::ExtAuthz::CheckStatus::Denied; + response.status_code = Http::Code::Unauthorized; + request_callbacks_->onComplete(std::make_unique(response)); + EXPECT_EQ(1U, decoder_filter_callbacks_.clusterInfo() + ->statsScope() + .counterFromString("ext_authz.denied") + .value()); + EXPECT_EQ(1U, config_->stats().denied_.value()); + EXPECT_EQ(0, decoder_filter_callbacks_.clusterInfo() + ->statsScope() + .counterFromString("upstream_rq_4xx") + .value()); +} + // Test that a denied response results in the connection closing with a 403 response to the client. TEST_P(HttpFilterTestParam, DeniedResponseWith403) { InSequence s; @@ -2760,6 +2972,143 @@ TEST_P(HttpFilterTestParam, NoCluster) { filter_->decodeHeaders(request_headers_, false); } +// Check that config validation for per-route filter works as expected. +TEST_F(HttpFilterTest, PerRouteCheckSettingsConfigCheck) { + // Set allow_partial_message to true and max_request_bytes to 5 on the per-route filter. + envoy::extensions::filters::http::ext_authz::v3::BufferSettings buffer_settings; + buffer_settings.set_max_request_bytes(5); // Set the max_request_bytes value + buffer_settings.set_allow_partial_message(true); // Set the allow_partial_message value + // Set the per-route filter config. + envoy::extensions::filters::http::ext_authz::v3::CheckSettings check_settings; + check_settings.mutable_with_request_body()->CopyFrom(buffer_settings); + check_settings.set_disable_request_body_buffering(true); + // Initialize the route's per filter config. + envoy::extensions::filters::http::ext_authz::v3::ExtAuthzPerRoute settings; + settings.mutable_check_settings()->CopyFrom(check_settings); + + // Expect an exception while initializing the route's per filter config. + EXPECT_THROW_WITH_MESSAGE((FilterConfigPerRoute(settings)), EnvoyException, + "Invalid configuration for check_settings. Only one of " + "disable_request_body_buffering or with_request_body can be set."); +} + +// Checks that the per-route filter can override the check_settings set on the main filter. +TEST_F(HttpFilterTest, PerRouteCheckSettingsWorks) { + InSequence s; + + initialize(R"EOF( + transport_api_version: V3 + grpc_service: + envoy_grpc: + cluster_name: "ext_authz_server" + failure_mode_allow: false + )EOF"); + + // Set allow_partial_message to true and max_request_bytes to 5 on the per-route filter. + envoy::extensions::filters::http::ext_authz::v3::BufferSettings buffer_settings; + buffer_settings.set_max_request_bytes(5); // Set the max_request_bytes value + buffer_settings.set_allow_partial_message(true); // Set the allow_partial_message value + // Set the per-route filter config. + envoy::extensions::filters::http::ext_authz::v3::CheckSettings check_settings; + check_settings.mutable_with_request_body()->CopyFrom(buffer_settings); + // Initialize the route's per filter config. + envoy::extensions::filters::http::ext_authz::v3::ExtAuthzPerRoute settings; + settings.mutable_check_settings()->CopyFrom(check_settings); + FilterConfigPerRoute auth_per_route(settings); + + ON_CALL(*decoder_filter_callbacks_.route_, mostSpecificPerFilterConfig(_)) + .WillByDefault(Return(&auth_per_route)); + ON_CALL(decoder_filter_callbacks_, connection()) + .WillByDefault(Return(OptRef{connection_})); + ON_CALL(decoder_filter_callbacks_, decodingBuffer()).WillByDefault(Return(&data_)); + ON_CALL(decoder_filter_callbacks_, addDecodedData(_, _)) + .WillByDefault(Invoke([&](Buffer::Instance& data, bool) { data_.add(data); })); + EXPECT_CALL(decoder_filter_callbacks_, setDecoderBufferLimit(_)).Times(0); + connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(addr_); + connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress(addr_); + EXPECT_CALL(*client_, check(_, _, _, _)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + + Buffer::OwnedImpl buffer1("foo"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, filter_->decodeData(buffer1, false)); + data_.add(buffer1.toString()); + + Buffer::OwnedImpl buffer2("bar"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(buffer2, true)); + data_.add(buffer2.toString()); + + Buffer::OwnedImpl buffer3("barfoo"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(buffer3, true)); + data_.add(buffer3.toString()); + + Buffer::OwnedImpl buffer4("more data after watermark is set is possible"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(buffer4, true)); + + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(request_trailers_)); +} + +// Checks that the per-route filter can override the check_settings set on the main filter. +TEST_F(HttpFilterTest, PerRouteCheckSettingsOverrideWorks) { + InSequence s; + + initialize(R"EOF( + transport_api_version: V3 + grpc_service: + envoy_grpc: + cluster_name: "ext_authz_server" + failure_mode_allow: false + with_request_body: + max_request_bytes: 1 + allow_partial_message: false + )EOF"); + + // Set allow_partial_message to true and max_request_bytes to 10 on the per-route filter. + envoy::extensions::filters::http::ext_authz::v3::BufferSettings buffer_settings; + buffer_settings.set_max_request_bytes(10); // Set the max_request_bytes value + buffer_settings.set_allow_partial_message(true); // Set the allow_partial_message value + // Set the per-route filter config. + envoy::extensions::filters::http::ext_authz::v3::CheckSettings check_settings; + check_settings.mutable_with_request_body()->CopyFrom(buffer_settings); + // Initialize the route's per filter config. + envoy::extensions::filters::http::ext_authz::v3::ExtAuthzPerRoute settings; + settings.mutable_check_settings()->CopyFrom(check_settings); + FilterConfigPerRoute auth_per_route(settings); + + ON_CALL(*decoder_filter_callbacks_.route_, mostSpecificPerFilterConfig(_)) + .WillByDefault(Return(&auth_per_route)); + ON_CALL(decoder_filter_callbacks_, connection()) + .WillByDefault(Return(OptRef{connection_})); + ON_CALL(decoder_filter_callbacks_, decodingBuffer()).WillByDefault(Return(&data_)); + ON_CALL(decoder_filter_callbacks_, addDecodedData(_, _)) + .WillByDefault(Invoke([&](Buffer::Instance& data, bool) { data_.add(data); })); + EXPECT_CALL(decoder_filter_callbacks_, setDecoderBufferLimit(_)).Times(0); + connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(addr_); + connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress(addr_); + EXPECT_CALL(*client_, check(_, _, _, _)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + + Buffer::OwnedImpl buffer1("foo"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, filter_->decodeData(buffer1, false)); + data_.add(buffer1.toString()); + + Buffer::OwnedImpl buffer2("bar"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, filter_->decodeData(buffer2, false)); + data_.add(buffer2.toString()); + + Buffer::OwnedImpl buffer3("barfoo"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(buffer3, true)); + data_.add(buffer3.toString()); + + Buffer::OwnedImpl buffer4("more data after watermark is set is possible"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(buffer4, true)); + + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(request_trailers_)); +} + // Verify that request body buffering can be skipped per route. TEST_P(HttpFilterTestParam, DisableRequestBodyBufferingOnRoute) { envoy::extensions::filters::http::ext_authz::v3::ExtAuthzPerRoute settings; diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD index 2fe5e47a3763..a99fde2191ee 100644 --- a/test/extensions/filters/http/ext_proc/BUILD +++ b/test/extensions/filters/http/ext_proc/BUILD @@ -118,7 +118,7 @@ envoy_extension_cc_test( size = "large", # This test can take a while under tsan. srcs = ["ext_proc_integration_test.cc"], extension_names = ["envoy.filters.http.ext_proc"], - shard_count = 2, + shard_count = 4, tags = [ "cpu:3", ], diff --git a/test/extensions/filters/http/ext_proc/config_test.cc b/test/extensions/filters/http/ext_proc/config_test.cc index 914e454ebd5b..6cfa1d9196a9 100644 --- a/test/extensions/filters/http/ext_proc/config_test.cc +++ b/test/extensions/filters/http/ext_proc/config_test.cc @@ -40,7 +40,8 @@ TEST(HttpExtProcConfigTest, CorrectConfig) { testing::NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.h b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.h index 836c5b7c7c61..a46639581e7c 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.h +++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz.h @@ -229,8 +229,11 @@ fuzzExtProcRun(const test::extensions::filters::http::ext_proc::ExtProcGrpcTestC FuzzedDataProvider downstream_provider( reinterpret_cast(input.downstream_data().data()), input.downstream_data().size()); - FuzzedDataProvider ext_proc_provider( - reinterpret_cast(input.ext_proc_data().data()), input.ext_proc_data().size()); + static std::unique_ptr ext_proc_data = nullptr; + ext_proc_data = std::make_unique(input.ext_proc_data()); + static std::unique_ptr ext_proc_provider = nullptr; + ext_proc_provider = std::make_unique( + reinterpret_cast(ext_proc_data->data()), ext_proc_data->size()); static std::unique_ptr fuzzer = nullptr; static std::unique_ptr fuzz_helper = nullptr; @@ -242,7 +245,7 @@ fuzzExtProcRun(const test::extensions::filters::http::ext_proc::ExtProcGrpcTestC TestEnvironment::getIpVersionsForTest()[0], TestEnvironment::getsGrpcVersionsForTest()[0]); } // Initialize fuzz_helper. - fuzz_helper = std::make_unique(&ext_proc_provider); + fuzz_helper = std::make_unique(ext_proc_provider.get()); // Initialize test server. if (fuzzCreateEnvoy(fuzz_exec_count, persistent_mode)) { @@ -295,6 +298,8 @@ fuzzExtProcRun(const test::extensions::filters::http::ext_proc::ExtProcGrpcTestC } else { fuzzer->tearDown(true); } + ext_proc_data.reset(); + ext_proc_provider.reset(); fuzz_helper.reset(); } diff --git a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc index 53ca4c68cd92..b565095349e7 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_grpc_fuzz_helper.cc @@ -107,7 +107,7 @@ ExtProcFuzzHelper::ExtProcFuzzHelper(FuzzedDataProvider* provider) { provider_ = std::string ExtProcFuzzHelper::consumeRepeatedString() { const uint32_t str_len = provider_->ConsumeIntegralInRange(0, ExtProcFuzzMaxDataSize); - return std::string(str_len, 'b'); + return {static_cast(str_len), 'b'}; } // Since FuzzedDataProvider requires enums to define a kMaxValue, we cannot @@ -147,7 +147,7 @@ void ExtProcFuzzHelper::logRequest(const ProcessingRequest* req) { grpc::Status ExtProcFuzzHelper::randomGrpcStatusWithMessage() { const grpc::StatusCode code = randomGrpcStatusCode(); ENVOY_LOG_MISC(trace, "Closing stream with StatusCode {}", code); - return grpc::Status(code, consumeRepeatedString()); + return {code, consumeRepeatedString()}; } // TODO(ikepolinsky): implement this function diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index 16625bbdfdb9..b75f2eb4e142 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -47,6 +47,7 @@ using namespace std::chrono_literals; struct ConfigOptions { bool valid_grpc_server = true; bool add_logging_filter = false; + bool http1_codec = false; }; // These tests exercise the ext_proc filter through Envoy's integration test @@ -109,7 +110,7 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, } // Construct a configuration proto for our filter and then re-write it // to JSON so that we can add it to the overall config - envoy::config::listener::v3::Filter ext_proc_filter; + envoy::extensions::filters::network::http_connection_manager::v3::HttpFilter ext_proc_filter; std::string ext_proc_filter_name = "envoy.filters.http.ext_proc"; ext_proc_filter.set_name(ext_proc_filter_name); ext_proc_filter.mutable_typed_config()->PackFrom(proto_config_); @@ -122,7 +123,7 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, test::integration::filters::LoggingTestFilterConfig logging_filter_config; logging_filter_config.set_logging_id(ext_proc_filter_name); logging_filter_config.set_upstream_cluster_name(valid_grpc_cluster_name); - envoy::config::listener::v3::Filter logging_filter; + envoy::extensions::filters::network::http_connection_manager::v3::HttpFilter logging_filter; logging_filter.set_name("logging-test-filter"); logging_filter.mutable_typed_config()->PackFrom(logging_filter_config); @@ -134,8 +135,14 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, config_helper_.addRuntimeOverride(Runtime::defer_processing_backedup_streams, deferredProcessing() ? "true" : "false"); }); - setUpstreamProtocol(Http::CodecType::HTTP2); - setDownstreamProtocol(Http::CodecType::HTTP2); + + if (config_option.http1_codec) { + setUpstreamProtocol(Http::CodecType::HTTP1); + setDownstreamProtocol(Http::CodecType::HTTP1); + } else { + setUpstreamProtocol(Http::CodecType::HTTP2); + setDownstreamProtocol(Http::CodecType::HTTP2); + } } void setPerRouteConfig(Route* route, const ExtProcPerRoute& cfg) { @@ -166,7 +173,8 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, IntegrationStreamDecoderPtr sendDownstreamRequestWithBody( absl::string_view body, - absl::optional> modify_headers) { + absl::optional> modify_headers, + bool add_content_length = false) { auto conn = makeClientConnection(lookupPort("http")); codec_client_ = makeHttpConnection(std::move(conn)); Http::TestRequestHeaderMapImpl headers; @@ -175,6 +183,10 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, if (modify_headers) { (*modify_headers)(headers); } + + if (add_content_length) { + headers.setContentLength(body.size()); + } return codec_client_->makeRequestWithBody(headers, std::string(body)); } @@ -184,12 +196,24 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, EXPECT_EQ(std::to_string(status_code), response.headers().getStatusValue()); } - void handleUpstreamRequest() { + void handleUpstreamRequest(bool add_content_length = false) { ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); - upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); - upstream_request_->encodeData(100, true); + Http::TestResponseHeaderMapImpl response_headers = + Http::TestResponseHeaderMapImpl{{":status", "200"}}; + uint64_t content_length = 100; + if (add_content_length) { + response_headers.setContentLength(content_length); + } + upstream_request_->encodeHeaders(response_headers, false); + upstream_request_->encodeData(content_length, true); + } + + void verifyChunkedEncoding(const Http::RequestOrResponseHeaderMap& headers) { + EXPECT_EQ(headers.ContentLength(), nullptr); + EXPECT_THAT(headers, HeaderValueOf(Http::Headers::get().TransferEncoding, + Http::Headers::get().TransferEncodingValues.Chunked)); } void handleUpstreamRequestWithTrailer() { @@ -422,64 +446,67 @@ class ExtProcIntegrationTest : public HttpIntegrationTest, } } - void addMutationRemoveHeaders(const int count, - envoy::service::ext_proc::v3::HeaderMutation& mutation) { - for (int i = 0; i < count; i++) { - mutation.add_remove_headers(absl::StrCat("x-test-header-internal-", std::to_string(i))); - } - } - - // Send response data with small chunk size in STREAMED mode. - void streamingDataWithSmallChunks(const int last_chunk_size, const bool mutate_last_chunk, - std::string response_body) { - proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::STREAMED); - proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); - proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); - initializeConfig(); + // Verify content-length header set by external processor is removed and chunked encoding is + // enabled. + void testWithHeaderMutation(ConfigOptions config_option) { + initializeConfig(config_option); HttpIntegrationTest::initialize(); - auto response = sendDownstreamRequest(absl::nullopt); - ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); - ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); - ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); - upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + auto response = sendDownstreamRequestWithBody("Replace this!", absl::nullopt); + processRequestHeadersMessage( + *grpc_upstreams_[0], true, [](const HttpHeaders&, HeadersResponse& headers_resp) { + auto* content_length = + headers_resp.mutable_response()->mutable_header_mutation()->add_set_headers(); + content_length->mutable_header()->set_key("content-length"); + content_length->mutable_header()->set_value("13"); + return true; + }); - // Send four chunks in total with last chunk end_stream flag set to be true.. - int chunk_number = 3; - for (int i = 0; i < chunk_number; i++) { - upstream_request_->encodeData(1, false); - } - upstream_request_->encodeData(last_chunk_size, true); + processRequestBodyMessage( + *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body("Hello, World!"); + return true; + }); + handleUpstreamRequest(); + // Verify that the content length header is removed and chunked encoding is enabled by http1 + // codec. + verifyChunkedEncoding(upstream_request_->headers()); + + EXPECT_EQ(upstream_request_->body().toString(), "Hello, World!"); + verifyDownstreamResponse(*response, 200); + } + + // Verify existing content-length header (i.e., no external processor mutation) is removed and + // chunked encoding is enabled. + void testWithoutHeaderMutation(ConfigOptions config_option) { + initializeConfig(config_option); + HttpIntegrationTest::initialize(); - // First chunk response. - processResponseBodyMessage( - *grpc_upstreams_[0], true, [](const HttpBody& body, BodyResponse& body_resp) { + auto response = + sendDownstreamRequestWithBody("test!", absl::nullopt, /*add_content_length=*/true); + processRequestHeadersMessage(*grpc_upstreams_[0], true, absl::nullopt); + processRequestBodyMessage( + *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); - body_mut->set_body(body.body() + " First "); + body_mut->set_body("Hello, World!"); return true; }); - for (int i = 0; i < chunk_number - 1; i++) { - processResponseBodyMessage( - *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { - auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); - body_mut->set_body(body.body() + " The Rest "); - return true; - }); - } + handleUpstreamRequest(); + verifyChunkedEncoding(upstream_request_->headers()); - if (mutate_last_chunk) { - processResponseBodyMessage( - *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { - auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); - body_mut->set_body(body.body() + " The Last "); - return true; - }); - } else { - processResponseBodyMessage(*grpc_upstreams_[0], false, absl::nullopt); - } + EXPECT_EQ(upstream_request_->body().toString(), "Hello, World!"); verifyDownstreamResponse(*response, 200); - EXPECT_EQ(response_body, response->body()); + } + + void addMutationRemoveHeaders(const int count, + envoy::service::ext_proc::v3::HeaderMutation& mutation) { + for (int i = 0; i < count; i++) { + mutation.add_remove_headers(absl::StrCat("x-test-header-internal-", std::to_string(i))); + } } envoy::extensions::filters::http::ext_proc::v3::ExternalProcessor proto_config_{}; @@ -562,7 +589,7 @@ TEST_P(ExtProcIntegrationTest, GetAndFailStreamWithLogging) { } // Test the filter connecting to an invalid ext_proc server that will result in open stream failure. -TEST_P(ExtProcIntegrationTest, GetAndFailStreamWithInvalidSever) { +TEST_P(ExtProcIntegrationTest, GetAndFailStreamWithInvalidServer) { ConfigOptions config_option = {}; config_option.valid_grpc_server = false; initializeConfig(config_option); @@ -575,6 +602,23 @@ TEST_P(ExtProcIntegrationTest, GetAndFailStreamWithInvalidSever) { std::chrono::milliseconds(25000))); } +TEST_P(ExtProcIntegrationTest, GetAndFailStreamWithInvalidServerOnResponse) { + proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); + proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::STREAMED); + + ConfigOptions config_option = {}; + config_option.valid_grpc_server = false; + config_option.http1_codec = true; + initializeConfig(config_option); + HttpIntegrationTest::initialize(); + + auto response = sendDownstreamRequestWithBody("Replace this!", absl::nullopt); + + handleUpstreamRequest(); + EXPECT_FALSE(grpc_upstreams_[0]->waitForHttpConnection(*dispatcher_, processor_connection_, + std::chrono::milliseconds(25000))); +} + // Test the filter using the default configuration by connecting to // an ext_proc server that responds to the request_headers message // successfully, but then sends a gRPC error. @@ -909,6 +953,160 @@ TEST_P(ExtProcIntegrationTest, GetBufferedButNoBodies) { verifyDownstreamResponse(*response, 200); } +TEST_P(ExtProcIntegrationTest, RemoveRequestContentLengthInStreamedMode) { + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + + ConfigOptions config_option = {}; + config_option.http1_codec = true; + testWithoutHeaderMutation(config_option); +} + +// Test the request content length is removed in BUFFERED BodySendMode + SKIP HeaderSendMode.. +TEST_P(ExtProcIntegrationTest, RemoveRequestContentLengthInBufferedMode) { + proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + + initializeConfig(); + HttpIntegrationTest::initialize(); + + auto response = + sendDownstreamRequestWithBody("test!", absl::nullopt, /*add_content_length=*/true); + processRequestBodyMessage( + *grpc_upstreams_[0], true, [](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body("Hello, World!"); + return true; + }); + + handleUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().ContentLength(), nullptr); + EXPECT_EQ(upstream_request_->body().toString(), "Hello, World!"); + verifyDownstreamResponse(*response, 200); +} + +TEST_P(ExtProcIntegrationTest, RemoveRequestContentLengthInBufferedPartialMode) { + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED_PARTIAL); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + + ConfigOptions config_option = {}; + config_option.http1_codec = true; + testWithoutHeaderMutation(config_option); +} + +TEST_P(ExtProcIntegrationTest, RemoveRequestContentLengthAfterStreamedProcessing) { + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + ConfigOptions config_option = {}; + config_option.http1_codec = true; + testWithHeaderMutation(config_option); +} + +TEST_P(ExtProcIntegrationTest, RemoveRequestContentLengthAfterBufferedPartialProcessing) { + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED_PARTIAL); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + ConfigOptions config_option = {}; + config_option.http1_codec = true; + testWithHeaderMutation(config_option); +} + +TEST_P(ExtProcIntegrationTest, RemoveResponseContentLength) { + proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); + proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::STREAMED); + + ConfigOptions config_option = {}; + config_option.http1_codec = true; + initializeConfig(config_option); + HttpIntegrationTest::initialize(); + + auto response = sendDownstreamRequestWithBody("test!", absl::nullopt); + + handleUpstreamRequest(/*add_content_length=*/true); + processResponseHeadersMessage(*grpc_upstreams_[0], true, absl::nullopt); + + processResponseBodyMessage( + *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body("Hello, World!"); + return true; + }); + + verifyDownstreamResponse(*response, 200); + verifyChunkedEncoding(response->headers()); + EXPECT_EQ(response->body(), "Hello, World!"); +} + +TEST_P(ExtProcIntegrationTest, RemoveResponseContentLengthAfterBodyProcessing) { + proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); + proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::STREAMED); + + ConfigOptions config_option = {}; + config_option.http1_codec = true; + initializeConfig(config_option); + HttpIntegrationTest::initialize(); + + auto response = sendDownstreamRequestWithBody("test!", absl::nullopt); + + handleUpstreamRequest(); + processResponseHeadersMessage( + *grpc_upstreams_[0], true, [](const HttpHeaders&, HeadersResponse& headers_resp) { + auto* content_length = + headers_resp.mutable_response()->mutable_header_mutation()->add_set_headers(); + content_length->mutable_header()->set_key("content-length"); + content_length->mutable_header()->set_value("13"); + return true; + }); + + processResponseBodyMessage( + *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body("Hello, World!"); + return true; + }); + + verifyDownstreamResponse(*response, 200); + verifyChunkedEncoding(response->headers()); + EXPECT_EQ(response->body(), "Hello, World!"); +} + +TEST_P(ExtProcIntegrationTest, MismatchedContentLengthAndBodyLength) { + proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED); + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + + ConfigOptions config_option = {}; + config_option.http1_codec = true; + initializeConfig(config_option); + HttpIntegrationTest::initialize(); + + auto response = sendDownstreamRequestWithBody("Replace this!", absl::nullopt); + std::string modified_body = "Hello, World!"; + // The content_length set by ext_proc server doesn't match the length of mutated body. + int set_content_length = modified_body.size() - 2; + processRequestHeadersMessage( + *grpc_upstreams_[0], true, [&](const HttpHeaders&, HeadersResponse& headers_resp) { + auto* content_length = + headers_resp.mutable_response()->mutable_header_mutation()->add_set_headers(); + content_length->mutable_header()->set_key("content-length"); + content_length->mutable_header()->set_value(absl::StrCat(set_content_length)); + return true; + }); + + processRequestBodyMessage( + *grpc_upstreams_[0], false, [&](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body(modified_body); + return true; + }); + EXPECT_FALSE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_, + std::chrono::milliseconds(25000))); + verifyDownstreamResponse(*response, 500); +} + // Test the filter using the default configuration by connecting to // an ext_proc server that responds to the response_headers message // by requesting to modify the response headers. @@ -1117,6 +1315,30 @@ TEST_P(ExtProcIntegrationTest, GetAndSetBodyAndHeadersOnResponse) { verifyDownstreamResponse(*response, 200); EXPECT_THAT(response->headers(), SingleHeaderValueIs("x-testing-response-header", "Yes")); + // Verify that the content length header in the response is set by external processor, + EXPECT_EQ(response->headers().getContentLengthValue(), "13"); + EXPECT_EQ("Hello, World!", response->body()); +} + +TEST_P(ExtProcIntegrationTest, GetAndSetBodyOnResponse) { + proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); + proto_config_.mutable_processing_mode()->set_response_body_mode(ProcessingMode::BUFFERED); + initializeConfig(); + HttpIntegrationTest::initialize(); + auto response = sendDownstreamRequest(absl::nullopt); + processRequestHeadersMessage(*grpc_upstreams_[0], true, absl::nullopt); + handleUpstreamRequest(); + + // Should get just one message with the body + processResponseBodyMessage( + *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { + EXPECT_TRUE(body.end_of_stream()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body("Hello, World!"); + return true; + }); + + verifyDownstreamResponse(*response, 200); EXPECT_EQ("Hello, World!", response->body()); } @@ -1129,8 +1351,15 @@ TEST_P(ExtProcIntegrationTest, GetAndSetBodyAndHeadersOnResponsePartialBuffered) auto response = sendDownstreamRequest(absl::nullopt); processRequestHeadersMessage(*grpc_upstreams_[0], true, absl::nullopt); handleUpstreamRequest(); - processResponseHeadersMessage(*grpc_upstreams_[0], false, absl::nullopt); + processResponseHeadersMessage( + *grpc_upstreams_[0], false, [](const HttpHeaders&, HeadersResponse& headers_resp) { + auto* content_length = + headers_resp.mutable_response()->mutable_header_mutation()->add_set_headers(); + content_length->mutable_header()->set_key("content-length"); + content_length->mutable_header()->set_value("100"); + return true; + }); // Should get just one message with the body processResponseBodyMessage( *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { @@ -1143,6 +1372,8 @@ TEST_P(ExtProcIntegrationTest, GetAndSetBodyAndHeadersOnResponsePartialBuffered) }); verifyDownstreamResponse(*response, 200); + // Verify that the content length header is removed in BUFFERED_PARTIAL BodySendMode. + EXPECT_EQ(response->headers().ContentLength(), nullptr); EXPECT_THAT(response->headers(), SingleHeaderValueIs("x-testing-response-header", "Yes")); } @@ -1582,6 +1813,29 @@ TEST_P(ExtProcIntegrationTest, GetAndRespondImmediatelyWithEnvoyHeaderMutation) EXPECT_THAT(response->headers(), HasNoHeader("x-envoy-foo")); } +TEST_P(ExtProcIntegrationTest, GetAndImmediateRespondMutationAllowEnvoy) { + filter_mutation_rule_ = "true"; + proto_config_.mutable_mutation_rules()->mutable_allow_envoy()->set_value(true); + proto_config_.mutable_mutation_rules()->mutable_allow_all_routing()->set_value(true); + + initializeConfig(); + HttpIntegrationTest::initialize(); + auto response = sendDownstreamRequest(absl::nullopt); + processAndRespondImmediately(*grpc_upstreams_[0], true, [](ImmediateResponse& immediate) { + immediate.mutable_status()->set_code(envoy::type::v3::StatusCode::Unauthorized); + auto* hdr = immediate.mutable_headers()->add_set_headers(); + hdr->mutable_header()->set_key("x-envoy-foo"); + hdr->mutable_header()->set_value("bar"); + auto* hdr1 = immediate.mutable_headers()->add_set_headers(); + hdr1->mutable_header()->set_key("host"); + hdr1->mutable_header()->set_value("test"); + }); + + verifyDownstreamResponse(*response, 401); + EXPECT_THAT(response->headers(), SingleHeaderValueIs("host", "test")); + EXPECT_THAT(response->headers(), SingleHeaderValueIs("x-envoy-foo", "bar")); +} + // Test the filter with request body buffering enabled using // an ext_proc server that responds to the request_body message // by modifying a header that should cause an error. @@ -2220,7 +2474,7 @@ TEST_P(ExtProcIntegrationTest, PerRouteGrpcService) { test::integration::filters::LoggingTestFilterConfig logging_filter_config; logging_filter_config.set_logging_id("envoy.filters.http.ext_proc"); logging_filter_config.set_upstream_cluster_name("ext_proc_server_1"); - envoy::config::listener::v3::Filter logging_filter; + envoy::extensions::filters::network::http_connection_manager::v3::HttpFilter logging_filter; logging_filter.set_name("logging-test-filter"); logging_filter.mutable_typed_config()->PackFrom(logging_filter_config); @@ -3004,93 +3258,6 @@ TEST_P(ExtProcIntegrationTest, SkipHeaderTrailerSendBodyClientSendAll) { verifyDownstreamResponse(*response, 200); } -// Send response data with small chunk size. -TEST_P(ExtProcIntegrationTest, StreamingResponseDataWithSmallChunks) { - streamingDataWithSmallChunks(1, true, "a First a The Rest a The Rest a The Last "); -} - -// Send response data with small chunk size terminated with an empty string. -TEST_P(ExtProcIntegrationTest, StreamingResponseDataSmallChunksTerminateEmptyString) { - streamingDataWithSmallChunks(0, true, "a First a The Rest a The Rest The Last "); -} - -// Send response data with small chunk size terminated with an empty string and no mutation -// on it. Since the last chunk data size after mutation is zero, Envoy won't inject it. -TEST_P(ExtProcIntegrationTest, StreamingResponseDataSmallChunksNoMutationLastChunk) { - streamingDataWithSmallChunks(0, false, "a First a The Rest a The Rest "); -} - -TEST_P(ExtProcIntegrationTest, StreamingRequestDataSmallChunks) { - proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED); - proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); - proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); - initializeConfig(); - HttpIntegrationTest::initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl headers; - HttpTestUtility::addDefaultHeaders(headers); - auto encoder_decoder = codec_client_->startRequest(headers); - request_encoder_ = &encoder_decoder.first; - IntegrationStreamDecoderPtr response = std::move(encoder_decoder.second); - - int chunk_number = 5; - for (int i = 0; i < chunk_number; i++) { - codec_client_->sendData(*request_encoder_, i + 1, false); - } - codec_client_->sendData(*request_encoder_, chunk_number + 1, true); - - processRequestBodyMessage(*grpc_upstreams_[0], true, absl::nullopt); - for (int i = 0; i < chunk_number - 1; i++) { - processRequestBodyMessage(*grpc_upstreams_[0], false, absl::nullopt); - } - // ext_proc server responds to clear the last chunk body. - processRequestBodyMessage( - *grpc_upstreams_[0], false, [](const HttpBody& body, BodyResponse& body_resp) { - EXPECT_TRUE(body.end_of_stream()); - auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); - body_mut->set_clear_body(true); - return true; - }); - - handleUpstreamRequest(); - verifyDownstreamResponse(*response, 200); -} - -TEST_P(ExtProcIntegrationTest, StreamingRequestBodyWithTrailer) { - proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::STREAMED); - proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SEND); - proto_config_.mutable_processing_mode()->set_response_header_mode(ProcessingMode::SKIP); - proto_config_.mutable_processing_mode()->set_request_header_mode(ProcessingMode::SKIP); - initializeConfig(); - HttpIntegrationTest::initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestRequestHeaderMapImpl headers; - HttpTestUtility::addDefaultHeaders(headers); - auto encoder_decoder = codec_client_->startRequest(headers); - request_encoder_ = &encoder_decoder.first; - IntegrationStreamDecoderPtr response = std::move(encoder_decoder.second); - - int chunk_number = 5; - // Sending streamed body. - for (int i = 0; i < chunk_number; i++) { - codec_client_->sendData(*request_encoder_, i + 1, false); - } - - Http::TestRequestTrailerMapImpl request_trailers{{"request", "trailer"}}; - codec_client_->sendTrailers(*request_encoder_, request_trailers); - - processRequestBodyMessage(*grpc_upstreams_[0], true, absl::nullopt); - for (int i = 0; i < chunk_number - 1; i++) { - processRequestBodyMessage(*grpc_upstreams_[0], false, absl::nullopt); - } - processRequestTrailersMessage(*grpc_upstreams_[0], false, absl::nullopt); - - handleUpstreamRequest(); - verifyDownstreamResponse(*response, 200); -} - TEST_P(ExtProcIntegrationTest, SendBodyBufferedPartialWithTrailer) { proto_config_.mutable_processing_mode()->set_request_body_mode(ProcessingMode::BUFFERED_PARTIAL); proto_config_.mutable_processing_mode()->set_request_trailer_mode(ProcessingMode::SEND); diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index ecf2893003cd..7e68bd4f1ac5 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -121,7 +121,8 @@ class HttpFilterTest : public testing::Test { if (!yaml.empty()) { TestUtility::loadFromYaml(yaml, proto_config); } - config_.reset(new FilterConfig(proto_config, 200ms, 10000, *stats_store_.rootScope(), "")); + config_ = + std::make_shared(proto_config, 200ms, 10000, *stats_store_.rootScope(), ""); filter_ = std::make_unique(config_, std::move(client_), proto_config.grpc_service()); filter_->setEncoderFilterCallbacks(encoder_callbacks_); EXPECT_CALL(encoder_callbacks_, encoderBufferLimit()).WillRepeatedly(Return(BufferSize)); @@ -453,6 +454,79 @@ class HttpFilterTest : public testing::Test { } } + void StreamingSmallChunksWithBodyMutation(bool empty_last_chunk, bool mutate_last_chunk) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + processing_mode: + request_header_mode: "SKIP" + response_header_mode: "SKIP" + request_body_mode: "NONE" + response_body_mode: "STREAMED" + request_trailer_mode: "SKIP" + response_trailer_mode: "SKIP" + )EOF"); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); + + Buffer::OwnedImpl first_chunk("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(first_chunk, false)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + response_headers_.addCopy(LowerCaseString(":status"), "200"); + response_headers_.addCopy(LowerCaseString("content-type"), "text/plain"); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + + Buffer::OwnedImpl want_response_body; + Buffer::OwnedImpl got_response_body; + EXPECT_CALL(encoder_callbacks_, injectEncodedDataToFilterChain(_, _)) + .WillRepeatedly(Invoke([&got_response_body](Buffer::Instance& data, Unused) { + got_response_body.move(data); + })); + uint32_t chunk_number = 3; + for (uint32_t i = 0; i < chunk_number; i++) { + Buffer::OwnedImpl resp_data(std::to_string(i)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(resp_data, false)); + processResponseBody( + [i, &want_response_body](const HttpBody& body, ProcessingResponse&, BodyResponse& resp) { + auto* body_mut = resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body(body.body() + " " + std::to_string(i) + " "); + want_response_body.add(body.body() + " " + std::to_string(i) + " "); + }, + false); + } + + std::string last_chunk_str = ""; + Buffer::OwnedImpl resp_data; + if (!empty_last_chunk) { + last_chunk_str = std::to_string(chunk_number); + } + resp_data.add(last_chunk_str); + EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_data, true)); + if (mutate_last_chunk) { + processResponseBody( + [&chunk_number, &want_response_body](const HttpBody& body, ProcessingResponse&, + BodyResponse& resp) { + auto* body_mut = resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body(body.body() + " " + std::to_string(chunk_number) + " "); + want_response_body.add(body.body() + " " + std::to_string(chunk_number) + " "); + }, + true); + } else { + processResponseBody(absl::nullopt, true); + want_response_body.add(last_chunk_str); + } + + EXPECT_EQ(want_response_body.toString(), got_response_body.toString()); + filter_->onDestroy(); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(4, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(4, config_->stats().stream_msgs_received_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); + } + // The metadata configured as part of ext_proc filter should be in the filter state. // In addition, bytes sent/received should also be stored. void expectFilterState(const Envoy::ProtobufWkt::Struct& expected_metadata) { @@ -809,11 +883,14 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBuffered) { )EOF"); // Create synthetic HTTP request + std::string request_body = "Replaced!"; + int request_body_length = request_body.size(); request_headers_.addCopy(LowerCaseString("content-type"), "text/plain"); - request_headers_.addCopy(LowerCaseString("content-length"), 100); + request_headers_.addCopy(LowerCaseString("content-length"), request_body_length); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); processRequestHeaders(true, absl::nullopt); + EXPECT_EQ(request_headers_.getContentLengthValue(), absl::StrCat(request_body_length)); Buffer::OwnedImpl req_data; TestUtility::feedBufferWithRandomCharacters(req_data, 100); @@ -822,28 +899,29 @@ TEST_F(HttpFilterTest, PostAndChangeRequestBodyBuffered) { // Testing the case where we just have one chunk of data and it is buffered. EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(req_data, true)); - processRequestBody( - [&buffered_data](const HttpBody& req_body, ProcessingResponse&, BodyResponse& body_resp) { - EXPECT_TRUE(req_body.end_of_stream()); - EXPECT_EQ(100, req_body.body().size()); - EXPECT_EQ(req_body.body(), buffered_data.toString()); - auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); - body_mut->set_body("Replaced!"); - }); + processRequestBody([&buffered_data, &request_body](const HttpBody& req_body, ProcessingResponse&, + BodyResponse& body_resp) { + EXPECT_TRUE(req_body.end_of_stream()); + EXPECT_EQ(100, req_body.body().size()); + EXPECT_EQ(req_body.body(), buffered_data.toString()); + auto* body_mut = body_resp.mutable_response()->mutable_body_mutation(); + body_mut->set_body(request_body); + }); // Expect that the original buffer is replaced. - EXPECT_EQ("Replaced!", buffered_data.toString()); + EXPECT_EQ(request_body, buffered_data.toString()); EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + std::string response_body = "bar"; response_headers_.addCopy(LowerCaseString(":status"), "200"); response_headers_.addCopy(LowerCaseString("content-type"), "text/plain"); - response_headers_.addCopy(LowerCaseString("content-length"), "3"); + response_headers_.addCopy(LowerCaseString("content-length"), absl::StrCat(response_body.size())); EXPECT_EQ(Filter1xxHeadersStatus::Continue, filter_->encode1xxHeaders(response_headers_)); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->encodeHeaders(response_headers_, false)); processResponseHeaders(false, absl::nullopt); Buffer::OwnedImpl resp_data; - resp_data.add("bar"); + resp_data.add(response_body); EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(resp_data, true)); EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); filter_->onDestroy(); @@ -1404,6 +1482,22 @@ TEST_F(HttpFilterTest, StreamingDataSmallChunk) { checkGrpcCallStatsAll(envoy::config::core::v3::TrafficDirection::OUTBOUND, 2 * chunk_number); } +TEST_F(HttpFilterTest, StreamingBodyMutateLastEmptyChunk) { + StreamingSmallChunksWithBodyMutation(true, true); +} + +TEST_F(HttpFilterTest, StreamingBodyNotMutateLastEmptyChunk) { + StreamingSmallChunksWithBodyMutation(true, false); +} + +TEST_F(HttpFilterTest, StreamingBodyMutateLastChunk) { + StreamingSmallChunksWithBodyMutation(false, true); +} + +TEST_F(HttpFilterTest, StreamingBodyNotMutateLastChunk) { + StreamingSmallChunksWithBodyMutation(false, false); +} + // gRPC call fails when streaming sends small chunk request data. TEST_F(HttpFilterTest, StreamingSendRequestDataGrpcFail) { initializeTestSendAll(); @@ -1639,6 +1733,8 @@ TEST_F(HttpFilterTest, PostStreamingBodies) { EXPECT_CALL(decoder_callbacks_, decodingBuffer()).WillRepeatedly(Return(nullptr)); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); processRequestHeaders(false, absl::nullopt); + // Test content-length header is removed in request in streamed mode. + EXPECT_EQ(request_headers_.ContentLength(), nullptr); bool decoding_watermarked = false; setUpDecodingWatermarking(decoding_watermarked); @@ -1667,6 +1763,8 @@ TEST_F(HttpFilterTest, PostStreamingBodies) { EXPECT_CALL(encoder_callbacks_, encodingBuffer()).WillRepeatedly(Return(nullptr)); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->encodeHeaders(response_headers_, false)); processResponseHeaders(false, absl::nullopt); + // Test content-length header is removed in response in streamed mode. + EXPECT_EQ(response_headers_.ContentLength(), nullptr); Buffer::OwnedImpl want_response_body; Buffer::OwnedImpl got_response_body; @@ -1879,14 +1977,18 @@ TEST_F(HttpFilterTest, GetStreamingBodyAndChangeMode) { // There should be two more messages outstanding, but not three, so respond // just to them. - for (int i = 0; i < 2; i++) { + for (int i = 0; i < 3; i++) { processResponseBody(absl::nullopt, false); } + EXPECT_CALL(encoder_callbacks_, injectEncodedDataToFilterChain(_, true)) + .WillRepeatedly(Invoke( + [&got_response_body](Buffer::Instance& data, Unused) { got_response_body.move(data); })); + // Close the stream Buffer::OwnedImpl last_resp_chunk; - EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(last_resp_chunk, true)); - processResponseBody(absl::nullopt, false); + EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(last_resp_chunk, true)); + processResponseBody(absl::nullopt, true); // At this point, the whole body should have been processed including things // that were rejected. @@ -1896,8 +1998,8 @@ TEST_F(HttpFilterTest, GetStreamingBodyAndChangeMode) { filter_->onDestroy(); EXPECT_EQ(1, config_->stats().streams_started_.value()); - EXPECT_EQ(5, config_->stats().stream_msgs_sent_.value()); - EXPECT_EQ(5, config_->stats().stream_msgs_received_.value()); + EXPECT_EQ(7, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(7, config_->stats().stream_msgs_received_.value()); EXPECT_EQ(1, config_->stats().streams_closed_.value()); } @@ -1970,8 +2072,7 @@ TEST_F(HttpFilterTest, GetStreamingBodyAndChangeModeDifferentOrder) { EXPECT_EQ(FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(resp_chunk, true)); got_response_body.move(resp_chunk); - // There should be two more messages outstanding, but not three, so respond - // just to them. + processResponseBody(absl::nullopt, false); processResponseBody(absl::nullopt, false); processResponseBody(absl::nullopt, true); @@ -1983,8 +2084,8 @@ TEST_F(HttpFilterTest, GetStreamingBodyAndChangeModeDifferentOrder) { filter_->onDestroy(); EXPECT_EQ(1, config_->stats().streams_started_.value()); - EXPECT_EQ(5, config_->stats().stream_msgs_sent_.value()); - EXPECT_EQ(5, config_->stats().stream_msgs_received_.value()); + EXPECT_EQ(6, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(6, config_->stats().stream_msgs_received_.value()); EXPECT_EQ(1, config_->stats().streams_closed_.value()); } @@ -2309,23 +2410,32 @@ TEST_F(HttpFilterTest, ProcessingModeRequestHeadersOnly) { EXPECT_EQ(1, config_->stats().streams_closed_.value()); } -// Use the default configuration, but then override the processing mode -// to disable processing of the response headers. +// Initial configuration is send everything, but after request header processing, +// override the processing mode to disable the rest. TEST_F(HttpFilterTest, ProcessingModeOverrideResponseHeaders) { initialize(R"EOF( grpc_service: envoy_grpc: cluster_name: "ext_proc_server" allow_mode_override: true + processing_mode: + request_body_mode: "STREAMED" + response_body_mode: "STREAMED" + request_trailer_mode: "SEND" + response_trailer_mode: "SEND" )EOF"); EXPECT_EQ(filter_->config().allowModeOverride(), true); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); - processRequestHeaders( - false, [](const HttpHeaders&, ProcessingResponse& response, HeadersResponse&) { - response.mutable_mode_override()->set_response_header_mode(ProcessingMode::SKIP); - }); - + processRequestHeaders(false, + [](const HttpHeaders&, ProcessingResponse& response, HeadersResponse&) { + auto mode = response.mutable_mode_override(); + mode->set_request_body_mode(ProcessingMode::NONE); + mode->set_request_trailer_mode(ProcessingMode::SKIP); + mode->set_response_header_mode(ProcessingMode::SKIP); + mode->set_response_body_mode(ProcessingMode::NONE); + mode->set_response_trailer_mode(ProcessingMode::SKIP); + }); Buffer::OwnedImpl first_chunk("foo"); EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(first_chunk, true)); EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); @@ -2663,6 +2773,7 @@ TEST_F(HttpFilterTest, ReplaceRequest) { response_headers_.addCopy(LowerCaseString("content-length"), "200"); EXPECT_EQ(FilterHeadersStatus::StopIteration, filter_->encodeHeaders(response_headers_, false)); processResponseHeaders(false, absl::nullopt); + EXPECT_EQ(response_headers_.getContentLengthValue(), "200"); Buffer::OwnedImpl resp_data_1; TestUtility::feedBufferWithRandomCharacters(resp_data_1, 100); diff --git a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc index b193e10e84c3..708d90c92414 100644 --- a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc +++ b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc @@ -224,6 +224,37 @@ TEST(MutationUtils, TestSetHeaderWithInvalidCharacter) { MutationUtils::applyHeaderMutations(mutation, headers, false, checker, rejections).ok()); } +TEST(MutationUtils, TestSetHeaderWithContentLength) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.send_header_raw_value", "false"}}); + Http::TestRequestHeaderMapImpl headers{ + {":scheme", "https"}, + {":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"host", "localhost:1000"}, + }; + // Use the default mutation rules + Checker checker(HeaderMutationRules::default_instance()); + Envoy::Stats::MockCounter rejections; + envoy::service::ext_proc::v3::HeaderMutation mutation; + auto* s = mutation.add_set_headers(); + // Test header key contains content_length. + s->mutable_header()->set_key("content-length"); + s->mutable_header()->set_value("10"); + + EXPECT_TRUE(MutationUtils::applyHeaderMutations(mutation, headers, false, checker, rejections, + /*remove_content_length=*/true) + .ok()); + // When `remove_content_length` is true, content_length headers is not added. + EXPECT_EQ(headers.ContentLength(), nullptr); + + EXPECT_TRUE(MutationUtils::applyHeaderMutations(mutation, headers, false, checker, rejections, + /*remove_content_length=*/false) + .ok()); + // When `remove_content_length` is false, content_length headers is added. + EXPECT_EQ(headers.getContentLengthValue(), "10"); +} + TEST(MutationUtils, TestRemoveHeaderWithInvalidCharacter) { Http::TestRequestHeaderMapImpl headers{ {":method", "GET"}, diff --git a/test/extensions/filters/http/ext_proc/ordering_test.cc b/test/extensions/filters/http/ext_proc/ordering_test.cc index 035be9801616..744692ef22ff 100644 --- a/test/extensions/filters/http/ext_proc/ordering_test.cc +++ b/test/extensions/filters/http/ext_proc/ordering_test.cc @@ -70,8 +70,8 @@ class OrderingTest : public testing::Test { if (cb) { (*cb)(proto_config); } - config_.reset(new FilterConfig(proto_config, kMessageTimeout, kMaxMessageTimeoutMs, - *stats_store_.rootScope(), "")); + config_ = std::make_shared(proto_config, kMessageTimeout, kMaxMessageTimeoutMs, + *stats_store_.rootScope(), ""); filter_ = std::make_unique(config_, std::move(client_), proto_config.grpc_service()); filter_->setEncoderFilterCallbacks(encoder_callbacks_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); diff --git a/test/extensions/filters/http/ext_proc/state_test.cc b/test/extensions/filters/http/ext_proc/state_test.cc index 28e384d9b5f7..314e2fa2af0a 100644 --- a/test/extensions/filters/http/ext_proc/state_test.cc +++ b/test/extensions/filters/http/ext_proc/state_test.cc @@ -14,7 +14,7 @@ TEST(StateTest, EmptyQueue) { Buffer::OwnedImpl out_data; EXPECT_TRUE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); - EXPECT_FALSE(queue.pop(false, out_data)); + EXPECT_FALSE(queue.pop(out_data)); } TEST(StateTest, BasicQueue) { @@ -23,15 +23,14 @@ TEST(StateTest, BasicQueue) { Buffer::OwnedImpl data1("Hello"); EXPECT_EQ(queue.receivedData().length(), 0); - queue.push(data1, false, false); + queue.push(data1, false); EXPECT_EQ(queue.receivedData().toString(), "Hello"); EXPECT_FALSE(queue.empty()); EXPECT_EQ(5, queue.bytesEnqueued()); - auto popped = queue.pop(false, out_data); + auto popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), "Hello"); - EXPECT_FALSE((*popped)->end_stream); - EXPECT_FALSE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 5); + EXPECT_FALSE(popped->end_stream); + EXPECT_EQ(popped->length, 5); EXPECT_TRUE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().length(), 0); @@ -43,41 +42,38 @@ TEST(StateTest, OkToPushEmptyDataToQueue) { Buffer::OwnedImpl data1(""); EXPECT_EQ(queue.receivedData().length(), 0); - queue.push(data1, true, true); + queue.push(data1, true); EXPECT_EQ(queue.receivedData().toString(), ""); EXPECT_FALSE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); - auto popped = queue.pop(false, out_data); + auto popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), ""); - EXPECT_TRUE((*popped)->end_stream); - EXPECT_TRUE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 0); + EXPECT_TRUE(popped->end_stream); + EXPECT_EQ(popped->length, 0); EXPECT_TRUE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().length(), 0); Buffer::OwnedImpl data2("Hello"); - queue.push(data2, false, false); + queue.push(data2, false); EXPECT_EQ(queue.receivedData().toString(), "Hello"); - queue.push(data1, true, true); + queue.push(data1, true); EXPECT_EQ(queue.receivedData().toString(), "Hello"); out_data.drain(out_data.length()); - popped = queue.pop(false, out_data); + popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), "Hello"); - EXPECT_FALSE((*popped)->end_stream); - EXPECT_FALSE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 5); + EXPECT_FALSE(popped->end_stream); + EXPECT_EQ(popped->length, 5); EXPECT_FALSE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().length(), 0); out_data.drain(out_data.length()); - popped = queue.pop(false, out_data); + popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), ""); - EXPECT_TRUE((*popped)->end_stream); - EXPECT_TRUE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 0); + EXPECT_TRUE(popped->end_stream); + EXPECT_EQ(popped->length, 0); EXPECT_TRUE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().length(), 0); @@ -87,48 +83,45 @@ TEST(StateTest, EnqueueDequeue) { ChunkQueue queue; Buffer::OwnedImpl out_data; Buffer::OwnedImpl data1("Hello"); - queue.push(data1, false, false); + queue.push(data1, false); Buffer::OwnedImpl data2(", World!"); - queue.push(data2, false, false); + queue.push(data2, false); EXPECT_EQ(queue.receivedData().toString(), "Hello, World!"); EXPECT_FALSE(queue.empty()); EXPECT_EQ(13, queue.bytesEnqueued()); - auto popped = queue.pop(false, out_data); + auto popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), "Hello"); - EXPECT_FALSE((*popped)->end_stream); - EXPECT_FALSE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 5); + EXPECT_FALSE(popped->end_stream); + EXPECT_EQ(popped->length, 5); EXPECT_FALSE(queue.empty()); EXPECT_EQ(8, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().length(), 8); EXPECT_EQ(queue.receivedData().toString(), ", World!"); Buffer::OwnedImpl data3("Bye"); - queue.push(data3, true, false); + queue.push(data3, true); EXPECT_EQ(11, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().toString(), ", World!Bye"); out_data.drain(out_data.length()); - popped = queue.pop(false, out_data); + popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), ", World!"); - EXPECT_FALSE((*popped)->end_stream); - EXPECT_FALSE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 8); + EXPECT_FALSE(popped->end_stream); + EXPECT_EQ(popped->length, 8); EXPECT_FALSE(queue.empty()); EXPECT_EQ(3, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().toString(), "Bye"); out_data.drain(out_data.length()); - popped = queue.pop(false, out_data); + popped = queue.pop(out_data); EXPECT_EQ(out_data.toString(), "Bye"); - EXPECT_TRUE((*popped)->end_stream); - EXPECT_FALSE((*popped)->delivered); - EXPECT_EQ((*popped)->length, 3); + EXPECT_TRUE(popped->end_stream); + EXPECT_EQ(popped->length, 3); EXPECT_TRUE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().length(), 0); out_data.drain(out_data.length()); - popped = queue.pop(false, out_data); + popped = queue.pop(out_data); EXPECT_FALSE(popped); EXPECT_TRUE(queue.empty()); EXPECT_EQ(0, queue.bytesEnqueued()); @@ -137,11 +130,11 @@ TEST(StateTest, EnqueueDequeue) { TEST(StateTest, ConsolidateThree) { ChunkQueue queue; Buffer::OwnedImpl data1("Hello"); - queue.push(data1, false, false); + queue.push(data1, false); Buffer::OwnedImpl data2(", "); - queue.push(data2, false, false); + queue.push(data2, false); Buffer::OwnedImpl data3("World!"); - queue.push(data3, false, false); + queue.push(data3, false); EXPECT_FALSE(queue.empty()); EXPECT_EQ(13, queue.bytesEnqueued()); const auto& chunk = queue.consolidate(); @@ -149,20 +142,18 @@ TEST(StateTest, ConsolidateThree) { EXPECT_EQ(13, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().toString(), "Hello, World!"); EXPECT_FALSE(chunk.end_stream); - EXPECT_TRUE(chunk.delivered); EXPECT_EQ(chunk.length, 13); } TEST(StateTest, ConsolidateOne) { ChunkQueue queue; Buffer::OwnedImpl data1("Hello"); - queue.push(data1, false, false); + queue.push(data1, false); const auto& chunk = queue.consolidate(); EXPECT_FALSE(queue.empty()); EXPECT_EQ(5, queue.bytesEnqueued()); EXPECT_EQ(queue.receivedData().toString(), "Hello"); EXPECT_FALSE(chunk.end_stream); - EXPECT_TRUE(chunk.delivered); EXPECT_EQ(chunk.length, 5); } diff --git a/test/extensions/filters/http/ext_proc/streaming_integration_test.cc b/test/extensions/filters/http/ext_proc/streaming_integration_test.cc index 274404740ee7..5ebbc8ab3225 100644 --- a/test/extensions/filters/http/ext_proc/streaming_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/streaming_integration_test.cc @@ -36,6 +36,7 @@ class StreamingIntegrationTest : public HttpIntegrationTest, protected: StreamingIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP2, ipVersion()) {} + ~StreamingIntegrationTest() { TearDown(); } void TearDown() override { cleanupUpstreamAndDownstream(); @@ -44,9 +45,13 @@ class StreamingIntegrationTest : public HttpIntegrationTest, void initializeConfig() { scoped_runtime_.mergeValues({{"envoy.reloadable_features.send_header_raw_value", "false"}}); + skip_tag_extraction_rule_check_ = true; // This enables a built-in automatic upstream server. autonomous_upstream_ = true; + autonomous_allow_incomplete_streams_ = true; proto_config_.set_allow_mode_override(true); + proto_config_.mutable_message_timeout()->set_seconds(2); + proto_config_.set_failure_mode_allow(true); config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // Create a cluster for our gRPC server pointing to the address that is running the gRPC // server. @@ -397,7 +402,7 @@ TEST_P(StreamingIntegrationTest, PostAndProcessStreamedRequestBodyAndClose) { // Do an HTTP GET that will return a body smaller than the buffer limit, which we process // in the processor. -TEST_P(StreamingIntegrationTest, GetAndProcessBufferedResponseBody) { +TEST_P(StreamingIntegrationTest, DISABLED_GetAndProcessBufferedResponseBody) { uint32_t response_size = 90000; test_processor_.start( @@ -435,7 +440,7 @@ TEST_P(StreamingIntegrationTest, GetAndProcessBufferedResponseBody) { // Do an HTTP GET that will return a body larger than the buffer limit, which we process // in the processor using streaming. -TEST_P(StreamingIntegrationTest, GetAndProcessStreamedResponseBody) { +TEST_P(StreamingIntegrationTest, DISABLED_GetAndProcessStreamedResponseBody) { uint32_t response_size = 170000; test_processor_.start( @@ -491,7 +496,7 @@ TEST_P(StreamingIntegrationTest, GetAndProcessStreamedResponseBody) { // that we got back what we expected. The processor itself must be written carefully // because once the request headers are delivered, the request and response body // chunks and the response headers can come in any order. -TEST_P(StreamingIntegrationTest, PostAndProcessStreamBothBodies) { +TEST_P(StreamingIntegrationTest, DISABLED_PostAndProcessStreamBothBodies) { const uint32_t send_chunks = 10; const uint32_t chunk_size = 11000; uint32_t request_size = send_chunks * chunk_size; @@ -579,7 +584,7 @@ TEST_P(StreamingIntegrationTest, PostAndProcessStreamBothBodies) { // Send a large HTTP POST, and expect back an equally large reply. Stream both and replace both // the request and response bodies with different bodies. -TEST_P(StreamingIntegrationTest, PostAndStreamAndTransformBothBodies) { +TEST_P(StreamingIntegrationTest, DISABLED_PostAndStreamAndTransformBothBodies) { const uint32_t send_chunks = 12; const uint32_t chunk_size = 10000; uint32_t response_size = 180000; @@ -654,7 +659,7 @@ TEST_P(StreamingIntegrationTest, PostAndStreamAndTransformBothBodies) { // Send a body that's larger than the buffer limit and have the processor // try to process it in buffered mode. The client should get an error. -TEST_P(StreamingIntegrationTest, PostAndProcessBufferedRequestBodyTooBig) { +TEST_P(StreamingIntegrationTest, DISABLED_PostAndProcessBufferedRequestBodyTooBig) { // Send just one chunk beyond the buffer limit -- integration // test framework can't handle anything else. const uint32_t num_chunks = 11; diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5854655254757376 b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5854655254757376 new file mode 100644 index 000000000000..b89ceaf3addb --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-5854655254757376 @@ -0,0 +1 @@ +config { grpc_service { envoy_grpc { cluster_name: " " } } request_attributes: "!" } request { } response { response_body { } } \ No newline at end of file diff --git a/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-6492125776445440.fuzz b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-6492125776445440.fuzz new file mode 100644 index 000000000000..1098365c84cc --- /dev/null +++ b/test/extensions/filters/http/ext_proc/unit_test_fuzz/ext_proc_corpus/clusterfuzz-testcase-minimized-ext_proc_unit_test_fuzz-6492125776445440.fuzz @@ -0,0 +1,17 @@ +config { + grpc_service { + envoy_grpc { + cluster_name: "2" + } + } + async_mode: true + request_attributes: "type.googleapis.com/test.fuzz.ProtoBody" + stat_prefix: "t. \"t.e...\n rewpo...1.e.pc0&...2..6..*..econ\"p\"\n}\nree.key: hod&\n }\n voy.t\331\236pe.m.v3.nree.\\00ree.\000{\000\000\000\"t.\n.e . .respo\\210\\252\\270\\354\\254\\254S....\\316\\254....\\3........... }\\\\n voy.tchtype.m.v3.nre.e.p\\\"p\\z\\n}\\nree.key: \\\":method\\\"\\n e.\\\\\\\\00ree.\\\\000{\\\\000\\\\000\\\\000\\\\\\\"t.e... \"t. \\\"t.e...\\n respo... \\\\\\\"t.espo...1.e.p\\\\\\\"p..ze. st.type.m.\\326\\261.Vee.ree\316\261.Vee.ree.M0..p^.I{.....z.nan.......\\\\\\\\31efixext_proc>/.[ethod\\\\x-envo..[.r..*.6..g.g&...r!..,..:p.-.`.s`.strevo3.ValueMpat3\\\\\\\\254ersle_f//////...............\\\\\\\\177\\\\\\\\177\\\\X177\\\\\\\\177\\\\\\\\177\\\\\\\\177\\\\\\\\177?.........Trve+......................envoy.reloazable_features.runtim........m\\177\\177\\177\\177\\177\\177H0\\\\\\\\\\\\\\\\000\\\\\\\\\\\\\\\\010.p\\\\\\\\\\\\\\\"\\\\\\\"\n stat_prefix: \"t. \\\"t.e...\\.M0e.p^.{...ze{.3.zep\"\n}\nrequest {\n}\nresponse {\n reqthod\"\n \322\202Hw\001 voy.tme.y.pv3.Va\\35;\\3eqn}entchervo3.ValueMatcher\"ValueMatcher\"\n \322\202H\\\":methodX\"\\n \\322\\202\\310\\215\\001 voy.type.m.v3.Va\\\\35;\\S\003e...........Q..........-A...........\316\247\322\234\002Q\002r\002P\001P\330\222\002P\001P\001t\302\277\002V\334\234\001\332\271\336\254\302\225\001P\001N\001q\335\201\002S\001|t\002r\302\203\002P\001N\327\202\302\240\001z~\330\265\332\275.......\\316\\205.................}.......nan.......\\\\316e: \"#\"\n }\n }\n processing_mode {\n response_header_mode: SEND\n request_trailer_mode: SEND\n n }\\n voy.type.m.4v.n1.\\\\n.\\\"e\\\"\\n}\\n }\\n voy.type.m.3v.nree.\\\\00re..ze. ste.\\000{\\000tat_prefix: }\\n voy.type.m.3v.nree.\\\\\\000tat_prefix: \\\"t. \\\\\\\"t\\003.e\\343\\220\\220QK.1.e.p\\\\\\\"p..ze. st.typVe..3m.vee.r.ee.M0e.p^.I{.....z.nan.../.\\\\1.63.\\\\\\\\\\\\256.......\\\\\\\\006.\\\\\\\\020\\\\\\\\001.........-.......7......{.Q.nfig {\\\\\\\\ngoogle.protobuf.Any ore.v...1.e.p\\\\\\\\\\\\\\\"p..ze. st.type.m.\\\\326\\\\261.Vee.ree.M0..p^.I{z#tatus\\\\\\\\n Bore.oy.configkcore.v.@....)....[.....c.ro.y.....or.y.config.c0.e.p\\\\010.\\\\\\\\\\\\..........\\\\316\\\\233....spo... }\\\\n xext_proc./.[r.....[*.6..g.g&...r!..,..:p.-.`.strevo3.ValueMp4\\000\\000E\\003~\\177\\177\\177\\000\\000\\000\\000v........>...equest_body {\\n }\\n}\\nonse_trailer_...NmQ............@/.\\\\..00\\356\\222\\274.p\\\\\\\"\\\\n}\\\\nre000\\\\\\\\@000te.p\\\\\\\"\\\\n}Z\\361\\265\\276\\245Q\\\\000{\\\\rb&..ec0&...2.-6..*..econC\302\207\001fig.core.v3.Back\\\\177\\\\177.n..7e.p2\\n}\\nrequest {nre/.\\\\\\\\000{\\\\000tat_prefix: \\\\\\\"t. \\\\\\\\\\\\\\\"t@003.e\\\\343\\\\220X220Q.\\\\\\\\n ore.body {\\n }\\n}\\nonse_.voy.type.m.3v.nree.\\\\00re..z\\\\\\\\\\\\\\\"t.e.....\\\\Ln }\\\\n .ype.m.v3.Va\\\\\\\\^S35;\\\\\\\\\\\\\\\\3 value: \\\\\\\"envmy.ytpe.matcher.veqn}entchervo3.Va].[cPpL0\\\\PS\\\\PL0].[c=z=z\\\\SVPL0].[.[c=z=z\\\\SVP\\\\3 value: \\\"envoy.type.matcher.veqn}entchervo6.Vaesponse {\n response_body {\n }w\\001 {\n response_bodyvoy.type.m.v3.Va\\\\35;\\\\3 ...................%.matche {\nr.veqn}ge_timeout {entchervo3.Value\n nMpatcanos: 102341 0176\n }\n}}\n}\n\n" + disable_immediate_response: true +} +request { +} +response { + request_body { + } +} diff --git a/test/extensions/filters/http/fault/config_test.cc b/test/extensions/filters/http/fault/config_test.cc index a0870ee5c9b8..d34165e4fedc 100644 --- a/test/extensions/filters/http/fault/config_test.cc +++ b/test/extensions/filters/http/fault/config_test.cc @@ -22,7 +22,7 @@ TEST(FaultFilterConfigTest, ValidateFail) { NiceMock context; envoy::extensions::filters::http::fault::v3::HTTPFault fault; fault.mutable_abort(); - EXPECT_THROW(FaultFilterFactory().createFilterFactoryFromProto(fault, "stats", context), + EXPECT_THROW(FaultFilterFactory().createFilterFactoryFromProto(fault, "stats", context).value(), ProtoValidationException); } @@ -40,7 +40,8 @@ TEST(FaultFilterConfigTest, FaultFilterCorrectJson) { const auto proto_config = convertYamlStrToProtoConfig(yaml_string); NiceMock context; FaultFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -77,7 +78,7 @@ TEST(FaultFilterConfigTest, FaultFilterCorrectProto) { NiceMock context; FaultFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -87,7 +88,8 @@ TEST(FaultFilterConfigTest, FaultFilterEmptyProto) { NiceMock context; FaultFilterFactory factory; Http::FilterFactoryCb cb = - factory.createFilterFactoryFromProto(*factory.createEmptyConfigProto(), "stats", context); + factory.createFilterFactoryFromProto(*factory.createEmptyConfigProto(), "stats", context) + .value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/file_system_buffer/config_test.cc b/test/extensions/filters/http/file_system_buffer/config_test.cc index 1444498d6337..9091afc327ee 100644 --- a/test/extensions/filters/http/file_system_buffer/config_test.cc +++ b/test/extensions/filters/http/file_system_buffer/config_test.cc @@ -52,7 +52,7 @@ class FileSystemBufferFilterConfigTest : public testing::Test { captureConfigFromProto(const ProtoFileSystemBufferFilterConfig& proto_config) { NiceMock context; Http::FilterFactoryCb cb = - factory()->createFilterFactoryFromProto(proto_config, "stats", context); + factory()->createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; std::shared_ptr config; EXPECT_CALL(filter_callback, addStreamFilter(_)).WillOnce(Invoke(captureConfig(&config))); diff --git a/test/extensions/filters/http/file_system_buffer/fragment_test.cc b/test/extensions/filters/http/file_system_buffer/fragment_test.cc index 0079da849557..662173729c62 100644 --- a/test/extensions/filters/http/file_system_buffer/fragment_test.cc +++ b/test/extensions/filters/http/file_system_buffer/fragment_test.cc @@ -19,7 +19,6 @@ using Extensions::Common::AsyncFiles::CancelFunction; using Extensions::Common::AsyncFiles::MockAsyncFileContext; using Extensions::Common::AsyncFiles::MockAsyncFileHandle; using StatusHelpers::HasStatusMessage; -using StatusHelpers::IsOk; using ::testing::_; using ::testing::Eq; using ::testing::HasSubstr; diff --git a/test/extensions/filters/http/gcp_authn/filter_config_test.cc b/test/extensions/filters/http/gcp_authn/filter_config_test.cc index d2c98d3a6dfc..5aaac73b58ec 100644 --- a/test/extensions/filters/http/gcp_authn/filter_config_test.cc +++ b/test/extensions/filters/http/gcp_authn/filter_config_test.cc @@ -35,7 +35,8 @@ TEST(GcpAuthnFilterConfigTest, GcpAuthnFilterWithCorrectProto) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); GcpAuthnFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(filter_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(filter_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/gcp_authn/gcp_authn_filter_test.cc b/test/extensions/filters/http/gcp_authn/gcp_authn_filter_test.cc index 0a5020d8f608..847aa8b7a94c 100644 --- a/test/extensions/filters/http/gcp_authn/gcp_authn_filter_test.cc +++ b/test/extensions/filters/http/gcp_authn/gcp_authn_filter_test.cc @@ -52,7 +52,7 @@ class GcpAuthnFilterTest : public testing::Test { } void setupMockObjects() { - EXPECT_CALL(context_.cluster_manager_, getThreadLocalCluster(_)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) .WillRepeatedly(Return(&thread_local_cluster_)); EXPECT_CALL(thread_local_cluster_.async_client_, send_(_, _, _)) .WillRepeatedly(Invoke([&](Envoy::Http::RequestMessagePtr& message, @@ -157,8 +157,11 @@ TEST_F(GcpAuthnFilterTest, NoCluster) { // The pointer of thread local cluster is expected to be nullptr and http async client is not // expected to be called since `cluster` is not configured. - EXPECT_CALL(context_.cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(nullptr)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_, httpAsyncClient()).Times(0); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + httpAsyncClient()) + .Times(0); EXPECT_CALL(request_callbacks_, onComplete(/*response_ptr=*/nullptr)); GcpAuthnFilterConfig config; diff --git a/test/extensions/filters/http/geoip/BUILD b/test/extensions/filters/http/geoip/BUILD index 03ce9172a8ff..6e626debcd4b 100644 --- a/test/extensions/filters/http/geoip/BUILD +++ b/test/extensions/filters/http/geoip/BUILD @@ -7,7 +7,6 @@ load( "//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_mock", "envoy_extension_cc_test", - "envoy_extension_cc_test_library", ) licenses(["notice"]) # Apache 2 @@ -24,9 +23,9 @@ envoy_extension_cc_test( size = "small", srcs = ["config_test.cc"], extension_names = ["envoy.filters.http.geoip"], + tags = ["skip_on_windows"], deps = [ ":geoip_mocks", - ":utils_lib", "//source/common/http:message_lib", "//source/extensions/filters/http/geoip:config", "//test/mocks/server:factory_context_mocks", @@ -41,7 +40,9 @@ envoy_extension_cc_test( size = "small", srcs = ["geoip_filter_test.cc"], extension_names = ["envoy.filters.http.geoip"], + tags = ["skip_on_windows"], deps = [ + ":dummy_cc_proto", ":geoip_mocks", "//source/extensions/filters/http/geoip:config", "//source/extensions/filters/http/geoip:geoip_filter_lib", @@ -53,19 +54,38 @@ envoy_extension_cc_test( ], ) +envoy_extension_cc_test( + name = "geoip_filter_integration_test", + size = "large", + srcs = select({ + "//bazel:linux": ["geoip_filter_integration_test.cc"], + "//conditions:default": [], + }), + data = [ + "//test/extensions/geoip_providers/maxmind/test_data:geolocation_databases", + ], + extension_names = [ + "envoy.filters.http.geoip", + ], + tags = ["skip_on_windows"], + deps = [ + "//source/extensions/filters/http/geoip:config", + "//source/extensions/filters/http/geoip:geoip_filter_lib", + "//source/extensions/geoip_providers/maxmind:config", + "//source/extensions/geoip_providers/maxmind:provider_impl", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/filters/http/geoip/v3:pkg_cc_proto", + ], +) + envoy_extension_cc_mock( name = "geoip_mocks", hdrs = ["mocks.h"], extension_names = ["envoy.filters.http.geoip"], + tags = ["skip_on_windows"], deps = [ ":dummy_cc_proto", - "//source/extensions/filters/http/geoip:provider_config", + "//envoy/geoip:geoip_provider_driver_interface", ], ) - -envoy_extension_cc_test_library( - name = "utils_lib", - hdrs = ["utils.h"], - extension_names = ["envoy.filters.http.geoip"], - deps = ["//source/extensions/filters/http/geoip:geoip_filter_lib"], -) diff --git a/test/extensions/filters/http/geoip/config_test.cc b/test/extensions/filters/http/geoip/config_test.cc index 7c27734d1b91..c516b2223d45 100644 --- a/test/extensions/filters/http/geoip/config_test.cc +++ b/test/extensions/filters/http/geoip/config_test.cc @@ -1,12 +1,11 @@ #include "envoy/extensions/filters/http/geoip/v3/geoip.pb.h" #include "envoy/extensions/filters/http/geoip/v3/geoip.pb.validate.h" +#include "envoy/geoip/geoip_provider_driver.h" #include "source/extensions/filters/http/geoip/config.h" #include "source/extensions/filters/http/geoip/geoip_filter.h" -#include "source/extensions/filters/http/geoip/geoip_provider_config.h" #include "test/extensions/filters/http/geoip/mocks.h" -#include "test/extensions/filters/http/geoip/utils.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/registry.h" #include "test/test_common/test_runtime.h" @@ -20,6 +19,14 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Geoip { + +class GeoipFilterPeer { +public: + static bool useXff(const GeoipFilter& filter) { return filter.config_->useXff(); } + static uint32_t xffNumTrustedHops(const GeoipFilter& filter) { + return filter.config_->xffNumTrustedHops(); + } +}; namespace { using GeoipFilterConfig = envoy::extensions::filters::http::geoip::v3::Geoip; @@ -44,61 +51,11 @@ MATCHER_P(HasXffNumTrustedHops, expected, "") { return false; } -MATCHER_P(HasGeoHeader, expected_header, "") { - auto filter = std::static_pointer_cast(arg); - auto geo_headers = GeoipFilterPeer::geoHeaders(*filter); - for (const auto& header : geo_headers) { - if (testing::Matches(expected_header)(header)) { - return true; - } - } - *result_listener << "expected header=" << expected_header - << " but header was not found in header map"; - return false; -} - -MATCHER_P(HasGeoHeadersSize, expected_size, "") { - auto filter = std::static_pointer_cast(arg); - auto geo_headers = GeoipFilterPeer::geoHeaders(*filter); - if (expected_size == static_cast(geo_headers.size())) { - return true; - } - *result_listener << "expected geo headers size=" << expected_size << " but was " - << geo_headers.size(); - return false; -} - -MATCHER_P(HasGeoAnonHeader, expected_header, "") { - auto filter = std::static_pointer_cast(arg); - auto geo_anon_headers = GeoipFilterPeer::geoAnonHeaders(*filter); - for (const auto& header : geo_anon_headers) { - if (testing::Matches(expected_header)(header)) { - return true; - } - } - *result_listener << "expected anon header=" << expected_header - << " but header was not found in header map"; - return false; -} - -MATCHER_P(HasAnonGeoHeadersSize, expected_size, "") { - auto filter = std::static_pointer_cast(arg); - auto geo_headers = GeoipFilterPeer::geoAnonHeaders(*filter); - if (expected_size == static_cast(geo_headers.size())) { - return true; - } - *result_listener << "expected geo anon headers size=" << expected_size << " but was " - << geo_headers.size(); - return false; -} - TEST(GeoipFilterConfigTest, GeoipFilterDefaultValues) { TestScopedRuntime scoped_runtime; DummyGeoipProviderFactory dummy_factory; - Registry::InjectFactory registered(dummy_factory); + Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( - geo_headers_to_add: - city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" typed_config: @@ -109,26 +66,21 @@ TEST(GeoipFilterConfigTest, GeoipFilterDefaultValues) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()).Times(2); GeoipFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(filter_config, "geoip", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(filter_config, "geoip", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, - addStreamDecoderFilter(AllOf(HasUseXff(false), HasXffNumTrustedHops(0), - HasGeoHeader("x-geo-city"), HasGeoHeadersSize(1), - HasAnonGeoHeadersSize(0)))); + addStreamDecoderFilter(AllOf(HasUseXff(false), HasXffNumTrustedHops(0)))); cb(filter_callback); } TEST(GeoipFilterConfigTest, GeoipFilterConfigWithCorrectProto) { TestScopedRuntime scoped_runtime; DummyGeoipProviderFactory dummy_factory; - Registry::InjectFactory registered(dummy_factory); + Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( xff_config: xff_num_trusted_hops: 1 - geo_headers_to_add: - country: "x-geo-country" - region: "x-geo-region" - anon_vpn: "x-anon-vpn" provider: name: "envoy.geoip_providers.dummy" typed_config: @@ -139,26 +91,21 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigWithCorrectProto) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()).Times(2); GeoipFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(filter_config, "geoip", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(filter_config, "geoip", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, - addStreamDecoderFilter( - AllOf(HasUseXff(true), HasXffNumTrustedHops(1), HasGeoHeader("x-geo-country"), - HasGeoHeader("x-geo-region"), HasGeoHeadersSize(2), - HasAnonGeoHeadersSize(1), HasGeoAnonHeader("x-anon-vpn")))); + addStreamDecoderFilter(AllOf(HasUseXff(true), HasXffNumTrustedHops(1)))); cb(filter_callback); } -TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingGeoHeaders) { +TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingProvider) { TestScopedRuntime scoped_runtime; DummyGeoipProviderFactory dummy_factory; - Registry::InjectFactory registered(dummy_factory); + Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( xff_config: xff_num_trusted_hops: 0 - provider: - typed_config: - "@type": type.googleapis.com/test.extensions.filters.http.geoip.DummyProvider )EOF"; GeoipFilterConfig filter_config; @@ -166,39 +113,16 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingGeoHeaders) { NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); GeoipFilterFactory factory; - EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(filter_config, "geoip", context), - ProtoValidationException, - "Proto constraint validation failed.*value is required.*"); -} - -TEST(GeoipFilterConfigTest, GeoipFilterConfigMissingProvider) { - TestScopedRuntime scoped_runtime; - DummyGeoipProviderFactory dummy_factory; - Registry::InjectFactory registered(dummy_factory); - std::string filter_config_yaml = R"EOF( - geo_headers_to_add: - country: "x-geo-country" - region: "x-geo-region" - )EOF"; - GeoipFilterConfig filter_config; - - TestUtility::loadFromYaml(filter_config_yaml, filter_config); - NiceMock context; - EXPECT_CALL(context, messageValidationVisitor()); - GeoipFilterFactory factory; - EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(filter_config, "geoip", context), - ProtoValidationException, - "Proto constraint validation failed.*value is required.*"); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(filter_config, "geoip", context).value(), + ProtoValidationException, "Proto constraint validation failed.*value is required.*"); } TEST(GeoipFilterConfigTest, GeoipFilterConfigUnknownProvider) { TestScopedRuntime scoped_runtime; DummyGeoipProviderFactory dummy_factory; - Registry::InjectFactory registered(dummy_factory); + Registry::InjectFactory registered(dummy_factory); std::string filter_config_yaml = R"EOF( - geo_headers_to_add: - country: "x-geo-country" - region: "x-geo-region" provider: name: "envoy.geoip_providers.unknown" )EOF"; @@ -206,7 +130,6 @@ TEST(GeoipFilterConfigTest, GeoipFilterConfigUnknownProvider) { TestUtility::loadFromYaml(filter_config_yaml, filter_config); NiceMock context; - EXPECT_CALL(context, messageValidationVisitor()); GeoipFilterFactory factory; EXPECT_THROW_WITH_MESSAGE( factory.createFilterFactoryFromProtoTyped(filter_config, "geoip", context), diff --git a/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc b/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc new file mode 100644 index 000000000000..d315b6078472 --- /dev/null +++ b/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc @@ -0,0 +1,205 @@ +#include "envoy/extensions/filters/http/geoip/v3/geoip.pb.h" + +#include "test/integration/http_integration.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::DoAll; +using testing::SaveArg; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Geoip { +namespace { + +const std::string DefaultConfig = R"EOF( +name: envoy.filters.http.geoip +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip + provider: + name: envoy.geoip_providers.maxmind + typed_config: + "@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + asn: "x-geo-asn" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + isp_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-ASN-Test.mmdb" +)EOF"; + +const std::string ConfigWithXff = R"EOF( +name: envoy.filters.http.geoip +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip + xff_config: + xff_num_trusted_hops: 1 + provider: + name: envoy.geoip_providers.maxmind + typed_config: + "@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + asn: "x-geo-asn" + is_anon: "x-geo-anon" + anon_vpn: "x-geo-anon-vpn" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + isp_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-ASN-Test.mmdb" + anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" +)EOF"; + +class GeoipFilterIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + GeoipFilterIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) {} + + absl::string_view headerValue(const absl::string_view& header_name) const { + return upstream_request_->headers() + .get(Http::LowerCaseString(header_name))[0] + ->value() + .getStringView(); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, GeoipFilterIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedNoXff) { + config_helper_.prependFilter(TestEnvironment::substitute(DefaultConfig)); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "host"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + EXPECT_EQ("Boxford", headerValue("x-geo-city")); + EXPECT_EQ("ENG", headerValue("x-geo-region")); + EXPECT_EQ("GB", headerValue("x-geo-country")); + EXPECT_EQ("15169", headerValue("x-geo-asn")); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 1); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.isp_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.hit")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.isp_db.hit")->value()); +} + +TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseXff) { + config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithXff)); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "1.2.0.0,9.10.11.12"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + EXPECT_EQ("Boxford", headerValue("x-geo-city")); + EXPECT_EQ("ENG", headerValue("x-geo-region")); + EXPECT_EQ("GB", headerValue("x-geo-country")); + EXPECT_EQ("15169", headerValue("x-geo-asn")); + EXPECT_EQ("true", headerValue("x-geo-anon")); + EXPECT_EQ("true", headerValue("x-geo-anon-vpn")); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 1); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.anon_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.anon_db.hit")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.hit")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.isp_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.isp_db.hit")->value()); +} + +TEST_P(GeoipFilterIntegrationTest, GeoHeadersOverridenInRequest) { + config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithXff)); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "81.2.69.142,9.10.11.12"}, + {"x-geo-city", "Berlin"}, + {"x-geo-country", "Germany"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + EXPECT_EQ("London", headerValue("x-geo-city")); + EXPECT_EQ("GB", headerValue("x-geo-country")); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 1); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.anon_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.anon_db.hit")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.hit")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.isp_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.isp_db.hit")->value()); +} + +TEST_P(GeoipFilterIntegrationTest, GeoDataNotPopulatedOnEmptyLookupResult) { + config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithXff)); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "10.10.10.10,9.10.11.12"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + // 10.10.10.10 is a private IP and is absent in test_data/GeoIP2-Anonymous-IP-Test.mmdb database. + ASSERT_TRUE(response->headers().get(Http::LowerCaseString("x-geo-anon")).empty()); + ASSERT_TRUE(response->headers().get(Http::LowerCaseString("x-geo-anon-vpn")).empty()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 1); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.anon_db.total")->value()); + EXPECT_EQ(nullptr, test_server_->counter("http.config_test.maxmind.anon_db.hit")); +} + +TEST_P(GeoipFilterIntegrationTest, GeoipFilterNoCrashOnLdsUpdate) { + config_helper_.prependFilter(TestEnvironment::substitute(DefaultConfig)); + initialize(); + + // LDS update to modify the listener and corresponding drain. + { + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); + new_config_helper.addConfigModifier( + [](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + listener->mutable_listener_filters_timeout()->set_seconds(10); + }); + new_config_helper.setLds("1"); + test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 1); + test_server_->waitForCounterEq("listener_manager.lds.update_success", 2); + test_server_->waitForGaugeEq("listener_manager.total_listeners_draining", 0); + } + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "host"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + EXPECT_EQ("Boxford", headerValue("x-geo-city")); + EXPECT_EQ("ENG", headerValue("x-geo-region")); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + auto response2 = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 2); + EXPECT_EQ(2, test_server_->counter("http.config_test.maxmind.city_db.total")->value()); + EXPECT_EQ(2, test_server_->counter("http.config_test.maxmind.city_db.hit")->value()); +} + +} // namespace +} // namespace Geoip +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/geoip/geoip_filter_test.cc b/test/extensions/filters/http/geoip/geoip_filter_test.cc index 035827901bc6..2a0aa027ffbd 100644 --- a/test/extensions/filters/http/geoip/geoip_filter_test.cc +++ b/test/extensions/filters/http/geoip/geoip_filter_test.cc @@ -45,8 +45,8 @@ MATCHER_P2(HasExpectedHeader, expected_header, expected_value, "") { class GeoipFilterTest : public testing::Test { public: GeoipFilterTest() - : dummy_factory_(new DummyGeoipProviderFactory()), dummy_driver_(dummy_factory_->getDriver()), - empty_response_(absl::nullopt) {} + : dummy_factory_(new DummyGeoipProviderFactory()), + dummy_driver_(dummy_factory_->getDriver()) {} void initializeFilter(const std::string& yaml) { ON_CALL(filter_callbacks_, dispatcher()).WillByDefault(::testing::ReturnRef(*dispatcher_)); @@ -59,12 +59,13 @@ class GeoipFilterTest : public testing::Test { void initializeProviderFactory() { TestScopedRuntime scoped_runtime; - Registry::InjectFactory registered(*dummy_factory_); + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); + Registry::InjectFactory registered(*dummy_factory_); } - void expectStats(const std::string& geo_header) { - EXPECT_CALL(stats_, counter(absl::StrCat("prefix.geoip.", geo_header, ".total"))); - EXPECT_CALL(stats_, counter(absl::StrCat("prefix.geoip.", geo_header, ".hit"))); + void expectStats(const uint32_t n_total = 1) { + EXPECT_CALL(stats_, counter("prefix.geoip.total")).Times(n_total); } NiceMock stats_; @@ -75,17 +76,14 @@ class GeoipFilterTest : public testing::Test { NiceMock filter_callbacks_; Api::ApiPtr api_ = Api::createApiForTest(); Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); - absl::optional empty_response_; - LookupRequest captured_rq_; - LookupGeoHeadersCallback captured_cb_; + Geolocation::LookupRequest captured_rq_; + Geolocation::LookupGeoHeadersCallback captured_cb_; Buffer::OwnedImpl data_; }; TEST_F(GeoipFilterTest, NoXffSuccessfulLookup) { initializeProviderFactory(); const std::string external_request_yaml = R"EOF( - geo_headers_to_add: - city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" typed_config: @@ -93,7 +91,7 @@ TEST_F(GeoipFilterTest, NoXffSuccessfulLookup) { )EOF"; initializeFilter(external_request_yaml); Http::TestRequestHeaderMapImpl request_headers; - expectStats("x-geo-city"); + expectStats(); Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddress("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( @@ -105,15 +103,12 @@ TEST_F(GeoipFilterTest, NoXffSuccessfulLookup) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, false)); - captured_cb_(LookupResult{{"x-geo-city", absl::make_optional("dummy-city")}}); + captured_cb_(Geolocation::LookupResult{{"x-geo-city", "dummy-city"}}); EXPECT_CALL(filter_callbacks_, continueDecoding()); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_EQ(1, request_headers.size()); EXPECT_THAT(request_headers, HasExpectedHeader("x-geo-city", "dummy-city")); EXPECT_EQ("1.2.3.4:0", captured_rq_.remoteAddress()->asString()); - EXPECT_EQ(0, captured_rq_.geoAnonHeaders().size()); - EXPECT_EQ(1, captured_rq_.geoHeaders().size()); - EXPECT_THAT(captured_rq_.geoHeaders(), testing::UnorderedElementsAre("x-geo-city")); ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); filter_->onDestroy(); } @@ -123,15 +118,13 @@ TEST_F(GeoipFilterTest, UseXffSuccessfulLookup) { const std::string external_request_yaml = R"EOF( xff_config: xff_num_trusted_hops: 1 - geo_headers_to_add: - region: "x-geo-region" provider: name: "envoy.geoip_providers.dummy" )EOF"; initializeFilter(external_request_yaml); Http::TestRequestHeaderMapImpl request_headers; request_headers.addCopy("x-forwarded-for", "10.0.0.1,10.0.0.2"); - expectStats("x-geo-region"); + expectStats(); Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddress("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( @@ -139,7 +132,7 @@ TEST_F(GeoipFilterTest, UseXffSuccessfulLookup) { EXPECT_CALL(*dummy_driver_, lookup(_, _)) .WillRepeatedly( DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { - captured_cb_(LookupResult{{"x-geo-region", absl::make_optional("dummy-region")}}); + captured_cb_(Geolocation::LookupResult{{"x-geo-region", "dummy-region"}}); }))); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, false)); @@ -148,9 +141,6 @@ TEST_F(GeoipFilterTest, UseXffSuccessfulLookup) { EXPECT_EQ(2, request_headers.size()); EXPECT_THAT(request_headers, HasExpectedHeader("x-geo-region", "dummy-region")); EXPECT_EQ("10.0.0.1:0", captured_rq_.remoteAddress()->asString()); - EXPECT_EQ(0, captured_rq_.geoAnonHeaders().size()); - EXPECT_EQ(1, captured_rq_.geoHeaders().size()); - EXPECT_THAT(captured_rq_.geoHeaders(), testing::UnorderedElementsAre("x-geo-region")); ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); filter_->onDestroy(); } @@ -158,9 +148,6 @@ TEST_F(GeoipFilterTest, UseXffSuccessfulLookup) { TEST_F(GeoipFilterTest, GeoHeadersOverridenForIncomingRequest) { initializeProviderFactory(); const std::string external_request_yaml = R"EOF( - geo_headers_to_add: - region: "x-geo-region" - city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" )EOF"; @@ -170,20 +157,16 @@ TEST_F(GeoipFilterTest, GeoHeadersOverridenForIncomingRequest) { request_headers.addCopy("x-geo-city", "ngnix_city"); std::map geo_headers = {{"x-geo-region", "dummy_region"}, {"x-geo-city", "dummy_city"}}; - for (auto& geo_header : geo_headers) { - auto& header = geo_header.first; - expectStats(header); - } + expectStats(); Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddress("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); EXPECT_CALL(*dummy_driver_, lookup(_, _)) - .WillRepeatedly( - DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { - captured_cb_(LookupResult{{"x-geo-city", absl::make_optional("dummy-city")}, - {"x-geo-region", absl::make_optional("dummy-region")}}); - }))); + .WillRepeatedly(DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { + captured_cb_(Geolocation::LookupResult{ + {"x-geo-city", "dummy-city"}, {"x-geo-region", "dummy-region"}}); + }))); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, false)); EXPECT_CALL(filter_callbacks_, continueDecoding()); @@ -192,10 +175,6 @@ TEST_F(GeoipFilterTest, GeoHeadersOverridenForIncomingRequest) { EXPECT_THAT(request_headers, HasExpectedHeader("x-geo-city", "dummy-city")); EXPECT_THAT(request_headers, HasExpectedHeader("x-geo-region", "dummy-region")); EXPECT_EQ("1.2.3.4:0", captured_rq_.remoteAddress()->asString()); - EXPECT_EQ(0, captured_rq_.geoAnonHeaders().size()); - EXPECT_EQ(2, captured_rq_.geoHeaders().size()); - EXPECT_THAT(captured_rq_.geoHeaders(), - testing::UnorderedElementsAre("x-geo-region", "x-geo-city")); ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); filter_->onDestroy(); } @@ -203,16 +182,6 @@ TEST_F(GeoipFilterTest, GeoHeadersOverridenForIncomingRequest) { TEST_F(GeoipFilterTest, AllHeadersPropagatedCorrectly) { initializeProviderFactory(); const std::string external_request_yaml = R"EOF( - geo_headers_to_add: - region: "x-geo-region" - country: "x-geo-country" - city: "x-geo-city" - asn: "x-geo-asn" - is_anon: "x-geo-anon" - anon_vpn: "x-geo-anon-vpn" - anon_hosting: "x-geo-anon-hosting" - anon_tor: "x-geo-anon-tor" - anon_proxy: "x-geo-anon-proxy" provider: name: "envoy.geoip_providers.dummy" )EOF"; @@ -227,29 +196,23 @@ TEST_F(GeoipFilterTest, AllHeadersPropagatedCorrectly) { {"x-geo-anon-hosting", "true"}, {"x-geo-anon-tor", "true"}, {"x-geo-anon-proxy", "true"}}; - for (auto& geo_header : geo_headers) { - auto& header = geo_header.first; - expectStats(header); - } - for (auto& geo_anon_header : geo_anon_headers) { - auto& header = geo_anon_header.first; - expectStats(header); - } + expectStats(); Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddress("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); EXPECT_CALL(*dummy_driver_, lookup(_, _)) .WillRepeatedly(DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { - captured_cb_(LookupResult{{"x-geo-city", "dummy-city"}, - {"x-geo-region", "dummy-region"}, - {"x-geo-country", "dummy-country"}, - {"x-geo-asn", "dummy-asn"}, - {"x-geo-anon", "true"}, - {"x-geo-anon-vpn", "false"}, - {"x-geo-anon-hosting", "true"}, - {"x-geo-anon-tor", "true"}, - {"x-geo-anon-proxy", "true"}}); + captured_cb_( + Geolocation::LookupResult{{"x-geo-city", "dummy-city"}, + {"x-geo-region", "dummy-region"}, + {"x-geo-country", "dummy-country"}, + {"x-geo-asn", "dummy-asn"}, + {"x-geo-anon", "true"}, + {"x-geo-anon-vpn", "false"}, + {"x-geo-anon-hosting", "true"}, + {"x-geo-anon-tor", "true"}, + {"x-geo-anon-proxy", "true"}}); }))); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, false)); @@ -257,14 +220,6 @@ TEST_F(GeoipFilterTest, AllHeadersPropagatedCorrectly) { dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_EQ("1.2.3.4:0", captured_rq_.remoteAddress()->asString()); EXPECT_EQ(9, request_headers.size()); - EXPECT_EQ(5, captured_rq_.geoAnonHeaders().size()); - EXPECT_EQ(4, captured_rq_.geoHeaders().size()); - EXPECT_THAT( - captured_rq_.geoHeaders(), - testing::UnorderedElementsAre("x-geo-region", "x-geo-city", "x-geo-country", "x-geo-asn")); - EXPECT_THAT(captured_rq_.geoAnonHeaders(), - testing::UnorderedElementsAre("x-geo-anon", "x-geo-anon-vpn", "x-geo-anon-hosting", - "x-geo-anon-tor", "x-geo-anon-proxy")); for (auto& geo_header : geo_headers) { auto& header = geo_header.first; auto& value = geo_header.second; @@ -282,26 +237,20 @@ TEST_F(GeoipFilterTest, AllHeadersPropagatedCorrectly) { TEST_F(GeoipFilterTest, GeoHeaderNotAppendedOnEmptyLookup) { initializeProviderFactory(); const std::string external_request_yaml = R"EOF( - geo_headers_to_add: - region: "x-geo-region" - city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" )EOF"; initializeFilter(external_request_yaml); Http::TestRequestHeaderMapImpl request_headers; - EXPECT_CALL(stats_, counter("prefix.geoip.x-geo-city.total")); - EXPECT_CALL(stats_, counter("prefix.geoip.x-geo-city.hit")).Times(0); - EXPECT_CALL(stats_, counter("prefix.geoip.x-geo-region.total")); - EXPECT_CALL(stats_, counter("prefix.geoip.x-geo-region.hit")); + expectStats(); Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddress("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); EXPECT_CALL(*dummy_driver_, lookup(_, _)) .WillRepeatedly(DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { - captured_cb_(LookupResult{{"x-geo-city", empty_response_}, - {"x-geo-region", "dummy-region"}}); + captured_cb_(Geolocation::LookupResult{ + {"x-geo-city", ""}, {"x-geo-region", "dummy-region"}}); }))); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, false)); @@ -309,8 +258,6 @@ TEST_F(GeoipFilterTest, GeoHeaderNotAppendedOnEmptyLookup) { dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_EQ("1.2.3.4:0", captured_rq_.remoteAddress()->asString()); EXPECT_EQ(1, request_headers.size()); - EXPECT_EQ(0, captured_rq_.geoAnonHeaders().size()); - EXPECT_EQ(2, captured_rq_.geoHeaders().size()); EXPECT_THAT(request_headers, HasExpectedHeader("x-geo-region", "dummy-region")); ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); filter_->onDestroy(); @@ -319,8 +266,6 @@ TEST_F(GeoipFilterTest, GeoHeaderNotAppendedOnEmptyLookup) { TEST_F(GeoipFilterTest, NoCrashIfFilterDestroyedBeforeCallbackCalled) { initializeProviderFactory(); const std::string external_request_yaml = R"EOF( - geo_headers_to_add: - city: "x-geo-city" provider: name: "envoy.geoip_providers.dummy" )EOF"; @@ -331,10 +276,9 @@ TEST_F(GeoipFilterTest, NoCrashIfFilterDestroyedBeforeCallbackCalled) { filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); EXPECT_CALL(*dummy_driver_, lookup(_, _)) - .WillRepeatedly( - DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { - captured_cb_(LookupResult{{"x-geo-city", absl::make_optional("dummy-city")}}); - }))); + .WillRepeatedly(DoAll(SaveArg<0>(&captured_rq_), SaveArg<1>(&captured_cb_), Invoke([this]() { + captured_cb_(Geolocation::LookupResult{{"x-geo-city", "dummy-city"}}); + }))); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, false)); filter_.reset(); diff --git a/test/extensions/filters/http/geoip/mocks.h b/test/extensions/filters/http/geoip/mocks.h index 34a93df3c48c..6f89272740b8 100644 --- a/test/extensions/filters/http/geoip/mocks.h +++ b/test/extensions/filters/http/geoip/mocks.h @@ -1,6 +1,6 @@ #pragma once -#include "source/extensions/filters/http/geoip/geoip_provider_config.h" +#include "envoy/geoip/geoip_provider_driver.h" #include "test/extensions/filters/http/geoip/dummy.pb.h" #include "test/extensions/filters/http/geoip/dummy.pb.validate.h" @@ -12,18 +12,21 @@ namespace Extensions { namespace HttpFilters { namespace Geoip { -class MockDriver : public Driver { +class MockDriver : public Geolocation::Driver { public: - MOCK_METHOD(void, lookup, (LookupRequest && request, LookupGeoHeadersCallback&&), (const)); + MOCK_METHOD(void, lookup, + (Geolocation::LookupRequest && request, Geolocation::LookupGeoHeadersCallback&&), + (const)); }; using MockDriverSharedPtr = std::shared_ptr; -class DummyGeoipProviderFactory : public GeoipProviderFactory { +class DummyGeoipProviderFactory : public Geolocation::GeoipProviderFactory { public: DummyGeoipProviderFactory() : driver_(new MockDriver()) {} - DriverSharedPtr createGeoipProviderDriver(const Protobuf::Message&, - GeoipProviderFactoryContextPtr&) override { + Geolocation::DriverSharedPtr + createGeoipProviderDriver(const Protobuf::Message&, const std::string&, + Server::Configuration::FactoryContext&) override { return driver_; } diff --git a/test/extensions/filters/http/grpc_field_extraction/filter_config_test.cc b/test/extensions/filters/http/grpc_field_extraction/filter_config_test.cc index 7d0e243ea225..55485685d229 100644 --- a/test/extensions/filters/http/grpc_field_extraction/filter_config_test.cc +++ b/test/extensions/filters/http/grpc_field_extraction/filter_config_test.cc @@ -51,8 +51,9 @@ using FilterConfigTestOk = FilterConfigTestBase; TEST_F(FilterConfigTestOk, DescriptorInline) { parseConfigProto(); *proto_config_.mutable_descriptor_set()->mutable_inline_bytes() = - api_->fileSystem().fileReadToEnd( - TestEnvironment::runfilesPath("test/proto/apikeys.descriptor")); + api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath("test/proto/apikeys.descriptor")) + .value(); filter_config_ = std::make_unique(proto_config_, std::make_unique(), *api_); EXPECT_EQ(filter_config_->findExtractor("undefined"), nullptr); @@ -232,8 +233,9 @@ TEST_F(FilterConfigTestException, ErrorParsingDescriptorFromFile) { TEST_F(FilterConfigTestException, UnsupportedDescriptorSourceTyep) { parseConfigProto(); *proto_config_.mutable_descriptor_set()->mutable_inline_string() = - api_->fileSystem().fileReadToEnd( - TestEnvironment::runfilesPath("test/proto/apikeys.descriptor")); + api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath("test/proto/apikeys.descriptor")) + .value(); EXPECT_THAT_THROWS_MESSAGE( std::make_unique(proto_config_, std::make_unique(), *api_), diff --git a/test/extensions/filters/http/grpc_field_extraction/filter_test.cc b/test/extensions/filters/http/grpc_field_extraction/filter_test.cc index 8f0dca099f03..886d72591247 100644 --- a/test/extensions/filters/http/grpc_field_extraction/filter_test.cc +++ b/test/extensions/filters/http/grpc_field_extraction/filter_test.cc @@ -45,8 +45,9 @@ class FilterTestBase : public ::testing::Test { ASSERT_TRUE(Protobuf::TextFormat::ParseFromString(config.empty() ? protoConfig() : config, &proto_config_)); *proto_config_.mutable_descriptor_set()->mutable_inline_bytes() = - api_->fileSystem().fileReadToEnd( - TestEnvironment::runfilesPath("test/proto/apikeys.descriptor")); + api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath("test/proto/apikeys.descriptor")) + .value(); ON_CALL(mock_decoder_callbacks_, decoderBufferLimit()) .WillByDefault(testing::Return(UINT32_MAX)); filter_config_ = std::make_unique( diff --git a/test/extensions/filters/http/grpc_http1_bridge/config_test.cc b/test/extensions/filters/http/grpc_http1_bridge/config_test.cc index f54d81de6f94..5e4f69d331f9 100644 --- a/test/extensions/filters/http/grpc_http1_bridge/config_test.cc +++ b/test/extensions/filters/http/grpc_http1_bridge/config_test.cc @@ -17,7 +17,7 @@ TEST(GrpcHttp1BridgeFilterConfigTest, GrpcHttp1BridgeFilter) { NiceMock context; GrpcHttp1BridgeFilterConfig factory; envoy::extensions::filters::http::grpc_http1_bridge::v3::Config config; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc index 96e584d502b5..af4da43613a1 100644 --- a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc +++ b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc @@ -31,9 +31,10 @@ namespace { class GrpcHttp1BridgeFilterTest : public testing::Test { protected: - void initialize(bool upgrade_protobuf = false) { + void initialize(bool upgrade_protobuf = false, bool ignore_query_params = false) { envoy::extensions::filters::http::grpc_http1_bridge::v3::Config config; config.set_upgrade_protobuf_to_grpc(upgrade_protobuf); + config.set_ignore_query_parameters(ignore_query_params); filter_ = std::make_unique(context_, config); filter_->setDecoderFilterCallbacks(decoder_callbacks_); filter_->setEncoderFilterCallbacks(encoder_callbacks_); @@ -288,7 +289,6 @@ TEST_F(GrpcHttp1BridgeFilterTest, ProtobufUpgradedHeaderSanitized) { Http::TestRequestHeaderMapImpl request_headers{{"content-type", "application/x-protobuf"}, {"content-length", "5"}, {":path", "/v1/spotify.Concat/Concat"}}; - Buffer::OwnedImpl data("hello"); EXPECT_CALL(decoder_callbacks_.downstream_callbacks_, clearRouteCache()); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); @@ -296,6 +296,17 @@ TEST_F(GrpcHttp1BridgeFilterTest, ProtobufUpgradedHeaderSanitized) { EXPECT_EQ("", request_headers.getContentLengthValue()); } +// Verifies that the query params in URL are removed when ignore_query_parameters is enabled +TEST_F(GrpcHttp1BridgeFilterTest, QueryParamsIgnored) { + initialize(false, true); + Http::TestRequestHeaderMapImpl request_headers{ + {"content-type", "application/grpc"}, + {":path", "/v1/spotify.Concat/Concat?timestamp=1701678591"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("/v1/spotify.Concat/Concat", request_headers.getPathValue()); +} + } // namespace } // namespace GrpcHttp1Bridge } // namespace HttpFilters diff --git a/test/extensions/filters/http/grpc_http1_reverse_bridge/config_test.cc b/test/extensions/filters/http/grpc_http1_reverse_bridge/config_test.cc index a23d934f7465..2123a6c3f0dd 100644 --- a/test/extensions/filters/http/grpc_http1_reverse_bridge/config_test.cc +++ b/test/extensions/filters/http/grpc_http1_reverse_bridge/config_test.cc @@ -29,7 +29,7 @@ withhold_grpc_frames: true NiceMock context; Config config_factory; Http::FilterFactoryCb cb = - config_factory.createFilterFactoryFromProto(proto_config, "stats", context); + config_factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc index 029d1153477c..a1ce6eaf34a8 100644 --- a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc +++ b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_integration_test.cc @@ -48,8 +48,7 @@ name: grpc_http1_reverse_bridge envoy::extensions::filters::http::grpc_http1_reverse_bridge::v3::FilterConfigPerRoute route_config; route_config.set_disabled(true); - (*vhost.mutable_routes(0) - ->mutable_typed_per_filter_config())["envoy.filters.http.grpc_http1_reverse_bridge"] + (*vhost.mutable_routes(0)->mutable_typed_per_filter_config())["grpc_http1_reverse_bridge"] .PackFrom(route_config); config_helper_.addVirtualHost(vhost); diff --git a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc index 566a1aee7e0c..b8bbd517e458 100644 --- a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc +++ b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc @@ -383,8 +383,9 @@ TEST_F(ReverseBridgeTest, GrpcRequestNoContentLength) { } } -// Regression tests that header-only responses do not get the content-length -// adjusted (https://github.com/envoyproxy/envoy/issues/11099) +// Regression tests that header-only responses get the gRPC frame appended and +// content-length header adjusted, as well as trailers added. +// (https://github.com/envoyproxy/envoy/issues/29989). TEST_F(ReverseBridgeTest, GrpcRequestHeaderOnlyResponse) { initialize(); decoder_callbacks_.is_grpc_request_ = true; @@ -424,11 +425,17 @@ TEST_F(ReverseBridgeTest, GrpcRequestHeaderOnlyResponse) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); } + // call should prefix the buffer with the gRPC frame header and insert the + // gRPC status into trailers. + EXPECT_CALL(encoder_callbacks_, addEncodedData(_, false)); + Http::TestResponseTrailerMapImpl trailers; + EXPECT_CALL(encoder_callbacks_, addEncodedTrailers()).WillOnce(ReturnRef(trailers)); + Http::TestResponseHeaderMapImpl headers( {{":status", "200"}, {"content-length", "0"}, {"content-type", "application/x-protobuf"}}); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, true)); EXPECT_THAT(headers, HeaderValueOf(Http::Headers::get().ContentType, "application/grpc")); - EXPECT_THAT(headers, HeaderValueOf(Http::Headers::get().ContentLength, "0")); + EXPECT_THAT(headers, HeaderValueOf(Http::Headers::get().ContentLength, "5")); } // Tests that a gRPC is downgraded to application/x-protobuf and upgraded back diff --git a/test/extensions/filters/http/grpc_json_transcoder/config_test.cc b/test/extensions/filters/http/grpc_json_transcoder/config_test.cc index 014bec89bbaf..923980666027 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/config_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/config_test.cc @@ -16,10 +16,13 @@ namespace { TEST(GrpcJsonTranscoderFilterConfigTest, ValidateFail) { NiceMock context; - EXPECT_THROW(GrpcJsonTranscoderFilterConfig().createFilterFactoryFromProto( - envoy::extensions::filters::http::grpc_json_transcoder::v3::GrpcJsonTranscoder(), - "stats", context), - ProtoValidationException); + EXPECT_THROW( + GrpcJsonTranscoderFilterConfig() + .createFilterFactoryFromProto( + envoy::extensions::filters::http::grpc_json_transcoder::v3::GrpcJsonTranscoder(), + "stats", context) + .value(), + ProtoValidationException); } } // namespace diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index 6fb01ce288c0..56515243009a 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -234,7 +234,7 @@ name: grpc_json_transcoder ->Mutable(0) ->mutable_typed_per_filter_config(); - (*config)["envoy.filters.http.grpc_json_transcoder"].PackFrom(per_route_config); + (*config)["grpc_json_transcoder"].PackFrom(per_route_config); }; config_helper_.addConfigModifier(modifier); diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 488de5895508..4b940cdac87a 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -69,8 +69,10 @@ class GrpcJsonTranscoderConfigTest : public testing::Test, public GrpcJsonTransc std::string makeProtoDescriptor(std::function process) { FileDescriptorSet descriptor_set; - descriptor_set.ParseFromString(api_->fileSystem().fileReadToEnd( - TestEnvironment::runfilesPath("test/proto/bookstore.descriptor"))); + descriptor_set.ParseFromString( + api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath("test/proto/bookstore.descriptor")) + .value()); process(descriptor_set); @@ -131,8 +133,10 @@ TEST_F(GrpcJsonTranscoderConfigTest, ParseConfigSkipRecalculating) { TEST_F(GrpcJsonTranscoderConfigTest, ParseBinaryConfig) { envoy::extensions::filters::http::grpc_json_transcoder::v3::GrpcJsonTranscoder proto_config; - proto_config.set_proto_descriptor_bin(api_->fileSystem().fileReadToEnd( - TestEnvironment::runfilesPath("test/proto/bookstore.descriptor"))); + proto_config.set_proto_descriptor_bin( + api_->fileSystem() + .fileReadToEnd(TestEnvironment::runfilesPath("test/proto/bookstore.descriptor")) + .value()); proto_config.add_services("bookstore.Bookstore"); EXPECT_NO_THROW(JsonTranscoderConfig config(proto_config, *api_)); } diff --git a/test/extensions/filters/http/grpc_stats/config_test.cc b/test/extensions/filters/http/grpc_stats/config_test.cc index ca324ae8d666..4346ec23aee8 100644 --- a/test/extensions/filters/http/grpc_stats/config_test.cc +++ b/test/extensions/filters/http/grpc_stats/config_test.cc @@ -15,7 +15,6 @@ using testing::_; using testing::Property; -using testing::Return; namespace Envoy { namespace Extensions { @@ -27,7 +26,8 @@ class GrpcStatsFilterConfigTest : public testing::Test { protected: void initialize() { GrpcStatsFilterConfigFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config_, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(config_, "stats", context_).value(); Http::MockFilterChainFactoryCallbacks filter_callback; ON_CALL(filter_callback, addStreamFilter(_)).WillByDefault(testing::SaveArg<0>(&filter_)); diff --git a/test/extensions/filters/http/grpc_web/config_test.cc b/test/extensions/filters/http/grpc_web/config_test.cc index 32be18ff29b7..0dd76f388759 100644 --- a/test/extensions/filters/http/grpc_web/config_test.cc +++ b/test/extensions/filters/http/grpc_web/config_test.cc @@ -17,7 +17,7 @@ TEST(GrpcWebFilterConfigTest, GrpcWebFilter) { NiceMock context; GrpcWebFilterConfig factory; envoy::extensions::filters::http::grpc_web::v3::GrpcWeb config; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc index f7986667f57c..8476cfebb8d1 100644 --- a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc +++ b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc @@ -57,28 +57,28 @@ class GrpcWebFilterTest : public testing::TestWithParam(GetParam()); } + const std::string& requestContentType() const { return std::get<0>(GetParam()); } - const std::string& request_accept() const { return std::get<1>(GetParam()); } + const std::string& requestAccept() const { return std::get<1>(GetParam()); } bool isTextRequest() const { - return request_content_type() == Http::Headers::get().ContentTypeValues.GrpcWebText || - request_content_type() == Http::Headers::get().ContentTypeValues.GrpcWebTextProto; + return requestContentType() == Http::Headers::get().ContentTypeValues.GrpcWebText || + requestContentType() == Http::Headers::get().ContentTypeValues.GrpcWebTextProto; } bool isBinaryRequest() const { - return request_content_type() == Http::Headers::get().ContentTypeValues.GrpcWeb || - request_content_type() == Http::Headers::get().ContentTypeValues.GrpcWebProto; + return requestContentType() == Http::Headers::get().ContentTypeValues.GrpcWeb || + requestContentType() == Http::Headers::get().ContentTypeValues.GrpcWebProto; } - bool accept_text_response() const { - return request_accept() == Http::Headers::get().ContentTypeValues.GrpcWebText || - request_accept() == Http::Headers::get().ContentTypeValues.GrpcWebTextProto; + bool acceptTextResponse() const { + return requestAccept() == Http::Headers::get().ContentTypeValues.GrpcWebText || + requestAccept() == Http::Headers::get().ContentTypeValues.GrpcWebTextProto; } - bool accept_binary_response() const { - return request_accept() == Http::Headers::get().ContentTypeValues.GrpcWeb || - request_accept() == Http::Headers::get().ContentTypeValues.GrpcWebProto; + bool acceptBinaryResponse() const { + return requestAccept() == Http::Headers::get().ContentTypeValues.GrpcWeb || + requestAccept() == Http::Headers::get().ContentTypeValues.GrpcWebProto; } bool doStatTracking() const { return filter_.doStatTracking(); } @@ -420,7 +420,7 @@ TEST_F(GrpcWebFilterTest, InvalidUpstreamResponseForTextWithTrailers) { TEST_P(GrpcWebFilterTest, StatsNoCluster) { Http::TestRequestHeaderMapImpl request_headers{ - {"content-type", request_content_type()}, + {"content-type", requestContentType()}, {":path", "/lyft.users.BadCompanions/GetBadCompanions"}}; EXPECT_CALL(decoder_callbacks_, clusterInfo()).WillOnce(Return(nullptr)); @@ -430,7 +430,7 @@ TEST_P(GrpcWebFilterTest, StatsNoCluster) { TEST_P(GrpcWebFilterTest, StatsNormalResponse) { Http::TestRequestHeaderMapImpl request_headers{ - {"content-type", request_content_type()}, + {"content-type", requestContentType()}, {":path", "/lyft.users.BadCompanions/GetBadCompanions"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); @@ -460,7 +460,7 @@ TEST_P(GrpcWebFilterTest, StatsNormalResponse) { TEST_P(GrpcWebFilterTest, StatsErrorResponse) { Http::TestRequestHeaderMapImpl request_headers{ - {"content-type", request_content_type()}, + {"content-type", requestContentType()}, {":path", "/lyft.users.BadCompanions/GetBadCompanions"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); Http::TestResponseHeaderMapImpl response_headers{ @@ -483,14 +483,14 @@ TEST_P(GrpcWebFilterTest, StatsErrorResponse) { TEST_P(GrpcWebFilterTest, ExternallyProvidedEncodingHeader) { Http::TestRequestHeaderMapImpl request_headers{ - {"grpc-accept-encoding", "foo"}, {":path", "/"}, {"content-type", request_accept()}}; + {"grpc-accept-encoding", "foo"}, {":path", "/"}, {"content-type", requestAccept()}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers_, false)); EXPECT_EQ("foo", request_headers.get_(Http::CustomHeaders::get().GrpcAcceptEncoding)); } TEST_P(GrpcWebFilterTest, MediaTypeWithParameter) { - Http::TestRequestHeaderMapImpl request_headers{{"content-type", request_content_type()}, + Http::TestRequestHeaderMapImpl request_headers{{"content-type", requestContentType()}, {":path", "/test.MediaTypes/GetParameter"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); Http::TestResponseHeaderMapImpl response_headers{ @@ -504,8 +504,8 @@ TEST_P(GrpcWebFilterTest, MediaTypeWithParameter) { TEST_P(GrpcWebFilterTest, Unary) { // Tests request headers. - request_headers_.addCopy(Http::Headers::get().ContentType, request_content_type()); - request_headers_.addCopy(Http::CustomHeaders::get().Accept, request_accept()); + request_headers_.addCopy(Http::Headers::get().ContentType, requestContentType()); + request_headers_.addCopy(Http::CustomHeaders::get().Accept, requestAccept()); request_headers_.addCopy(Http::Headers::get().ContentLength, uint64_t(8)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers_, false)); @@ -541,7 +541,7 @@ TEST_P(GrpcWebFilterTest, Unary) { } EXPECT_EQ(std::string(TEXT_MESSAGE, TEXT_MESSAGE_SIZE), decoded_buffer.toString()); } else { - FAIL() << "Unsupported gRPC-Web request content-type: " << request_content_type(); + FAIL() << "Unsupported gRPC-Web request content-type: " << requestContentType(); } // Tests request trailers, they are passed through. @@ -559,18 +559,18 @@ TEST_P(GrpcWebFilterTest, Unary) { Http::Headers::get().ContentTypeValues.GrpcWebProto); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.encodeHeaders(response_headers, false)); EXPECT_EQ("200", response_headers.get_(Http::Headers::get().Status.get())); - if (accept_binary_response()) { + if (acceptBinaryResponse()) { EXPECT_EQ(Http::Headers::get().ContentTypeValues.GrpcWebProto, response_headers.getContentTypeValue()); - } else if (accept_text_response()) { + } else if (acceptTextResponse()) { EXPECT_EQ(Http::Headers::get().ContentTypeValues.GrpcWebTextProto, response_headers.getContentTypeValue()); } else { - FAIL() << "Unsupported gRPC-Web request accept: " << request_accept(); + FAIL() << "Unsupported gRPC-Web request accept: " << requestAccept(); } // Tests response data. - if (accept_binary_response()) { + if (acceptBinaryResponse()) { Buffer::OwnedImpl response_buffer; Buffer::OwnedImpl encoded_buffer; for (size_t i = 0; i < MESSAGE_SIZE; i++) { @@ -579,7 +579,7 @@ TEST_P(GrpcWebFilterTest, Unary) { encoded_buffer.move(response_buffer); } EXPECT_EQ(std::string(MESSAGE, MESSAGE_SIZE), encoded_buffer.toString()); - } else if (accept_text_response()) { + } else if (acceptTextResponse()) { Buffer::OwnedImpl response_buffer; Buffer::OwnedImpl encoded_buffer; for (size_t i = 0; i < TEXT_MESSAGE_SIZE; i++) { @@ -606,9 +606,9 @@ TEST_P(GrpcWebFilterTest, Unary) { response_trailers.addCopy(Http::Headers::get().GrpcStatus, "0"); response_trailers.addCopy(Http::Headers::get().GrpcMessage, "ok"); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.encodeTrailers(response_trailers)); - if (accept_binary_response()) { + if (acceptBinaryResponse()) { EXPECT_EQ(std::string(TRAILERS, TRAILERS_SIZE), trailers_buffer.toString()); - } else if (accept_text_response()) { + } else if (acceptTextResponse()) { EXPECT_EQ(std::string(TRAILERS, TRAILERS_SIZE), Base64::decode(trailers_buffer.toString())); } else { FAIL() << "Unsupported gRPC-Web response content-type: " diff --git a/test/extensions/filters/http/header_mutation/config_test.cc b/test/extensions/filters/http/header_mutation/config_test.cc index 330d132d3cd3..38210e1929bb 100644 --- a/test/extensions/filters/http/header_mutation/config_test.cc +++ b/test/extensions/filters/http/header_mutation/config_test.cc @@ -49,7 +49,8 @@ TEST(FactoryTest, FactoryTest) { testing::NiceMock mock_factory_context; - auto cb = factory->createFilterFactoryFromProto(proto_config, "test", mock_factory_context); + auto cb = + factory->createFilterFactoryFromProto(proto_config, "test", mock_factory_context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamFilter(_)); cb(filter_callbacks); diff --git a/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc b/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc index 92e62124603c..7d5c0a866583 100644 --- a/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc +++ b/test/extensions/filters/http/header_mutation/header_mutation_integration_test.cc @@ -37,6 +37,27 @@ name: donwstream-header-mutation key: "downstream-global-flag-header" value: "downstream-global-flag-header-value" append_action: APPEND_IF_EXISTS_OR_ADD +)EOF", + true); + + config_helper_.prependFilter(R"EOF( +name: downstream-header-mutation-disabled-by-default +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.header_mutation.v3.HeaderMutation + mutations: + request_mutations: + - append: + header: + key: "downstream-request-global-flag-header-disabled-by-default" + value: "downstream-request-global-flag-header-value-disabled-by-default" + append_action: APPEND_IF_EXISTS_OR_ADD + response_mutations: + - append: + header: + key: "downstream-global-flag-header-disabled-by-default" + value: "downstream-global-flag-header-value-disabled-by-default" + append_action: APPEND_IF_EXISTS_OR_ADD +disabled: true )EOF", true); @@ -77,51 +98,64 @@ name: upstream-header-mutation route->mutable_match()->set_path("/default/route"); another_route->mutable_match()->set_path("/disable/filter/route"); - // Per route header mutation for downstream. - PerRouteProtoConfig header_mutation_1; - auto response_mutation = - header_mutation_1.mutable_mutations()->mutable_response_mutations()->Add(); - response_mutation->mutable_append()->mutable_header()->set_key( - "downstream-per-route-flag-header"); - response_mutation->mutable_append()->mutable_header()->set_value( - "downstream-per-route-flag-header-value"); - response_mutation->mutable_append()->set_append_action( - envoy::config::core::v3::HeaderValueOption::APPEND_IF_EXISTS_OR_ADD); - - ProtobufWkt::Any per_route_config_1; - per_route_config_1.PackFrom(header_mutation_1); - - route->mutable_typed_per_filter_config()->insert( - {"donwstream-header-mutation", per_route_config_1}); - - // Per route header mutation for upstream. - PerRouteProtoConfig header_mutation_2; - response_mutation = - header_mutation_2.mutable_mutations()->mutable_response_mutations()->Add(); - response_mutation->mutable_append()->mutable_header()->set_key( - "upstream-per-route-flag-header"); - response_mutation->mutable_append()->mutable_header()->set_value( - "upstream-per-route-flag-header-value"); - response_mutation->mutable_append()->set_append_action( - envoy::config::core::v3::HeaderValueOption::APPEND_IF_EXISTS_OR_ADD); - - ProtobufWkt::Any per_route_config_2; - per_route_config_2.PackFrom(header_mutation_2); - - route->mutable_typed_per_filter_config()->insert( - {"upstream-header-mutation", per_route_config_2}); - - // Per route disable downstream header mutation. - envoy::config::route::v3::FilterConfig filter_config; - filter_config.mutable_config(); - filter_config.set_disabled(true); - ProtobufWkt::Any per_route_config_3; - per_route_config_3.PackFrom(filter_config); - another_route->mutable_typed_per_filter_config()->insert( - {"donwstream-header-mutation", per_route_config_3}); - // Try disable upstream header mutation but this is not supported and should not work. - another_route->mutable_typed_per_filter_config()->insert( - {"upstream-header-mutation", per_route_config_3}); + { + // Per route header mutation for downstream. + PerRouteProtoConfig header_mutation; + auto response_mutation = + header_mutation.mutable_mutations()->mutable_response_mutations()->Add(); + response_mutation->mutable_append()->mutable_header()->set_key( + "downstream-per-route-flag-header"); + response_mutation->mutable_append()->mutable_header()->set_value( + "downstream-per-route-flag-header-value"); + response_mutation->mutable_append()->set_append_action( + envoy::config::core::v3::HeaderValueOption::APPEND_IF_EXISTS_OR_ADD); + ProtobufWkt::Any per_route_config; + per_route_config.PackFrom(header_mutation); + route->mutable_typed_per_filter_config()->insert( + {"donwstream-header-mutation", per_route_config}); + } + + { + // Per route header mutation for upstream. + PerRouteProtoConfig header_mutation; + auto response_mutation = + header_mutation.mutable_mutations()->mutable_response_mutations()->Add(); + response_mutation->mutable_append()->mutable_header()->set_key( + "upstream-per-route-flag-header"); + response_mutation->mutable_append()->mutable_header()->set_value( + "upstream-per-route-flag-header-value"); + response_mutation->mutable_append()->set_append_action( + envoy::config::core::v3::HeaderValueOption::APPEND_IF_EXISTS_OR_ADD); + ProtobufWkt::Any per_route_config; + per_route_config.PackFrom(header_mutation); + route->mutable_typed_per_filter_config()->insert( + {"upstream-header-mutation", per_route_config}); + } + + { + // Per route enable the filter that is disabled by default. + envoy::config::route::v3::FilterConfig filter_config; + filter_config.mutable_config()->PackFrom(PerRouteProtoConfig()); + filter_config.set_disabled(false); + ProtobufWkt::Any per_route_config; + per_route_config.PackFrom(filter_config); + // Try enable the filter that is disabled by default. + route->mutable_typed_per_filter_config()->insert( + {"downstream-header-mutation-disabled-by-default", per_route_config}); + } + + { + // Per route disable downstream header mutation. + envoy::config::route::v3::FilterConfig filter_config; + filter_config.set_disabled(true); + ProtobufWkt::Any per_route_config; + per_route_config.PackFrom(filter_config); + another_route->mutable_typed_per_filter_config()->insert( + {"donwstream-header-mutation", per_route_config}); + // Try disable upstream header mutation but this is not supported and should not work. + another_route->mutable_typed_per_filter_config()->insert( + {"upstream-header-mutation", per_route_config}); + } }); HttpIntegrationTest::initialize(); } @@ -144,6 +178,12 @@ TEST_P(HeaderMutationIntegrationTest, TestHeaderMutation) { .get(Http::LowerCaseString("downstream-request-global-flag-header"))[0] ->value() .getStringView()); + EXPECT_EQ("downstream-request-global-flag-header-value-disabled-by-default", + upstream_request_->headers() + .get(Http::LowerCaseString( + "downstream-request-global-flag-header-disabled-by-default"))[0] + ->value() + .getStringView()); EXPECT_EQ("upstream-request-global-flag-header-value", upstream_request_->headers() .get(Http::LowerCaseString("upstream-request-global-flag-header"))[0] @@ -160,6 +200,11 @@ TEST_P(HeaderMutationIntegrationTest, TestHeaderMutation) { .get(Http::LowerCaseString("downstream-global-flag-header"))[0] ->value() .getStringView()); + EXPECT_EQ("downstream-global-flag-header-value-disabled-by-default", + response->headers() + .get(Http::LowerCaseString("downstream-global-flag-header-disabled-by-default"))[0] + ->value() + .getStringView()); EXPECT_EQ("downstream-per-route-flag-header-value", response->headers() .get(Http::LowerCaseString("downstream-per-route-flag-header"))[0] @@ -193,6 +238,10 @@ TEST_P(HeaderMutationIntegrationTest, TestDisableDownstreamHeaderMutation) { EXPECT_EQ(0, upstream_request_->headers() .get(Http::LowerCaseString("downstream-request-global-flag-header")) .size()); + EXPECT_EQ(0, upstream_request_->headers() + .get(Http::LowerCaseString("downstream-request-global-flag-header-disabled-by-" + "default")) + .size()); EXPECT_EQ("upstream-request-global-flag-header-value", upstream_request_->headers() @@ -208,6 +257,9 @@ TEST_P(HeaderMutationIntegrationTest, TestDisableDownstreamHeaderMutation) { EXPECT_EQ(0, response->headers().get(Http::LowerCaseString("downstream-global-flag-header")).size()); + EXPECT_EQ(0, response->headers() + .get(Http::LowerCaseString("downstream-global-flag-header-disabled-by-default")) + .size()); EXPECT_EQ( 0, response->headers().get(Http::LowerCaseString("downstream-per-route-flag-header")).size()); diff --git a/test/extensions/filters/http/header_mutation/header_mutation_test.cc b/test/extensions/filters/http/header_mutation/header_mutation_test.cc index 0db520ef0528..7e61f30bcc75 100644 --- a/test/extensions/filters/http/header_mutation/header_mutation_test.cc +++ b/test/extensions/filters/http/header_mutation/header_mutation_test.cc @@ -161,10 +161,10 @@ TEST(HeaderMutationFilterTest, HeaderMutationFilterTest) { {":status", "200"}, }; - const Http::RequestHeaderMap* request_headers_pointer = + Http::RequestHeaderMap* request_headers_pointer = Http::StaticEmptyHeaders::get().request_headers.get(); - EXPECT_CALL(encoder_callbacks.stream_info_, getRequestHeaders()) - .WillOnce(testing::Return(request_headers_pointer)); + EXPECT_CALL(encoder_callbacks, requestHeaders()) + .WillOnce(testing::Return(makeOptRefFromPtr(request_headers_pointer))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter.encodeHeaders(headers, true)); @@ -196,8 +196,8 @@ TEST(HeaderMutationFilterTest, HeaderMutationFilterTest) { {":status", "200"}, }; - EXPECT_CALL(encoder_callbacks.stream_info_, getRequestHeaders()) - .WillOnce(testing::Return(nullptr)); + EXPECT_CALL(encoder_callbacks, requestHeaders()) + .WillOnce(testing::Return(Http::RequestHeaderMapOptRef{})); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter.encodeHeaders(headers, true)); diff --git a/test/extensions/filters/http/header_to_metadata/config_test.cc b/test/extensions/filters/http/header_to_metadata/config_test.cc index aea26a17b6c6..3f4661335820 100644 --- a/test/extensions/filters/http/header_to_metadata/config_test.cc +++ b/test/extensions/filters/http/header_to_metadata/config_test.cc @@ -28,7 +28,7 @@ void testForbiddenConfig(const std::string& yaml) { testing::NiceMock context; HeaderToMetadataConfig factory; - EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "stats", context), + EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "stats", context).value(), EnvoyException); } @@ -84,7 +84,8 @@ TEST(HeaderToMetadataFilterConfigTest, SimpleConfig) { testing::NiceMock context; HeaderToMetadataConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamFilter(_)); cb(filter_callbacks); @@ -112,7 +113,8 @@ TEST(HeaderToMetadataFilterConfigTest, SimpleCookieConfig) { testing::NiceMock context; HeaderToMetadataConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamFilter(_)); cb(filter_callbacks); diff --git a/test/extensions/filters/http/health_check/config_test.cc b/test/extensions/filters/http/health_check/config_test.cc index 256efe8968a1..19d46bd48762 100644 --- a/test/extensions/filters/http/health_check/config_test.cc +++ b/test/extensions/filters/http/health_check/config_test.cc @@ -34,7 +34,8 @@ TEST(HealthCheckFilterConfig, HealthCheckFilter) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; HealthCheckFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -70,7 +71,9 @@ TEST(HealthCheckFilterConfig, FailsWhenNotPassThroughButTimeoutSetYaml) { HealthCheckFilterConfig factory; NiceMock context; - EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "dummy_stats_prefix", context), + EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "dummy_stats_prefix", context) + .status() + .IgnoreError(), EnvoyException); } @@ -90,8 +93,9 @@ TEST(HealthCheckFilterConfig, NotFailingWhenNotPassThroughAndTimeoutNotSetYaml) HealthCheckFilterConfig factory; NiceMock context; - EXPECT_NO_THROW( - factory.createFilterFactoryFromProto(proto_config, "dummy_stats_prefix", context)); + EXPECT_TRUE(factory.createFilterFactoryFromProto(proto_config, "dummy_stats_prefix", context) + .status() + .ok()); } TEST(HealthCheckFilterConfig, FailsWhenNotPassThroughButTimeoutSetProto) { @@ -106,7 +110,9 @@ TEST(HealthCheckFilterConfig, FailsWhenNotPassThroughButTimeoutSetProto) { header.mutable_string_match()->set_exact("foo"); EXPECT_THROW( - healthCheckFilterConfig.createFilterFactoryFromProto(config, "dummy_stats_prefix", context), + healthCheckFilterConfig.createFilterFactoryFromProto(config, "dummy_stats_prefix", context) + .status() + .IgnoreError(), EnvoyException); } @@ -119,7 +125,10 @@ TEST(HealthCheckFilterConfig, NotFailingWhenNotPassThroughAndTimeoutNotSetProto) envoy::config::route::v3::HeaderMatcher& header = *config.add_headers(); header.set_name(":path"); header.mutable_string_match()->set_exact("foo"); - healthCheckFilterConfig.createFilterFactoryFromProto(config, "dummy_stats_prefix", context); + EXPECT_TRUE( + healthCheckFilterConfig.createFilterFactoryFromProto(config, "dummy_stats_prefix", context) + .status() + .ok()); } TEST(HealthCheckFilterConfig, HealthCheckFilterWithEmptyProto) { @@ -133,7 +142,10 @@ TEST(HealthCheckFilterConfig, HealthCheckFilterWithEmptyProto) { envoy::config::route::v3::HeaderMatcher& header = *config.add_headers(); header.set_name(":path"); header.mutable_string_match()->set_exact("foo"); - healthCheckFilterConfig.createFilterFactoryFromProto(config, "dummy_stats_prefix", context); + EXPECT_TRUE( + healthCheckFilterConfig.createFilterFactoryFromProto(config, "dummy_stats_prefix", context) + .status() + .ok()); } void testHealthCheckHeaderMatch( @@ -149,7 +161,8 @@ void testHealthCheckHeaderMatch( *config = input_config; Http::FilterFactoryCb cb = - healthCheckFilterConfig.createFilterFactoryFromProto(*config, "dummy_stats_prefix", context); + healthCheckFilterConfig.createFilterFactoryFromProto(*config, "dummy_stats_prefix", context) + .value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; Http::StreamFilterSharedPtr health_check_filter; diff --git a/test/extensions/filters/http/health_check/health_check_test.cc b/test/extensions/filters/http/health_check/health_check_test.cc index e706e82f7a5e..0513d731b658 100644 --- a/test/extensions/filters/http/health_check/health_check_test.cc +++ b/test/extensions/filters/http/health_check/health_check_test.cc @@ -56,7 +56,7 @@ class HealthCheckFilterTest : public testing::Test { filter_->setDecoderFilterCallbacks(callbacks_); } - NiceMock context_; + NiceMock context_; Event::MockTimer* cache_timer_{}; Event::MockDispatcher dispatcher_; HealthCheckCacheManagerSharedPtr cache_manager_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 4b73a023f473..5ef71cd603e7 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -16,7 +16,6 @@ #include "gtest/gtest.h" using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/json_to_metadata/config_test.cc b/test/extensions/filters/http/json_to_metadata/config_test.cc index 36c9929dcc13..6d11fac96282 100644 --- a/test/extensions/filters/http/json_to_metadata/config_test.cc +++ b/test/extensions/filters/http/json_to_metadata/config_test.cc @@ -11,7 +11,7 @@ namespace HttpFilters { namespace JsonToMetadata { TEST(Factory, Basic) { - const std::string yaml = R"( + const std::string yaml_request = R"( request_rules: rules: - selectors: @@ -31,37 +31,73 @@ TEST(Factory, Basic) { preserve_existing_metadata_value: true )"; + const std::string yaml_response = R"( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: version + value: 'unknown' + preserve_existing_metadata_value: true + on_error: + metadata_namespace: envoy.lb + key: version + value: 'error' + preserve_existing_metadata_value: true + )"; + JsonToMetadataConfig factory; ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); - TestUtility::loadFromYaml(yaml, *proto_config); + TestUtility::loadFromYaml(yaml_request, *proto_config); NiceMock context; - auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); callback(filter_callback); + + TestUtility::loadFromYaml(yaml_response, *proto_config); + callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); + EXPECT_CALL(filter_callback, addStreamFilter(_)); + callback(filter_callback); } TEST(Factory, NoOnPresentOnMissing) { - const std::string yaml = R"( + const std::string yaml_request = R"( request_rules: rules: - selectors: - key: version )"; + const std::string yaml_response = R"( +response_rules: + rules: + - selectors: + - key: version + )"; + JsonToMetadataConfig factory; ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); - TestUtility::loadFromYaml(yaml, *proto_config); + TestUtility::loadFromYaml(yaml_request, *proto_config); NiceMock context; - EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(*proto_config, "stats", context), - EnvoyException, - "json to metadata filter: neither `on_present` nor `on_missing` set"); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json to metadata filter: neither `on_present` nor `on_missing` set"); + TestUtility::loadFromYaml(yaml_response, *proto_config); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json to metadata filter: neither `on_present` nor `on_missing` set"); } TEST(Factory, NoValueIntOnMissing) { - const std::string yaml = R"( + const std::string yaml_request = R"( request_rules: rules: - selectors: @@ -74,17 +110,34 @@ TEST(Factory, NoValueIntOnMissing) { key: version )"; + const std::string yaml_response = R"( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: version + )"; + JsonToMetadataConfig factory; ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); - TestUtility::loadFromYaml(yaml, *proto_config); + TestUtility::loadFromYaml(yaml_request, *proto_config); NiceMock context; EXPECT_THROW_WITH_REGEX( - factory.createFilterFactoryFromProto(*proto_config, "stats", context), EnvoyException, - "json to metadata filter: cannot specify on_missing rule with empty value"); + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json to metadata filter: cannot specify on_missing rule with empty value"); + TestUtility::loadFromYaml(yaml_response, *proto_config); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json to metadata filter: cannot specify on_missing rule with empty value"); } TEST(Factory, NoValueIntOnError) { - const std::string yaml = R"( + const std::string yaml_request = R"( request_rules: rules: - selectors: @@ -97,13 +150,42 @@ TEST(Factory, NoValueIntOnError) { key: version )"; + const std::string yaml_response = R"( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + on_error: + metadata_namespace: envoy.lb + key: version + )"; + JsonToMetadataConfig factory; ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); - TestUtility::loadFromYaml(yaml, *proto_config); + TestUtility::loadFromYaml(yaml_request, *proto_config); NiceMock context; - EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(*proto_config, "stats", context), - EnvoyException, - "json to metadata filter: cannot specify on_error rule with empty value"); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json to metadata filter: cannot specify on_error rule with empty value"); + TestUtility::loadFromYaml(yaml_response, *proto_config); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json to metadata filter: cannot specify on_error rule with empty value"); +} + +TEST(Factory, NoRule) { + const std::string yaml_empty = R"({})"; + + JsonToMetadataConfig factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyRouteConfigProto(); + TestUtility::loadFromYaml(yaml_empty, *proto_config); + NiceMock context; + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "json_to_metadata_filter: Per filter configs must at least specify"); } } // namespace JsonToMetadata diff --git a/test/extensions/filters/http/json_to_metadata/filter_test.cc b/test/extensions/filters/http/json_to_metadata/filter_test.cc index 900765802e3a..e2bd9365cfe7 100644 --- a/test/extensions/filters/http/json_to_metadata/filter_test.cc +++ b/test/extensions/filters/http/json_to_metadata/filter_test.cc @@ -47,7 +47,7 @@ class FilterTest : public testing::Test { public: FilterTest() = default; - const std::string request_config_yaml_ = R"EOF( + const std::string config_yaml_ = R"EOF( request_rules: rules: - selectors: @@ -65,6 +65,23 @@ class FilterTest : public testing::Test { key: version value: 'error' preserve_existing_metadata_value: true +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: version + value: 'unknown' + preserve_existing_metadata_value: true + on_error: + metadata_namespace: envoy.lb + key: version + value: 'error' + preserve_existing_metadata_value: true )EOF"; void initializeFilter(const std::string& yaml) { @@ -73,6 +90,7 @@ class FilterTest : public testing::Test { config_ = std::make_shared(config, *scope_.rootScope()); filter_ = std::make_shared(config_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); + filter_->setEncoderFilterCallbacks(encoder_callbacks_); } void sendData(const std::vector& data_vector) { @@ -104,8 +122,18 @@ class FilterTest : public testing::Test { EXPECT_EQ(expected_result, filter_->decodeData(buffer_, end_stream)); } + void + testResponseWithBody(const std::string& body, bool end_stream = true, + Http::FilterDataStatus expected_result = Http::FilterDataStatus::Continue) { + buffer_.add(body); + ON_CALL(encoder_callbacks_, encodingBuffer()).WillByDefault(Return(&buffer_)); + + EXPECT_EQ(expected_result, filter_->encodeData(buffer_, end_stream)); + } + NiceMock scope_; NiceMock decoder_callbacks_; + NiceMock encoder_callbacks_; NiceMock stream_info_; envoy::config::core::v3::Metadata dynamic_metadata_; std::shared_ptr config_; @@ -113,10 +141,12 @@ class FilterTest : public testing::Test { Buffer::OwnedImpl buffer_; Http::TestRequestHeaderMapImpl incoming_headers_{ {":path", "/ping"}, {":method", "GET"}, {"Content-Type", "application/json"}}; + Http::TestResponseHeaderMapImpl response_headers_{{":status", "200"}, + {"Content-Type", "application/json"}}; }; TEST_F(FilterTest, BasicStringMatch) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter( {"version":"1.0.0", @@ -137,14 +167,38 @@ TEST_F(FilterTest, BasicStringMatch) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, BasicResponseStringMatch) { + initializeFilter(config_yaml_); + const std::string response_body = + R"delimiter( + {"version":"1.0.0", + "messages":[ + {"role":"user","content":"content A"}, + {"role":"assistant","content":"content B"}, + {"role":"user","content":"content C"}, + {"role":"assistant","content":"content D"}, + {"role":"user","content":"content E"}], + "stream":true})delimiter"; + const std::map expected = {{"version", "1.0.0"}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + testResponseWithBody(response_body); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, BasicBoolMatch) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter({"version":true})delimiter"; std::map expected = {{"version", true}}; @@ -159,14 +213,14 @@ TEST_F(FilterTest, BasicBoolMatch) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, BasicIntegerMatch) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter({"version":1})delimiter"; std::map expected = {{"version", 1.0}}; @@ -181,14 +235,14 @@ TEST_F(FilterTest, BasicIntegerMatch) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, BasicDoubleMatch) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter({"version":1.0})delimiter"; std::map expected = {{"version", 1.0}}; @@ -203,10 +257,10 @@ TEST_F(FilterTest, BasicDoubleMatch) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, TrailerSupport) { @@ -235,10 +289,42 @@ TEST_F(FilterTest, TrailerSupport) { Http::TestRequestTrailerMapImpl trailers{{"some", "trailer"}}; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, ResponseTrailerSupport) { + initializeFilter(R"EOF( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version +)EOF"); + const std::string response_body = R"delimiter({"version":"good version"})delimiter"; + const std::map expected = {{"version", "good version"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers_, false)); + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + testResponseWithBody(response_body, false, Http::FilterDataStatus::StopIterationAndBuffer); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers_, false)); + + Http::TestResponseTrailerMapImpl trailers{{"some", "trailer"}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, StringToString) { @@ -262,10 +348,10 @@ TEST_F(FilterTest, StringToString) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, StringToNumber) { @@ -293,10 +379,10 @@ TEST_F(FilterTest, StringToNumber) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, BadStringToNumber) { @@ -328,10 +414,10 @@ TEST_F(FilterTest, BadStringToNumber) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, NumberToString) { @@ -355,10 +441,10 @@ TEST_F(FilterTest, NumberToString) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, NumberToNumber) { @@ -386,10 +472,10 @@ TEST_F(FilterTest, NumberToNumber) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, IntegerToString) { @@ -413,10 +499,10 @@ TEST_F(FilterTest, IntegerToString) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, IntegerToNumber) { @@ -444,10 +530,10 @@ TEST_F(FilterTest, IntegerToNumber) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, BoolToString) { @@ -471,10 +557,10 @@ TEST_F(FilterTest, BoolToString) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, BoolToNumber) { @@ -502,10 +588,10 @@ TEST_F(FilterTest, BoolToNumber) { }))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnPresentWithValueSet) { @@ -529,10 +615,10 @@ TEST_F(FilterTest, OnPresentWithValueSet) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, NoApplyOnMissingWhenPayloadIsPresent) { @@ -561,10 +647,10 @@ TEST_F(FilterTest, NoApplyOnMissingWhenPayloadIsPresent) { EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)).Times(0); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, DefaultNamespaceTest) { @@ -587,14 +673,14 @@ TEST_F(FilterTest, DefaultNamespaceTest) { setDynamicMetadata("envoy.filters.http.json_to_metadata", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, DecodeTwoDataStreams) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body1 = R"delimiter( @@ -618,10 +704,41 @@ TEST_F(FilterTest, DecodeTwoDataStreams) { testRequestWithBody(request_body1, false, Http::FilterDataStatus::StopIterationAndBuffer); testRequestWithBody(request_body2); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, EncodeTwoDataStreams) { + initializeFilter(config_yaml_); + + const std::string response_body1 = + R"delimiter( + {"version":"1.0.0", + "messages":[ + {"role":"user","content":"content A"}, + {"role":"assis)delimiter"; + const std::string response_body2 = + R"delimiter(tant","content":"content B"}, + {"role":"user","content":"content C"}, + {"role":"assistant","content":"content D"}, + {"role":"user","content":"content E"}], + "stream":true})delimiter"; + const std::map expected = {{"version", "1.0.0"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers_, false)); + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + testResponseWithBody(response_body1, false, Http::FilterDataStatus::StopIterationAndBuffer); + testResponseWithBody(response_body2); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, SecondLayerMatch) { @@ -650,10 +767,10 @@ TEST_F(FilterTest, SecondLayerMatch) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnMissingFirstLayer) { @@ -683,10 +800,10 @@ TEST_F(FilterTest, OnMissingFirstLayer) { EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)).Times(0); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnMissingSecondLayer) { @@ -716,10 +833,10 @@ TEST_F(FilterTest, OnMissingSecondLayer) { EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)).Times(0); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnMissingForArray) { @@ -749,10 +866,10 @@ TEST_F(FilterTest, OnMissingForArray) { EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)).Times(0); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnMissingSecondLayerString) { @@ -783,14 +900,14 @@ TEST_F(FilterTest, OnMissingSecondLayerString) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, NoRequestContentType) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); Http::TestRequestHeaderMapImpl mismatched_incoming_headers{{":path", "/ping"}, {":method", "GET"}}; @@ -799,14 +916,14 @@ TEST_F(FilterTest, NoRequestContentType) { filter_->decodeHeaders(mismatched_incoming_headers, false)); testRequestWithBody("{}"); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MismatchedRequestContentType) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); Http::TestRequestHeaderMapImpl mismatched_incoming_headers{ {":path", "/ping"}, {":method", "GET"}, {"Content-Type", "application/not-a-json"}}; @@ -815,39 +932,64 @@ TEST_F(FilterTest, MismatchedRequestContentType) { filter_->decodeHeaders(mismatched_incoming_headers, false)); testRequestWithBody("Peter picked a peck of pickled peppers"); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, NoRequestBody) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(incoming_headers_, true)); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, NoResponseBody) { + initializeFilter(config_yaml_); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, EmptyPayloadValue) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(incoming_headers_, false)); testRequestWithBody(""); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, ResponseEmptyPayloadValue) { + initializeFilter(config_yaml_); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(response_headers_, false)); + + testResponseWithBody(""); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, InvalidJsonPayload) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); // missing right-most curly brace const std::string request_body = R"delimiter( @@ -867,14 +1009,14 @@ TEST_F(FilterTest, InvalidJsonPayload) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 1); } TEST_F(FilterTest, OnMissingQuotedString) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter("")delimiter"; const std::map expected = {{"version", "unknown"}}; @@ -884,14 +1026,14 @@ TEST_F(FilterTest, OnMissingQuotedString) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnMissingQuotedJsonObject) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter("{\"model\": \"gpt-3.5-turbo\",\"temperature\": 0.2,\"stream\": false}")delimiter"; const std::map expected = {{"version", "unknown"}}; @@ -902,14 +1044,14 @@ TEST_F(FilterTest, OnMissingQuotedJsonObject) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, OnMissingPureNumber) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter(5566)delimiter"; const std::map expected = {{"version", "unknown"}}; @@ -919,15 +1061,15 @@ TEST_F(FilterTest, OnMissingPureNumber) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } // TODO(kuochunghsu): Planned to support trimming. TEST_F(FilterTest, InvalidJsonForAdditionalPrefixSuffix) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); // missing right-most curly brace const std::string request_body = R"delimiter(data: {"id":"ID","object":"chat.completion.chunk","created":1686100940,"version":"1.0.0-0301"}\n\ndata: [DONE]\n\n)delimiter"; @@ -939,10 +1081,10 @@ TEST_F(FilterTest, InvalidJsonForAdditionalPrefixSuffix) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 1); } TEST_F(FilterTest, EmptyStringValue) { @@ -970,10 +1112,10 @@ TEST_F(FilterTest, EmptyStringValue) { Buffer::OwnedImpl buffer(request_body); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, PayloadValueTooLong) { @@ -1003,10 +1145,10 @@ TEST_F(FilterTest, PayloadValueTooLong) { Buffer::OwnedImpl buffer(request_body); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, PayloadValueTooLongValueTypeString) { @@ -1037,14 +1179,14 @@ TEST_F(FilterTest, PayloadValueTooLongValueTypeString) { Buffer::OwnedImpl buffer(request_body); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MissingMetadataKeyAndFallbackValue) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter( {"messages":[ @@ -1061,10 +1203,10 @@ TEST_F(FilterTest, MissingMetadataKeyAndFallbackValue) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MissingMetadataKeyWithNoFallbackValue) { @@ -1095,14 +1237,14 @@ TEST_F(FilterTest, MissingMetadataKeyWithNoFallbackValue) { EXPECT_CALL(decoder_callbacks_, streamInfo()).Times(0); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MissingMetadataKeyWithExistingMetadata) { - initializeFilter(request_config_yaml_); + initializeFilter(config_yaml_); const std::string request_body = R"delimiter( {"messages":[ @@ -1123,10 +1265,10 @@ TEST_F(FilterTest, MissingMetadataKeyWithExistingMetadata) { EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)).Times(0); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MultipleRules) { @@ -1156,10 +1298,10 @@ TEST_F(FilterTest, MultipleRules) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MultipleRulesInSamePath) { @@ -1189,10 +1331,10 @@ TEST_F(FilterTest, MultipleRulesInSamePath) { EXPECT_CALL(stream_info_, setDynamicMetadata("another.namespace", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, MultipleRulesSecondLayer) { @@ -1232,10 +1374,10 @@ TEST_F(FilterTest, MultipleRulesSecondLayer) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); } TEST_F(FilterTest, CustomRequestAllowContentTypeAccepted) { @@ -1262,10 +1404,40 @@ TEST_F(FilterTest, CustomRequestAllowContentTypeAccepted) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, CustomResponseAllowContentTypeAccepted) { + initializeFilter(R"EOF( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + allow_content_types: + - "application/better-json" +)EOF"); + const std::string response_body = R"delimiter({"version":"good version"})delimiter"; + const std::map expected = {{"version", "good version"}}; + + Http::TestResponseHeaderMapImpl matched_incoming_headers{ + {":path", "/ping"}, {"Content-Type", "application/better-json"}}; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(matched_incoming_headers, false)); + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + testResponseWithBody(response_body); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, CustomRequestAllowContentTypeRejected) { @@ -1284,10 +1456,32 @@ TEST_F(FilterTest, CustomRequestAllowContentTypeRejected) { testRequestWithBody("{}"); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, CustomResponseAllowContentTypeRejected) { + initializeFilter(R"EOF( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + allow_content_types: + - "application/non-json" +)EOF"); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + + testResponseWithBody("{}"); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } TEST_F(FilterTest, RequestAllowEmptyContentType) { @@ -1312,10 +1506,90 @@ TEST_F(FilterTest, RequestAllowEmptyContentType) { EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); testRequestWithBody(request_body); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_success"), 1); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_mismatched_content_type"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_no_body"), 0); - EXPECT_EQ(getCounterValue("json_to_metadata.rq_invalid_json_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, ResponseAllowEmptyContentType) { + initializeFilter(R"EOF( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + allow_empty_content_type: true +)EOF"); + const std::string response_body = R"delimiter({"version":"good version"})delimiter"; + const std::map expected = {{"version", "good version"}}; + + Http::TestResponseHeaderMapImpl matched_incoming_headers{{":path", "/ping"}, {":method", "GET"}}; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->encodeHeaders(matched_incoming_headers, false)); + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(stream_info_)); + EXPECT_CALL(stream_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + testResponseWithBody(response_body); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 1); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); +} + +TEST_F(FilterTest, RequestBodyWithResponseRule) { + initializeFilter(R"EOF( +response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + type: STRING +)EOF"); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(incoming_headers_, false)); + + const std::string request_body = R"delimiter({"version":"good version"})delimiter"; + + testRequestWithBody(request_body); + + Http::TestRequestTrailerMapImpl trailers{{"some", "trailer"}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(trailers)); + + EXPECT_EQ(getCounterValue("json_to_metadata.rq.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.rq.invalid_json_body"), 0); +} + +TEST_F(FilterTest, ResponseBodyWithRequestRule) { + initializeFilter(R"EOF( +request_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + type: STRING +)EOF"); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); + + const std::string response_body = R"delimiter({"version":"good version"})delimiter"; + + testResponseWithBody(response_body); + + Http::TestResponseTrailerMapImpl trailers{{"some", "trailer"}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(trailers)); + + EXPECT_EQ(getCounterValue("json_to_metadata.resp.success"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.mismatched_content_type"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.no_body"), 0); + EXPECT_EQ(getCounterValue("json_to_metadata.resp.invalid_json_body"), 0); } } // namespace JsonToMetadata diff --git a/test/extensions/filters/http/json_to_metadata/integration_test.cc b/test/extensions/filters/http/json_to_metadata/integration_test.cc index 3c4cab72157b..4c607914e2bd 100644 --- a/test/extensions/filters/http/json_to_metadata/integration_test.cc +++ b/test/extensions/filters/http/json_to_metadata/integration_test.cc @@ -12,6 +12,7 @@ class JsonToMetadataIntegrationTest : public Event::TestUsingSimulatedTime, } void runTest(const Http::RequestHeaderMap& request_headers, const std::string& request_body, + const Http::ResponseHeaderMap& response_headers, const std::string& response_body, const size_t chunk_size = 5, bool has_trailer = false) { codec_client_ = makeHttpConnection(lookupPort("http")); IntegrationStreamDecoderPtr response; @@ -39,7 +40,24 @@ class JsonToMetadataIntegrationTest : public Event::TestUsingSimulatedTime, ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); - upstream_request_->encodeHeaders(default_response_headers_, true); + if (response_body.empty()) { + upstream_request_->encodeHeaders(response_headers, true); + } else { + upstream_request_->encodeHeaders(response_headers, false); + size_t i = 0; + for (; i < response_body.length() / chunk_size; i++) { + Buffer::OwnedImpl buffer(response_body.substr(i * chunk_size, chunk_size)); + upstream_request_->encodeData(buffer, false); + } + // Send the last chunk flagged as end_stream. + Buffer::OwnedImpl buffer( + response_body.substr(i * chunk_size, response_body.length() % chunk_size)); + upstream_request_->encodeData(buffer, !has_trailer); + + if (has_trailer) { + upstream_request_->encodeTrailers(response_trailers_); + } + } ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); @@ -70,6 +88,23 @@ name: envoy.filters.http.json_to_metadata key: version value: 'error' preserve_existing_metadata_value: true + response_rules: + rules: + - selectors: + - key: version + on_present: + metadata_namespace: envoy.lb + key: version + on_missing: + metadata_namespace: envoy.lb + key: version + value: 'unknown' + preserve_existing_metadata_value: true + on_error: + metadata_namespace: envoy.lb + key: version + value: 'error' + preserve_existing_metadata_value: true )EOF"; Http::TestRequestHeaderMapImpl incoming_headers_{{":scheme", "http"}, @@ -79,6 +114,10 @@ name: envoy.filters.http.json_to_metadata {"Content-Type", "application/json"}}; Http::TestRequestTrailerMapImpl incoming_trailers_{{"request1", "trailer1"}, {"request2", "trailer2"}}; + Http::TestResponseHeaderMapImpl response_headers_{{":status", "200"}, + {"Content-Type", "application/json"}}; + Http::TestResponseTrailerMapImpl response_trailers_{{"request1", "trailer1"}, + {"request2", "trailer2"}}; const std::string request_body_ = R"delimiter( @@ -90,6 +129,16 @@ name: envoy.filters.http.json_to_metadata {"role":"assistant","content":"content D"}, {"role":"user","content":"content E"}], "stream":true})delimiter"; + const std::string response_body_ = + R"delimiter( + {"version":"1.0.0", + "messages":[ + {"role":"user","content":"content A"}, + {"role":"assistant","content":"content B"}, + {"role":"user","content":"content C"}, + {"role":"assistant","content":"content D"}, + {"role":"user","content":"content E"}], + "stream":true})delimiter"; }; // TODO(#26236): Fix test suite for HTTP/3. @@ -101,34 +150,49 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(JsonToMetadataIntegrationTest, Basic) { initializeFilter(); - runTest(incoming_headers_, request_body_); + runTest(incoming_headers_, request_body_, response_headers_, response_body_); + + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.invalid_json_body")->value()); - EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq_success")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_mismatched_content_type")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_no_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_invalid_json_body")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.resp.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.invalid_json_body")->value()); } TEST_P(JsonToMetadataIntegrationTest, BasicOneChunk) { initializeFilter(); - runTest(incoming_headers_, request_body_, 1); + runTest(incoming_headers_, request_body_, response_headers_, response_body_, 1); - EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq_success")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_mismatched_content_type")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_no_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_invalid_json_body")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.invalid_json_body")->value()); + + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.resp.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.invalid_json_body")->value()); } TEST_P(JsonToMetadataIntegrationTest, Trailer) { initializeFilter(); - runTest(incoming_headers_, request_body_, 5, true); + runTest(incoming_headers_, request_body_, response_headers_, response_body_, 5, true); + + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.invalid_json_body")->value()); - EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq_success")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_mismatched_content_type")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_no_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_invalid_json_body")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.resp.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.invalid_json_body")->value()); } TEST_P(JsonToMetadataIntegrationTest, MismatchedContentType) { @@ -139,35 +203,52 @@ TEST_P(JsonToMetadataIntegrationTest, MismatchedContentType) { {":method", "GET"}, {":authority", "host"}, {"Content-Type", "application/x-thrift"}}; + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, + {"Content-Type", "application/x-thrift"}}; + + runTest(incoming_headers, request_body_, response_headers, response_body_); - runTest(incoming_headers, request_body_); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.success")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.invalid_json_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_success")->value()); - EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq_mismatched_content_type")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_no_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_invalid_json_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.success")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.resp.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.invalid_json_body")->value()); } TEST_P(JsonToMetadataIntegrationTest, NoBody) { initializeFilter(); - runTest(incoming_headers_, ""); + runTest(incoming_headers_, "", response_headers_, ""); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_success")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_mismatched_content_type")->value()); - EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq_no_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_invalid_json_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.mismatched_content_type")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.invalid_json_body")->value()); + + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.mismatched_content_type")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.resp.no_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.invalid_json_body")->value()); } TEST_P(JsonToMetadataIntegrationTest, InvalidJson) { initializeFilter(); - runTest(incoming_headers_, "it's not a json body"); + runTest(incoming_headers_, "it's not a json body", response_headers_, "it's not a json body"); + + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq.no_body")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq.invalid_json_body")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_success")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_mismatched_content_type")->value()); - EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.rq_no_body")->value()); - EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.rq_invalid_json_body")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.success")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.mismatched_content_type")->value()); + EXPECT_EQ(0UL, test_server_->counter("json_to_metadata.resp.no_body")->value()); + EXPECT_EQ(1UL, test_server_->counter("json_to_metadata.resp.invalid_json_body")->value()); } } // namespace diff --git a/test/extensions/filters/http/jwt_authn/BUILD b/test/extensions/filters/http/jwt_authn/BUILD index 81fc0657cf03..9b84849c432f 100644 --- a/test/extensions/filters/http/jwt_authn/BUILD +++ b/test/extensions/filters/http/jwt_authn/BUILD @@ -158,7 +158,10 @@ envoy_extension_cc_test( name = "filter_integration_test", size = "large", srcs = ["filter_integration_test.cc"], - extension_names = ["envoy.filters.http.jwt_authn"], + extension_names = [ + "envoy.filters.http.jwt_authn", + "envoy.filters.http.set_filter_state", + ], shard_count = 4, tags = [ "cpu:3", @@ -166,12 +169,13 @@ envoy_extension_cc_test( deps = [ "//source/common/router:string_accessor_lib", "//source/extensions/filters/http/jwt_authn:config", + "//source/extensions/filters/http/set_filter_state:config", "//test/config:utility_lib", "//test/extensions/filters/http/jwt_authn:test_common_lib", "//test/integration:http_protocol_integration_lib", - "//test/integration/filters:header_to_filter_state_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/http/set_filter_state/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/jwt_authn/authenticator_test.cc b/test/extensions/filters/http/jwt_authn/authenticator_test.cc index 4ebe0f9a3feb..9a58370bb3a4 100644 --- a/test/extensions/filters/http/jwt_authn/authenticator_test.cc +++ b/test/extensions/filters/http/jwt_authn/authenticator_test.cc @@ -1,6 +1,7 @@ #include "envoy/config/core/v3/http_uri.pb.h" #include "envoy/extensions/filters/http/jwt_authn/v3/config.pb.h" +#include "source/common/common/base64.h" #include "source/common/http/message_impl.h" #include "source/common/protobuf/utility.h" #include "source/extensions/filters/http/common/jwks_fetcher.h" @@ -57,7 +58,8 @@ class AuthenticatorTest : public testing::Test { EXPECT_TRUE(jwks_->getStatus() == Status::Ok); } - void expectVerifyStatus(Status expected_status, Http::RequestHeaderMap& headers) { + void expectVerifyStatus(Status expected_status, Http::RequestHeaderMap& headers, + bool expect_clear_route = false) { std::function on_complete_cb = [&expected_status](const Status& status) { ASSERT_EQ(status, expected_status); }; @@ -67,8 +69,10 @@ class AuthenticatorTest : public testing::Test { }; initTokenExtractor(); auto tokens = extractor_->extract(headers); + bool clear_route = false; auth_->verify(headers, parent_span_, std::move(tokens), std::move(set_extracted_jwt_data_cb), - std::move(on_complete_cb)); + std::move(on_complete_cb), [&clear_route] { clear_route = true; }); + EXPECT_EQ(expect_clear_route, clear_route); } void initTokenExtractor() { @@ -153,6 +157,48 @@ TEST_F(AuthenticatorTest, TestClaimToHeader) { EXPECT_EQ(headers.get_("x-jwt-claim-nested"), "value1"); EXPECT_EQ(headers.get_("x-jwt-bool-claim"), "true"); EXPECT_EQ(headers.get_("x-jwt-int-claim"), "9999"); + + // This check verifies whether the claim with non-primitive type are + // successfully serialized and added to headers. + std::string expected_json = "[\"str1\",\"str2\"]"; + + ASSERT_EQ(headers.get_("x-jwt-claim-object-key"), + Envoy::Base64::encode(expected_json.data(), expected_json.size())); +} + +// This test verifies whether the claim is successfully added to header or not +TEST_F(AuthenticatorTest, TestClaimToHeaderWithClearRouteCache) { + TestUtility::loadFromYaml(ClaimToHeadersConfig, proto_config_); + createAuthenticator(); + EXPECT_CALL(*raw_fetcher_, fetch(_, _)) + .WillOnce(Invoke([this](Tracing::Span&, JwksFetcher::JwksReceiver& receiver) { + receiver.onJwksSuccess(std::move(jwks_)); + })); + + { + Http::TestRequestHeaderMapImpl headers{ + {"Authorization", "Bearer " + std::string(NestedGoodToken)}}; + expectVerifyStatus(Status::Ok, headers, true); + EXPECT_EQ(headers.get_("x-jwt-claim-nested"), "value1"); + } + + { + Http::TestRequestHeaderMapImpl headers{{"Authorization", "Bearer " + std::string(GoodToken)}}; + expectVerifyStatus(Status::Ok, headers, false); + EXPECT_FALSE(headers.has("x-jwt-claim-nested")); + } +} + +TEST_F(AuthenticatorTest, TestPayloadInMetadataWithClearRouteCache) { + TestUtility::loadFromYaml(PayloadClearRouteCacheConfig, proto_config_); + createAuthenticator(); + EXPECT_CALL(*raw_fetcher_, fetch(_, _)) + .WillOnce(Invoke([this](Tracing::Span&, JwksFetcher::JwksReceiver& receiver) { + receiver.onJwksSuccess(std::move(jwks_)); + })); + Http::TestRequestHeaderMapImpl headers{ + {"Authorization", "Bearer " + std::string(NestedGoodToken)}}; + expectVerifyStatus(Status::Ok, headers, true); } // This test verifies when wrong claim is passed in claim_to_headers @@ -224,6 +270,38 @@ TEST_F(AuthenticatorTest, TestSetPayload) { TestUtility::protoEqual(expected_payload, out_extracted_data_.fields().at("my_payload"))); } +// This test verifies the JWT payload is set. +TEST_F(AuthenticatorTest, TestSetPayloadWithSpaces) { + // Config payload_in_metadata flag + (*proto_config_.mutable_providers())[std::string(ProviderName)].set_payload_in_metadata( + "my_payload"); + auto* normalize_payload = (*proto_config_.mutable_providers())[std::string(ProviderName)] + .mutable_normalize_payload_in_metadata(); + normalize_payload->add_space_delimited_claims("scope"); + normalize_payload->add_space_delimited_claims("test_string"); + normalize_payload->add_space_delimited_claims("test_num"); + + createAuthenticator(); + EXPECT_CALL(*raw_fetcher_, fetch(_, _)) + .WillOnce(Invoke([this](Tracing::Span&, JwksFetcher::JwksReceiver& receiver) { + receiver.onJwksSuccess(std::move(jwks_)); + })); + + // Test OK pubkey and its cache + Http::TestRequestHeaderMapImpl headers{ + {"Authorization", "Bearer " + std::string(GoodTokenWithSpaces)}}; + + expectVerifyStatus(Status::Ok, headers); + + // Only one field is set. + EXPECT_EQ(1, out_extracted_data_.fields().size()); + + ProtobufWkt::Value expected_payload; + TestUtility::loadFromJson(ExpectedPayloadJSONWithSpaces, expected_payload); + EXPECT_TRUE( + TestUtility::protoEqual(expected_payload, out_extracted_data_.fields().at("my_payload"))); +} + // This test verifies setting only the extracted header to metadata. TEST_F(AuthenticatorTest, TestSetHeader) { // Set the extracted header to metadata. @@ -712,7 +790,9 @@ TEST_F(AuthenticatorTest, TestInvalidPrefix) { // This test verifies when a JWT is non-expiring without audience specified, JwtAudienceNotAllowed // is returned. TEST_F(AuthenticatorTest, TestNonExpiringJWT) { - EXPECT_CALL(mock_factory_ctx_.cluster_manager_.thread_local_cluster_, httpAsyncClient()).Times(0); + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_, + httpAsyncClient()) + .Times(0); Http::TestRequestHeaderMapImpl headers{ {"Authorization", "Bearer " + std::string(NonExpiringToken)}}; @@ -814,7 +894,8 @@ TEST_F(AuthenticatorTest, TestOnDestroy) { auto tokens = extractor_->extract(headers); // callback should not be called. std::function on_complete_cb = [](const Status&) { FAIL(); }; - auth_->verify(headers, parent_span_, std::move(tokens), nullptr, std::move(on_complete_cb)); + auth_->verify(headers, parent_span_, std::move(tokens), nullptr, std::move(on_complete_cb), + nullptr); // Destroy the authenticating process. auth_->onDestroy(); @@ -973,7 +1054,7 @@ class AuthenticatorJwtCacheTest : public testing::Test { }; auto tokens = extractor_->extract(headers); auth_->verify(headers, parent_span_, std::move(tokens), set_extracted_jwt_data_cb, - on_complete_cb); + on_complete_cb, nullptr); } ::google::jwt_verify::JwksPtr jwks_; diff --git a/test/extensions/filters/http/jwt_authn/extractor_runtime_guard_test.cc b/test/extensions/filters/http/jwt_authn/extractor_runtime_guard_test.cc index be32b1bb8042..137c693b7589 100644 --- a/test/extensions/filters/http/jwt_authn/extractor_runtime_guard_test.cc +++ b/test/extensions/filters/http/jwt_authn/extractor_runtime_guard_test.cc @@ -7,7 +7,6 @@ #include "test/test_common/utility.h" using envoy::extensions::filters::http::jwt_authn::v3::JwtAuthentication; -using envoy::extensions::filters::http::jwt_authn::v3::JwtProvider; using Envoy::Http::TestRequestHeaderMapImpl; namespace Envoy { diff --git a/test/extensions/filters/http/jwt_authn/filter_config_test.cc b/test/extensions/filters/http/jwt_authn/filter_config_test.cc index 6fcc2204c301..b3969774d375 100644 --- a/test/extensions/filters/http/jwt_authn/filter_config_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_config_test.cc @@ -166,37 +166,24 @@ TEST(HttpJwtAuthnFilterConfigTest, VerifyTLSLifetime) { provider_name: provider1 )"; - NiceMock server_context; - // Make sure that the thread callbacks are not invoked inline. - server_context.thread_local_.defer_data_ = true; - { - // Scope in all the things that the filter depends on, so they are destroyed as we leave the - // scope. - NiceMock context; - // The threadLocal, dispatcher and api that are used by the filter config, actually belong to - // the server factory context that who's lifetime is longer. We simulate that by returning - // their instances from outside the scope. - ON_CALL(context, mainThreadDispatcher()) - .WillByDefault(ReturnRef(server_context.mainThreadDispatcher())); - ON_CALL(context, api()).WillByDefault(ReturnRef(server_context.api())); - ON_CALL(context, threadLocal()).WillByDefault(ReturnRef(server_context.threadLocal())); - - JwtAuthentication proto_config; - TestUtility::loadFromYaml(config, proto_config); - auto filter_conf = std::make_unique(proto_config, "", context); - } + NiceMock context; + context.server_factory_context_.thread_local_.defer_data_ = true; + + JwtAuthentication proto_config; + TestUtility::loadFromYaml(config, proto_config); + auto filter_conf = std::make_unique(proto_config, "", context); // Even though filter_conf is now de-allocated, using a reference to it might still work, as its // memory was not cleared. This leads to a false positive in this test when run normally. The // test should fail under asan if the code uses invalid reference. // Make sure the filter scheduled a callback - EXPECT_EQ(1, server_context.thread_local_.deferred_data_.size()); + EXPECT_EQ(1, context.server_factory_context_.thread_local_.deferred_data_.size()); // Simulate a situation where the callback is called after the filter config is destroyed. // call the tls callback. we want to make sure that it doesn't depend on objects // that are out of scope. - EXPECT_NO_THROW(server_context.thread_local_.call()); + EXPECT_NO_THROW(context.server_factory_context_.thread_local_.call()); } TEST(HttpJwtAuthnFilterConfigTest, FindByFilterState) { diff --git a/test/extensions/filters/http/jwt_authn/filter_factory_test.cc b/test/extensions/filters/http/jwt_authn/filter_factory_test.cc index 5159fa09af7d..a783973f1370 100644 --- a/test/extensions/filters/http/jwt_authn/filter_factory_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_factory_test.cc @@ -27,7 +27,8 @@ TEST(HttpJwtAuthnFilterFactoryTest, GoodRemoteJwks) { NiceMock context; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -41,7 +42,8 @@ TEST(HttpJwtAuthnFilterFactoryTest, GoodLocalJwks) { NiceMock context; FilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -55,7 +57,7 @@ TEST(HttpJwtAuthnFilterFactoryTest, BadLocalJwks) { NiceMock context; FilterFactory factory; - EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "stats", context), + EXPECT_THROW(factory.createFilterFactoryFromProto(proto_config, "stats", context).value(), EnvoyException); } @@ -67,7 +69,8 @@ TEST(HttpJwtAuthnFilterFactoryTest, ProviderWithoutIssuer) { NiceMock context; FilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc index 1be88f1d9692..d8972fa5111c 100644 --- a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc @@ -1,11 +1,11 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/extensions/filters/http/jwt_authn/v3/config.pb.h" +#include "envoy/extensions/filters/http/set_filter_state/v3/set_filter_state.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "source/common/router/string_accessor_impl.h" #include "test/extensions/filters/http/jwt_authn/test_common.h" -#include "test/integration/filters/header_to_filter_state.pb.h" #include "test/integration/http_protocol_integration.h" #include "test/test_common/registry.h" @@ -208,6 +208,17 @@ TEST_P(LocalJwksIntegrationTest, CorsPreflight) { EXPECT_EQ("200", response->headers().getStatusValue()); } +class TestObjectFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "jwt_selector"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +REGISTER_FACTORY(TestObjectFactory, StreamInfo::FilterState::ObjectFactory); + // This test verifies JwtRequirement specified from filter state rules TEST_P(LocalJwksIntegrationTest, FilterStateRequirement) { // A config with metadata rules. @@ -228,10 +239,13 @@ TEST_P(LocalJwksIntegrationTest, FilterStateRequirement) { config_helper_.prependFilter(R"( name: header-to-filter-state typed_config: - "@type": type.googleapis.com/test.integration.filters.HeaderToFilterStateFilterConfig - header_name: jwt_selector - state_name: jwt_selector - read_only: true + "@type": type.googleapis.com/envoy.extensions.filters.http.set_filter_state.v3.Config + on_request_headers: + - object_key: jwt_selector + format_string: + text_format_source: + inline_string: "%REQ(jwt_selector)%" + read_only: true )"); initialize(); diff --git a/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc b/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc index 91105bdd232f..1b82d0996160 100644 --- a/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc +++ b/test/extensions/filters/http/jwt_authn/jwks_async_fetcher_test.cc @@ -4,7 +4,6 @@ #include "test/mocks/server/factory_context.h" using envoy::extensions::filters::http::jwt_authn::v3::RemoteJwks; -using Envoy::Extensions::HttpFilters::Common::JwksFetcher; using Envoy::Extensions::HttpFilters::Common::JwksFetcherPtr; namespace Envoy { @@ -57,7 +56,7 @@ class JwksAsyncFetcherTest : public testing::TestWithParam { // if async_fetch is enabled, timer is created if (config_.has_async_fetch()) { - timer_ = new NiceMock(&context_.dispatcher_); + timer_ = new NiceMock(&context_.server_factory_context_.dispatcher_); } async_fetcher_ = std::make_unique( diff --git a/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc b/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc index d33de67890b9..bd27f7417f70 100644 --- a/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc +++ b/test/extensions/filters/http/jwt_authn/jwks_cache_test.cc @@ -101,7 +101,7 @@ TEST_F(JwksCacheTest, TestSetRemoteJwks) { EXPECT_FALSE(jwks->isExpired()); // cache duration is 1 second, sleep two seconds to expire it - context_.time_system_.advanceTimeWait(std::chrono::seconds(2)); + context_.server_factory_context_.time_system_.advanceTimeWait(std::chrono::seconds(2)); EXPECT_TRUE(jwks->isExpired()); } diff --git a/test/extensions/filters/http/jwt_authn/jwt_authn_corpus/large_timeout b/test/extensions/filters/http/jwt_authn/jwt_authn_corpus/large_timeout new file mode 100644 index 000000000000..78a98a70280a --- /dev/null +++ b/test/extensions/filters/http/jwt_authn/jwt_authn_corpus/large_timeout @@ -0,0 +1,35 @@ +config { + providers { + key: "\005" + value { + remote_jwks { + http_uri { + uri: "https:/pubkey_server/pubkey_phttps:/pubkey_server/pubkey_pathath" + cluster: "2" + timeout { + seconds: 68719476736 + } + } + } + } + } + rules { + match { + prefix: "" + } + requires { + allow_missing { + } + } + } +} +request_data { + headers { + headers { + key: "authorization" + value: "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwczovL2V4cGxlLmNvbSIsImF1ZCI6ImV4YW1wbGVfc2VydmljZSIsImV4cCI6MjAwMTAwMTAwMX0.n45uWZfIBZwCIPiL0K8Ca3tmm-ZlsDrC79_vXCspPwk5oxdSn983tuC9GfVWKXWUMHe11DsB02b19Ow-fmoEzooTFn65Ml7G34nW07amyM6lETiMhNzyiunctplOr6xKKJHmzTUhfTirvDeG-q9n24-8lH7GP8GgHvDlgSM9OY7TGp81bRcnZBmxim_UzHoYO3_c8OP4ZX3xG5PfihVk5G0g6wcHrO70w0_64JgkKRCrLHMJSrhIgp9NHel_CNOnL0AjQKe9IGblJrMuouqYYS0zEWwmOVUWUSxQkoLpldQUVefcfjQeGjz8IlvktRa77FYexfP590ACPyXrivtsxg" + } + } +} +remote_jwks: "{\"keys\":[{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"62a93512c9ee4c7f8067b5a216dade2763d32a47\",\"n\":\"0YWnm_eplO9BFtUUUUUUU\0055H8mYDW11HolzZmTQpRoLV8ZoHbHEaTfqX_aYahIw\",\"e\":\"AQAB\"},{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"b3319a147514df7ee5e4bcdee51350cc890cc89e\",\"n\":\"qDi7Tx4DhNDDdk0iUtJSPZliUHJBI_pj8M-2Mn_oA8jBuI8YKwBqYkZCN2I95Q\",\"e\":\"AQAB\"}]}" +num_calls: 4 diff --git a/test/extensions/filters/http/jwt_authn/jwt_authn_fuzz_test.cc b/test/extensions/filters/http/jwt_authn/jwt_authn_fuzz_test.cc index 880489505dbf..926fbaacdd46 100644 --- a/test/extensions/filters/http/jwt_authn/jwt_authn_fuzz_test.cc +++ b/test/extensions/filters/http/jwt_authn/jwt_authn_fuzz_test.cc @@ -88,10 +88,12 @@ DEFINE_PROTO_FUZZER(const JwtAuthnFuzzInput& input) { // The jwt token in the corpus files expired at 2001001001 (at year 2033). if (input.force_jwt_expired()) { // 20 years == 615168000 seconds. - mock_factory_ctx.time_system_.advanceTimeWait(Envoy::Seconds(615168000)); + mock_factory_ctx.server_factory_context_.time_system_.advanceTimeWait( + Envoy::Seconds(615168000)); } - MockJwksUpstream mock_jwks(mock_factory_ctx.cluster_manager_, input.remote_jwks()); + MockJwksUpstream mock_jwks(mock_factory_ctx.server_factory_context_.cluster_manager_, + input.remote_jwks()); // Mock per route config. std::unique_ptr mock_per_route; diff --git a/test/extensions/filters/http/jwt_authn/mock.h b/test/extensions/filters/http/jwt_authn/mock.h index 36263bd6dbcc..e3a792c765ba 100644 --- a/test/extensions/filters/http/jwt_authn/mock.h +++ b/test/extensions/filters/http/jwt_authn/mock.h @@ -35,8 +35,8 @@ class MockAuthenticator : public Authenticator { void verify(Http::HeaderMap& headers, Tracing::Span& parent_span, std::vector&& tokens, - SetExtractedJwtDataCallback set_extracted_jwt_data_cb, - AuthenticatorCallback callback) override { + SetExtractedJwtDataCallback set_extracted_jwt_data_cb, AuthenticatorCallback callback, + ClearRouteCacheCallback) override { doVerify(headers, parent_span, &tokens, std::move(set_extracted_jwt_data_cb), std::move(callback)); } @@ -47,6 +47,7 @@ class MockAuthenticator : public Authenticator { class MockVerifierCallbacks : public Verifier::Callbacks { public: MOCK_METHOD(void, setExtractedData, (const ProtobufWkt::Struct& payload)); + MOCK_METHOD(void, clearRouteCache, ()); MOCK_METHOD(void, onComplete, (const Status& status)); }; diff --git a/test/extensions/filters/http/jwt_authn/provider_verifier_test.cc b/test/extensions/filters/http/jwt_authn/provider_verifier_test.cc index 6457e5b6b188..fa17239d1ce4 100644 --- a/test/extensions/filters/http/jwt_authn/provider_verifier_test.cc +++ b/test/extensions/filters/http/jwt_authn/provider_verifier_test.cc @@ -34,7 +34,8 @@ ProtobufWkt::Struct getExpectedPayload(const std::string& name) { class ProviderVerifierTest : public testing::Test { public: ProviderVerifierTest() { - mock_factory_ctx_.cluster_manager_.initializeThreadLocalClusters({"pubkey_cluster"}); + mock_factory_ctx_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"pubkey_cluster"}); } void createVerifier() { @@ -57,7 +58,7 @@ TEST_F(ProviderVerifierTest, TestOkJWT) { (*proto_config_.mutable_providers())[std::string(ProviderName)].set_payload_in_metadata( "my_payload"); createVerifier(); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, PublicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, PublicKey); EXPECT_CALL(mock_cb_, setExtractedData(_)) .WillOnce(Invoke([](const ProtobufWkt::Struct& payload) { @@ -88,7 +89,7 @@ TEST_F(ProviderVerifierTest, TestOkJWTWithExtractedHeaderAndPayload) { (*proto_config_.mutable_providers())[std::string(ProviderName)].set_header_in_metadata( "my_header"); createVerifier(); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, PublicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, PublicKey); EXPECT_CALL(mock_cb_, setExtractedData(_)) .WillOnce(Invoke([](const ProtobufWkt::Struct& payload) { @@ -115,7 +116,7 @@ TEST_F(ProviderVerifierTest, TestSpanPassedDown) { (*proto_config_.mutable_providers())[std::string(ProviderName)].set_payload_in_metadata( "my_payload"); createVerifier(); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, PublicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, PublicKey); EXPECT_CALL(mock_cb_, setExtractedData(_)) .WillOnce(Invoke([](const ProtobufWkt::Struct& payload) { @@ -128,7 +129,8 @@ TEST_F(ProviderVerifierTest, TestSpanPassedDown) { .setTimeout(std::chrono::milliseconds(5 * 1000)) .setParentSpan(parent_span_) .setChildSpanName("JWT Remote PubKey Fetch"); - EXPECT_CALL(mock_factory_ctx_.cluster_manager_.thread_local_cluster_.async_client_, + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .async_client_, send_(_, _, Eq(options))); auto headers = Http::TestRequestHeaderMapImpl{ @@ -212,7 +214,7 @@ TEST_F(ProviderVerifierTest, TestRequiresProviderWithAudiences) { provider_and_audiences->set_provider_name("example_provider"); provider_and_audiences->add_audiences("invalid_service"); createVerifier(); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, PublicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, PublicKey); EXPECT_CALL(mock_cb_, onComplete(_)) .WillOnce( @@ -264,7 +266,7 @@ TEST_P(ProviderVerifiersJwtCacheTest, TestRequirementsWithAudiences) { VerifierConstPtr verifier2 = Verifier::create(require2, proto_config_.providers(), *filter_config_); - MockUpstream mock_pubkey(mock_factory_ctx_.cluster_manager_, PublicKey); + MockUpstream mock_pubkey(mock_factory_ctx_.server_factory_context_.cluster_manager_, PublicKey); { // First call, audience matched, so it is good. diff --git a/test/extensions/filters/http/jwt_authn/test_common.h b/test/extensions/filters/http/jwt_authn/test_common.h index 95199667d03d..2c583604d20b 100644 --- a/test/extensions/filters/http/jwt_authn/test_common.h +++ b/test/extensions/filters/http/jwt_authn/test_common.h @@ -97,6 +97,64 @@ const char ExampleConfig[] = R"( claim_name: "nested.nested-2.key-3" - header_name: "x-jwt-int-claim" claim_name: "nested.nested-2.key-4" + - header_name: "x-jwt-claim-object-key" + claim_name: "nested.nested-2.key-5" +rules: +- match: + path: "/" + requires: + provider_name: "example_provider" +bypass_cors_preflight: true +)"; + +// Config with claim_to_headers and clear_route_cache. +const char ClaimToHeadersConfig[] = R"( +providers: + example_provider: + issuer: https://example.com + audiences: + - example_service + - http://example_service1 + - https://example_service2/ + remote_jwks: + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster + timeout: + seconds: 5 + cache_duration: + seconds: 600 + claim_to_headers: + - header_name: "x-jwt-claim-nested" + claim_name: "nested.key-1" + clear_route_cache: true +rules: +- match: + path: "/" + requires: + provider_name: "example_provider" +bypass_cors_preflight: true +)"; + +// Config with payload_in_metadata and clear_route_cache. +const char PayloadClearRouteCacheConfig[] = R"( +providers: + example_provider: + issuer: https://example.com + audiences: + - example_service + - http://example_service1 + - https://example_service2/ + remote_jwks: + http_uri: + uri: https://pubkey_server/pubkey_path + cluster: pubkey_cluster + timeout: + seconds: 5 + cache_duration: + seconds: 600 + payload_in_metadata: test_payload + clear_route_cache: true rules: - match: path: "/" @@ -149,6 +207,26 @@ const char GoodToken[] = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwc "EprqSZUzi_ZzzYzqBNVhIJujcNWij7JRra2sXXiSAfKjtxHQoxrX8n4V1ySWJ3_1T" "H_cJcdfS_RKP7YgXRWC0L16PNF5K7iqRqmjKALNe83ZFnFIw"; +// Payload: +// { +// "iss": "https://example.com", +// "sub": "test@example.com", +// "exp": 2001001001, +// "aud": "example_service", +// "scope": "read write", +// "test_string": "test_value", +// "test_num": 1337 +// } +const char GoodTokenWithSpaces[] = + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJodHRwczovL2V4YW1wbGUuY29tIiwic3ViIjoidGVzdEBleGFtcGxlLmNvbSIsImV4cCI6MjAwMTAwMTAwMS" + "wiYXVkIjoiZXhhbXBsZV9zZXJ2aWNlIiwic2NvcGUiOiJyZWFkIHdyaXRlIiwidGVzdF9zdHJpbmciOiJ0ZXN0X3ZhbHVl" + "IiwidGVzdF9udW0iOjEzMzd9.cKTwWSJgS0TZ3Ajc9QrAA50Me7j1zVv9YzDT_" + "2UE5jlCs5vWkdWjUb2r7MYaqximXj3affDZdDsUxMaqqR7lWT2EbxOoEceBkCMmakgSs8tjZ210w0YTU0OyhrrxsyxUpsp" + "PeRzPIHQTUdN7zU_KkMcUU1yDSlnJxqlYXyTL9E-DhTnLwoOdgFGiQs-md_QJfdOFgXQqU71EZ-" + "Ofxen8EFl10wbzHubMHGLJqVfFzK-iuVr2P0OZ0ymWvPGwQdlVMojHx3P0Yb8MRbhdW04hCJq-_" + "fTE1RNb6ja1JBFQbyGcQTtWVSdkHZ_C8syd8s-aK4C8_VhwNEDviOVrHPbztw"; + // Payload: // {"iss":"https://example.com","sub":"test@example.com","exp":null} const char NonExpiringToken[] = @@ -269,6 +347,19 @@ const char ExpectedPayloadJSON[] = R"( } )"; +// Base64 decoded Payload with space-delimited claims JSON +const char ExpectedPayloadJSONWithSpaces[] = R"( +{ + "iss":"https://example.com", + "sub":"test@example.com", + "exp":2001001001, + "aud":"example_service", + "scope":["read","write"], + "test_string":["test_value"], + "test_num":1337 +} +)"; + const char ExpectedHeaderJSON[] = R"( { "alg": "RS256", diff --git a/test/extensions/filters/http/kill_request/kill_request_config_test.cc b/test/extensions/filters/http/kill_request/kill_request_config_test.cc index af20c428f1e9..060905252a8b 100644 --- a/test/extensions/filters/http/kill_request/kill_request_config_test.cc +++ b/test/extensions/filters/http/kill_request/kill_request_config_test.cc @@ -23,7 +23,8 @@ TEST(KillRequestConfigTest, KillRequestFilterWithCorrectProto) { NiceMock context; KillRequestFilterFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(kill_request, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(kill_request, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -33,7 +34,8 @@ TEST(KillRequestConfigTest, KillRequestFilterWithEmptyProto) { NiceMock context; KillRequestFilterFactory factory; Http::FilterFactoryCb cb = - factory.createFilterFactoryFromProto(*factory.createEmptyConfigProto(), "stats", context); + factory.createFilterFactoryFromProto(*factory.createEmptyConfigProto(), "stats", context) + .value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/local_ratelimit/config_test.cc b/test/extensions/filters/http/local_ratelimit/config_test.cc index 3985837745e1..37c3a991e256 100644 --- a/test/extensions/filters/http/local_ratelimit/config_test.cc +++ b/test/extensions/filters/http/local_ratelimit/config_test.cc @@ -22,8 +22,8 @@ stat_prefix: test NiceMock context; - EXPECT_CALL(context.dispatcher_, createTimer_(_)).Times(0); - auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + EXPECT_CALL(context.server_factory_context_.dispatcher_, createTimer_(_)).Times(0); + auto callback = factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); callback(filter_callback); diff --git a/test/extensions/filters/http/local_ratelimit/filter_test.cc b/test/extensions/filters/http/local_ratelimit/filter_test.cc index 4e519ea9181f..9cacb3d7e953 100644 --- a/test/extensions/filters/http/local_ratelimit/filter_test.cc +++ b/test/extensions/filters/http/local_ratelimit/filter_test.cc @@ -17,6 +17,7 @@ namespace LocalRateLimitFilter { static const std::string config_yaml = R"( stat_prefix: test +rate_limited_as_resource_exhausted: {} token_bucket: max_tokens: {} tokens_per_fill: 1 @@ -105,17 +106,17 @@ class FilterTest : public testing::Test { }; TEST_F(FilterTest, Runtime) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "false", "\"OFF\""), false, false); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "false", "\"OFF\""), false, false); EXPECT_EQ(&runtime_, &(config_->runtime())); } TEST_F(FilterTest, ToErrorCode) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "false", "\"OFF\""), false, false); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "false", "\"OFF\""), false, false); EXPECT_EQ(Http::Code::BadRequest, toErrorCode(400)); } TEST_F(FilterTest, Disabled) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "false", "\"OFF\""), false, false); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "false", "\"OFF\""), false, false); auto headers = Http::TestRequestHeaderMapImpl(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.enabled")); @@ -123,7 +124,7 @@ TEST_F(FilterTest, Disabled) { } TEST_F(FilterTest, RequestOk) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "false", "\"OFF\"")); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "false", "\"OFF\"")); auto headers = Http::TestRequestHeaderMapImpl(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_2_->decodeHeaders(headers, false)); @@ -134,7 +135,7 @@ TEST_F(FilterTest, RequestOk) { } TEST_F(FilterTest, RequestOkPerConnection) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "true", "\"OFF\"")); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "true", "\"OFF\"")); auto headers = Http::TestRequestHeaderMapImpl(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_2_->decodeHeaders(headers, false)); @@ -145,7 +146,7 @@ TEST_F(FilterTest, RequestOkPerConnection) { } TEST_F(FilterTest, RequestRateLimited) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "false", "\"OFF\"")); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "false", "\"OFF\"")); EXPECT_CALL(decoder_callbacks_2_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)) .WillOnce(Invoke([](Http::Code code, absl::string_view body, @@ -165,7 +166,6 @@ TEST_F(FilterTest, RequestRateLimited) { EXPECT_EQ("123", response_headers.get(Http::LowerCaseString("test-resp-req-id"))[0] ->value() .getStringView()); - EXPECT_EQ(grpc_status, absl::nullopt); EXPECT_EQ(details, "local_rate_limited"); })); @@ -186,6 +186,48 @@ TEST_F(FilterTest, RequestRateLimited) { EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); } +TEST_F(FilterTest, RequestRateLimitedResourceExhausted) { + setup(fmt::format(fmt::runtime(config_yaml), "true", "1", "false", "\"OFF\"")); + + EXPECT_CALL(decoder_callbacks_2_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)) + .WillOnce(Invoke([](Http::Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details) { + EXPECT_EQ(Http::Code::TooManyRequests, code); + EXPECT_EQ("local_rate_limited", body); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + modify_headers(response_headers); + EXPECT_EQ("true", response_headers.get(Http::LowerCaseString("x-test-rate-limit"))[0] + ->value() + .getStringView()); + // Make sure that generated local reply headers contain a value dynamically + // generated by header formatter REQ(test-req-id) + EXPECT_EQ("123", response_headers.get(Http::LowerCaseString("test-resp-req-id"))[0] + ->value() + .getStringView()); + EXPECT_EQ(grpc_status, + absl::make_optional(Grpc::Status::WellKnownGrpcStatus::ResourceExhausted)); + EXPECT_EQ(details, "local_rate_limited"); + })); + + // Add a custom header to the request. + // Locally generated reply is configured to refer to this value. + Http::TestRequestHeaderMapImpl request_headers{{"test-req-id", "123"}}; + NiceMock stream_info; + + EXPECT_CALL(decoder_callbacks_2_, streamInfo).WillRepeatedly(testing::ReturnRef(stream_info)); + EXPECT_CALL(stream_info, getRequestHeaders).WillRepeatedly(Return(&request_headers)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_2_->decodeHeaders(request_headers, false)); + EXPECT_EQ(2U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.ok")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); +} + /* This test sets 'local_rate_limit_per_downstream_connection' to true. Doing this enables per connection rate limiting and even though 'max_token' is set to 1, it allows 2 requests to go through @@ -193,7 +235,7 @@ connection rate limiting and even though 'max_token' is set to 1, it allows 2 re allowed (across the process) for the same configuration. */ TEST_F(FilterTest, RequestRateLimitedPerConnection) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "true", "\"OFF\"")); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "true", "\"OFF\"")); EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)) .WillOnce(Invoke([](Http::Code code, absl::string_view body, @@ -230,7 +272,7 @@ TEST_F(FilterTest, RequestRateLimitedPerConnection) { } TEST_F(FilterTest, RequestRateLimitedButNotEnforced) { - setup(fmt::format(fmt::runtime(config_yaml), "0", "false", "\"OFF\""), true, false); + setup(fmt::format(fmt::runtime(config_yaml), "false", "0", "false", "\"OFF\""), true, false); EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::TooManyRequests, _, _, _, _)).Times(0); @@ -246,7 +288,7 @@ TEST_F(FilterTest, RequestRateLimitedButNotEnforced) { } TEST_F(FilterTest, RequestRateLimitedXRateLimitHeaders) { - setup(fmt::format(fmt::runtime(config_yaml), "1", "false", "DRAFT_VERSION_03")); + setup(fmt::format(fmt::runtime(config_yaml), "false", "1", "false", "DRAFT_VERSION_03")); auto request_headers = Http::TestRequestHeaderMapImpl(); auto response_headers = Http::TestResponseHeaderMapImpl(); @@ -311,6 +353,49 @@ enable_x_ratelimit_headers: {} stage: {} )"; +static const std::string consume_default_token_config_yaml = R"( +stat_prefix: test +token_bucket: + max_tokens: {} + tokens_per_fill: 1 + fill_interval: 60s +filter_enabled: + runtime_key: test_enabled + default_value: + numerator: 100 + denominator: HUNDRED +filter_enforced: + runtime_key: test_enforced + default_value: + numerator: 100 + denominator: HUNDRED +response_headers_to_add: + - append_action: OVERWRITE_IF_EXISTS_OR_ADD + header: + key: x-test-rate-limit + value: 'true' +local_rate_limit_per_downstream_connection: true +always_consume_default_token_bucket: {} +descriptors: +- entries: + - key: hello + value: world + - key: foo + value: bar + token_bucket: + max_tokens: 10 + tokens_per_fill: 10 + fill_interval: 60s +- entries: + - key: foo2 + value: bar2 + token_bucket: + max_tokens: {} + tokens_per_fill: 1 + fill_interval: 60s +stage: {} + )"; + static const std::string descriptor_vh_config_yaml = R"( stat_prefix: test token_bucket: @@ -469,6 +554,70 @@ TEST_F(DescriptorFilterTest, RouteDescriptorNotFound) { EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.rate_limited")); } +TEST_F(DescriptorFilterTest, RouteDescriptorNotFoundWithConsumeDefaultTokenTrue) { + setUpTest(fmt::format(fmt::runtime(consume_default_token_config_yaml), "0", "true", "1", "0")); + + EXPECT_CALL(decoder_callbacks_.route_->route_entry_.rate_limit_policy_, + getApplicableRateLimit(0)); + + EXPECT_CALL(route_rate_limit_, populateLocalDescriptors(_, _, _, _)) + .WillOnce(testing::SetArgReferee<0>(descriptor_not_found_)); + + auto headers = Http::TestRequestHeaderMapImpl(); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); +} + +TEST_F(DescriptorFilterTest, RouteDescriptorWithConsumeDefaultTokenTrue) { + setUpTest(fmt::format(fmt::runtime(consume_default_token_config_yaml), "0", "true", "1", "0")); + + EXPECT_CALL(decoder_callbacks_.route_->route_entry_.rate_limit_policy_, + getApplicableRateLimit(0)); + + EXPECT_CALL(route_rate_limit_, populateLocalDescriptors(_, _, _, _)) + .WillOnce(testing::SetArgReferee<0>(descriptor_)); + + auto headers = Http::TestRequestHeaderMapImpl(); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); +} + +TEST_F(DescriptorFilterTest, RouteDescriptorWithConsumeDefaultTokenFalse) { + setUpTest(fmt::format(fmt::runtime(consume_default_token_config_yaml), "0", "false", "1", "0")); + + EXPECT_CALL(decoder_callbacks_.route_->route_entry_.rate_limit_policy_, + getApplicableRateLimit(0)); + + EXPECT_CALL(route_rate_limit_, populateLocalDescriptors(_, _, _, _)) + .WillOnce(testing::SetArgReferee<0>(descriptor_)); + + auto headers = Http::TestRequestHeaderMapImpl(); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(0U, findCounter("test.http_local_rate_limit.rate_limited")); +} + +TEST_F(DescriptorFilterTest, RouteDescriptorNotFoundWithConsumeDefaultTokenFalse) { + setUpTest(fmt::format(fmt::runtime(consume_default_token_config_yaml), "0", "false", "1", "0")); + + EXPECT_CALL(decoder_callbacks_.route_->route_entry_.rate_limit_policy_, + getApplicableRateLimit(0)); + + EXPECT_CALL(route_rate_limit_, populateLocalDescriptors(_, _, _, _)) + .WillOnce(testing::SetArgReferee<0>(descriptor_not_found_)); + + auto headers = Http::TestRequestHeaderMapImpl(); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enabled")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.enforced")); + EXPECT_EQ(1U, findCounter("test.http_local_rate_limit.rate_limited")); +} + TEST_F(DescriptorFilterTest, RouteDescriptorBothMatch) { // Request should also be rate limited as it should match both descriptors and global token. setUpTest(fmt::format(fmt::runtime(descriptor_config_yaml), "0", "\"OFF\"", "0", "0")); diff --git a/test/extensions/filters/http/lua/config_test.cc b/test/extensions/filters/http/lua/config_test.cc index 652a35cdadac..19328d4e471a 100644 --- a/test/extensions/filters/http/lua/config_test.cc +++ b/test/extensions/filters/http/lua/config_test.cc @@ -21,8 +21,11 @@ namespace { TEST(LuaFilterConfigTest, ValidateEmptyConfigNotFail) { NiceMock context; - EXPECT_NO_THROW(LuaFilterConfig().createFilterFactoryFromProto( - envoy::extensions::filters::http::lua::v3::Lua(), "stats", context)); + EXPECT_NO_THROW(LuaFilterConfig() + .createFilterFactoryFromProto( + envoy::extensions::filters::http::lua::v3::Lua(), "stats", context) + .status() + .IgnoreError()); } TEST(LuaFilterConfigTest, LuaFilterWithDefaultSourceCode) { @@ -38,7 +41,8 @@ TEST(LuaFilterConfigTest, LuaFilterWithDefaultSourceCode) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; LuaFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -54,7 +58,8 @@ TEST(LuaFilterConfigTest, LuaFilterInJson) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; LuaFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -72,7 +77,8 @@ TEST(LuaFilterConfigTest, LuaFilterWithDeprecatedInlineCode) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; LuaFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -96,7 +102,8 @@ TEST(LuaFilterConfigTest, LuaFilterWithBothDeprecatedInlineCodeAndDefaultSourceC NiceMock context; LuaFilterConfig factory; EXPECT_THROW_WITH_MESSAGE( - factory.createFilterFactoryFromProto(proto_config, "stats", context), EnvoyException, + factory.createFilterFactoryFromProto(proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "Error: Only one of `inline_code` or `default_source_code` can be set for the Lua filter."); } #endif diff --git a/test/extensions/filters/http/lua/lua_integration_test.cc b/test/extensions/filters/http/lua/lua_integration_test.cc index f60ac3d40118..970d1cd81112 100644 --- a/test/extensions/filters/http/lua/lua_integration_test.cc +++ b/test/extensions/filters/http/lua/lua_integration_test.cc @@ -311,6 +311,8 @@ name: lua request_handle:streamInfo():downstreamLocalAddress()) request_handle:headers():add("request_downstream_directremote_address_value", request_handle:streamInfo():downstreamDirectRemoteAddress()) + request_handle:headers():add("request_downstream_remote_address_value", + request_handle:streamInfo():downstreamRemoteAddress()) request_handle:headers():add("request_requested_server_name", request_handle:streamInfo():requestedServerName()) end @@ -420,6 +422,12 @@ name: lua .getStringView(), GetParam() == Network::Address::IpVersion::v4 ? "127.0.0.1:" : "[::1]:")); + EXPECT_EQ("10.0.0.1:0", + upstream_request_->headers() + .get(Http::LowerCaseString("request_downstream_remote_address_value"))[0] + ->value() + .getStringView()); + EXPECT_EQ("", upstream_request_->headers() .get(Http::LowerCaseString("request_requested_server_name"))[0] ->value() @@ -864,7 +872,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute disabled: true - match: @@ -872,7 +880,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute name: hello.lua - match: @@ -880,7 +888,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute name: byebye.lua - match: @@ -888,7 +896,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute source_code: inline_string: | @@ -900,7 +908,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute name: nocode.lua )EOF"; @@ -917,7 +925,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute source_code: inline_string: | @@ -929,7 +937,7 @@ name: basic_lua_routes route: cluster: lua_cluster typed_per_filter_config: - envoy.filters.http.lua: + lua: "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute source_code: inline_string: | diff --git a/test/extensions/filters/http/lua/wrappers_test.cc b/test/extensions/filters/http/lua/wrappers_test.cc index 85f5c13b31a9..cc761915ea79 100644 --- a/test/extensions/filters/http/lua/wrappers_test.cc +++ b/test/extensions/filters/http/lua/wrappers_test.cc @@ -12,7 +12,6 @@ using testing::Expectation; using testing::InSequence; using testing::ReturnPointee; -using testing::ReturnRef; namespace Envoy { namespace Extensions { @@ -349,6 +348,7 @@ TEST_F(LuaStreamInfoWrapperTest, ReturnCurrentDownstreamAddresses) { function callMe(object) testPrint(object:downstreamLocalAddress()) testPrint(object:downstreamDirectRemoteAddress()) + testPrint(object:downstreamRemoteAddress()) end )EOF"}; @@ -360,13 +360,17 @@ TEST_F(LuaStreamInfoWrapperTest, ReturnCurrentDownstreamAddresses) { new Network::Address::Ipv4Instance("127.0.0.1", 8000)}; auto downstream_direct_remote = Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance("8.8.8.8", 3000)}; + auto downstream_remote = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance("10.1.2.3", 5000)}; stream_info.downstream_connection_info_provider_->setLocalAddress(address); stream_info.downstream_connection_info_provider_->setDirectRemoteAddressForTest( downstream_direct_remote); + stream_info.downstream_connection_info_provider_->setRemoteAddress(downstream_remote); Filters::Common::Lua::LuaDeathRef wrapper( StreamInfoWrapper::create(coroutine_->luaState(), stream_info), true); EXPECT_CALL(printer_, testPrint(address->asString())); EXPECT_CALL(printer_, testPrint(downstream_direct_remote->asString())); + EXPECT_CALL(printer_, testPrint(downstream_remote->asString())); start("callMe"); wrapper.reset(); } diff --git a/test/extensions/filters/http/oauth2/config_test.cc b/test/extensions/filters/http/oauth2/config_test.cc index a44afe3fbd4a..09d1a8cd0edd 100644 --- a/test/extensions/filters/http/oauth2/config_test.cc +++ b/test/extensions/filters/http/oauth2/config_test.cc @@ -62,14 +62,16 @@ void expectInvalidSecretConfig(const std::string& failed_secret_name, TestUtility::loadFromYaml(yaml, *proto_config); NiceMock context; - auto& secret_manager = context.cluster_manager_.cluster_manager_factory_.secretManager(); + auto& secret_manager = + context.server_factory_context_.cluster_manager_.cluster_manager_factory_.secretManager(); ON_CALL(secret_manager, findStaticGenericSecretProvider(failed_secret_name == "token" ? "hmac" : "token")) .WillByDefault(Return(std::make_shared( envoy::extensions::transport_sockets::tls::v3::GenericSecret()))); - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(*proto_config, "stats", context), - EnvoyException, exception_message); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, exception_message); } } // namespace @@ -114,25 +116,27 @@ TEST(ConfigTest, CreateFilter) { OAuth2Config factory; ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); TestUtility::loadFromYaml(yaml, *proto_config); - Server::Configuration::MockFactoryContext context; - context.cluster_manager_.initializeClusters({"foo"}, {}); + NiceMock context; + context.server_factory_context_.cluster_manager_.initializeClusters({"foo"}, {}); // This returns non-nullptr for token_secret and hmac_secret. - auto& secret_manager = context.cluster_manager_.cluster_manager_factory_.secretManager(); + auto& secret_manager = + context.server_factory_context_.cluster_manager_.cluster_manager_factory_.secretManager(); ON_CALL(secret_manager, findStaticGenericSecretProvider(_)) .WillByDefault(Return(std::make_shared( envoy::extensions::transport_sockets::tls::v3::GenericSecret()))); EXPECT_CALL(context, messageValidationVisitor()); - EXPECT_CALL(context, clusterManager()); + EXPECT_CALL(context.server_factory_context_, clusterManager()); EXPECT_CALL(context, scope()); - EXPECT_CALL(context, timeSource()); - EXPECT_CALL(context, api()); + EXPECT_CALL(context.server_factory_context_, timeSource()); + EXPECT_CALL(context.server_factory_context_, api()); EXPECT_CALL(context, initManager()).Times(2); EXPECT_CALL(context, getTransportSocketFactoryContext()); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(*proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; - EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); + EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); } @@ -193,8 +197,9 @@ TEST(ConfigTest, WrongCookieName) { TestUtility::loadFromYaml(yaml, *proto_config); NiceMock context; - EXPECT_THROW_WITH_REGEX(factory.createFilterFactoryFromProto(*proto_config, "stats", context), - EnvoyException, "value does not match regex pattern"); + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, "value does not match regex pattern"); } } // namespace Oauth2 diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index 97038bff5be4..3ac4819bc635 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -10,6 +10,7 @@ #include "source/common/http/message_impl.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/utility.h" +#include "source/common/runtime/runtime_protos.h" #include "source/common/secret/secret_manager_impl.h" #include "source/extensions/filters/http/oauth2/filter.h" @@ -59,6 +60,8 @@ class MockOAuth2CookieValidator : public CookieValidator { MOCK_METHOD(std::string&, username, (), (const)); MOCK_METHOD(std::string&, token, (), (const)); MOCK_METHOD(std::string&, refreshToken, (), (const)); + + MOCK_METHOD(bool, canUpdateTokenByRefreshToken, (), (const)); MOCK_METHOD(bool, isValid, (), (const)); MOCK_METHOD(void, setParams, (const Http::RequestHeaderMap& headers, const std::string& secret)); }; @@ -74,12 +77,17 @@ class MockOAuth2Client : public OAuth2Client { MOCK_METHOD(void, asyncGetAccessToken, (const std::string&, const std::string&, const std::string&, const std::string&, Envoy::Extensions::HttpFilters::Oauth2::AuthType)); + + MOCK_METHOD(void, asyncRefreshAccessToken, + (const std::string&, const std::string&, const std::string&, + Envoy::Extensions::HttpFilters::Oauth2::AuthType)); }; class OAuth2Test : public testing::TestWithParam { public: OAuth2Test() : request_(&cm_.thread_local_cluster_.async_client_) { - factory_context_.cluster_manager_.initializeClusters({"auth.example.com"}, {}); + factory_context_.server_factory_context_.cluster_manager_.initializeClusters( + {"auth.example.com"}, {}); init(); } @@ -93,12 +101,17 @@ class OAuth2Test : public testing::TestWithParam { config_ = config; filter_ = std::make_shared(config_, std::move(oauth_client_ptr), test_time_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); + filter_->setEncoderFilterCallbacks(encoder_callbacks_); validator_ = std::make_shared(); filter_->validator_ = validator_; } // Set up proto fields with standard config. - FilterConfigSharedPtr getConfig(bool forward_bearer_token = true) { + FilterConfigSharedPtr + getConfig(bool forward_bearer_token = true, bool use_refresh_token = false, + ::envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType auth_type = + ::envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType:: + OAuth2Config_AuthType_URL_ENCODED_BODY) { envoy::extensions::filters::http::oauth2::v3::OAuth2Config p; auto* endpoint = p.mutable_token_endpoint(); endpoint->set_cluster("auth.example.com"); @@ -109,6 +122,10 @@ class OAuth2Test : public testing::TestWithParam { p.set_authorization_endpoint("https://auth.example.com/oauth/authorize/"); p.mutable_signout_path()->mutable_path()->set_exact("/_signout"); p.set_forward_bearer_token(forward_bearer_token); + + auto* useRefreshToken = p.mutable_use_refresh_token(); + useRefreshToken->set_value(use_refresh_token); + p.set_auth_type(auth_type); p.add_auth_scopes("user"); p.add_auth_scopes("openid"); p.add_auth_scopes("email"); @@ -129,8 +146,9 @@ class OAuth2Test : public testing::TestWithParam { // Create filter config. auto secret_reader = std::make_shared(); - FilterConfigSharedPtr c = std::make_shared(p, factory_context_.cluster_manager_, - secret_reader, scope_, "test."); + FilterConfigSharedPtr c = + std::make_shared(p, factory_context_.server_factory_context_.cluster_manager_, + secret_reader, scope_, "test."); return c; } @@ -169,11 +187,13 @@ class OAuth2Test : public testing::TestWithParam { auto cookie_validator = std::make_shared(test_time_, cookie_names); EXPECT_EQ(cookie_validator->token(), ""); + EXPECT_EQ(cookie_validator->refreshToken(), ""); cookie_validator->setParams(request_headers, "mock-secret"); EXPECT_TRUE(cookie_validator->hmacIsValid()); EXPECT_TRUE(cookie_validator->timestampIsValid()); EXPECT_TRUE(cookie_validator->isValid()); + EXPECT_FALSE(cookie_validator->canUpdateTokenByRefreshToken()); // If we advance time beyond 10s the timestamp should no longer be valid. test_time_.advanceTimeWait(std::chrono::seconds(11)); @@ -185,6 +205,7 @@ class OAuth2Test : public testing::TestWithParam { NiceMock* attachmentTimeout_timer_{}; NiceMock factory_context_; NiceMock decoder_callbacks_; + NiceMock encoder_callbacks_; NiceMock cm_; std::shared_ptr validator_; std::shared_ptr filter_; @@ -273,7 +294,7 @@ name: client } // Verifies that we fail constructing the filter if the configured cluster doesn't exist. TEST_F(OAuth2Test, InvalidCluster) { - ON_CALL(factory_context_.cluster_manager_, clusters()) + ON_CALL(factory_context_.server_factory_context_.cluster_manager_, clusters()) .WillByDefault(Return(Upstream::ClusterManager::ClusterInfoMaps())); EXPECT_THROW_WITH_MESSAGE(init(), EnvoyException, @@ -291,8 +312,8 @@ TEST_F(OAuth2Test, InvalidAuthorizationEndpoint) { auto secret_reader = std::make_shared(); EXPECT_THROW_WITH_MESSAGE( - std::make_shared(p, factory_context_.cluster_manager_, secret_reader, scope_, - "test."), + std::make_shared(p, factory_context_.server_factory_context_.cluster_manager_, + secret_reader, scope_, "test."), EnvoyException, "OAuth2 filter: invalid authorization endpoint URL 'INVALID_URL' in config."); } @@ -324,8 +345,8 @@ TEST_F(OAuth2Test, DefaultAuthScope) { // Create the OAuth config. auto secret_reader = std::make_shared(); FilterConfigSharedPtr test_config_; - test_config_ = std::make_shared(p, factory_context_.cluster_manager_, secret_reader, - scope_, "test."); + test_config_ = std::make_shared( + p, factory_context_.server_factory_context_.cluster_manager_, secret_reader, scope_, "test."); // resource is optional EXPECT_EQ(test_config_->encodedResourceQueryParams(), ""); @@ -382,8 +403,8 @@ TEST_F(OAuth2Test, PreservesQueryParametersInAuthorizationEndpoint) { // Create the OAuth config. auto secret_reader = std::make_shared(); FilterConfigSharedPtr test_config_; - test_config_ = std::make_shared(p, factory_context_.cluster_manager_, secret_reader, - scope_, "test."); + test_config_ = std::make_shared( + p, factory_context_.server_factory_context_.cluster_manager_, secret_reader, scope_, "test."); init(test_config_); Http::TestRequestHeaderMapImpl request_headers{ {Http::Headers::get().Path.get(), "/not/_oauth"}, @@ -438,8 +459,8 @@ TEST_F(OAuth2Test, PreservesQueryParametersInAuthorizationEndpointWithUrlEncodin // Create the OAuth config. auto secret_reader = std::make_shared(); FilterConfigSharedPtr test_config_; - test_config_ = std::make_shared(p, factory_context_.cluster_manager_, secret_reader, - scope_, "test."); + test_config_ = std::make_shared( + p, factory_context_.server_factory_context_.cluster_manager_, secret_reader, scope_, "test."); init(test_config_); Http::TestRequestHeaderMapImpl request_headers{ {Http::Headers::get().Path.get(), "/not/_oauth"}, @@ -971,6 +992,25 @@ TEST_F(OAuth2Test, CookieValidatorInvalidExpiresAt) { } } +// Validates the behavior of the cookie validator when the expires_at value is not a valid integer. +TEST_F(OAuth2Test, CookieValidatorCanUpdateToken) { + Http::TestRequestHeaderMapImpl request_headers{ + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Path.get(), "/anypath"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Get}, + {Http::Headers::get().Cookie.get(), "OauthExpires=notanumber;version=test"}, + {Http::Headers::get().Cookie.get(), + "BearerToken=xyztoken;version=test;RefreshToken=dsdtoken;"}, + }; + + auto cookie_validator = std::make_shared( + test_time_, + CookieNames("BearerToken", "OauthHMAC", "OauthExpires", "IdToken", "RefreshToken")); + cookie_validator->setParams(request_headers, "mock-secret"); + + EXPECT_TRUE(cookie_validator->canUpdateTokenByRefreshToken()); +} + // Verify that we 401 the request if the state query param doesn't contain a valid URL. TEST_F(OAuth2Test, OAuthTestInvalidUrlInStateQueryParam) { Http::TestRequestHeaderMapImpl request_headers{ @@ -1365,6 +1405,97 @@ TEST_F(OAuth2Test, OAuthTestFullFlowPostWithParametersLegacyEncoding) { } } +TEST_F(OAuth2Test, OAuthTestFullFlowPostWithParametersFillRefreshAndIdToken) { + // First construct the initial request to the oauth filter with URI parameters. + Http::TestRequestHeaderMapImpl first_request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + // This is the immediate response - a redirect to the auth cluster. + Http::TestResponseHeaderMapImpl first_response_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().Location.get(), + "https://auth.example.com/oauth/" + "authorize/?client_id=" + + TEST_CLIENT_ID + + "&redirect_uri=https%3A%2F%2Ftraffic.example.com%2F_oauth" + "&response_type=code" + "&scope=" + + TEST_ENCODED_AUTH_SCOPES + + "&state=https%3A%2F%2Ftraffic.example.com%2Ftest%3Fname%3Dadmin%26level%3Dtrace" + "&resource=oauth2-resource&resource=http%3A%2F%2Fexample.com" + "&resource=https%3A%2F%2Fexample.com%2Fsome%2Fpath%252F..%252F%2Futf8%C3%83%3Bfoo%3Dbar%" + "3Fvar1%3D1%26var2%3D2"}, + }; + + // Fail the validation to trigger the OAuth flow. + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + + // Check that the redirect includes the escaped parameter characters, '?', '&' and '='. + EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&first_response_headers), true)); + + // This represents the beginning of the OAuth filter. + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(first_request_headers, false)); + + // This represents the callback request from the authorization server. + Http::TestRequestHeaderMapImpl second_request_headers{ + {Http::Headers::get().Path.get(), "/_oauth?code=123&state=https%3A%2F%2Ftraffic.example.com%" + "2Ftest%3Fname%3Dadmin%26level%3Dtrace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Get}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + // Deliberately fail the HMAC validation check. + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + + EXPECT_CALL(*oauth_client_, asyncGetAccessToken("123", TEST_CLIENT_ID, "asdf_client_secret_fdsa", + "https://traffic.example.com" + TEST_CALLBACK, + AuthType::UrlEncodedBody)); + + // Invoke the callback logic. As a side effect, state_ will be populated. + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndBuffer, + filter_->decodeHeaders(second_request_headers, false)); + + EXPECT_EQ(1, config_->stats().oauth_unauthorized_rq_.value()); + EXPECT_EQ(config_->clusterName(), "auth.example.com"); + + // Set SystemTime to a fixed point so we get consistent HMAC encodings between test runs. + test_time_.setSystemTime(SystemTime(std::chrono::seconds(0))); + const std::chrono::seconds expiredTime(10); + filter_->updateTokens("accessToken", "idToken", "refreshToken", expiredTime); + + // Expected response after the callback & validation is complete - verifying we kept the + // state and method of the original request, including the query string parameters. + Http::TestRequestHeaderMapImpl second_response_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().SetCookie.get(), "OauthHMAC=" + "OYnODPsSGabEpZ2LAiPxyjAFgN/7/5Xg24G7jUoUbyI=;" + "version=1;path=/;Max-Age=10;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "OauthExpires=10;version=1;path=/;Max-Age=10;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=accessToken;version=1;path=/;Max-Age=10;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "IdToken=idToken;version=1;path=/;Max-Age=10;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "RefreshToken=refreshToken;version=1;path=/;Max-Age=10;secure;HttpOnly"}, + {Http::Headers::get().Location.get(), + "https://traffic.example.com/test?name=admin&level=trace"}, + }; + + EXPECT_CALL(decoder_callbacks_, + encodeHeaders_(HeaderMapEqualRef(&second_response_headers), true)); + + filter_->finishGetAccessTokenFlow(); +} + // This test adds %-encoded UTF-8 characters to the URL and shows that // the new decoding correctly handles that case. TEST_F(OAuth2Test, OAuthTestFullFlowPostWithParameters) { @@ -1801,6 +1932,335 @@ TEST_P(OAuth2Test, CookieValidatorInTransition) { EXPECT_TRUE(cookie_validator->hmacIsValid()); } +// - The filter receives the initial request +// - The filter redirects a user to the authorization endpoint +// - The filter receives the callback request from the authorization endpoint +// - The filter gets a bearer and refresh tokens from the authorization endpoint +// - The filter redirects a user to the user agent with actual authorization data +// - The filter receives an other request when a bearer token is expired +// - The filter tries to update a bearer token via the refresh token instead of redirect user to the +// authorization endpoint +// - The filter gets a new bearer and refresh tokens via the current refresh token +// - The filter continues to handler the request without redirection to the user agent +TEST_F(OAuth2Test, OAuthTestFullFlowWithUseRefreshToken) { + init(getConfig(true /* forward_bearer_token */, true /* use_refresh_token */)); + // First construct the initial request to the oauth filter with URI parameters. + Http::TestRequestHeaderMapImpl first_request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + // This is the immediate response - a redirect to the auth cluster. + Http::TestResponseHeaderMapImpl first_response_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().Location.get(), + "https://auth.example.com/oauth/" + "authorize/?client_id=" + + TEST_CLIENT_ID + + "&redirect_uri=https%3A%2F%2Ftraffic.example.com%2F_oauth" + "&response_type=code" + "&scope=" + + TEST_ENCODED_AUTH_SCOPES + + "&state=https%3A%2F%2Ftraffic.example.com%2Ftest%3Fname%3Dadmin%26level%3Dtrace" + "&resource=oauth2-resource&resource=http%3A%2F%2Fexample.com" + "&resource=https%3A%2F%2Fexample.com%2Fsome%2Fpath%252F..%252F%2Futf8%C3%83%3Bfoo%3Dbar%" + "3Fvar1%3D1%26var2%3D2"}, + }; + + // Fail the validation to trigger the OAuth flow. + + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_CALL(*validator_, canUpdateTokenByRefreshToken()).WillOnce(Return(false)); + + // Check that the redirect includes the escaped parameter characters, '?', '&' and '='. + EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&first_response_headers), true)); + + // This represents the beginning of the OAuth filter. + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(first_request_headers, false)); + + // This represents the callback request from the authorization server. + Http::TestRequestHeaderMapImpl second_request_headers{ + {Http::Headers::get().Path.get(), "/_oauth?code=123&state=https%3A%2F%2Ftraffic.example.com%" + "2Ftest%3Fname%3Dadmin%26level%3Dtrace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Get}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + // Deliberately fail the HMAC validation check. + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + + EXPECT_CALL(*oauth_client_, asyncGetAccessToken("123", TEST_CLIENT_ID, "asdf_client_secret_fdsa", + "https://traffic.example.com" + TEST_CALLBACK, + AuthType::UrlEncodedBody)); + + // Invoke the callback logic. As a side effect, state_ will be populated. + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndBuffer, + filter_->decodeHeaders(second_request_headers, false)); + + EXPECT_EQ(1, config_->stats().oauth_unauthorized_rq_.value()); + EXPECT_EQ(config_->clusterName(), "auth.example.com"); + + // Expected response after the callback & validation is complete - verifying we kept the + // state and method of the original request, including the query string parameters. + Http::TestRequestHeaderMapImpl second_response_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().SetCookie.get(), "OauthHMAC=" + "fV62OgLipChTQQC3UFgDp+l5sCiSb3zt7nCoJiVivWw=;" + "version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "OauthExpires=;version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=;version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().Location.get(), + "https://traffic.example.com/test?name=admin&level=trace"}, + }; + + EXPECT_CALL(decoder_callbacks_, + encodeHeaders_(HeaderMapEqualRef(&second_response_headers), true)); + + filter_->finishGetAccessTokenFlow(); + + // the third request to the oauth filter with URI parameters. + Http::TestRequestHeaderMapImpl third_request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + std::string legit_refresh_token{"legit_refresh_token"}; + EXPECT_CALL(*validator_, refreshToken()).WillRepeatedly(ReturnRef(legit_refresh_token)); + + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_CALL(*validator_, canUpdateTokenByRefreshToken()).WillOnce(Return(true)); + + EXPECT_CALL(*oauth_client_, + asyncRefreshAccessToken(legit_refresh_token, TEST_CLIENT_ID, + "asdf_client_secret_fdsa", AuthType::UrlEncodedBody)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(third_request_headers, false)); + + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + + filter_->finishRefreshAccessTokenFlow(); + EXPECT_EQ(1, config_->stats().oauth_refreshtoken_success_.value()); + EXPECT_EQ(2, config_->stats().oauth_success_.value()); +} + +TEST_F(OAuth2Test, OAuthTestRefreshAccessTokenSuccess) { + + init(getConfig(true /* forward_bearer_token */, true /* use_refresh_token */)); + // First construct the initial request to the oauth filter with URI parameters. + Http::TestRequestHeaderMapImpl first_request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + std::string legit_token{"legit_token"}; + EXPECT_CALL(*validator_, token()).WillRepeatedly(ReturnRef(legit_token)); + + std::string legit_refresh_token{"legit_refresh_token"}; + EXPECT_CALL(*validator_, refreshToken()).WillRepeatedly(ReturnRef(legit_refresh_token)); + + // Fail the validation to trigger the OAuth flow with trying to get the access token using by + // refresh token. + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_CALL(*validator_, canUpdateTokenByRefreshToken()).WillOnce(Return(true)); + + EXPECT_CALL(*oauth_client_, + asyncRefreshAccessToken(legit_refresh_token, TEST_CLIENT_ID, + "asdf_client_secret_fdsa", AuthType::UrlEncodedBody)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(first_request_headers, false)); + + Http::TestResponseHeaderMapImpl redirect_response_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().Location.get(), + "https://auth.example.com/oauth/" + "authorize/?client_id=" + + TEST_CLIENT_ID + + "&redirect_uri=https%3A%2F%2Ftraffic.example.com%2F_oauth" + "&response_type=code" + "&scope=" + + TEST_ENCODED_AUTH_SCOPES + + "&state=https%3A%2F%2Ftraffic.example.com%2Ftest%3Fname%3Dadmin%26level%3Dtrace" + "&resource=oauth2-resource&resource=http%3A%2F%2Fexample.com" + "&resource=https%3A%2F%2Fexample.com"}, + }; + + // Check that the redirect includes the escaped parameter characters, '?', '&' and '='. + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + + filter_->onRefreshAccessTokenSuccess("", "", "", std::chrono::seconds(10)); + + EXPECT_EQ(1, config_->stats().oauth_refreshtoken_success_.value()); + EXPECT_EQ(1, config_->stats().oauth_success_.value()); +} + +TEST_F(OAuth2Test, OAuthTestRefreshAccessTokenFail) { + + init(getConfig(true /* forward_bearer_token */, true /* use_refresh_token */)); + // First construct the initial request to the oauth filter with URI parameters. + Http::TestRequestHeaderMapImpl first_request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + std::string legit_token{"legit_token"}; + EXPECT_CALL(*validator_, token()).WillRepeatedly(ReturnRef(legit_token)); + + std::string legit_refresh_token{"legit_refresh_token"}; + EXPECT_CALL(*validator_, refreshToken()).WillRepeatedly(ReturnRef(legit_refresh_token)); + + // Fail the validation to trigger the OAuth flow with trying to get the access token using by + // refresh token. + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_CALL(*validator_, canUpdateTokenByRefreshToken()).WillOnce(Return(true)); + + EXPECT_CALL(*oauth_client_, + asyncRefreshAccessToken(legit_refresh_token, TEST_CLIENT_ID, + "asdf_client_secret_fdsa", AuthType::UrlEncodedBody)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(first_request_headers, false)); + + Http::TestResponseHeaderMapImpl redirect_response_headers{ + {Http::Headers::get().Status.get(), "302"}, + {Http::Headers::get().Location.get(), + "https://auth.example.com/oauth/" + "authorize/?client_id=" + + TEST_CLIENT_ID + + "&redirect_uri=https%3A%2F%2Ftraffic.example.com%2F_oauth" + "&response_type=code" + "&scope=" + + TEST_ENCODED_AUTH_SCOPES + + "&state=https%3A%2F%2Ftraffic.example.com%2Ftest%3Fname%3Dadmin%26level%3Dtrace" + "&resource=oauth2-resource&resource=http%3A%2F%2Fexample.com" + "&resource=https%3A%2F%2Fexample.com%2Fsome%2Fpath%252F..%252F%2Futf8%C3%83%3Bfoo%3Dbar%" + "3Fvar1%3D1%26var2%3D2"}, + }; + + // Check that the redirect includes the escaped parameter characters, '?', '&' and '='. + EXPECT_CALL(decoder_callbacks_, + encodeHeaders_(HeaderMapEqualRef(&redirect_response_headers), true)); + + filter_->onRefreshAccessTokenFailure(); + + EXPECT_EQ(1, config_->stats().oauth_unauthorized_rq_.value()); + EXPECT_EQ(1, config_->stats().oauth_refreshtoken_failure_.value()); +} + +TEST_F(OAuth2Test, OAuthTestSetCookiesAfterRefreshAccessToken) { + + init(getConfig(true /* forward_bearer_token */, true /* use_refresh_token */)); + + // the third request to the oauth filter with URI parameters. + Http::TestRequestHeaderMapImpl request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + std::string legit_refresh_token{"legit_refresh_token"}; + EXPECT_CALL(*validator_, refreshToken()).WillRepeatedly(ReturnRef(legit_refresh_token)); + + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_CALL(*validator_, canUpdateTokenByRefreshToken()).WillOnce(Return(true)); + + EXPECT_CALL(*oauth_client_, + asyncRefreshAccessToken(legit_refresh_token, TEST_CLIENT_ID, + "asdf_client_secret_fdsa", AuthType::UrlEncodedBody)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers, false)); + + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + + filter_->finishRefreshAccessTokenFlow(); + + Http::TestResponseHeaderMapImpl response_headers{}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + + Http::TestResponseHeaderMapImpl expected_response_headers{ + {Http::Headers::get().SetCookie.get(), "OauthHMAC=" + "fV62OgLipChTQQC3UFgDp+l5sCiSb3zt7nCoJiVivWw=;" + "version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "OauthExpires=;version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=;version=1;path=/;Max-Age=;secure;HttpOnly"}, + }; + + EXPECT_THAT(response_headers, HeaderMapEqualRef(&expected_response_headers)); +} + +TEST_F(OAuth2Test, OAuthTestSetCookiesAfterRefreshAccessTokenWithBasicAuth) { + + init(getConfig(true /* forward_bearer_token */, true /* use_refresh_token */, + ::envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType:: + OAuth2Config_AuthType_BASIC_AUTH + /* authType */)); + + Http::TestRequestHeaderMapImpl request_headers{ + {Http::Headers::get().Path.get(), "/test?name=admin&level=trace"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Post}, + {Http::Headers::get().Scheme.get(), "https"}, + }; + + std::string legit_refresh_token{"legit_refresh_token"}; + EXPECT_CALL(*validator_, refreshToken()).WillRepeatedly(ReturnRef(legit_refresh_token)); + + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(false)); + EXPECT_CALL(*validator_, canUpdateTokenByRefreshToken()).WillOnce(Return(true)); + + EXPECT_CALL(*oauth_client_, + asyncRefreshAccessToken(legit_refresh_token, TEST_CLIENT_ID, + "asdf_client_secret_fdsa", AuthType::BasicAuth)); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers, false)); + + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + + filter_->finishRefreshAccessTokenFlow(); + + Http::TestResponseHeaderMapImpl response_headers{}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + + Http::TestResponseHeaderMapImpl expected_response_headers{ + {Http::Headers::get().SetCookie.get(), "OauthHMAC=" + "fV62OgLipChTQQC3UFgDp+l5sCiSb3zt7nCoJiVivWw=;" + "version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "OauthExpires=;version=1;path=/;Max-Age=;secure;HttpOnly"}, + {Http::Headers::get().SetCookie.get(), + "BearerToken=;version=1;path=/;Max-Age=;secure;HttpOnly"}, + }; + + EXPECT_THAT(response_headers, HeaderMapEqualRef(&expected_response_headers)); +} + } // namespace Oauth2 } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/oauth2/oauth_integration_test.cc b/test/extensions/filters/http/oauth2/oauth_integration_test.cc index ce4e68dfe4db..883a43bdfcc9 100644 --- a/test/extensions/filters/http/oauth2/oauth_integration_test.cc +++ b/test/extensions/filters/http/oauth2/oauth_integration_test.cc @@ -175,6 +175,21 @@ class OauthIntegrationTest : public HttpIntegrationTest, addFakeUpstream(Http::CodecType::HTTP2); } + void cleanup() { + codec_client_->close(); + if (fake_oauth2_connection_ != nullptr) { + AssertionResult result = fake_oauth2_connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = fake_oauth2_connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + } + if (fake_upstream_connection_ != nullptr) { + AssertionResult result = fake_upstream_connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = fake_upstream_connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + } + } virtual void setOauthConfig() { // This config is same as when the 'auth_type: "URL_ENCODED_BODY"' is set as it's the default // value @@ -209,6 +224,7 @@ name: oauth path_config_source: path: "{{ test_tmpdir }}/hmac_secret.yaml" resource_api_version: V3 + use_refresh_token: true auth_scopes: - user - openid @@ -228,6 +244,8 @@ name: oauth Http::Utility::parseSetCookieValue(headers, default_cookie_names_.bearer_token_); std::string hmac = Http::Utility::parseSetCookieValue(headers, default_cookie_names_.oauth_hmac_); + std::string refreshToken = + Http::Utility::parseSetCookieValue(headers, default_cookie_names_.refresh_token_); Http::TestRequestHeaderMapImpl validate_headers{{":authority", std::string(host)}}; @@ -239,18 +257,49 @@ name: oauth validate_headers.addReferenceKey(Http::Headers::get().Cookie, absl::StrCat(default_cookie_names_.bearer_token_, "=", token)); + validate_headers.addReferenceKey( + Http::Headers::get().Cookie, + absl::StrCat(default_cookie_names_.refresh_token_, "=", refreshToken)); + OAuth2CookieValidator validator{api_->timeSource(), default_cookie_names_}; validator.setParams(validate_headers, std::string(hmac_secret)); return validator.isValid(); } virtual void checkClientSecretInRequest(absl::string_view token_secret) { - std::string request_body = upstream_request_->body().toString(); - const auto query_parameters = Http::Utility::parseFromBody(request_body); - auto it = query_parameters.find("client_secret"); + std::string request_body = oauth2_request_->body().toString(); + const auto query_parameters = + Http::Utility::QueryParamsMulti::parseParameters(request_body, 0, true); + auto secret = query_parameters.getFirstValue("client_secret"); + + ASSERT_TRUE(secret.has_value()); + EXPECT_EQ(secret.value(), token_secret); + } + + void waitForOAuth2Response(absl::string_view token_secret) { + AssertionResult result = + fake_upstreams_.back()->waitForHttpConnection(*dispatcher_, fake_oauth2_connection_); + RELEASE_ASSERT(result, result.message()); + result = fake_oauth2_connection_->waitForNewStream(*dispatcher_, oauth2_request_); + RELEASE_ASSERT(result, result.message()); + result = oauth2_request_->waitForEndStream(*dispatcher_); + RELEASE_ASSERT(result, result.message()); + + ASSERT_TRUE(oauth2_request_->waitForHeadersComplete()); + + checkClientSecretInRequest(token_secret); + + oauth2_request_->encodeHeaders( + Http::TestRequestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + false); + + envoy::extensions::http_filters::oauth2::OAuthResponse oauth_response; + oauth_response.mutable_access_token()->set_value("bar"); + oauth_response.mutable_refresh_token()->set_value("foo"); + oauth_response.mutable_expires_in()->set_value(DateUtil::nowToSeconds(api_->timeSource()) + 10); - ASSERT_TRUE(it != query_parameters.end()); - EXPECT_EQ(it->second, token_secret); + Buffer::OwnedImpl buffer(MessageUtil::getJsonStringFromMessageOrError(oauth_response)); + oauth2_request_->encodeData(buffer, true); } void doAuthenticationFlow(absl::string_view token_secret, absl::string_view hmac_secret) { @@ -268,22 +317,7 @@ name: oauth request_encoder_ = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); - waitForNextUpstreamRequest(std::vector({0, 1, 2, 3})); - - ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); - - checkClientSecretInRequest(token_secret); - - upstream_request_->encodeHeaders( - Http::TestRequestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, - false); - - envoy::extensions::http_filters::oauth2::OAuthResponse oauth_response; - oauth_response.mutable_access_token()->set_value("bar"); - oauth_response.mutable_expires_in()->set_value(DateUtil::nowToSeconds(api_->timeSource()) + 10); - - Buffer::OwnedImpl buffer(MessageUtil::getJsonStringFromMessageOrError(oauth_response)); - upstream_request_->encodeData(buffer, true); + waitForOAuth2Response(token_secret); // We should get an immediate redirect back. response->waitForHeaders(); @@ -298,7 +332,7 @@ name: oauth response->headers(), default_cookie_names_.oauth_expires_); RELEASE_ASSERT(response->waitForEndStream(), "unexpected timeout"); - codec_client_->close(); + cleanup(); // Now try sending the cookies back codec_client_ = makeHttpConnection(lookupPort("http")); @@ -319,7 +353,43 @@ name: oauth EXPECT_EQ("http://traffic.example.com/not/_oauth", response->headers().Location()->value().getStringView()); RELEASE_ASSERT(response->waitForEndStream(), "unexpected timeout"); - codec_client_->close(); + cleanup(); + } + + void doRefreshTokenFlow(absl::string_view token_secret, absl::string_view hmac_secret) { + codec_client_ = makeHttpConnection(lookupPort("http")); + + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, {":path", "/request1"}, + {":scheme", "http"}, {"x-forwarded-proto", "http"}, + {":authority", "authority"}, {"Cookie", "RefreshToken=efddf321;BearerToken=ff1234fc"}, + {":authority", "authority"}, {"authority", "Bearer token"}}; + + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + + waitForOAuth2Response(token_secret); + + AssertionResult result = + fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_); + RELEASE_ASSERT(result, result.message()); + result = fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_); + RELEASE_ASSERT(result, result.message()); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request_->encodeData(response_size_, true); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + + EXPECT_TRUE( + validateHmac(response->headers(), headers.Host()->value().getStringView(), hmac_secret)); + + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ(response_size_, response->body().size()); + + cleanup(); } const CookieNames default_cookie_names_{"BearerToken", "OauthHMAC", "OauthExpires", "IdToken", @@ -328,6 +398,10 @@ name: oauth std::string listener_name_{"http"}; FakeHttpConnectionPtr lds_connection_; FakeStreamPtr lds_stream_{}; + const uint64_t response_size_ = 512; + + FakeHttpConnectionPtr fake_oauth2_connection_{}; + FakeStreamPtr oauth2_request_{}; }; INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, OauthIntegrationTest, @@ -354,6 +428,8 @@ TEST_P(OauthIntegrationTest, UnauthenticatedFlow) { // We should get an immediate redirect back. response->waitForHeaders(); EXPECT_EQ("302", response->headers().getStatusValue()); + + cleanup(); } TEST_P(OauthIntegrationTest, AuthenticationFlow) { @@ -380,6 +456,30 @@ TEST_P(OauthIntegrationTest, AuthenticationFlow) { doAuthenticationFlow("token_secret_1", "hmac_secret_1"); } +TEST_P(OauthIntegrationTest, RefreshTokenFlow) { + on_server_init_function_ = [&]() { + createLdsStream(); + sendLdsResponse({MessageUtil::getYamlStringFromMessage(listener_config_)}, "initial"); + }; + + initialize(); + + // 1. Do one authentication flow. + doRefreshTokenFlow("token_secret", "hmac_secret"); + + // 2. Reload secrets. + EXPECT_EQ(test_server_->counter("sds.token.update_success")->value(), 1); + EXPECT_EQ(test_server_->counter("sds.hmac.update_success")->value(), 1); + TestEnvironment::renameFile(TestEnvironment::temporaryPath("token_secret_1.yaml"), + TestEnvironment::temporaryPath("token_secret.yaml")); + test_server_->waitForCounterEq("sds.token.update_success", 2, std::chrono::milliseconds(5000)); + TestEnvironment::renameFile(TestEnvironment::temporaryPath("hmac_secret_1.yaml"), + TestEnvironment::temporaryPath("hmac_secret.yaml")); + test_server_->waitForCounterEq("sds.hmac.update_success", 2, std::chrono::milliseconds(5000)); + // 3. Do another one refresh token flow. + doRefreshTokenFlow("token_secret_1", "hmac_secret_1"); +} + // Regression test(issue #22678) where (incorrectly)using server's init manager(initialized state) // to add init target by the secret manager led to the assertion failure. TEST_P(OauthIntegrationTest, LoadListenerAfterServerIsInitialized) { @@ -481,13 +581,12 @@ name: oauth } void checkClientSecretInRequest(absl::string_view token_secret) override { - EXPECT_FALSE( - upstream_request_->headers().get(Http::CustomHeaders::get().Authorization).empty()); + EXPECT_FALSE(oauth2_request_->headers().get(Http::CustomHeaders::get().Authorization).empty()); const std::string basic_auth_token = absl::StrCat("foo:", token_secret); const std::string encoded_token = Base64::encode(basic_auth_token.data(), basic_auth_token.size()); const auto token_secret_expected = absl::StrCat("Basic ", encoded_token); - EXPECT_EQ(token_secret_expected, upstream_request_->headers() + EXPECT_EQ(token_secret_expected, oauth2_request_->headers() .get(Http::CustomHeaders::get().Authorization)[0] ->value() .getStringView()); @@ -527,6 +626,8 @@ TEST_P(OauthIntegrationTest, MissingStateParam) { // contain the state param. response->waitForHeaders(); EXPECT_EQ("401", response->headers().getStatusValue()); + + cleanup(); } } // namespace diff --git a/test/extensions/filters/http/oauth2/oauth_test.cc b/test/extensions/filters/http/oauth2/oauth_test.cc index 114d09367868..388aedf8aeb7 100644 --- a/test/extensions/filters/http/oauth2/oauth_test.cc +++ b/test/extensions/filters/http/oauth2/oauth_test.cc @@ -30,6 +30,9 @@ class MockCallbacks : public FilterCallbacks { MOCK_METHOD(void, sendUnauthorizedResponse, ()); MOCK_METHOD(void, onGetAccessTokenSuccess, (const std::string&, const std::string&, const std::string&, std::chrono::seconds)); + MOCK_METHOD(void, onRefreshAccessTokenSuccess, + (const std::string&, const std::string&, const std::string&, std::chrono::seconds)); + MOCK_METHOD(void, onRefreshAccessTokenFailure, ()); }; class OAuth2ClientTest : public testing::Test { @@ -191,6 +194,113 @@ TEST_F(OAuth2ClientTest, RequestAccessTokenInvalidResponse) { [&](auto* callback) { callback->onSuccess(request, std::move(mock_response)); })); } +TEST_F(OAuth2ClientTest, RequestRefreshAccessTokenSuccess) { + std::string json = R"EOF( + { + "access_token": "golden ticket", + "expires_in": 1000 + } + )EOF"; + Http::ResponseHeaderMapPtr mock_response_headers{new Http::TestResponseHeaderMapImpl{ + {Http::Headers::get().Status.get(), "200"}, + {Http::Headers::get().ContentType.get(), "application/json"}, + }}; + Http::ResponseMessagePtr mock_response( + new Http::ResponseMessageImpl(std::move(mock_response_headers))); + mock_response->body().add(json); + + EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillRepeatedly(Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& cb, + const Http::AsyncClient::RequestOptions&) + -> Http::AsyncClient::Request* { + EXPECT_EQ(Http::Headers::get().MethodValues.Post, + message->headers().Method()->value().getStringView()); + EXPECT_EQ(Http::Headers::get().ContentTypeValues.FormUrlEncoded, + message->headers().ContentType()->value().getStringView()); + EXPECT_NE("", message->headers().getContentLengthValue()); + EXPECT_TRUE( + !message->headers().get(Http::CustomHeaders::get().Accept).empty() && + message->headers().get(Http::CustomHeaders::get().Accept)[0]->value().getStringView() == + Http::Headers::get().ContentTypeValues.Json); + callbacks_.push_back(&cb); + return &request_; + })); + + client_->setCallbacks(*mock_callbacks_); + client_->asyncRefreshAccessToken("a", "b", "c"); + EXPECT_EQ(1, callbacks_.size()); + EXPECT_CALL(*mock_callbacks_, onRefreshAccessTokenSuccess(_, _, _, _)); + Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); + ASSERT_TRUE(popPendingCallback( + [&](auto* callback) { callback->onSuccess(request, std::move(mock_response)); })); +} + +TEST_F(OAuth2ClientTest, RequestSuccessBasicAuthType) { + std::string json = R"EOF( +{ + "access_token": "golden ticket", + "expires_in": 1000 +} +)EOF"; + Http::ResponseHeaderMapPtr mock_response_headers{new Http::TestResponseHeaderMapImpl{ + {Http::Headers::get().Status.get(), "200"}, + {Http::Headers::get().ContentType.get(), "application/json"}, + }}; + Http::ResponseMessagePtr mock_response( + new Http::ResponseMessageImpl(std::move(mock_response_headers))); + mock_response->body().add(json); + + EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillRepeatedly(Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& cb, + const Http::AsyncClient::RequestOptions&) + -> Http::AsyncClient::Request* { + EXPECT_EQ(Http::Headers::get().MethodValues.Post, + message->headers().Method()->value().getStringView()); + EXPECT_EQ(Http::Headers::get().ContentTypeValues.FormUrlEncoded, + message->headers().ContentType()->value().getStringView()); + EXPECT_NE("", message->headers().getContentLengthValue()); + EXPECT_TRUE( + !message->headers().get(Http::CustomHeaders::get().Accept).empty() && + message->headers().get(Http::CustomHeaders::get().Accept)[0]->value().getStringView() == + Http::Headers::get().ContentTypeValues.Json); + callbacks_.push_back(&cb); + return &request_; + })); + + client_->setCallbacks(*mock_callbacks_); + client_->asyncRefreshAccessToken("a", "b", "c", AuthType::BasicAuth); + EXPECT_EQ(1, callbacks_.size()); + EXPECT_CALL(*mock_callbacks_, onRefreshAccessTokenSuccess(_, _, _, _)); + Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); + ASSERT_TRUE(popPendingCallback( + [&](auto* callback) { callback->onSuccess(request, std::move(mock_response)); })); +} + +TEST_F(OAuth2ClientTest, RequestErrorResponse) { + Http::ResponseHeaderMapPtr mock_response_headers{new Http::TestResponseHeaderMapImpl{ + {Http::Headers::get().Status.get(), "500"}, + {Http::Headers::get().ContentType.get(), "application/json"}, + }}; + Http::ResponseMessagePtr mock_response( + new Http::ResponseMessageImpl(std::move(mock_response_headers))); + + EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillRepeatedly( + Invoke([&](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& cb, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks_.push_back(&cb); + return &request_; + })); + + client_->setCallbacks(*mock_callbacks_); + client_->asyncRefreshAccessToken("a", "b", "c"); + EXPECT_EQ(1, callbacks_.size()); + EXPECT_CALL(*mock_callbacks_, onRefreshAccessTokenFailure()); + Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); + ASSERT_TRUE(popPendingCallback( + [&](auto* callback) { callback->onSuccess(request, std::move(mock_response)); })); +} + TEST_F(OAuth2ClientTest, NetworkError) { EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) .WillRepeatedly( @@ -211,6 +321,48 @@ TEST_F(OAuth2ClientTest, NetworkError) { })); } +TEST_F(OAuth2ClientTest, UpdateTokenNetworkError) { + EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillRepeatedly( + Invoke([&](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& cb, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks_.push_back(&cb); + return &request_; + })); + + client_->setCallbacks(*mock_callbacks_); + client_->asyncRefreshAccessToken("a", "b", "c"); + EXPECT_EQ(1, callbacks_.size()); + + EXPECT_CALL(*mock_callbacks_, onRefreshAccessTokenFailure()); + Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); + ASSERT_TRUE(popPendingCallback([&](auto* callback) { + callback->onFailure(request, Http::AsyncClient::FailureReason::Reset); + })); +} + +TEST_F(OAuth2ClientTest, NetworkErrorDoubleCallStateInvalid) { + EXPECT_CALL(cm_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillRepeatedly( + Invoke([&](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& cb, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks_.push_back(&cb); + return &request_; + })); + + client_->setCallbacks(*mock_callbacks_); + client_->asyncRefreshAccessToken("a", "b", "c"); + EXPECT_EQ(1, callbacks_.size()); + + EXPECT_CALL(*mock_callbacks_, onRefreshAccessTokenFailure()); + Http::MockAsyncClientRequest request(&cm_.thread_local_cluster_.async_client_); + ASSERT_TRUE(popPendingCallback([&](auto* callback) { + callback->onFailure(request, Http::AsyncClient::FailureReason::Reset); + EXPECT_DEATH(callback->onFailure(request, Http::AsyncClient::FailureReason::Reset), + "Malformed oauth client state"); + })); +} + TEST_F(OAuth2ClientTest, NoCluster) { ON_CALL(cm_, getThreadLocalCluster("auth")).WillByDefault(Return(nullptr)); client_->setCallbacks(*mock_callbacks_); diff --git a/test/extensions/filters/http/original_src/original_src_config_factory_test.cc b/test/extensions/filters/http/original_src/original_src_config_factory_test.cc index 8830e2d0a1e7..9474eb8b0eea 100644 --- a/test/extensions/filters/http/original_src/original_src_config_factory_test.cc +++ b/test/extensions/filters/http/original_src/original_src_config_factory_test.cc @@ -30,7 +30,8 @@ TEST(OriginalSrcHttpConfigFactoryTest, TestCreateFactory) { NiceMock context; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(*proto_config, "", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; Http::StreamDecoderFilterSharedPtr added_filter; diff --git a/test/extensions/filters/http/rate_limit_quota/client_test_utils.h b/test/extensions/filters/http/rate_limit_quota/client_test_utils.h index 95855541ed41..2f19f9b48b22 100644 --- a/test/extensions/filters/http/rate_limit_quota/client_test_utils.h +++ b/test/extensions/filters/http/rate_limit_quota/client_test_utils.h @@ -52,17 +52,21 @@ class RateLimitTestClient { // Note, we need to set it through `MockFactoryContext` rather than `MockAsyncClientManager` // directly because the rate limit client object below requires context argument as the input. if (external_inited_) { - EXPECT_CALL(context.cluster_manager_.async_client_manager_, + EXPECT_CALL(context.server_factory_context_.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClient(_, _, _)) .Times(2) .WillRepeatedly(Invoke(this, &RateLimitTestClient::mockCreateAsyncClient)); } else { - EXPECT_CALL(context.cluster_manager_.async_client_manager_, - getOrCreateRawAsyncClient(_, _, _)) + EXPECT_CALL(context.server_factory_context_.cluster_manager_.async_client_manager_, + getOrCreateRawAsyncClientWithHashKey(_, _, _)) .WillOnce(Invoke(this, &RateLimitTestClient::mockCreateAsyncClient)); } - client_ = createRateLimitClient(context, grpc_service, &callbacks_, bucket_cache_, domain_); + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key = + Grpc::GrpcServiceConfigWithHashKey(grpc_service); + + client_ = + createRateLimitClient(context, &callbacks_, bucket_cache_, domain_, config_with_hash_key); } Grpc::RawAsyncClientSharedPtr mockCreateAsyncClient(Unused, Unused, Unused) { diff --git a/test/extensions/filters/http/rate_limit_quota/filter_test.cc b/test/extensions/filters/http/rate_limit_quota/filter_test.cc index d1d84668bc18..86f3cfc036ef 100644 --- a/test/extensions/filters/http/rate_limit_quota/filter_test.cc +++ b/test/extensions/filters/http/rate_limit_quota/filter_test.cc @@ -85,8 +85,10 @@ class FilterTest : public testing::Test { void createFilter(bool set_callback = true) { filter_config_ = std::make_shared(config_); + Grpc::GrpcServiceConfigWithHashKey config_with_hash_key = + Grpc::GrpcServiceConfigWithHashKey(filter_config_->rlqs_server()); filter_ = std::make_unique(filter_config_, context_, bucket_cache_, - thread_local_client_); + thread_local_client_, config_with_hash_key); if (set_callback) { filter_->setDecoderFilterCallbacks(decoder_callbacks_); } diff --git a/test/extensions/filters/http/rate_limit_quota/integration_test.cc b/test/extensions/filters/http/rate_limit_quota/integration_test.cc index f52547399fcd..afec9fc7de35 100644 --- a/test/extensions/filters/http/rate_limit_quota/integration_test.cc +++ b/test/extensions/filters/http/rate_limit_quota/integration_test.cc @@ -412,6 +412,86 @@ TEST_P(RateLimitQuotaIntegrationTest, BasicFlowPeriodicalReport) { rlqs_stream_->sendGrpcMessage(rlqs_response2); } +TEST_P(RateLimitQuotaIntegrationTest, MultiRequestWithTokenBucketThrottling) { + initializeConfig(); + HttpIntegrationTest::initialize(); + absl::flat_hash_map custom_headers = {{"environment", "staging"}, + {"group", "envoy"}}; + int max_token = 1; + int tokens_per_fill = 30; + int fill_interval_sec = 60; + int fill_one_token_in_ms = fill_interval_sec / tokens_per_fill * 1000; + // First request: allowed; fail-open, default no assignment policy. + // Second request: allowed; one token remaining, token bucket's max_token is 1. + // Third request: allowed; token bucket has been refilled by advancing 2s. + // Fourth request: rejected; no token left and no token refilled. + // Fifth request: allowed; token bucket has been refilled by advancing 2s. + // Sixth request: rejected; no token left and no token refilled. + for (int i = 0; i < 6; ++i) { + // We advance time by 2s for 3rd and 5th requests so that token bucket can + // be refilled. + if (i == 2 || i == 4) { + simTime().advanceTimeAndRun(std::chrono::milliseconds(fill_one_token_in_ms), *dispatcher_, + Envoy::Event::Dispatcher::RunType::NonBlock); + } + // Send downstream client request to upstream. + sendClientRequest(&custom_headers); + + // Only first downstream client request will trigger the reports to RLQS + // server as the subsequent requests will find the entry in the cache. + if (i == 0) { + // Start the gRPC stream to RLQS server. + ASSERT_TRUE(grpc_upstreams_[0]->waitForHttpConnection(*dispatcher_, rlqs_connection_)); + ASSERT_TRUE(rlqs_connection_->waitForNewStream(*dispatcher_, rlqs_stream_)); + + envoy::service::rate_limit_quota::v3::RateLimitQuotaUsageReports reports; + ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports)); + rlqs_stream_->startGrpcStream(); + + // Build the response. + envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse rlqs_response; + absl::flat_hash_map custom_headers_cpy = custom_headers; + custom_headers_cpy.insert({"name", "prod"}); + auto* bucket_action = rlqs_response.add_bucket_action(); + for (const auto& [key, value] : custom_headers_cpy) { + (*bucket_action->mutable_bucket_id()->mutable_bucket()).insert({key, value}); + auto* quota_assignment = bucket_action->mutable_quota_assignment_action(); + quota_assignment->mutable_assignment_time_to_live()->set_seconds(120); + auto* strategy = quota_assignment->mutable_rate_limit_strategy(); + auto* token_bucket = strategy->mutable_token_bucket(); + token_bucket->set_max_tokens(max_token); + token_bucket->mutable_tokens_per_fill()->set_value(30); + token_bucket->mutable_fill_interval()->set_seconds(60); + } + + // Send the response from RLQS server. + rlqs_stream_->sendGrpcMessage(rlqs_response); + } + + // 4th and 6th request are throttled. + if (i == 3 || i == 5) { + ASSERT_TRUE(response_->waitForEndStream()); + EXPECT_TRUE(response_->complete()); + EXPECT_EQ(response_->headers().getStatusValue(), "429"); + } else { + // Handle the request received by upstream. + ASSERT_TRUE( + fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request_->encodeData(100, true); + + // Verify the response to downstream. + ASSERT_TRUE(response_->waitForEndStream()); + EXPECT_TRUE(response_->complete()); + EXPECT_EQ(response_->headers().getStatusValue(), "200"); + } + + cleanUp(); + } +} + } // namespace } // namespace RateLimitQuota } // namespace HttpFilters diff --git a/test/extensions/filters/http/ratelimit/config_test.cc b/test/extensions/filters/http/ratelimit/config_test.cc index 51c6c17b8db0..ab704a95307d 100644 --- a/test/extensions/filters/http/ratelimit/config_test.cc +++ b/test/extensions/filters/http/ratelimit/config_test.cc @@ -23,8 +23,9 @@ TEST(RateLimitFilterConfigTest, ValidateFail) { envoy::extensions::filters::http::ratelimit::v3::RateLimit config; config.mutable_rate_limit_service()->set_transport_api_version( envoy::config::core::v3::ApiVersion::V3); - EXPECT_THROW(RateLimitFilterConfig().createFilterFactoryFromProto(config, "stats", context), - ProtoValidationException); + EXPECT_THROW( + RateLimitFilterConfig().createFilterFactoryFromProto(config, "stats", context).value(), + ProtoValidationException); } TEST(RateLimitFilterConfigTest, RatelimitCorrectProto) { @@ -43,13 +44,15 @@ TEST(RateLimitFilterConfigTest, RatelimitCorrectProto) { NiceMock context; - EXPECT_CALL(context.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClient(_, _, _)) - .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { + EXPECT_CALL(context.server_factory_context_.cluster_manager_.async_client_manager_, + getOrCreateRawAsyncClientWithHashKey(_, _, _)) + .WillOnce(Invoke([](const Grpc::GrpcServiceConfigWithHashKey&, Stats::Scope&, bool) { return std::make_unique>(); })); RateLimitFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamFilter(_)); cb(filter_callback); @@ -65,7 +68,7 @@ TEST(RateLimitFilterConfigTest, RateLimitFilterEmptyProto) { *dynamic_cast( factory.createEmptyConfigProto().get()); - EXPECT_THROW(factory.createFilterFactoryFromProto(empty_proto_config, "stats", context), + EXPECT_THROW(factory.createFilterFactoryFromProto(empty_proto_config, "stats", context).value(), EnvoyException); } diff --git a/test/extensions/filters/http/ratelimit/ratelimit_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_test.cc index aec2472e7b3d..93da09c9f600 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_test.cc @@ -49,7 +49,7 @@ class HttpRateLimitFilterTest : public testing::Test { .WillByDefault(Return(true)); } - void SetUpTest(const std::string& yaml) { + void setUpTest(const std::string& yaml) { envoy::extensions::filters::http::ratelimit::v3::RateLimit proto_config{}; TestUtility::loadFromYaml(yaml, proto_config); @@ -73,6 +73,13 @@ class HttpRateLimitFilterTest : public testing::Test { failure_mode_deny: true )EOF"; + const std::string fail_close_with_custom_status_code_config_ = R"EOF( + domain: foo + failure_mode_deny: true + status_on_error: + code: 503 + )EOF"; + const std::string enable_x_ratelimit_headers_config_ = R"EOF( domain: foo enable_x_ratelimit_headers: DRAFT_VERSION_03 @@ -99,6 +106,11 @@ class HttpRateLimitFilterTest : public testing::Test { code: 200 )EOF"; + const std::string stat_prefix_config_ = R"EOF( + domain: foo + stat_prefix: with_stat_prefix + )EOF"; + Filters::Common::RateLimit::MockClient* client_; NiceMock filter_callbacks_; Stats::StatNamePool pool_{filter_callbacks_.clusterInfo()->statsScope().symbolTable()}; @@ -130,7 +142,7 @@ class HttpRateLimitFilterTest : public testing::Test { }; TEST_F(HttpRateLimitFilterTest, NoRoute) { - SetUpTest(filter_config_); + setUpTest(filter_config_); EXPECT_CALL(*filter_callbacks_.route_, routeEntry()).WillOnce(Return(nullptr)); @@ -146,7 +158,7 @@ TEST_F(HttpRateLimitFilterTest, NoRoute) { } TEST_F(HttpRateLimitFilterTest, NoCluster) { - SetUpTest(filter_config_); + setUpTest(filter_config_); ON_CALL(filter_callbacks_, clusterInfo()).WillByDefault(Return(nullptr)); @@ -160,7 +172,7 @@ TEST_F(HttpRateLimitFilterTest, NoCluster) { } TEST_F(HttpRateLimitFilterTest, NoApplicableRateLimit) { - SetUpTest(filter_config_); + setUpTest(filter_config_); filter_callbacks_.route_->route_entry_.rate_limit_policy_.rate_limit_policy_entry_.clear(); EXPECT_CALL(*client_, limit(_, _, _, _, _, 0)).Times(0); @@ -174,7 +186,7 @@ TEST_F(HttpRateLimitFilterTest, NoApplicableRateLimit) { } TEST_F(HttpRateLimitFilterTest, NoDescriptor) { - SetUpTest(filter_config_); + setUpTest(filter_config_); EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)); EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)); @@ -189,7 +201,7 @@ TEST_F(HttpRateLimitFilterTest, NoDescriptor) { } TEST_F(HttpRateLimitFilterTest, RuntimeDisabled) { - SetUpTest(filter_config_); + setUpTest(filter_config_); EXPECT_CALL(runtime_.snapshot_, featureEnabled("ratelimit.http_filter_enabled", 100)) .WillOnce(Return(false)); @@ -203,7 +215,7 @@ TEST_F(HttpRateLimitFilterTest, RuntimeDisabled) { } TEST_F(HttpRateLimitFilterTest, OkResponse) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -247,7 +259,7 @@ TEST_F(HttpRateLimitFilterTest, OkResponse) { } TEST_F(HttpRateLimitFilterTest, OkResponseWithHeaders) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -303,7 +315,7 @@ TEST_F(HttpRateLimitFilterTest, OkResponseWithHeaders) { } TEST_F(HttpRateLimitFilterTest, OkResponseWithFilterHeaders) { - SetUpTest(enable_x_ratelimit_headers_config_); + setUpTest(enable_x_ratelimit_headers_config_); InSequence s; EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -360,7 +372,7 @@ TEST_F(HttpRateLimitFilterTest, OkResponseWithFilterHeaders) { } TEST_F(HttpRateLimitFilterTest, ImmediateOkResponse) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) @@ -390,7 +402,7 @@ TEST_F(HttpRateLimitFilterTest, ImmediateOkResponse) { } TEST_F(HttpRateLimitFilterTest, ImmediateErrorResponse) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) @@ -425,7 +437,7 @@ TEST_F(HttpRateLimitFilterTest, ImmediateErrorResponse) { } TEST_F(HttpRateLimitFilterTest, ErrorResponse) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -459,7 +471,7 @@ TEST_F(HttpRateLimitFilterTest, ErrorResponse) { } TEST_F(HttpRateLimitFilterTest, ErrorResponseWithFailureModeAllowOff) { - SetUpTest(fail_close_config_); + setUpTest(fail_close_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -473,12 +485,54 @@ TEST_F(HttpRateLimitFilterTest, ErrorResponseWithFailureModeAllowOff) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); + EXPECT_CALL(filter_callbacks_.stream_info_, + setResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError)); + + EXPECT_CALL(filter_callbacks_, encodeHeaders_(_, true)) + .WillOnce(Invoke([&](const Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.getStatusValue(), + std::to_string(enumToInt(Http::Code::InternalServerError))); + })); + request_callbacks_->complete(Filters::Common::RateLimit::LimitStatus::Error, nullptr, nullptr, nullptr, "", nullptr); + EXPECT_EQ( + 1U, + filter_callbacks_.clusterInfo()->statsScope().counterFromStatName(ratelimit_error_).value()); + EXPECT_EQ(0U, filter_callbacks_.clusterInfo() + ->statsScope() + .counterFromStatName(ratelimit_failure_mode_allowed_) + .value()); + EXPECT_EQ("rate_limiter_error", filter_callbacks_.details()); +} + +TEST_F(HttpRateLimitFilterTest, ErrorResponseWithFailureModeAllowOffAndCustomStatusOn) { + setUpTest(fail_close_with_custom_status_code_config_); + InSequence s; + + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); + EXPECT_CALL(*client_, limit(_, _, _, _, _, 0)) + .WillOnce( + WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { + request_callbacks_ = &callbacks; + }))); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + EXPECT_CALL(filter_callbacks_.stream_info_, - setResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError)) - .Times(0); + setResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError)); + + EXPECT_CALL(filter_callbacks_, encodeHeaders_(_, true)) + .WillOnce(Invoke([&](const Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.getStatusValue(), + std::to_string(enumToInt(Http::Code::ServiceUnavailable))); + })); + + request_callbacks_->complete(Filters::Common::RateLimit::LimitStatus::Error, nullptr, nullptr, + nullptr, "", nullptr); EXPECT_EQ( 1U, @@ -491,7 +545,7 @@ TEST_F(HttpRateLimitFilterTest, ErrorResponseWithFailureModeAllowOff) { } TEST_F(HttpRateLimitFilterTest, LimitResponse) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -532,7 +586,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponse) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithDynamicMetadata) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -585,7 +639,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithDynamicMetadata) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithHeaders) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -637,7 +691,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithHeaders) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithBody) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -700,7 +754,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithBody) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithBodyAndContentType) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -769,7 +823,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithBodyAndContentType) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithFilterHeaders) { - SetUpTest(enable_x_ratelimit_headers_config_); + setUpTest(enable_x_ratelimit_headers_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -821,7 +875,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithFilterHeaders) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithoutEnvoyRateLimitedHeader) { - SetUpTest(disable_x_envoy_ratelimited_header_config_); + setUpTest(disable_x_envoy_ratelimited_header_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -860,7 +914,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithoutEnvoyRateLimitedHeader) { } TEST_F(HttpRateLimitFilterTest, LimitResponseRuntimeDisabled) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -901,7 +955,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseRuntimeDisabled) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithRateLimitedStatus) { - SetUpTest(rate_limited_status_config_); + setUpTest(rate_limited_status_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -942,7 +996,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithRateLimitedStatus) { } TEST_F(HttpRateLimitFilterTest, LimitResponseWithInvalidRateLimitedStatus) { - SetUpTest(invalid_rate_limited_status_config_); + setUpTest(invalid_rate_limited_status_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -983,7 +1037,7 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithInvalidRateLimitedStatus) { } TEST_F(HttpRateLimitFilterTest, ResetDuringCall) { - SetUpTest(filter_config_); + setUpTest(filter_config_); InSequence s; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) @@ -1003,7 +1057,7 @@ TEST_F(HttpRateLimitFilterTest, ResetDuringCall) { TEST_F(HttpRateLimitFilterTest, RouteRateLimitDisabledForRouteKey) { route_rate_limit_.disable_key_ = "test_key"; - SetUpTest(filter_config_); + setUpTest(filter_config_); ON_CALL(runtime_.snapshot_, featureEnabled("ratelimit.test_key.http_filter_enabled", 100)) .WillByDefault(Return(false)); @@ -1022,7 +1076,7 @@ TEST_F(HttpRateLimitFilterTest, RouteRateLimitDisabledForRouteKey) { TEST_F(HttpRateLimitFilterTest, VirtualHostRateLimitDisabledForRouteKey) { vh_rate_limit_.disable_key_ = "test_vh_key"; - SetUpTest(filter_config_); + setUpTest(filter_config_); ON_CALL(runtime_.snapshot_, featureEnabled("ratelimit.test_vh_key.http_filter_enabled", 100)) .WillByDefault(Return(false)); @@ -1046,7 +1100,7 @@ TEST_F(HttpRateLimitFilterTest, IncorrectRequestType) { "request_type" : "internal" } )EOF"; - SetUpTest(internal_filter_config); + setUpTest(internal_filter_config); EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); @@ -1065,7 +1119,7 @@ TEST_F(HttpRateLimitFilterTest, IncorrectRequestType) { "request_type" : "external" } )EOF"; - SetUpTest(external_filter_config); + setUpTest(external_filter_config); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); @@ -1087,7 +1141,7 @@ TEST_F(HttpRateLimitFilterTest, InternalRequestType) { "request_type" : "internal" } )EOF"; - SetUpTest(internal_filter_config); + setUpTest(internal_filter_config); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; InSequence s; @@ -1130,7 +1184,7 @@ TEST_F(HttpRateLimitFilterTest, ExternalRequestType) { "request_type" : "external" } )EOF"; - SetUpTest(external_filter_config); + setUpTest(external_filter_config); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "false"}}; InSequence s; @@ -1171,7 +1225,7 @@ TEST_F(HttpRateLimitFilterTest, DEPRECATED_FEATURE_TEST(ExcludeVirtualHost)) { "domain": "foo" } )EOF"; - SetUpTest(external_filter_config); + setUpTest(external_filter_config); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute vh_settings; vh_settings.clear_vh_rate_limits(); FilterConfigPerRoute per_route_config_(vh_settings); @@ -1220,7 +1274,7 @@ TEST_F(HttpRateLimitFilterTest, DEPRECATED_FEATURE_TEST(ExcludeVirtualHost)) { // Tests that the route rate limit is used when VhRateLimitsOptions::OVERRIDE and route rate limit // is set TEST_F(HttpRateLimitFilterTest, OverrideVHRateLimitOptionWithRouteRateLimitSet) { - SetUpTest(filter_config_); + setUpTest(filter_config_); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_vh_rate_limits( envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute::OVERRIDE); @@ -1270,7 +1324,7 @@ TEST_F(HttpRateLimitFilterTest, OverrideVHRateLimitOptionWithRouteRateLimitSet) // Tests that the virtual host rate limit is used when VhRateLimitsOptions::OVERRIDE is set and // route rate limit is empty TEST_F(HttpRateLimitFilterTest, OverrideVHRateLimitOptionWithoutRouteRateLimit) { - SetUpTest(filter_config_); + setUpTest(filter_config_); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_vh_rate_limits( envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute::OVERRIDE); @@ -1320,7 +1374,7 @@ TEST_F(HttpRateLimitFilterTest, OverrideVHRateLimitOptionWithoutRouteRateLimit) // Tests that the virtual host rate limit is used when VhRateLimitsOptions::INCLUDE is set and route // rate limit is empty TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithOnlyVHRateLimitSet) { - SetUpTest(filter_config_); + setUpTest(filter_config_); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_vh_rate_limits( envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute::INCLUDE); @@ -1367,7 +1421,7 @@ TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithOnlyVHRateLimitSet) // Tests that the virtual host rate limit is used when VhRateLimitsOptions::INCLUDE and route rate // limit is set TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithRouteAndVHRateLimitSet) { - SetUpTest(filter_config_); + setUpTest(filter_config_); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_vh_rate_limits( envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute::INCLUDE); @@ -1416,7 +1470,7 @@ TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithRouteAndVHRateLimitS // Tests that the route rate limit is used when VhRateLimitsOptions::IGNORE and route rate limit is // set TEST_F(HttpRateLimitFilterTest, IgnoreVHRateLimitOptionWithRouteRateLimitSet) { - SetUpTest(filter_config_); + setUpTest(filter_config_); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_vh_rate_limits( envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute::IGNORE); @@ -1463,7 +1517,7 @@ TEST_F(HttpRateLimitFilterTest, IgnoreVHRateLimitOptionWithRouteRateLimitSet) { // Tests that no rate limit is used when VhRateLimitsOptions::IGNORE is set and route rate limit // empty TEST_F(HttpRateLimitFilterTest, IgnoreVHRateLimitOptionWithOutRouteRateLimit) { - SetUpTest(filter_config_); + setUpTest(filter_config_); envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_vh_rate_limits( envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute::IGNORE); @@ -1496,7 +1550,7 @@ TEST_F(HttpRateLimitFilterTest, IgnoreVHRateLimitOptionWithOutRouteRateLimit) { // Tests that the domain is properly overridden when set at the per-route level TEST_F(HttpRateLimitFilterTest, PerRouteDomainSet) { - SetUpTest(filter_config_); + setUpTest(filter_config_); const std::string per_route_domain = "bar"; envoy::extensions::filters::http::ratelimit::v3::RateLimitPerRoute settings; settings.set_domain(per_route_domain); @@ -1545,7 +1599,7 @@ TEST_F(HttpRateLimitFilterTest, ConfigValueTest) { } )EOF"; - SetUpTest(stage_filter_config); + setUpTest(stage_filter_config); EXPECT_EQ(5UL, config_->stage()); EXPECT_EQ("foo", config_->domain()); @@ -1559,13 +1613,65 @@ TEST_F(HttpRateLimitFilterTest, DefaultConfigValueTest) { } )EOF"; - SetUpTest(stage_filter_config); + setUpTest(stage_filter_config); EXPECT_EQ(0UL, config_->stage()); EXPECT_EQ("foo", config_->domain()); EXPECT_EQ(FilterRequestType::Both, config_->requestType()); } +// Test that defining stat_prefix appends an additional prefix to the emitted statistics names. +TEST_F(HttpRateLimitFilterTest, StatsWithPrefix) { + const std::string stat_prefix = "with_stat_prefix"; + const std::string over_limit_counter_name_with_prefix = + absl::StrCat("ratelimit.", stat_prefix, ".over_limit"); + const std::string over_limit_counter_name_without_prefix = "ratelimit.over_limit"; + + setUpTest(stat_prefix_config_); + InSequence s; + + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); + EXPECT_CALL(*client_, limit(_, _, _, _, _, 0)) + .WillOnce( + WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { + request_callbacks_ = &callbacks; + }))); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_CALL(filter_callbacks_.stream_info_, + setResponseFlag(StreamInfo::ResponseFlag::RateLimited)); + + Http::ResponseHeaderMapPtr h{new Http::TestResponseHeaderMapImpl()}; + Http::TestResponseHeaderMapImpl response_headers{ + {":status", "429"}, + {"x-envoy-ratelimited", Http::Headers::get().EnvoyRateLimitedValues.True}}; + EXPECT_CALL(filter_callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); + EXPECT_CALL(filter_callbacks_, continueDecoding()).Times(0); + + request_callbacks_->complete(Filters::Common::RateLimit::LimitStatus::OverLimit, nullptr, + std::move(h), nullptr, "", nullptr); + + EXPECT_EQ(1U, filter_callbacks_.clusterInfo() + ->statsScope() + .counterFromString(over_limit_counter_name_with_prefix) + .value()); + + EXPECT_EQ(0U, filter_callbacks_.clusterInfo() + ->statsScope() + .counterFromString(over_limit_counter_name_without_prefix) + .value()); + EXPECT_EQ( + 1U, + filter_callbacks_.clusterInfo()->statsScope().counterFromStatName(upstream_rq_4xx_).value()); + EXPECT_EQ( + 1U, + filter_callbacks_.clusterInfo()->statsScope().counterFromStatName(upstream_rq_429_).value()); + EXPECT_EQ("request_rate_limited", filter_callbacks_.details()); +} + } // namespace } // namespace RateLimitFilter } // namespace HttpFilters diff --git a/test/extensions/filters/http/rbac/config_test.cc b/test/extensions/filters/http/rbac/config_test.cc index d4caee34aa75..6a0f1175f599 100644 --- a/test/extensions/filters/http/rbac/config_test.cc +++ b/test/extensions/filters/http/rbac/config_test.cc @@ -30,7 +30,7 @@ TEST(RoleBasedAccessControlFilterConfigFactoryTest, ValidProto) { NiceMock context; RoleBasedAccessControlFilterConfigFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamDecoderFilter(_)); cb(filter_callbacks); @@ -50,7 +50,7 @@ TEST(RoleBasedAccessControlFilterConfigFactoryTest, ValidMatcherProto) { NiceMock context; RoleBasedAccessControlFilterConfigFactory factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context); + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamDecoderFilter(_)); cb(filter_callbacks); diff --git a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc index 9c24d339683a..988e955c428c 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc @@ -504,7 +504,7 @@ TEST_P(RBACIntegrationTest, RouteOverride) { ->Mutable(0) ->mutable_typed_per_filter_config(); - (*config)["envoy.filters.http.rbac"].PackFrom(per_route_config); + (*config)["rbac"].PackFrom(per_route_config); }); config_helper_.prependFilter(RBAC_CONFIG); @@ -899,7 +899,7 @@ TEST_P(RBACIntegrationTest, MatcherRouteOverride) { ->Mutable(0) ->mutable_typed_per_filter_config(); - (*config)["envoy.filters.http.rbac"].PackFrom(per_route_config); + (*config)["rbac"].PackFrom(per_route_config); }); config_helper_.prependFilter(RBAC_MATCHER_CONFIG); diff --git a/test/extensions/filters/http/router/config_test.cc b/test/extensions/filters/http/router/config_test.cc index 0ecdbe4e0e8b..febf44a31f09 100644 --- a/test/extensions/filters/http/router/config_test.cc +++ b/test/extensions/filters/http/router/config_test.cc @@ -21,6 +21,22 @@ namespace RouterFilter { namespace { TEST(RouterFilterConfigTest, SimpleRouterFilterConfig) { + const std::string yaml_string = R"EOF( + dynamic_stats: true + )EOF"; + + envoy::extensions::filters::http::router::v3::Router proto_config; + TestUtility::loadFromYaml(yaml_string, proto_config); + NiceMock context; + RouterFilterConfig factory; + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats.", context).value(); + Http::MockFilterChainFactoryCallbacks filter_callback; + EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); + cb(filter_callback); +} + +TEST(RouterFilterConfigTest, DEPRECATED_FEATURE_TEST(SimpleRouterFilterConfigWithChildSpan)) { const std::string yaml_string = R"EOF( dynamic_stats: true start_child_span: true @@ -30,7 +46,8 @@ TEST(RouterFilterConfigTest, SimpleRouterFilterConfig) { TestUtility::loadFromYaml(yaml_string, proto_config); NiceMock context; RouterFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats.", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats.", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -58,7 +75,7 @@ TEST(RouterFilterConfigTest, RouterFilterWithUnsupportedStrictHeaderCheck) { NiceMock context; RouterFilterConfig factory; EXPECT_THROW_WITH_REGEX( - factory.createFilterFactoryFromProto(router_config, "stats.", context), + factory.createFilterFactoryFromProto(router_config, "stats.", context).value(), ProtoValidationException, "Proto constraint validation failed \\(RouterValidationError.StrictCheckHeaders"); } @@ -69,7 +86,8 @@ TEST(RouterFilterConfigTest, RouterV2Filter) { NiceMock context; RouterFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(router_config, "stats.", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(router_config, "stats.", context).value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); @@ -79,7 +97,8 @@ TEST(RouterFilterConfigTest, RouterFilterWithEmptyProtoConfig) { NiceMock context; RouterFilterConfig factory; Http::FilterFactoryCb cb = - factory.createFilterFactoryFromProto(*factory.createEmptyConfigProto(), "stats.", context); + factory.createFilterFactoryFromProto(*factory.createEmptyConfigProto(), "stats.", context) + .value(); Http::MockFilterChainFactoryCallbacks filter_callback; EXPECT_CALL(filter_callback, addStreamDecoderFilter(_)); cb(filter_callback); diff --git a/test/extensions/filters/http/set_filter_state/BUILD b/test/extensions/filters/http/set_filter_state/BUILD new file mode 100644 index 000000000000..a3c19b44b22e --- /dev/null +++ b/test/extensions/filters/http/set_filter_state/BUILD @@ -0,0 +1,28 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "integration_test", + srcs = [ + "integration_test.cc", + ], + extension_names = ["envoy.filters.http.set_filter_state"], + deps = [ + "//source/common/router:string_accessor_lib", + "//source/extensions/filters/http/set_filter_state:config", + "//test/mocks/http:http_mocks", + "//test/mocks/server:factory_context_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/set_filter_state/integration_test.cc b/test/extensions/filters/http/set_filter_state/integration_test.cc new file mode 100644 index 000000000000..73c3763d8cd6 --- /dev/null +++ b/test/extensions/filters/http/set_filter_state/integration_test.cc @@ -0,0 +1,91 @@ +#include + +#include "source/common/protobuf/protobuf.h" +#include "source/common/router/string_accessor_impl.h" +#include "source/extensions/filters/http/set_filter_state/config.h" +#include "source/server/generic_factory_context.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace SetFilterState { + +class ObjectFooFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "foo"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +REGISTER_FACTORY(ObjectFooFactory, StreamInfo::FilterState::ObjectFactory); + +class SetMetadataIntegrationTest : public testing::Test { +public: + SetMetadataIntegrationTest() = default; + + void runFilter(const std::string& yaml_config) { + envoy::extensions::filters::http::set_filter_state::v3::Config proto_config; + TestUtility::loadFromYaml(yaml_config, proto_config); + + // Test the factory method. + { + SetFilterStateConfig factory; + auto cb_1 = factory.createFilterFactoryFromProto(proto_config, "", context_); + auto cb_2 = factory.createFilterFactoryFromProtoWithServerContext( + proto_config, "", context_.server_factory_context_); + + NiceMock filter_chain_factory_callbacks; + + EXPECT_CALL(filter_chain_factory_callbacks, addStreamDecoderFilter(_)).Times(2); + cb_1.value()(filter_chain_factory_callbacks); + cb_2(filter_chain_factory_callbacks); + } + + Server::GenericFactoryContextImpl generic_context(context_); + + auto config = std::make_shared( + proto_config.on_request_headers(), StreamInfo::FilterState::LifeSpan::FilterChain, + generic_context); + auto filter = std::make_shared(config); + NiceMock decoder_callbacks; + filter->setDecoderFilterCallbacks(decoder_callbacks); + EXPECT_CALL(decoder_callbacks, streamInfo()).WillRepeatedly(ReturnRef(info_)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter->decodeHeaders(headers_, true)); + } + + NiceMock context_; + Http::TestRequestHeaderMapImpl headers_{{"test-header", "test-value"}}; + NiceMock info_; +}; + +TEST_F(SetMetadataIntegrationTest, FromHeader) { + const std::string yaml_config = R"EOF( + on_request_headers: + - object_key: foo + format_string: + text_format_source: + inline_string: "%REQ(test-header)%" + )EOF"; + runFilter(yaml_config); + const auto* foo = info_.filterState()->getDataReadOnly("foo"); + ASSERT_NE(nullptr, foo); + EXPECT_EQ(foo->serializeAsString(), "test-value"); +} + +} // namespace SetFilterState +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/set_metadata/config_test.cc b/test/extensions/filters/http/set_metadata/config_test.cc index a5930f446cd4..f54552002155 100644 --- a/test/extensions/filters/http/set_metadata/config_test.cc +++ b/test/extensions/filters/http/set_metadata/config_test.cc @@ -22,12 +22,21 @@ using SetMetadataProtoConfig = envoy::extensions::filters::http::set_metadata::v TEST(SetMetadataFilterConfigTest, SimpleConfig) { const std::string yaml = R"EOF( -metadata_namespace: thenamespace -value: - mynumber: 20 - mylist: ["b"] - tags: - mytag1: 1 +metadata: +- metadata_namespace: thenamespace + value: + mynumber: 20 + mylist: ["b"] + tags: + mytag1: 1 + allow_overwrite: true +- metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: foo_namespace + value: + foo: bar + allow_overwrite: true )EOF"; SetMetadataProtoConfig proto_config; @@ -36,7 +45,8 @@ metadata_namespace: thenamespace testing::NiceMock context; SetMetadataConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamDecoderFilter(_)); cb(filter_callbacks); @@ -44,12 +54,21 @@ metadata_namespace: thenamespace TEST(SetMetadataFilterConfigTest, SimpleConfigServerContext) { const std::string yaml = R"EOF( -metadata_namespace: thenamespace -value: - mynumber: 20 - mylist: ["b"] - tags: - mytag1: 1 +metadata: +- metadata_namespace: thenamespace + value: + mynumber: 20 + mylist: ["b"] + tags: + mytag1: 1 + allow_overwrite: true +- metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: foo_namespace + value: + foo: bar + allow_overwrite: true )EOF"; SetMetadataProtoConfig proto_config; diff --git a/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc b/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc index 32fb9318b800..5ada203e0d78 100644 --- a/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc +++ b/test/extensions/filters/http/set_metadata/set_metadata_filter_test.cc @@ -18,42 +18,50 @@ namespace Extensions { namespace HttpFilters { namespace SetMetadataFilter { -class SetMetadataIntegrationTest : public testing::Test { +using FilterSharedPtr = std::shared_ptr; + +class SetMetadataFilterTest : public testing::Test { public: - SetMetadataIntegrationTest() = default; + SetMetadataFilterTest() = default; void runFilter(envoy::config::core::v3::Metadata& metadata, const std::string& yaml_config) { - envoy::extensions::filters::http::set_metadata::v3::Config ext_config; - TestUtility::loadFromYaml(yaml_config, ext_config); - auto config = std::make_shared(ext_config); - auto filter = std::make_shared(config); + envoy::extensions::filters::http::set_metadata::v3::Config proto_config; + TestUtility::loadFromYaml(yaml_config, proto_config); + config_ = std::make_shared(proto_config, *stats_store_.rootScope(), ""); + filter_ = std::make_shared(config_); Http::TestRequestHeaderMapImpl headers; NiceMock decoder_callbacks; NiceMock req_info; - filter->setDecoderFilterCallbacks(decoder_callbacks); + filter_->setDecoderFilterCallbacks(decoder_callbacks); EXPECT_CALL(decoder_callbacks, streamInfo()).WillRepeatedly(ReturnRef(req_info)); EXPECT_CALL(req_info, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter->decodeHeaders(headers, true)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, true)); Buffer::OwnedImpl buffer; - EXPECT_EQ(Http::FilterDataStatus::Continue, filter->decodeData(buffer, true)); - filter->onDestroy(); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(buffer, true)); + filter_->onDestroy(); } - void checkKeyInt(const ProtobufWkt::Struct& s, const char* key, int val) { + void checkKeyInt(const ProtobufWkt::Struct& s, std::string key, int val) { const auto& fields = s.fields(); const auto it = fields.find(key); - ASSERT_TRUE(it != fields.end()); + ASSERT_NE(it, fields.end()); const auto& pbval = it->second; ASSERT_EQ(pbval.kind_case(), ProtobufWkt::Value::kNumberValue); EXPECT_EQ(pbval.number_value(), val); } + + ConfigSharedPtr config_; + FilterSharedPtr filter_; + +private: + NiceMock stats_store_; }; -TEST_F(SetMetadataIntegrationTest, TestTagsHeaders) { +TEST_F(SetMetadataFilterTest, DeprecatedSimple) { const std::string yaml_config = R"EOF( metadata_namespace: thenamespace value: @@ -64,19 +72,19 @@ TEST_F(SetMetadataIntegrationTest, TestTagsHeaders) { envoy::config::core::v3::Metadata metadata; runFilter(metadata, yaml_config); - // Verify that `metadata` contains `{"thenamespace": {"tags": {"mytag0": 1}}}` + // Verify that `metadata` contains `{"thenamespace": {"tags": {"mytag0": 1}}}`. const auto& filter_metadata = metadata.filter_metadata(); const auto it_namespace = filter_metadata.find("thenamespace"); - ASSERT_TRUE(it_namespace != filter_metadata.end()); + ASSERT_NE(it_namespace, filter_metadata.end()); const auto& fields = it_namespace->second.fields(); const auto it_tags = fields.find("tags"); - ASSERT_TRUE(it_tags != fields.end()); + ASSERT_NE(it_tags, fields.end()); const auto& tags = it_tags->second; ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); checkKeyInt(tags.struct_value(), "mytag0", 1); } -TEST_F(SetMetadataIntegrationTest, TestTagsHeadersUpdate) { +TEST_F(SetMetadataFilterTest, DeprecatedWithMerge) { envoy::config::core::v3::Metadata metadata; { @@ -105,17 +113,17 @@ TEST_F(SetMetadataIntegrationTest, TestTagsHeadersUpdate) { } // Verify that `metadata` contains: - // ``{"thenamespace": {number: 20, mylist: ["a","b"], "tags": {"mytag0": 1, "mytag1": 1}}}`` + // ``{"thenamespace": {number: 20, mylist: ["a","b"], "tags": {"mytag0": 1, "mytag1": 1}}}``. const auto& filter_metadata = metadata.filter_metadata(); const auto it_namespace = filter_metadata.find("thenamespace"); - ASSERT_TRUE(it_namespace != filter_metadata.end()); - const auto& namespace_ = it_namespace->second; + ASSERT_NE(it_namespace, filter_metadata.end()); + const auto& namespaced_md = it_namespace->second; - checkKeyInt(namespace_, "mynumber", 20); + checkKeyInt(namespaced_md, "mynumber", 20); - const auto& fields = namespace_.fields(); + const auto& fields = namespaced_md.fields(); const auto it_mylist = fields.find("mylist"); - ASSERT_TRUE(it_mylist != fields.end()); + ASSERT_NE(it_mylist, fields.end()); const auto& mylist = it_mylist->second; ASSERT_EQ(mylist.kind_case(), ProtobufWkt::Value::kListValue); const auto& vals = mylist.list_value().values(); @@ -126,7 +134,7 @@ TEST_F(SetMetadataIntegrationTest, TestTagsHeadersUpdate) { EXPECT_EQ(vals[1].string_value(), "b"); const auto it_tags = fields.find("tags"); - ASSERT_TRUE(it_tags != fields.end()); + ASSERT_NE(it_tags, fields.end()); const auto& tags = it_tags->second; ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); const auto& tags_struct = tags.struct_value(); @@ -135,6 +143,312 @@ TEST_F(SetMetadataIntegrationTest, TestTagsHeadersUpdate) { checkKeyInt(tags_struct, "mytag1", 1); } +TEST_F(SetMetadataFilterTest, UntypedSimple) { + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + value: + tags: + mytag0: 1 + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains `{"thenamespace": {"tags": {"mytag0": 1}}}`. + const auto& filter_metadata = metadata.filter_metadata(); + const auto it_namespace = filter_metadata.find("thenamespace"); + ASSERT_NE(it_namespace, filter_metadata.end()); + const auto& fields = it_namespace->second.fields(); + const auto it_tags = fields.find("tags"); + ASSERT_NE(it_tags, fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + checkKeyInt(tags.struct_value(), "mytag0", 1); +} + +TEST_F(SetMetadataFilterTest, TypedSimple) { + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: foo_namespace + value: + foo: bar + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains our typed Config. + const auto& typed_metadata = metadata.typed_filter_metadata(); + const auto it_namespace2 = typed_metadata.find("thenamespace"); + ASSERT_NE(typed_metadata.end(), it_namespace2); + const auto any_val = it_namespace2->second; + ASSERT_EQ("type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config", + any_val.type_url()); + envoy::extensions::filters::http::set_metadata::v3::Config test_cfg; + ASSERT_TRUE(any_val.UnpackTo(&test_cfg)); + EXPECT_EQ("foo_namespace", test_cfg.metadata_namespace()); +} + +TEST_F(SetMetadataFilterTest, UntypedWithAllowOverwrite) { + envoy::config::core::v3::Metadata metadata; + + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + value: + mynumber: 10 + mylist: ["a"] + tags: + mytag0: 1 + - metadata_namespace: thenamespace + value: + mynumber: 20 + mylist: ["b"] + tags: + mytag1: 1 + allow_overwrite: true + )EOF"; + + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains: + // ``{"thenamespace": {number: 20, mylist: ["a","b"], "tags": {"mytag0": 1, "mytag1": 1}}}``. + const auto& filter_metadata = metadata.filter_metadata(); + const auto it_namespace = filter_metadata.find("thenamespace"); + ASSERT_NE(it_namespace, filter_metadata.end()); + const auto& namespaced_md = it_namespace->second; + + checkKeyInt(namespaced_md, "mynumber", 20); + + const auto& fields = namespaced_md.fields(); + const auto it_mylist = fields.find("mylist"); + ASSERT_NE(it_mylist, fields.end()); + const auto& mylist = it_mylist->second; + ASSERT_EQ(mylist.kind_case(), ProtobufWkt::Value::kListValue); + const auto& vals = mylist.list_value().values(); + ASSERT_EQ(vals.size(), 2); + ASSERT_EQ(vals[0].kind_case(), ProtobufWkt::Value::kStringValue); + EXPECT_EQ(vals[0].string_value(), "a"); + ASSERT_EQ(vals[1].kind_case(), ProtobufWkt::Value::kStringValue); + EXPECT_EQ(vals[1].string_value(), "b"); + + const auto it_tags = fields.find("tags"); + ASSERT_NE(it_tags, fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + const auto& tags_struct = tags.struct_value(); + + checkKeyInt(tags_struct, "mytag0", 1); + checkKeyInt(tags_struct, "mytag1", 1); +} + +TEST_F(SetMetadataFilterTest, UntypedWithNoAllowOverwrite) { + envoy::config::core::v3::Metadata metadata; + + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + value: + mynumber: 10 + mylist: ["a"] + tags: + mytag0: 1 + - metadata_namespace: thenamespace + value: + mynumber: 20 + mylist: ["b"] + tags: + mytag1: 1 + )EOF"; + + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains: + // ``{"thenamespace": {number: 10, mylist: ["a"], "tags": {"mytag0": 1}}}``. + const auto& filter_metadata = metadata.filter_metadata(); + const auto it_namespace = filter_metadata.find("thenamespace"); + ASSERT_NE(it_namespace, filter_metadata.end()); + const auto& namespaced_md = it_namespace->second; + + checkKeyInt(namespaced_md, "mynumber", 10); + + const auto& fields = namespaced_md.fields(); + const auto it_mylist = fields.find("mylist"); + ASSERT_NE(it_mylist, fields.end()); + const auto& mylist = it_mylist->second; + ASSERT_EQ(mylist.kind_case(), ProtobufWkt::Value::kListValue); + const auto& vals = mylist.list_value().values(); + ASSERT_EQ(vals.size(), 1); + ASSERT_EQ(vals[0].kind_case(), ProtobufWkt::Value::kStringValue); + EXPECT_EQ(vals[0].string_value(), "a"); + + const auto it_tags = fields.find("tags"); + ASSERT_NE(it_tags, fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + const auto& tags_struct = tags.struct_value(); + + checkKeyInt(tags_struct, "mytag0", 1); + EXPECT_EQ(1, config_->stats().overwrite_denied_.value()); +} + +TEST_F(SetMetadataFilterTest, TypedWithAllowOverwrite) { + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: foo_namespace + value: + foo: bar + - metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: bat_namespace + value: + bat: baz + allow_overwrite: true + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains our typed Config. + const auto& typed_metadata = metadata.typed_filter_metadata(); + const auto it_namespace2 = typed_metadata.find("thenamespace"); + ASSERT_NE(typed_metadata.end(), it_namespace2); + const auto any_val = it_namespace2->second; + ASSERT_EQ("type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config", + any_val.type_url()); + envoy::extensions::filters::http::set_metadata::v3::Config test_cfg; + ASSERT_TRUE(any_val.UnpackTo(&test_cfg)); + EXPECT_EQ("bat_namespace", test_cfg.metadata_namespace()); + ASSERT_TRUE(test_cfg.has_value()); + EXPECT_TRUE(test_cfg.value().fields().contains("bat")); +} + +TEST_F(SetMetadataFilterTest, TypedWithNoAllowOverwrite) { + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: foo_namespace + value: + foo: bar + - metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: bat_namespace + value: + bat: baz + allow_overwrite: false + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains our typed Config. + const auto& typed_metadata = metadata.typed_filter_metadata(); + const auto it_namespace2 = typed_metadata.find("thenamespace"); + ASSERT_NE(typed_metadata.end(), it_namespace2); + const auto any_val = it_namespace2->second; + ASSERT_EQ("type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config", + any_val.type_url()); + envoy::extensions::filters::http::set_metadata::v3::Config test_cfg; + ASSERT_TRUE(any_val.UnpackTo(&test_cfg)); + EXPECT_EQ("foo_namespace", test_cfg.metadata_namespace()); + ASSERT_TRUE(test_cfg.has_value()); + EXPECT_TRUE(test_cfg.value().fields().contains("foo")); + EXPECT_EQ(1, config_->stats().overwrite_denied_.value()); +} + +TEST_F(SetMetadataFilterTest, UntypedWithDeprecated) { + const std::string yaml_config = R"EOF( + metadata_namespace: thenamespace + value: + tags: + mytag0: 0 + metadata: + - metadata_namespace: thenamespace + value: + tags: + mytag0: 1 + allow_overwrite: true + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains `{"thenamespace": {"tags": {"mytag0": 1}}}`. + const auto& filter_metadata = metadata.filter_metadata(); + const auto it_namespace = filter_metadata.find("thenamespace"); + ASSERT_NE(it_namespace, filter_metadata.end()); + const auto& fields = it_namespace->second.fields(); + const auto it_tags = fields.find("tags"); + ASSERT_NE(it_tags, fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + checkKeyInt(tags.struct_value(), "mytag0", 1); +} + +TEST_F(SetMetadataFilterTest, TypedWithDeprecated) { + const std::string yaml_config = R"EOF( + metadata_namespace: thenamespace + value: + tags: + mytag0: 0 + metadata: + - metadata_namespace: thenamespace + typed_value: + '@type': type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config + metadata_namespace: foo_namespace + value: + foo: bar + )EOF"; + + envoy::config::core::v3::Metadata metadata; + runFilter(metadata, yaml_config); + + // Verify that `metadata` contains `{"thenamespace": {"tags": {"mytag0": 0}}}`. + const auto& untyped_metadata = metadata.filter_metadata(); + const auto it_namespace = untyped_metadata.find("thenamespace"); + ASSERT_NE(untyped_metadata.end(), it_namespace); + const auto& fields = it_namespace->second.fields(); + const auto it_tags = fields.find("tags"); + ASSERT_NE(it_tags, fields.end()); + const auto& tags = it_tags->second; + ASSERT_EQ(tags.kind_case(), ProtobufWkt::Value::kStructValue); + checkKeyInt(tags.struct_value(), "mytag0", 0); + + // Verify that `metadata` contains our typed Config. + const auto& typed_metadata = metadata.typed_filter_metadata(); + const auto it_namespace2 = typed_metadata.find("thenamespace"); + ASSERT_NE(typed_metadata.end(), it_namespace2); + const auto any_val = it_namespace2->second; + ASSERT_EQ("type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config", + any_val.type_url()); + envoy::extensions::filters::http::set_metadata::v3::Config test_cfg; + ASSERT_TRUE(any_val.UnpackTo(&test_cfg)); + EXPECT_EQ("foo_namespace", test_cfg.metadata_namespace()); +} + +TEST_F(SetMetadataFilterTest, LogsErrorWhenNoValueConfigured) { + const std::string yaml_config = R"EOF( + metadata: + - metadata_namespace: thenamespace + )EOF"; + envoy::config::core::v3::Metadata metadata; + EXPECT_LOG_CONTAINS( + "warn", + "set_metadata filter configuration contains metadata entries without value or typed_value", + runFilter(metadata, yaml_config)); +} + } // namespace SetMetadataFilter } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/stateful_session/BUILD b/test/extensions/filters/http/stateful_session/BUILD index fa8f8310f2ff..babe2a210533 100644 --- a/test/extensions/filters/http/stateful_session/BUILD +++ b/test/extensions/filters/http/stateful_session/BUILD @@ -19,6 +19,7 @@ envoy_extension_cc_test( extension_names = ["envoy.filters.http.stateful_session"], deps = [ "//source/extensions/filters/http/stateful_session:config", + "//source/server:generic_factory_context_lib", "//test/mocks/api:api_mocks", "//test/mocks/http:http_mocks", "//test/mocks/http:stateful_session_mock", diff --git a/test/extensions/filters/http/stateful_session/config_test.cc b/test/extensions/filters/http/stateful_session/config_test.cc index 45626b6d0fba..0be82eb43443 100644 --- a/test/extensions/filters/http/stateful_session/config_test.cc +++ b/test/extensions/filters/http/stateful_session/config_test.cc @@ -66,7 +66,8 @@ TEST(StatefulSessionFactoryConfigTest, SimpleConfigTest) { testing::NiceMock server_context; StatefulSessionFactoryConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context).value(); Http::MockFilterChainFactoryCallbacks filter_callbacks; EXPECT_CALL(filter_callbacks, addStreamFilter(_)); cb(filter_callbacks); @@ -81,7 +82,9 @@ TEST(StatefulSessionFactoryConfigTest, SimpleConfigTest) { EnvoyException, "Didn't find a registered implementation for name: 'envoy.http.stateful_session.not_exist'"); - EXPECT_NO_THROW(factory.createFilterFactoryFromProto(empty_proto_config, "stats", context)); + EXPECT_NO_THROW(factory.createFilterFactoryFromProto(empty_proto_config, "stats", context) + .status() + .IgnoreError()); EXPECT_NO_THROW(factory.createRouteSpecificFilterConfig(empty_proto_route_config, server_context, context.messageValidationVisitor())); } diff --git a/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc b/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc index a35a9a59d642..6d1f05dc50ee 100644 --- a/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc +++ b/test/extensions/filters/http/stateful_session/stateful_session_integration_test.cc @@ -148,6 +148,10 @@ name: envoy.filters.http.stateful_session "@type": type.googleapis.com/envoy.extensions.http.stateful_session.header.v3.HeaderBasedSessionState name: session-header )EOF"; +static const std::string STATEFUL_SESSION_STRICT_MODE = + R"EOF( + strict: true +)EOF"; static const std::string DISABLE_STATEFUL_SESSION = R"EOF( @@ -439,6 +443,156 @@ TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionCooki } } +// Test verifies cookie-based upstream host selection with strict mode. +// When requested upstream host is valid, it should be chosen. +// When requested upstream host is invalid, Envoy should return 503. +TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionCookieStrict) { + initializeFilterAndRoute(STATEFUL_SESSION_FILTER + STATEFUL_SESSION_STRICT_MODE, ""); + + // When request does not contain a cookie, cluster should select endpoint using an LB and cookie + // should be returned. In other words, it should work normally even when 'strict' mode is enabled. + { + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + waitForNextUpstreamRequest({0, 1, 2, 3}); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + // set-cookie header should be added. + EXPECT_FALSE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + + cleanupUpstreamAndDownstream(); + } + + { + + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + + envoy::Cookie cookie; + std::string cookie_string; + cookie.set_address(std::string( + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&cookie_string); + + std::string encoded_address = Envoy::Base64::encode(cookie_string.data(), cookie_string.size()); + + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + // The upstream with index 1 should be selected. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 1); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("set-cookie")).empty()); + + cleanupUpstreamAndDownstream(); + } + { + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(2, endpoint); + envoy::Cookie cookie; + std::string cookie_string; + cookie.set_address(std::string( + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()))); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&cookie_string); + + std::string encoded_address = Envoy::Base64::encode(cookie_string.data(), cookie_string.size()); + + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + // The upstream with index 2 should be selected. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 2); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("session-header")).empty()); + + cleanupUpstreamAndDownstream(); + } + + // Upstream endpoint encoded in the cookie points to unknown server address. + { + codec_client_ = makeHttpConnection(lookupPort("http")); + envoy::Cookie cookie; + std::string cookie_string; + cookie.set_address(std::string("127.0.0.7:50000")); + cookie.set_expires(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + + 120); + cookie.SerializeToString(&cookie_string); + + std::string encoded_address = Envoy::Base64::encode(cookie_string.data(), cookie_string.size()); + + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"cookie", fmt::format("global-session-cookie=\"{}\"", encoded_address)}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(response->complete()); + + EXPECT_EQ("503", response->headers().getStatusValue()); + + cleanupUpstreamAndDownstream(); + } +} + TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionHeader) { initializeFilterAndRoute(STATEFUL_SESSION_HEADER_FILTER, ""); @@ -511,7 +665,7 @@ TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionHeade cleanupUpstreamAndDownstream(); } - // Test the case that stateful session header with unknown server address. + // Upstream endpoint encoded in stateful session header points to unknown server address. { codec_client_ = makeHttpConnection(lookupPort("http")); Http::TestRequestHeaderMapImpl request_headers{ @@ -550,6 +704,102 @@ TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionHeade } } +// Test verifies header-based upstream host selection with strict mode. +// When requested upstream host is valid, it should be chosen. +// When requested upstream host is invalid, Envoy should return 503. +TEST_F(StatefulSessionIntegrationTest, DownstreamRequestWithStatefulSessionHeaderStrict) { + initializeFilterAndRoute(STATEFUL_SESSION_HEADER_FILTER + STATEFUL_SESSION_STRICT_MODE, ""); + + { + + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(1, endpoint); + const std::string address_string = + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); + + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"session-header", encoded_address}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + // The upstream with index 1 should be selected. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 1); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("session-header")).empty()); + + cleanupUpstreamAndDownstream(); + } + { + envoy::config::endpoint::v3::LbEndpoint endpoint; + setUpstreamAddress(2, endpoint); + const std::string address_string = + fmt::format("127.0.0.1:{}", endpoint.endpoint().address().socket_address().port_value()); + const std::string encoded_address = + Envoy::Base64::encode(address_string.data(), address_string.size()); + + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"session-header", encoded_address}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + // The upstream with index 2 should be selected. + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2, 3}); + EXPECT_EQ(upstream_index.value(), 2); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + + // No response header to be added. + EXPECT_TRUE(response->headers().get(Http::LowerCaseString("session-header")).empty()); + + cleanupUpstreamAndDownstream(); + } + + // Upstream endpoint encoded in stateful session header points to unknown server address. + { + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "stateful.session.com"}, + {"session-header", Envoy::Base64::encode("127.0.0.7:50000", 15)}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + ASSERT_TRUE(response->waitForEndStream()); + + EXPECT_TRUE(response->complete()); + + EXPECT_EQ("503", response->headers().getStatusValue()); + + cleanupUpstreamAndDownstream(); + } +} + TEST_F(StatefulSessionIntegrationTest, StatefulSessionDisabledByRoute) { initializeFilterAndRoute(STATEFUL_SESSION_FILTER, DISABLE_STATEFUL_SESSION); diff --git a/test/extensions/filters/http/stateful_session/stateful_session_test.cc b/test/extensions/filters/http/stateful_session/stateful_session_test.cc index 82a5fc145239..c30e68eed967 100644 --- a/test/extensions/filters/http/stateful_session/stateful_session_test.cc +++ b/test/extensions/filters/http/stateful_session/stateful_session_test.cc @@ -1,6 +1,7 @@ #include #include "source/extensions/filters/http/stateful_session/stateful_session.h" +#include "source/server/generic_factory_context.h" #include "test/mocks/http/mocks.h" #include "test/mocks/http/stateful_session.h" @@ -32,8 +33,9 @@ class StatefulSessionTest : public testing::Test { ASSERT(!config.empty()); ProtoConfig proto_config; TestUtility::loadFromYaml(std::string(config), proto_config); + Envoy::Server::GenericFactoryContextImpl generic_context(context_); - config_ = std::make_shared(proto_config, context_); + config_ = std::make_shared(proto_config, generic_context); filter_ = std::make_shared(config_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); @@ -49,7 +51,8 @@ class StatefulSessionTest : public testing::Test { .WillOnce(Return(route_factory_)); } - route_config_ = std::make_shared(proto_route_config, context_); + route_config_ = + std::make_shared(proto_route_config, generic_context); ON_CALL(*decoder_callbacks_.route_, mostSpecificPerFilterConfig(_)) .WillByDefault(Return(route_config_.get())); @@ -101,7 +104,9 @@ TEST_F(StatefulSessionTest, NormalSessionStateTest) { EXPECT_CALL(*raw_session_state, upstreamAddress()) .WillOnce(Return(absl::make_optional("1.2.3.4"))); EXPECT_CALL(decoder_callbacks_, setUpstreamOverrideHost(_)) - .WillOnce(testing::Invoke([&](absl::string_view host) { EXPECT_EQ("1.2.3.4", host); })); + .WillOnce(testing::Invoke([&](Upstream::LoadBalancerContext::OverrideHost host) { + EXPECT_EQ("1.2.3.4", host.first); + })); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); @@ -139,7 +144,9 @@ TEST_F(StatefulSessionTest, SessionStateOverrideByRoute) { EXPECT_CALL(*raw_session_state, upstreamAddress()) .WillOnce(Return(absl::make_optional("1.2.3.4"))); EXPECT_CALL(decoder_callbacks_, setUpstreamOverrideHost(_)) - .WillOnce(testing::Invoke([&](absl::string_view host) { EXPECT_EQ("1.2.3.4", host); })); + .WillOnce(testing::Invoke([&](Upstream::LoadBalancerContext::OverrideHost host) { + EXPECT_EQ("1.2.3.4", host.first); + })); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); @@ -181,7 +188,9 @@ TEST_F(StatefulSessionTest, NoUpstreamHost) { EXPECT_CALL(*raw_session_state, upstreamAddress()) .WillOnce(Return(absl::make_optional("1.2.3.4"))); EXPECT_CALL(decoder_callbacks_, setUpstreamOverrideHost(_)) - .WillOnce(testing::Invoke([&](absl::string_view host) { EXPECT_EQ("1.2.3.4", host); })); + .WillOnce(testing::Invoke([&](Upstream::LoadBalancerContext::OverrideHost host) { + EXPECT_EQ("1.2.3.4", host.first); + })); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); @@ -208,9 +217,9 @@ TEST_F(StatefulSessionTest, NullSessionState) { TEST(EmpytProtoConfigTest, EmpytProtoConfigTest) { ProtoConfig empty_proto_config; - testing::NiceMock server_context; + testing::NiceMock generic_context; - StatefulSessionConfig config(empty_proto_config, server_context); + StatefulSessionConfig config(empty_proto_config, generic_context); Http::TestRequestHeaderMapImpl request_headers{ {":path", "/"}, {":method", "GET"}, {":authority", "test.com"}}; diff --git a/test/extensions/filters/http/tap/BUILD b/test/extensions/filters/http/tap/BUILD index 9b4b3064126f..4e834ea9281e 100644 --- a/test/extensions/filters/http/tap/BUILD +++ b/test/extensions/filters/http/tap/BUILD @@ -45,6 +45,7 @@ envoy_extension_cc_test( "//source/extensions/filters/http/tap:tap_config_impl", "//test/extensions/common/tap:common", "//test/mocks:common_lib", + "//test/mocks/network:network_mocks", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], diff --git a/test/extensions/filters/http/tap/common.h b/test/extensions/filters/http/tap/common.h index 358b7e542efd..f262820a2699 100644 --- a/test/extensions/filters/http/tap/common.h +++ b/test/extensions/filters/http/tap/common.h @@ -13,8 +13,9 @@ class MockHttpTapConfig : public HttpTapConfig { public: HttpPerRequestTapperPtr createPerRequestTapper(const envoy::extensions::filters::http::tap::v3::Tap& tap_config, - uint64_t stream_id) override { - return HttpPerRequestTapperPtr{createPerRequestTapper_(tap_config, stream_id)}; + uint64_t stream_id, + OptRef connection) override { + return HttpPerRequestTapperPtr{createPerRequestTapper_(tap_config, stream_id, connection)}; } Extensions::Common::Tap::PerTapSinkHandleManagerPtr @@ -24,8 +25,8 @@ class MockHttpTapConfig : public HttpTapConfig { } MOCK_METHOD(HttpPerRequestTapper*, createPerRequestTapper_, - (const envoy::extensions::filters::http::tap::v3::Tap& tap_config, - uint64_t stream_id)); + (const envoy::extensions::filters::http::tap::v3::Tap& tap_config, uint64_t stream_id, + OptRef)); MOCK_METHOD(Extensions::Common::Tap::PerTapSinkHandleManager*, createPerTapSinkHandleManager_, (uint64_t trace_id)); MOCK_METHOD(uint32_t, maxBufferedRxBytes, (), (const)); diff --git a/test/extensions/filters/http/tap/tap_config_impl_test.cc b/test/extensions/filters/http/tap/tap_config_impl_test.cc index eb44585abd65..f413af5ca371 100644 --- a/test/extensions/filters/http/tap/tap_config_impl_test.cc +++ b/test/extensions/filters/http/tap/tap_config_impl_test.cc @@ -1,8 +1,12 @@ +#include "source/common/network/address_impl.h" +#include "source/common/network/socket_impl.h" +#include "source/common/network/utility.h" #include "source/extensions/filters/http/tap/tap_config_impl.h" #include "test/extensions/common/tap/common.h" #include "test/extensions/filters/http/tap/common.h" #include "test/mocks/common.h" +#include "test/mocks/network/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -31,7 +35,8 @@ class HttpPerRequestTapperImplTest : public testing::Test { EXPECT_CALL(*config_, timeSource()).WillRepeatedly(ReturnRef(time_system_)); time_system_.setSystemTime(std::chrono::seconds(0)); EXPECT_CALL(matcher_, onNewStream(_)).WillOnce(SaveArgAddress(&statuses_)); - tapper_ = std::make_unique(config_, tap_config_, 1); + tapper_ = std::make_unique(config_, tap_config_, 1, + OptRef{}); } std::shared_ptr config_{std::make_shared()}; @@ -327,9 +332,20 @@ class HttpPerRequestTapperImplForSpecificConfigTest : public testing::Test { time_system_.setSystemTime(std::chrono::seconds(0)); EXPECT_CALL(matcher_, onNewStream(_)).WillOnce(SaveArgAddress(&statuses_)); - // Make sure the record_req_resp_msg_caught_time is true tap_config_.set_record_headers_received_time(true); - tapper_ = std::make_unique(config_, tap_config_, 1); + tap_config_.set_record_downstream_connection(true); + + connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress( + std::make_shared("127.0.0.1", 1234)); + connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + std::make_shared("127.0.0.1", 4321)); + + tapper_ = std::make_unique(config_, tap_config_, 1, connection_); + + Network::ConnectionInfoProviderSharedPtr local_connection_info_provider = + std::make_shared( + Network::Utility::getCanonicalIpv4LoopbackAddress(), + Network::Utility::getCanonicalIpv4LoopbackAddress()); } std::shared_ptr config_{std::make_shared()}; @@ -346,11 +362,11 @@ class HttpPerRequestTapperImplForSpecificConfigTest : public testing::Test { const Http::TestResponseHeaderMapImpl response_headers_{{"e", "f"}}; const Http::TestResponseTrailerMapImpl response_trailers_{{"g", "h"}}; Event::SimulatedTimeSystem time_system_; + NiceMock connection_; }; // Buffered tap with a match and with record_headers_received_time is true. -TEST_F(HttpPerRequestTapperImplForSpecificConfigTest, - BufferedFlowTapWithRecordHeadersReceivedtimeTrue) { +TEST_F(HttpPerRequestTapperImplForSpecificConfigTest, BufferedFlowTapWithSpecificConfig) { EXPECT_CALL(*config_, streaming()).WillRepeatedly(Return(false)); EXPECT_CALL(*config_, maxBufferedRxBytes()).WillRepeatedly(Return(1024)); EXPECT_CALL(*config_, maxBufferedTxBytes()).WillRepeatedly(Return(1024)); @@ -394,10 +410,18 @@ TEST_F(HttpPerRequestTapperImplForSpecificConfigTest, - key: g value: h headers_received_time: 1970-01-01T00:00:00Z + downstream_connection: + local_address: + socket_address: + address: 127.0.0.1 + port_value: 1234 + remote_address: + socket_address: + address: 127.0.0.1 + port_value: 4321 )EOF"))); EXPECT_TRUE(tapper_->onDestroyLog()); } -// New test for HTTP specific config } // namespace } // namespace TapFilter diff --git a/test/extensions/filters/http/tap/tap_filter_integration_test.cc b/test/extensions/filters/http/tap/tap_filter_integration_test.cc index 69adb2aa257d..84437f154839 100644 --- a/test/extensions/filters/http/tap/tap_filter_integration_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_integration_test.cc @@ -1,5 +1,3 @@ -#include - #include "envoy/config/core/v3/base.pb.h" #include "envoy/data/tap/v3/wrapper.pb.h" @@ -1167,6 +1165,48 @@ name: tap EXPECT_EQ(1UL, test_server_->counter("http.config_test.tap.rq_tapped")->value()); } +// Verify option record_downstream_connection +// when a request header is matched in a static configuration +TEST_P(TapIntegrationTest, StaticFilePerHttpBufferTraceTapDownstreamConnection) { + constexpr absl::string_view filter_config = + R"EOF( +name: tap +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.tap.v3.Tap + common_config: + static_config: + match: + http_request_headers_match: + headers: + - name: foo + string_match: + exact: bar + output_config: + sinks: + - format: PROTO_BINARY_LENGTH_DELIMITED + file_per_tap: + path_prefix: {} + record_downstream_connection: true +)EOF"; + + const std::string path_prefix = getTempPathPrefix(); + initializeFilter(fmt::format(filter_config, path_prefix)); + + // Initial request/response with tap. + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + makeRequest(request_headers_tap_, {"hello"}, &request_trailers_, response_headers_no_tap_, + {"world"}, &response_trailers_); + codec_client_->close(); + test_server_->waitForCounterGe("http.config_test.downstream_cx_destroy", 1); + + std::vector traces = + Extensions::Common::Tap::readTracesFromPath(path_prefix); + ASSERT_EQ(1, traces.size()); + EXPECT_TRUE(traces[0].has_http_buffered_trace()); + + EXPECT_EQ(1UL, test_server_->counter("http.config_test.tap.rq_tapped")->value()); +} + // Verify that body matching works. TEST_P(TapIntegrationTest, AdminBodyMatching) { initializeFilter(admin_filter_config_); diff --git a/test/extensions/filters/http/tap/tap_filter_test.cc b/test/extensions/filters/http/tap/tap_filter_test.cc index 94d2e6dbf7b3..893d26281de7 100644 --- a/test/extensions/filters/http/tap/tap_filter_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_test.cc @@ -55,8 +55,9 @@ class TapFilterTest : public testing::Test { if (has_config) { EXPECT_CALL(callbacks_, streamId()); + EXPECT_CALL(callbacks_, connection()); http_per_request_tapper_ = new MockHttpPerRequestTapper(); - EXPECT_CALL(*http_tap_config_, createPerRequestTapper_(_, _)) + EXPECT_CALL(*http_tap_config_, createPerRequestTapper_(_, _, _)) .WillOnce(Return(http_per_request_tapper_)); } @@ -93,8 +94,7 @@ TEST_F(TapFilterTest, NoConfig) { Http::MetadataMap metadata; EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata)); - filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_, - AccessLog::AccessLogType::NotSet); + filter_->log({&request_headers, &response_headers, &response_trailers}, stream_info_); } // Verify filter functionality when there is a tap config. @@ -124,8 +124,7 @@ TEST_F(TapFilterTest, Config) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers)); EXPECT_CALL(*http_per_request_tapper_, onDestroyLog()).WillOnce(Return(true)); - filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_, - AccessLog::AccessLogType::NotSet); + filter_->log({&request_headers, &response_headers, &response_trailers}, stream_info_); EXPECT_EQ(1UL, filter_config_->stats_.rq_tapped_.value()); // Workaround InSequence/shared_ptr mock leak. @@ -149,7 +148,7 @@ TEST(TapFilterConfigTest, InvalidProto) { TestUtility::loadFromYaml(filter_config, config); NiceMock context; TapFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context).value(), EnvoyException, "Error: Specifying admin streaming output without configuring admin."); } @@ -171,7 +170,7 @@ TEST(TapFilterConfigTest, NeitherMatchNorMatchConfig) { NiceMock context; TapFilterFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context).value(), EnvoyException, fmt::format("Neither match nor match_config is set in TapConfig: {}", config.common_config().static_config().DebugString())); diff --git a/test/extensions/filters/http/wasm/BUILD b/test/extensions/filters/http/wasm/BUILD index dfaa4a054170..cfbddf2e7712 100644 --- a/test/extensions/filters/http/wasm/BUILD +++ b/test/extensions/filters/http/wasm/BUILD @@ -67,6 +67,7 @@ envoy_extension_cc_test( "//source/extensions/common/wasm:wasm_lib", "//source/extensions/filters/http/wasm:config", "//test/extensions/common/wasm:wasm_runtime", + "//test/mocks/network:network_mocks", "//test/mocks/server:server_mocks", "//test/test_common:environment_lib", "@envoy_api//envoy/extensions/filters/http/wasm/v3:pkg_cc_proto", diff --git a/test/extensions/filters/http/wasm/config_test.cc b/test/extensions/filters/http/wasm/config_test.cc index f5ab74c1d2f3..8e5086d741a9 100644 --- a/test/extensions/filters/http/wasm/config_test.cc +++ b/test/extensions/filters/http/wasm/config_test.cc @@ -13,6 +13,7 @@ #include "test/extensions/common/wasm/wasm_runtime.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/network/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/environment.h" @@ -34,12 +35,15 @@ class WasmFilterConfigTest : public Event::TestUsingSimulatedTime, public testing::TestWithParam> { protected: WasmFilterConfigTest() : api_(Api::createApiForTest(stats_store_)) { - ON_CALL(context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(context_.server_factory_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(context_, scope()).WillByDefault(ReturnRef(stats_scope_)); - ON_CALL(context_, listenerMetadata()).WillByDefault(ReturnRef(listener_metadata_)); + ON_CALL(context_, listenerInfo()).WillByDefault(ReturnRef(listener_info_)); + ON_CALL(listener_info_, metadata()).WillByDefault(ReturnRef(listener_metadata_)); EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager_)); - ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(context_, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(context_.server_factory_context_, clusterManager()) + .WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(context_.server_factory_context_, mainThreadDispatcher()) + .WillByDefault(ReturnRef(dispatcher_)); } void SetUp() override { Envoy::Extensions::Common::Wasm::clearCodeCacheForTesting(); } @@ -53,6 +57,7 @@ class WasmFilterConfigTest : public Event::TestUsingSimulatedTime, })); } + NiceMock listener_info_; NiceMock context_; Stats::IsolatedStoreImpl stats_store_; Stats::Scope& stats_scope_{*stats_store_.rootScope()}; @@ -94,7 +99,8 @@ TEST_P(WasmFilterConfigTest, JsonLoadFromFileWasm) { envoy::extensions::filters::http::wasm::v3::Wasm proto_config; TestUtility::loadFromJson(json, proto_config); WasmFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -128,7 +134,7 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasm) { { WasmFilterConfig factory; Http::FilterFactoryCb cb = - factory.createFilterFactoryFromProto(proto_config, "stats", context_); + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -165,7 +171,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasmFailOpenOk) { envoy::extensions::filters::http::wasm::v3::Wasm proto_config; TestUtility::loadFromYaml(yaml, proto_config); WasmFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -196,8 +203,9 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasmInvalidConfig) { envoy::extensions::filters::http::wasm::v3::Wasm proto_config; TestUtility::loadFromYaml(invalid_yaml, proto_config); WasmFilterConfig factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); const std::string valid_yaml = TestEnvironment::substitute(absl::StrCat(R"EOF( config: @@ -215,7 +223,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromFileWasmInvalidConfig) { value: "valid" )EOF")); TestUtility::loadFromYaml(valid_yaml, proto_config); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -241,7 +250,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadInlineWasm) { envoy::extensions::filters::http::wasm::v3::Wasm proto_config; TestUtility::loadFromYaml(yaml, proto_config); WasmFilterConfig factory; - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -265,8 +275,9 @@ TEST_P(WasmFilterConfigTest, YamlLoadInlineBadCode) { envoy::extensions::filters::http::wasm::v3::Wasm proto_config; TestUtility::loadFromYaml(yaml, proto_config); WasmFilterConfig factory; - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); } TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasm) { @@ -308,7 +319,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasm) { return &request; })); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -358,8 +370,9 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailOnUncachedThenSucceed) { return &request; })); - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); @@ -370,7 +383,7 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailOnUncachedThenSucceed) { EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager2)); - auto cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + auto cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher2, ready()); init_manager2.initialize(init_watcher2); @@ -434,11 +447,13 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { })); // Case 1: fail and fetch in the background, got 503, cache failure. - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); // Fail a second time because we are in-progress. - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); async_callbacks->onSuccess( request, Http::ResponseMessagePtr{new Http::ResponseMessageImpl(Http::ResponseHeaderMapPtr{ new Http::TestResponseHeaderMapImpl{{":status", "503"}}})}); @@ -452,8 +467,9 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { Init::ExpectableWatcherImpl init_watcher2; EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager2)); - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); EXPECT_CALL(init_watcher2, ready()); init_manager2.initialize(init_watcher2); @@ -480,8 +496,9 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager3)); - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); EXPECT_CALL(init_watcher3, ready()); init_manager3.initialize(init_watcher3); @@ -493,7 +510,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager4)); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher4, ready()); init_manager4.initialize(init_watcher4); @@ -537,8 +555,9 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager5)); - EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(proto_config2, "stats", context_), - WasmException, "Unable to create Wasm HTTP filter "); + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(proto_config2, "stats", context_).status().IgnoreError(), + WasmException, "Unable to create Wasm HTTP filter "); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); @@ -550,7 +569,7 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager6)); - factory.createFilterFactoryFromProto(proto_config, "stats", context_); + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher6, ready()); init_manager6.initialize(init_watcher6); @@ -562,7 +581,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmFailCachedThenSucceed) { EXPECT_CALL(context_, initManager()).WillRepeatedly(ReturnRef(init_manager7)); - Http::FilterFactoryCb cb2 = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb2 = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher7, ready()); init_manager7.initialize(init_watcher7); @@ -614,7 +634,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteConnectionReset) { return &request; })); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); } @@ -659,7 +680,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteSuccessWith503) { return &request; })); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); } @@ -704,7 +726,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteSuccessIncorrectSha256) { return &request; })); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); } @@ -772,7 +795,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteMultipleRetries) { })); EXPECT_CALL(*retry_timer_, disableTimer()); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); EXPECT_EQ(context_.initManager().state(), Init::Manager::State::Initialized); @@ -820,7 +844,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteSuccessBadcode) { return nullptr; })); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); @@ -890,7 +915,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteSuccessBadcodeFailOpen) { return nullptr; })); - Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context_); + Http::FilterFactoryCb cb = + factory.createFilterFactoryFromProto(proto_config, "stats", context_).value(); EXPECT_CALL(init_watcher_, ready()); context_.initManager().initialize(init_watcher_); Http::MockFilterChainFactoryCallbacks filter_callback; @@ -936,7 +962,8 @@ TEST_P(WasmFilterConfigTest, YamlLoadFromRemoteWasmCreateFilter) { return &request; })); NiceMock threadlocal; - EXPECT_CALL(context_, threadLocal()).WillRepeatedly(ReturnRef(threadlocal)); + EXPECT_CALL(context_.server_factory_context_, threadLocal()) + .WillRepeatedly(ReturnRef(threadlocal)); threadlocal.registered_ = false; auto filter_config = std::make_unique(proto_config, context_); EXPECT_EQ(filter_config->createFilter(), nullptr); @@ -969,7 +996,7 @@ TEST_P(WasmFilterConfigTest, FailedToGetThreadLocalPlugin) { envoy::extensions::filters::http::wasm::v3::Wasm proto_config; TestUtility::loadFromYaml(yaml, proto_config); - EXPECT_CALL(context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); + EXPECT_CALL(context_.server_factory_context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); threadlocal.registered_ = true; auto filter_config = std::make_unique(proto_config, context_); ASSERT_EQ(threadlocal.current_slot_, 1); diff --git a/test/extensions/filters/http/wasm/test_data/test_cpp.cc b/test/extensions/filters/http/wasm/test_data/test_cpp.cc index 3ddb83924630..2bb3b9d1f8eb 100644 --- a/test/extensions/filters/http/wasm/test_data/test_cpp.cc +++ b/test/extensions/filters/http/wasm/test_data/test_cpp.cc @@ -280,7 +280,7 @@ FilterHeadersStatus TestContext::onRequestHeaders(uint32_t, bool) { FilterTrailersStatus TestContext::onRequestTrailers(uint32_t) { auto request_trailer = getRequestTrailer("bogus-trailer"); - if (request_trailer && request_trailer->view() != "") { + if (request_trailer && !request_trailer->view().empty()) { logWarn("request bogus-trailer found"); } CHECK_RESULT(replaceRequestTrailer("new-trailer", "value")); @@ -288,7 +288,7 @@ FilterTrailersStatus TestContext::onRequestTrailers(uint32_t) { // Not available yet. replaceResponseTrailer("new-trailer", "value"); auto response_trailer = getResponseTrailer("bogus-trailer"); - if (response_trailer && response_trailer->view() != "") { + if (response_trailer && !response_trailer->view().empty()) { logWarn("request bogus-trailer found"); } return FilterTrailersStatus::Continue; @@ -305,7 +305,7 @@ FilterHeadersStatus TestContext::onResponseHeaders(uint32_t, bool) { FilterTrailersStatus TestContext::onResponseTrailers(uint32_t) { auto value = getResponseTrailer("bogus-trailer"); - if (value && value->view() != "") { + if (value && !value->view().empty()) { logWarn("response bogus-trailer found"); } CHECK_RESULT(replaceResponseTrailer("new-trailer", "value")); @@ -362,15 +362,15 @@ void TestContext::onLog() { logWarn("onLog " + std::to_string(id()) + " " + std::string(path->view()) + " " + std::string(status->view())); auto response_header = getResponseHeader("bogus-header"); - if (response_header && response_header->view() != "") { + if (response_header && !response_header->view().empty()) { logWarn("response bogus-header found"); } auto response_trailer = getResponseTrailer("bogus-trailer"); - if (response_trailer && response_trailer->view() != "") { + if (response_trailer && !response_trailer->view().empty()) { logWarn("response bogus-trailer found"); } auto request_trailer = getRequestTrailer("error-details"); - if (request_trailer && request_trailer->view() != "") { + if (request_trailer && !request_trailer->view().empty()) { logWarn("request bogus-trailer found"); } } else if (test == "cluster_metadata") { @@ -537,7 +537,7 @@ void TestContext::onLog() { // validate null field std::string b; - if (!getValue({"protobuf_state", "b"}, &b) || b != "") { + if (!getValue({"protobuf_state", "b"}, &b) || !b.empty()) { logWarn("null field returned " + b); } @@ -618,16 +618,16 @@ void TestContext::onDone() { } void TestRootContext::onTick() { - if (test_ == "headers") { + if (test_ == "headers") { // NOLINT(clang-analyzer-optin.portability.UnixAPI) getContext(stream_context_id_)->setEffectiveContext(); replaceRequestHeader("server", "envoy-wasm-continue"); continueRequest(); - if (getBufferBytes(WasmBufferType::PluginConfiguration, 0, 1)->view() != "") { + if (!getBufferBytes(WasmBufferType::PluginConfiguration, 0, 1)->view().empty()) { logDebug("unexpectd success of getBufferBytes PluginConfiguration"); } - } else if (test_ == "metadata") { + } else if (test_ == "metadata") { // NOLINT(clang-analyzer-optin.portability.UnixAPI) std::string value; - if (!getValue({"node", "metadata", "wasm_node_get_key"}, &value)) { + if (!getValue({"node", "metadata", "wasm_node_get_key"}, &value)) { // NOLINT(clang-analyzer-optin.portability.UnixAPI) logDebug("missing node metadata"); } logDebug(std::string("onTick ") + value); diff --git a/test/extensions/filters/http/wasm/test_data/test_grpc_call_cpp.cc b/test/extensions/filters/http/wasm/test_data/test_grpc_call_cpp.cc index 6de2f6fa268d..de21031728c2 100644 --- a/test/extensions/filters/http/wasm/test_data/test_grpc_call_cpp.cc +++ b/test/extensions/filters/http/wasm/test_data/test_grpc_call_cpp.cc @@ -13,7 +13,7 @@ START_WASM_PLUGIN(HttpWasmTestCpp) class MyGrpcCallHandler : public GrpcCallHandler { public: - MyGrpcCallHandler() : GrpcCallHandler() {} + MyGrpcCallHandler() = default; void onSuccess(size_t body_size) override { if (call_done_) { proxy_done(); diff --git a/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc b/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc index a7bd41884a7f..bb8fb690c8b8 100644 --- a/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc +++ b/test/extensions/filters/http/wasm/test_data/test_grpc_stream_cpp.cc @@ -31,7 +31,7 @@ static RegisterContextFactory register_GrpcStreamContextProto(CONTEXT_FACTORY(Gr class MyGrpcStreamHandler : public GrpcStreamHandler { public: - MyGrpcStreamHandler() : GrpcStreamHandler() {} + MyGrpcStreamHandler() = default; void onReceiveInitialMetadata(uint32_t) override { auto h = getHeaderMapValue(WasmHeaderMapType::GrpcReceiveInitialMetadata, "test"); if (h->view() == "reset") { diff --git a/test/extensions/filters/http/wasm/wasm_filter_test.cc b/test/extensions/filters/http/wasm/wasm_filter_test.cc index 5ca67af6eb6f..1cffc3a67b8c 100644 --- a/test/extensions/filters/http/wasm/wasm_filter_test.cc +++ b/test/extensions/filters/http/wasm/wasm_filter_test.cc @@ -678,8 +678,7 @@ TEST_P(WasmHttpFilterTest, AccessLog) { StreamInfo::MockStreamInfo log_stream_info; EXPECT_CALL(log_stream_info, requestComplete()) .WillRepeatedly(testing::Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, &response_headers, &response_trailers, log_stream_info, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers, &response_headers, &response_trailers}, log_stream_info); filter().onDestroy(); } @@ -698,8 +697,7 @@ TEST_P(WasmHttpFilterTest, AccessLogClientDisconnected) { StreamInfo::MockStreamInfo log_stream_info; EXPECT_CALL(log_stream_info, requestComplete()) .WillRepeatedly(testing::Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, nullptr, nullptr, log_stream_info, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers}, log_stream_info); filter().onDestroy(); } @@ -716,8 +714,7 @@ TEST_P(WasmHttpFilterTest, AccessLogDisabledForIncompleteStream) { StreamInfo::MockStreamInfo log_stream_info; EXPECT_CALL(log_stream_info, requestComplete()).WillRepeatedly(testing::Return(absl::nullopt)); - filter().log(&request_headers, nullptr, nullptr, log_stream_info, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers}, log_stream_info); filter().onDestroy(); } @@ -733,8 +730,7 @@ TEST_P(WasmHttpFilterTest, AccessLogCreate) { Http::TestRequestHeaderMapImpl request_headers{{":path", "/"}}; Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; Http::TestResponseTrailerMapImpl response_trailers{}; - filter().log(&request_headers, &response_headers, &response_trailers, log_stream_info, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers, &response_headers, &response_trailers}, log_stream_info); filter().onDestroy(); } @@ -1763,8 +1759,7 @@ TEST_P(WasmHttpFilterTest, Metadata) { Buffer::OwnedImpl data("hello"); EXPECT_EQ(Http::FilterDataStatus::Continue, filter().decodeData(data, true)); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers}, request_stream_info_); const auto* result = request_stream_info_.filterState()->getDataReadOnly( @@ -1831,8 +1826,7 @@ TEST_P(WasmHttpFilterTest, Property) { request_stream_info_.upstreamInfo()->setUpstreamHost(host_description); EXPECT_CALL(request_stream_info_, requestComplete()) .WillRepeatedly(Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers}, request_stream_info_); } TEST_P(WasmHttpFilterTest, ClusterMetadata) { @@ -1863,16 +1857,14 @@ TEST_P(WasmHttpFilterTest, ClusterMetadata) { request_stream_info_.upstreamInfo()->setUpstreamHost(host_description); EXPECT_CALL(request_stream_info_, requestComplete) .WillRepeatedly(Return(std::chrono::milliseconds(30))); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers}, request_stream_info_); // If upstream host is empty, fallback to upstream cluster info for cluster metadata. request_stream_info_.upstreamInfo()->setUpstreamHost(nullptr); EXPECT_CALL(request_stream_info_, upstreamClusterInfo()).WillRepeatedly(Return(cluster)); EXPECT_CALL(filter(), log_(spdlog::level::warn, Eq(absl::string_view("cluster metadata: cluster")))); - filter().log(&request_headers, nullptr, nullptr, request_stream_info_, - AccessLog::AccessLogType::NotSet); + filter().log({&request_headers}, request_stream_info_); } TEST_P(WasmHttpFilterTest, SharedData) { diff --git a/test/extensions/filters/listener/common/fuzz/BUILD b/test/extensions/filters/listener/common/fuzz/BUILD index 3e21c9b2600d..95035e090d24 100644 --- a/test/extensions/filters/listener/common/fuzz/BUILD +++ b/test/extensions/filters/listener/common/fuzz/BUILD @@ -22,8 +22,8 @@ envoy_cc_test_library( ":listener_filter_fakes", ":listener_filter_fuzzer_proto_cc_proto", "//envoy/network:filter_interface", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:connection_balancer_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/network:network_mocks", "//test/test_common:network_utility_lib", "//test/test_common:threadsafe_singleton_injector_lib", diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc index 45605699b1c2..428577783279 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc @@ -34,7 +34,8 @@ ListenerFilterWithDataFuzzer::ListenerFilterWithDataFuzzer() Network::Test::getCanonicalLoopbackAddress(Network::Address::IpVersion::v4))), connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), - init_manager_(nullptr) { + init_manager_(nullptr), + listener_info_(std::make_shared>()) { socket_factories_.emplace_back(std::make_unique()); EXPECT_CALL(*static_cast(socket_factories_[0].get()), socketType()) @@ -45,7 +46,7 @@ ListenerFilterWithDataFuzzer::ListenerFilterWithDataFuzzer() EXPECT_CALL(*static_cast(socket_factories_[0].get()), getListenSocket(_)) .WillOnce(Return(socket_)); - connection_handler_->addListener(absl::nullopt, *this, runtime_); + connection_handler_->addListener(absl::nullopt, *this, runtime_, random_); conn_ = dispatcher_->createClientConnection( socket_->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr, nullptr); diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h index 91487a97ee59..f72a794a65df 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h @@ -2,11 +2,12 @@ #include "envoy/network/filter.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/connection_balancer_impl.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" #include "test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h" #include "test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.pb.validate.h" +#include "test/mocks/common.h" #include "test/mocks/event/mocks.h" #include "test/mocks/network/mocks.h" #include "test/test_common/network_utility.h" @@ -60,14 +61,10 @@ class ListenerFilterWithDataFuzzer : public Network::ListenerConfig, uint64_t listenerTag() const override { return 1; } ResourceLimit& openConnections() override { return open_connections_; } const std::string& name() const override { return name_; } - Network::UdpListenerConfigOptRef udpListenerConfig() override { - return Network::UdpListenerConfigOptRef(); - } - Network::InternalListenerConfigOptRef internalListenerConfig() override { - return Network::InternalListenerConfigOptRef(); - } - envoy::config::core::v3::TrafficDirection direction() const override { - return envoy::config::core::v3::UNSPECIFIED; + Network::UdpListenerConfigOptRef udpListenerConfig() override { return {}; } + Network::InternalListenerConfigOptRef internalListenerConfig() override { return {}; } + const Network::ListenerInfoConstSharedPtr& listenerInfo() const override { + return listener_info_; } Network::ConnectionBalancer& connectionBalancer(const Network::Address::Instance&) override { return connection_balancer_; @@ -101,6 +98,7 @@ class ListenerFilterWithDataFuzzer : public Network::ListenerConfig, void disconnect(); testing::NiceMock runtime_; + testing::NiceMock random_; Stats::TestUtil::TestStore stats_store_; Api::ApiPtr api_; BasicResourceLimitImpl open_connections_; @@ -118,6 +116,7 @@ class ListenerFilterWithDataFuzzer : public Network::ListenerConfig, const std::vector empty_access_logs_; std::unique_ptr init_manager_; bool connection_established_{}; + const Network::ListenerInfoConstSharedPtr listener_info_; }; } // namespace ListenerFilters diff --git a/test/extensions/filters/listener/original_dst/BUILD b/test/extensions/filters/listener/original_dst/BUILD index dfe51cca02f8..5d2c2a4ac1a9 100644 --- a/test/extensions/filters/listener/original_dst/BUILD +++ b/test/extensions/filters/listener/original_dst/BUILD @@ -39,3 +39,16 @@ envoy_cc_test( "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "original_dst_test", + srcs = ["original_dst_test.cc"], + deps = [ + "//source/common/network:filter_state_dst_address_lib", + "//source/common/network:listener_filter_buffer_lib", + "//source/common/network:utility_lib", + "//source/extensions/filters/listener/original_dst:original_dst_lib", + "//test/mocks/network:network_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/listener/original_dst/original_dst_test.cc b/test/extensions/filters/listener/original_dst/original_dst_test.cc new file mode 100644 index 000000000000..d79d5f791a6c --- /dev/null +++ b/test/extensions/filters/listener/original_dst/original_dst_test.cc @@ -0,0 +1,105 @@ +#include "source/common/network/filter_state_dst_address.h" +#include "source/common/network/listener_filter_buffer_impl.h" +#include "source/common/network/utility.h" +#include "source/extensions/filters/listener/original_dst/original_dst.h" + +#include "test/mocks/network/mocks.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace OriginalDst { +namespace { + +using testing::Return; +using testing::ReturnRef; + +class OriginalDstTest : public testing::Test { +public: + OriginalDstTest() : filter_(envoy::config::core::v3::TrafficDirection::OUTBOUND) { + EXPECT_CALL(cb_, socket()).WillRepeatedly(ReturnRef(socket_)); + } + + void expectInternalAddress() { + EXPECT_CALL(socket_, addressType()) + .WillRepeatedly(Return(Network::Address::Type::EnvoyInternal)); + EXPECT_CALL(cb_, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata_)); + EXPECT_CALL(cb_, filterState()).Times(testing::AtLeast(1)); + } + + OriginalDstFilter filter_; + Network::MockListenerFilterCallbacks cb_; + Network::MockConnectionSocket socket_; + envoy::config::core::v3::Metadata metadata_; +}; + +TEST_F(OriginalDstTest, Pipe) { + EXPECT_EQ(0, filter_.maxReadBytes()); + EXPECT_CALL(socket_, addressType()).WillRepeatedly(Return(Network::Address::Type::Pipe)); + filter_.onAccept(cb_); + EXPECT_FALSE(socket_.connectionInfoProvider().localAddressRestored()); + + NiceMock io_handle; + NiceMock dispatcher; + Network::ListenerFilterBufferImpl buffer( + io_handle, dispatcher, [](bool) {}, [](Network::ListenerFilterBuffer&) {}, 1); + EXPECT_EQ(Network::FilterStatus::Continue, filter_.onData(buffer)); +} + +TEST_F(OriginalDstTest, InternalNone) { + expectInternalAddress(); + filter_.onAccept(cb_); + EXPECT_FALSE(socket_.connectionInfoProvider().localAddressRestored()); +} + +TEST_F(OriginalDstTest, InternalDynamicMetadata) { + expectInternalAddress(); + TestUtility::loadFromYaml(R"EOF( + filter_metadata: + envoy.filters.listener.original_dst: + local: 127.0.0.1:8080 + )EOF", + metadata_); + filter_.onAccept(cb_); + EXPECT_TRUE(socket_.connectionInfoProvider().localAddressRestored()); + EXPECT_EQ("127.0.0.1:8080", socket_.connectionInfoProvider().localAddress()->asString()); +} + +TEST_F(OriginalDstTest, InternalDynamicMetadataFailure) { + expectInternalAddress(); + TestUtility::loadFromYaml(R"EOF( + filter_metadata: + envoy.filters.listener.original_dst: + local: aaabb + )EOF", + metadata_); + filter_.onAccept(cb_); + EXPECT_FALSE(socket_.connectionInfoProvider().localAddressRestored()); +} + +TEST_F(OriginalDstTest, InternalFilterState) { + expectInternalAddress(); + const auto local = Network::Utility::parseInternetAddress("10.20.30.40", 456, false); + const auto remote = Network::Utility::parseInternetAddress("127.0.0.1", 8000, false); + cb_.filter_state_.setData("envoy.filters.listener.original_dst.local_ip", + std::make_shared(local), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + cb_.filter_state_.setData("envoy.filters.listener.original_dst.remote_ip", + std::make_shared(remote), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + filter_.onAccept(cb_); + EXPECT_TRUE(socket_.connectionInfoProvider().localAddressRestored()); + EXPECT_EQ(local->asString(), socket_.connectionInfoProvider().localAddress()->asString()); + EXPECT_EQ(remote->asString(), socket_.connectionInfoProvider().remoteAddress()->asString()); +} + +} // namespace +} // namespace OriginalDst +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/proxy_protocol/BUILD b/test/extensions/filters/listener/proxy_protocol/BUILD index e82aca1a38d9..0babe857b97b 100644 --- a/test/extensions/filters/listener/proxy_protocol/BUILD +++ b/test/extensions/filters/listener/proxy_protocol/BUILD @@ -1,6 +1,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_fuzz_test", + "envoy_cc_test_library", "envoy_package", "envoy_proto_library", ) @@ -21,13 +22,13 @@ envoy_extension_cc_test( "//source/common/buffer:buffer_lib", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:listener_lib", "//source/common/network:utility_lib", "//source/common/stats:stats_lib", "//source/extensions/filters/listener/proxy_protocol:config", "//source/extensions/filters/listener/proxy_protocol:proxy_protocol_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/api:api_mocks", "//test/mocks/buffer:buffer_mocks", "//test/mocks/network:network_mocks", @@ -60,15 +61,19 @@ envoy_cc_fuzz_test( ], ) +envoy_cc_test_library( + name = "proxy_proto_integration_test_lib", + hdrs = ["proxy_proto_integration_test.h"], + deps = ["@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto"], +) + envoy_extension_cc_test( name = "proxy_proto_integration_test", size = "large", - srcs = [ - "proxy_proto_integration_test.cc", - "proxy_proto_integration_test.h", - ], + srcs = ["proxy_proto_integration_test.cc"], extension_names = ["envoy.filters.listener.proxy_protocol"], deps = [ + ":proxy_proto_integration_test_lib", "//source/common/buffer:buffer_lib", "//source/common/http:codec_client_lib", "//source/extensions/access_loggers/file:config", diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc index 9aaf9fee88d2..cc365938b0ef 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc @@ -238,7 +238,7 @@ TEST_P(ProxyProtoTcpIntegrationTest, AccessLog) { std::string log_result; // Access logs only get flushed to disk periodically, so poll until the log is non-empty do { - log_result = api_->fileSystem().fileReadToEnd(access_log_path); + log_result = api_->fileSystem().fileReadToEnd(access_log_path).value(); } while (log_result.empty()); EXPECT_EQ(log_result, "remote=1.2.3.4:12345 local=254.254.254.254:1234"); diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index 38efd6256d4c..af16be2cff70 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -10,6 +10,7 @@ #include "source/common/api/os_sys_calls_impl.h" #include "source/common/buffer/buffer_impl.h" #include "source/common/event/dispatcher_impl.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/listen_socket_impl.h" #include "source/common/network/proxy_protocol_filter_state.h" @@ -17,7 +18,6 @@ #include "source/common/network/tcp_listener_impl.h" #include "source/common/network/utility.h" #include "source/extensions/filters/listener/proxy_protocol/proxy_protocol.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" #include "test/mocks/api/mocks.h" #include "test/mocks/buffer/mocks.h" @@ -65,7 +65,8 @@ class ProxyProtocolTest : public testing::TestWithParam>()) { socket_factories_.emplace_back(std::make_unique()); EXPECT_CALL(*static_cast(socket_factories_[0].get()), socketType()) @@ -76,7 +77,7 @@ class ProxyProtocolTest : public testing::TestWithParam(socket_factories_[0].get()), getListenSocket(_)) .WillOnce(Return(socket_)); - connection_handler_->addListener(absl::nullopt, *this, runtime_); + connection_handler_->addListener(absl::nullopt, *this, runtime_, random_); conn_ = dispatcher_->createClientConnection(socket_->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr, @@ -99,14 +100,10 @@ class ProxyProtocolTest : public testing::TestWithParam runtime_; + testing::NiceMock random_; Stats::TestUtil::TestStore stats_store_; Api::ApiPtr api_; BasicResourceLimitImpl open_connections_; @@ -230,6 +228,7 @@ class ProxyProtocolTest : public testing::TestWithParam empty_access_logs_; std::unique_ptr init_manager_; + const Network::ListenerInfoConstSharedPtr listener_info_; }; // Parameterize the listener socket address version. @@ -1973,7 +1972,8 @@ class WildcardProxyProtocolTest : public testing::TestWithParamconnectionInfoProvider().localAddress()->ip()->port())), connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), - init_manager_(nullptr) { + init_manager_(nullptr), + listener_info_(std::make_shared>()) { socket_factories_.emplace_back(std::make_unique()); EXPECT_CALL(*static_cast(socket_factories_[0].get()), socketType()) @@ -1984,7 +1984,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParam(socket_factories_[0].get()), getListenSocket(_)) .WillOnce(Return(socket_)); - connection_handler_->addListener(absl::nullopt, *this, runtime_); + connection_handler_->addListener(absl::nullopt, *this, runtime_, random_); conn_ = dispatcher_->createClientConnection( local_dst_address_, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr, nullptr); @@ -2016,14 +2016,10 @@ class WildcardProxyProtocolTest : public testing::TestWithParam runtime_; + testing::NiceMock random_; Stats::IsolatedStoreImpl stats_store_; Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; @@ -2107,6 +2104,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParam empty_access_logs_; std::unique_ptr init_manager_; + const Network::ListenerInfoConstSharedPtr listener_info_; }; // Parameterize the listener socket address version. diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc index 70fdd0dea880..e3ff14c9654b 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc @@ -65,7 +65,7 @@ class FastMockOsSysCalls : public Api::MockOsSysCalls { const std::vector client_hello_; }; -static void BM_TlsInspector(benchmark::State& state) { +static void bmTlsInspector(benchmark::State& state) { NiceMock os_sys_calls(Tls::Test::generateClientHello( Config::TLS_MIN_SUPPORTED_VERSION, Config::TLS_MAX_SUPPORTED_VERSION, "example.com", "\x02h2\x08http/1.1")); @@ -100,7 +100,7 @@ static void BM_TlsInspector(benchmark::State& state) { } } -BENCHMARK(BM_TlsInspector)->Unit(benchmark::kMicrosecond); +BENCHMARK(bmTlsInspector)->Unit(benchmark::kMicrosecond); } // namespace TlsInspector } // namespace ListenerFilters diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc index 0bd45b131c53..8dffc9ddc05e 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc @@ -22,7 +22,9 @@ using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; +#ifdef WIN32 using testing::Return; +#endif using testing::ReturnNew; using testing::ReturnRef; using testing::SaveArg; diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/hcm_header_value_extractor_key_name b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/hcm_header_value_extractor_key_name new file mode 100644 index 000000000000..dfa5b48243c1 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/hcm_header_value_extractor_key_name @@ -0,0 +1,12 @@ +config { + name: "envoy.filters.network.http_connection_manager" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + value: "\022\002--B\002\020\001\250\001\001\372\001\310\001\n\001Z\022\257\001\n\254\001\n\251\001\n\240\001$config {\n name: \"envoy.filters.network.http_connection_manager\"\n typed_config {\n type_url: \"type.googleapis.com/envoy.extensions.filters.network.http_conn\022\002T+\030&\"\021\n\017\n\001e\032\010\n\006\n\004$v21*\000\350\002\004\270\003\001" + } +} +actions { + on_data { + data: "PUT /2.0nnnnnn\n\n\n\n\022r\n" + } +} diff --git a/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc b/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc index f697e8964060..f324641b5fbe 100644 --- a/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc +++ b/test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc @@ -54,7 +54,7 @@ void UberFilterFuzzer::perFilterSetup(const std::string& filter_name) { return async_request_.get(); }))); - ON_CALL(factory_context_.cluster_manager_.async_client_manager_, + ON_CALL(factory_context_.server_factory_context_.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClient(_, _, _)) .WillByDefault(Invoke([&](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { return async_client_; })); @@ -84,7 +84,7 @@ void UberFilterFuzzer::perFilterSetup(const std::string& filter_name) { return async_request_.get(); }))); - ON_CALL(factory_context_.cluster_manager_.async_client_manager_, + ON_CALL(factory_context_.server_factory_context_.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClient(_, _, _)) .WillByDefault(Invoke([&](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { return async_client_; })); @@ -139,7 +139,7 @@ void UberFilterFuzzer::checkInvalidInputForFuzzer(const std::string& filter_name // Sanity check on connection_keepalive interval and timeout. try { PROTOBUF_GET_MS_REQUIRED(config.http2_protocol_options().connection_keepalive(), interval); - } catch (const DurationUtil::OutOfRangeException& e) { + } catch (const EnvoyException& e) { throw EnvoyException( absl::StrCat("In http2_protocol_options.connection_keepalive interval shall not be " "negative. Exception {}", @@ -147,7 +147,7 @@ void UberFilterFuzzer::checkInvalidInputForFuzzer(const std::string& filter_name } try { PROTOBUF_GET_MS_REQUIRED(config.http2_protocol_options().connection_keepalive(), timeout); - } catch (const DurationUtil::OutOfRangeException& e) { + } catch (const EnvoyException& e) { throw EnvoyException( absl::StrCat("In http2_protocol_options.connection_keepalive timeout shall not be " "negative. Exception {}", diff --git a/test/extensions/filters/network/common/fuzz/uber_readfilter.cc b/test/extensions/filters/network/common/fuzz/uber_readfilter.cc index 819a36b1151e..98d54be9a556 100644 --- a/test/extensions/filters/network/common/fuzz/uber_readfilter.cc +++ b/test/extensions/filters/network/common/fuzz/uber_readfilter.cc @@ -27,7 +27,7 @@ void UberFilterFuzzer::reset() { Event::MockDispatcher& mock_dispatcher = dynamic_cast(read_filter_callbacks_->connection_.dispatcher_); mock_dispatcher.clearDeferredDeleteList(); - factory_context_.admin_.config_tracker_.config_tracker_callbacks_.clear(); + factory_context_.server_factory_context_.admin_.config_tracker_.config_tracker_callbacks_.clear(); read_filter_.reset(); } @@ -62,22 +62,24 @@ void UberFilterFuzzer::fuzzerSetup() { async_request_ = std::make_unique(); // Set featureEnabled for mongo_proxy - ON_CALL(factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.proxy_enabled", 100)) + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled("mongo.proxy_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.connection_logging_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.logging_enabled", 100)) + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled("mongo.logging_enabled", 100)) .WillByDefault(Return(true)); // Set featureEnabled for thrift_proxy - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.thrift_filter_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.thrift_filter_enforcing", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.test_key.thrift_filter_enabled", 100)) .WillByDefault(Return(true)); @@ -85,8 +87,9 @@ void UberFilterFuzzer::fuzzerSetup() { // TODO(mattklein123): This should not be required and we should be able to test different // variations here, however the fuzz test fails without this change and the overall change is // large enough as it is so this will be revisited. - ON_CALL(factory_context_.cluster_manager_, getThreadLocalCluster(_)) - .WillByDefault(Return(&factory_context_.cluster_manager_.thread_local_cluster_)); + ON_CALL(factory_context_.server_factory_context_.cluster_manager_, getThreadLocalCluster(_)) + .WillByDefault( + Return(&factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_)); } UberFilterFuzzer::UberFilterFuzzer() : time_source_(factory_context_.simulatedTimeSystem()) { @@ -140,7 +143,8 @@ void UberFilterFuzzer::fuzz( case test::extensions::filters::network::Action::kAdvanceTime: { time_source_.advanceTimeAndRun( std::chrono::milliseconds(action.advance_time().milliseconds()), - factory_context_.mainThreadDispatcher(), Event::Dispatcher::RunType::NonBlock); + factory_context_.server_factory_context_.mainThreadDispatcher(), + Event::Dispatcher::RunType::NonBlock); break; } default: { diff --git a/test/extensions/filters/network/common/fuzz/uber_writefilter.cc b/test/extensions/filters/network/common/fuzz/uber_writefilter.cc index 941d5b121acc..03451eb23a30 100644 --- a/test/extensions/filters/network/common/fuzz/uber_writefilter.cc +++ b/test/extensions/filters/network/common/fuzz/uber_writefilter.cc @@ -45,22 +45,24 @@ void UberWriteFilterFuzzer::fuzzerSetup() { factory_context_.prepareSimulatedSystemTime(); // Set featureEnabled for mongo_proxy - ON_CALL(factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.proxy_enabled", 100)) + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled("mongo.proxy_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.connection_logging_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.logging_enabled", 100)) + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled("mongo.logging_enabled", 100)) .WillByDefault(Return(true)); // Set featureEnabled for thrift_proxy - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.thrift_filter_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.thrift_filter_enforcing", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context_.runtime_loader_.snapshot_, + ON_CALL(factory_context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.test_key.thrift_filter_enabled", 100)) .WillByDefault(Return(true)); } @@ -103,7 +105,8 @@ void UberWriteFilterFuzzer::fuzz( case test::extensions::filters::network::WriteAction::kAdvanceTime: { time_source_.advanceTimeAndRun( std::chrono::milliseconds(action.advance_time().milliseconds()), - factory_context_.mainThreadDispatcher(), Event::Dispatcher::RunType::NonBlock); + factory_context_.server_factory_context_.mainThreadDispatcher(), + Event::Dispatcher::RunType::NonBlock); break; } default: { diff --git a/test/extensions/filters/network/common/fuzz/utils/BUILD b/test/extensions/filters/network/common/fuzz/utils/BUILD index 6c231c2a185f..4fd962cee805 100644 --- a/test/extensions/filters/network/common/fuzz/utils/BUILD +++ b/test/extensions/filters/network/common/fuzz/utils/BUILD @@ -12,6 +12,7 @@ envoy_cc_test_library( name = "network_filter_fuzzer_fakes_lib", hdrs = ["fakes.h"], deps = [ + "//source/common/config:metadata_lib", "//test/mocks/server:factory_context_mocks", ], ) diff --git a/test/extensions/filters/network/common/fuzz/utils/fakes.h b/test/extensions/filters/network/common/fuzz/utils/fakes.h index 4a4e134867af..152381c9cbe8 100644 --- a/test/extensions/filters/network/common/fuzz/utils/fakes.h +++ b/test/extensions/filters/network/common/fuzz/utils/fakes.h @@ -1,44 +1,55 @@ #pragma once +#include "source/common/config/metadata.h" + #include "test/mocks/server/factory_context.h" namespace Envoy { namespace Server { namespace Configuration { -class FakeFactoryContext : public MockFactoryContext { + +class FakeListenerInfo : public Network::ListenerInfo { public: - void prepareSimulatedSystemTime() { - api_ = Api::createApiForTest(time_system_); - dispatcher_ = api_->allocateDispatcher("test_thread"); + const envoy::config::core::v3::Metadata& metadata() const override { + return metadata_.proto_metadata_; } - AccessLog::AccessLogManager& accessLogManager() override { return access_log_manager_; } - Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } - Event::Dispatcher& mainThreadDispatcher() override { return *dispatcher_; } + const Envoy::Config::TypedMetadata& typedMetadata() const override { + return metadata_.typed_metadata_; + } + envoy::config::core::v3::TrafficDirection direction() const override { + return envoy::config::core::v3::UNSPECIFIED; + } + bool isQuic() const override { return false; } + +private: + Envoy::Config::MetadataPack metadata_; +}; + +class FakeFactoryContext : public MockFactoryContext { +public: const Network::DrainDecision& drainDecision() override { return drain_manager_; } Init::Manager& initManager() override { return init_manager_; } - ServerLifecycleNotifier& lifecycleNotifier() override { return lifecycle_notifier_; } - const LocalInfo::LocalInfo& localInfo() const override { return local_info_; } - Envoy::Runtime::Loader& runtime() override { return runtime_loader_; } Stats::Scope& scope() override { return scope_; } - Singleton::Manager& singletonManager() override { return *singleton_manager_; } - ThreadLocal::Instance& threadLocal() override { return thread_local_; } - OptRef admin() override { return admin_; } Stats::Scope& listenerScope() override { return listener_scope_; } - Api::Api& api() override { return *api_; } - TimeSource& timeSource() override { return time_system_; } - OverloadManager& overloadManager() override { return overload_manager_; } - ProtobufMessage::ValidationContext& messageValidationContext() override { - return validation_context_; - } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + ProtobufMessage::ValidationVisitor& messageValidationVisitor() const override { return ProtobufMessage::getStrictValidationVisitor(); } + const Network::ListenerInfo& listenerInfo() const override { return listener_info_; } + + void prepareSimulatedSystemTime() { + api_ = Api::createApiForTest(time_system_); + dispatcher_ = api_->allocateDispatcher("test_thread"); + + ON_CALL(server_factory_context_, timeSource()).WillByDefault(testing::ReturnRef(time_system_)); + ON_CALL(server_factory_context_, api()).WillByDefault(testing::ReturnRef(*api_)); + ON_CALL(server_factory_context_, mainThreadDispatcher()) + .WillByDefault(testing::ReturnRef(*dispatcher_)); + } Event::SimulatedTimeSystem& simulatedTimeSystem() { return dynamic_cast(time_system_); } - Event::TestTimeSystem& timeSystem() { return time_system_; } - Grpc::Context& grpcContext() override { return grpc_context_; } - Http::Context& httpContext() override { return http_context_; } + + FakeListenerInfo listener_info_; Event::DispatcherPtr dispatcher_; Event::SimulatedTimeSystem time_system_; diff --git a/test/extensions/filters/network/connection_limit/connection_limit_integration_test.cc b/test/extensions/filters/network/connection_limit/connection_limit_integration_test.cc index 23c55b125872..ea589aa50fb9 100644 --- a/test/extensions/filters/network/connection_limit/connection_limit_integration_test.cc +++ b/test/extensions/filters/network/connection_limit/connection_limit_integration_test.cc @@ -8,13 +8,7 @@ class ConnectionLimitIntegrationTest : public Event::TestUsingSimulatedTime, public BaseIntegrationTest { public: ConnectionLimitIntegrationTest() - : BaseIntegrationTest(GetParam(), ConfigHelper::tcpProxyConfig()) { - // TODO(ggreenway): add tag extraction rules. - // Missing stat tag-extraction rule for stat - // 'connection_limit.connection_limit_stats.limited_connections' and stat_prefix - // 'connection_limit_stats'. - skip_tag_extraction_rule_check_ = true; - } + : BaseIntegrationTest(GetParam(), ConfigHelper::tcpProxyConfig()) {} void setup(const std::string& filter_yaml) { config_helper_.addNetworkFilter(filter_yaml); diff --git a/test/extensions/filters/network/dubbo_proxy/config_test.cc b/test/extensions/filters/network/dubbo_proxy/config_test.cc index a6c7bcf71710..3cd2ba47fad8 100644 --- a/test/extensions/filters/network/dubbo_proxy/config_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/config_test.cc @@ -76,7 +76,7 @@ TEST_F(DubboFilterConfigTest, ValidProtoConfiguration) { NiceMock context; DubboProxyFilterConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, context); - EXPECT_TRUE(factory.isTerminalFilterByProto(config, context.getServerFactoryContext())); + EXPECT_TRUE(factory.isTerminalFilterByProto(config, context.serverFactoryContext())); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); cb(connection); @@ -206,8 +206,8 @@ version_info: "1" EXPECT_TRUE(context_.server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ ->onConfigUpdate(decoded_resources.refvec_, response.version_info()) .ok()); - auto message_ptr = context_.admin_.config_tracker_.config_tracker_callbacks_["drds_routes"]( - universal_name_matcher); + auto message_ptr = context_.server_factory_context_.admin_.config_tracker_ + .config_tracker_callbacks_["drds_routes"](universal_name_matcher); const auto& dump = TestUtility::downcastAndValidate(*message_ptr); EXPECT_EQ(1, dump.dynamic_route_configs().size()); diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index 2b6e7230a370..60650be44870 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -121,14 +121,16 @@ class ConnectionManagerTest : public testing::Test { : stats_(DubboFilterStats::generateStats("test.", *store_.rootScope())), engine_(std::make_unique()) { - route_config_provider_manager_ = - std::make_unique(factory_context_.admin_); + route_config_provider_manager_ = std::make_unique( + factory_context_.server_factory_context_.admin_); } ~ConnectionManagerTest() override { filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); } - TimeSource& timeSystem() { return factory_context_.mainThreadDispatcher().timeSource(); } + TimeSource& timeSystem() { + return factory_context_.server_factory_context_.mainThreadDispatcher().timeSource(); + } void initializeFilter() { initializeFilter(""); } diff --git a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc index 396cc35188cc..8a3ab4740e53 100644 --- a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc @@ -42,7 +42,7 @@ class DecoderStateMachineTestBase { context->setBodySize(body_size); metadata->setMessageType(type); - return std::pair(context, true); + return {context, true}; })); } @@ -177,7 +177,7 @@ TEST_F(DubboDecoderTest, NeedMoreDataForProtocolHeader) { EXPECT_CALL(protocol_, decodeHeader(_, _)) .WillOnce(Invoke( [](Buffer::Instance&, MessageMetadataSharedPtr) -> std::pair { - return std::pair(nullptr, false); + return {nullptr, false}; })); RequestDecoder decoder(protocol_, request_callbacks_); @@ -196,7 +196,7 @@ TEST_F(DubboDecoderTest, NeedMoreDataForProtocolBody) { auto context = std::make_shared(); context->setHeaderSize(16); context->setBodySize(10); - return std::pair(context, true); + return {context, true}; })); EXPECT_CALL(protocol_, decodeData(_, _, _)) .WillOnce(Invoke([&](Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr) -> bool { @@ -230,7 +230,7 @@ TEST_F(DubboDecoderTest, DecodeResponseMessage) { auto context = std::make_shared(); context->setHeaderSize(16); context->setBodySize(10); - return std::pair(context, true); + return {context, true}; })); EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(true)); EXPECT_CALL(response_callbacks_, newStream()).WillOnce(ReturnRef(handler_)); @@ -253,7 +253,7 @@ TEST_F(DubboDecoderTest, DecodeResponseMessage) { auto context = std::make_shared(); context->setHeaderSize(16); context->setBodySize(10); - return std::pair(context, true); + return {context, true}; })); EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(true)); EXPECT_CALL(response_callbacks_, newStream()).WillOnce(ReturnRef(handler_)); diff --git a/test/extensions/filters/network/dubbo_proxy/router_test.cc b/test/extensions/filters/network/dubbo_proxy/router_test.cc index c4bac322dabb..8f26e582a172 100644 --- a/test/extensions/filters/network/dubbo_proxy/router_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/router_test.cc @@ -94,7 +94,7 @@ class DubboRouterTestBase { return protocol_; }), serializer_register_(serializer_factory_), protocol_register_(protocol_factory_) { - context_.cluster_manager_.initializeThreadLocalClusters({"cluster"}); + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"cluster"}); } void verifyMetadataMatchCriteriaFromRequest(bool route_entry_has_match) { @@ -235,7 +235,7 @@ class DubboRouterTestBase { route_ = new NiceMock(); route_ptr_.reset(route_); - router_ = std::make_unique(context_.clusterManager()); + router_ = std::make_unique(context_.server_factory_context_.clusterManager()); EXPECT_EQ(nullptr, router_->downstreamConnection()); @@ -283,20 +283,23 @@ class DubboRouterTestBase { } void connectUpstream() { - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; })); conn_state_.reset(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); EXPECT_CALL(callbacks_, continueDecoding()); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); EXPECT_NE(nullptr, upstream_callbacks_); } @@ -308,7 +311,8 @@ class DubboRouterTestBase { initializeMetadata(msg_type); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; @@ -326,12 +330,15 @@ class DubboRouterTestBase { EXPECT_CALL(callbacks_, protocolType()).WillOnce(Return(ProtocolType::Dubbo)); EXPECT_CALL(callbacks_, continueDecoding()).Times(0); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); } @@ -351,8 +358,9 @@ class DubboRouterTestBase { EXPECT_CALL(callbacks_, upstreamData(Ref(buffer))) .WillOnce(Return(DubboFilters::UpstreamResponseStatus::Complete)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, - released(Ref(upstream_connection_))); + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + released(Ref(upstream_connection_))); upstream_callbacks_->onUpstreamData(buffer, false); } @@ -407,12 +415,12 @@ TEST_F(DubboRouterTest, PoolRemoteConnectionFailure) { })); startRequest(MessageType::Request); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); } TEST_F(DubboRouterTest, PoolLocalConnectionFailure) { @@ -427,12 +435,12 @@ TEST_F(DubboRouterTest, PoolLocalConnectionFailure) { })); startRequest(MessageType::Request); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::LocalConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::LocalConnectionFailure); } TEST_F(DubboRouterTest, PoolTimeout) { @@ -447,8 +455,8 @@ TEST_F(DubboRouterTest, PoolTimeout) { })); startRequest(MessageType::Request); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::Timeout); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::Timeout); } TEST_F(DubboRouterTest, PoolOverflowFailure) { @@ -463,8 +471,8 @@ TEST_F(DubboRouterTest, PoolOverflowFailure) { })); startRequest(MessageType::Request); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::Overflow); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::Overflow); } TEST_F(DubboRouterTest, ClusterMaintenanceMode) { @@ -474,7 +482,9 @@ TEST_F(DubboRouterTest, ClusterMaintenanceMode) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, maintenanceMode()) + EXPECT_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + maintenanceMode()) .WillOnce(Return(true)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) @@ -494,7 +504,8 @@ TEST_F(DubboRouterTest, NoHealthyHosts) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) @@ -509,7 +520,7 @@ TEST_F(DubboRouterTest, NoHealthyHosts) { } TEST_F(DubboRouterTest, PoolConnectionFailureWithOnewayMessage) { - context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); initializeRouter(); initializeMetadata(MessageType::Oneway); @@ -519,8 +530,8 @@ TEST_F(DubboRouterTest, PoolConnectionFailureWithOnewayMessage) { EXPECT_CALL(callbacks_, resetStream()); EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); destroyRouter(); } @@ -547,7 +558,8 @@ TEST_F(DubboRouterTest, NoCluster) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_, getThreadLocalCluster(Eq(cluster_name_))) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq(cluster_name_))) .WillOnce(Return(nullptr)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) .WillOnce(Invoke([&](const DubboFilters::DirectResponse& response, bool end_stream) -> void { @@ -669,8 +681,9 @@ TEST_F(DubboRouterTest, OneWay) { initializeRouter(); initializeMetadata(MessageType::Oneway); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, - released(Ref(upstream_connection_))); + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + released(Ref(upstream_connection_))); startRequest(MessageType::Oneway); connectUpstream(); @@ -790,7 +803,9 @@ TEST_F(DubboRouterTest, DestroyWhileConnecting) { initializeMetadata(MessageType::Request); NiceMock conn_pool_handle; - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce(Invoke([&](Tcp::ConnectionPool::Callbacks&) -> Tcp::ConnectionPool::Cancellable* { return &conn_pool_handle; })); @@ -833,9 +848,9 @@ TEST_F(DubboRouterTest, ResponseOk) { response_meta->setMessageType(MessageType::Response); response_meta->setResponseStatus(ResponseStatus::Ok); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); EXPECT_EQ(FilterStatus::Continue, router_->onMessageEncoded(response_meta, message_context_)); destroyRouter(); @@ -850,9 +865,9 @@ TEST_F(DubboRouterTest, ResponseException) { response_meta->setMessageType(MessageType::Exception); response_meta->setResponseStatus(ResponseStatus::Ok); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); EXPECT_EQ(FilterStatus::Continue, router_->onMessageEncoded(response_meta, message_context_)); destroyRouter(); @@ -867,9 +882,9 @@ TEST_F(DubboRouterTest, ResponseServerTimeout) { response_meta->setMessageType(MessageType::Response); response_meta->setResponseStatus(ResponseStatus::ServerTimeout); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginTimeout, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginTimeout, _)); EXPECT_EQ(FilterStatus::Continue, router_->onMessageEncoded(response_meta, message_context_)); destroyRouter(); @@ -884,9 +899,9 @@ TEST_F(DubboRouterTest, ResponseServerError) { response_meta->setMessageType(MessageType::Response); response_meta->setResponseStatus(ResponseStatus::ServiceError); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); EXPECT_EQ(FilterStatus::Continue, router_->onMessageEncoded(response_meta, message_context_)); destroyRouter(); diff --git a/test/extensions/filters/network/ext_authz/config_test.cc b/test/extensions/filters/network/ext_authz/config_test.cc index 1bd80d53d6ff..094736caab36 100644 --- a/test/extensions/filters/network/ext_authz/config_test.cc +++ b/test/extensions/filters/network/ext_authz/config_test.cc @@ -36,10 +36,8 @@ void expectCorrectProto() { TestUtility::loadFromYaml(yaml, *proto_config); NiceMock context; - testing::StrictMock server_context; - EXPECT_CALL(context, getServerFactoryContext()) - .WillRepeatedly(testing::ReturnRef(server_context)); - EXPECT_CALL(context.cluster_manager_.async_client_manager_, factoryForGrpcService(_, _, _)) + EXPECT_CALL(context.server_factory_context_.cluster_manager_.async_client_manager_, + factoryForGrpcService(_, _, _)) .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { return std::make_unique>(); })); @@ -52,9 +50,6 @@ void expectCorrectProto() { TEST(ExtAuthzFilterConfigTest, ValidateFail) { NiceMock context; - testing::StrictMock server_context; - EXPECT_CALL(context, getServerFactoryContext()) - .WillRepeatedly(testing::ReturnRef(server_context)); envoy::extensions::filters::network::ext_authz::v3::ExtAuthz config; config.set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); EXPECT_THROW(ExtAuthzConfigFactory().createFilterFactoryFromProto(config, context), diff --git a/test/extensions/filters/network/http_connection_manager/BUILD b/test/extensions/filters/network/http_connection_manager/BUILD index dcdd0ab23ef9..9cc6b5d4c323 100644 --- a/test/extensions/filters/network/http_connection_manager/BUILD +++ b/test/extensions/filters/network/http_connection_manager/BUILD @@ -20,7 +20,7 @@ envoy_proto_library( envoy_extension_cc_test_library( name = "config_test_base", - srcs = ["config_test_base.h"], + hdrs = ["config_test_base.h"], extension_names = ["envoy.filters.network.http_connection_manager"], deps = [ ":config_cc_proto", diff --git a/test/extensions/filters/network/http_connection_manager/config_filter_chain_test.cc b/test/extensions/filters/network/http_connection_manager/config_filter_chain_test.cc index dd22340f0b87..1c42d6a1f420 100644 --- a/test/extensions/filters/network/http_connection_manager/config_filter_chain_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_filter_chain_test.cc @@ -10,7 +10,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Eq; using testing::Return; namespace Envoy { @@ -56,6 +55,72 @@ TEST_F(FilterChainTest, CreateFilterChain) { config.createFilterChain(manager); } +TEST_F(FilterChainTest, CreateFilterChainWithDisabledFilter) { + const std::string config_yaml = R"EOF( +codec_type: http1 +server_name: foo +stat_prefix: router +route_config: + virtual_hosts: + - name: service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: cluster +http_filters: +- name: encoder-decoder-buffer-filter + disabled: true +- name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(basic_config_), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_, tracer_manager_, + filter_config_provider_manager_); + + NiceMock manager; + EXPECT_CALL(manager.callbacks_, addStreamDecoderFilter(_)); // Router + config.createFilterChain(manager); +} + +TEST_F(FilterChainTest, CreateFilterChainWithDisabledTerminalFilter) { + const std::string config_yaml = R"EOF( +codec_type: http1 +server_name: foo +stat_prefix: router +route_config: + virtual_hosts: + - name: service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: cluster +http_filters: +- name: encoder-decoder-buffer-filter +- name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + disabled: true + )EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromYaml(config_yaml), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_, tracer_manager_, + filter_config_provider_manager_), + EnvoyException, + "Error: the last (terminal) filter (envoy.filters.http.router) in the chain cannot be " + "disabled by default."); +} + TEST_F(FilterChainTest, CreateDynamicFilterChain) { const std::string yaml_string = R"EOF( codec_type: http1 diff --git a/test/extensions/filters/network/http_connection_manager/config_filter_dependencies_test.cc b/test/extensions/filters/network/http_connection_manager/config_filter_dependencies_test.cc index 597834fdd805..7d03a24e9fe4 100644 --- a/test/extensions/filters/network/http_connection_manager/config_filter_dependencies_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_filter_dependencies_test.cc @@ -15,7 +15,6 @@ using envoy::extensions::filters::common::dependency::v3::Dependency; using envoy::extensions::filters::common::dependency::v3::FilterDependencies; using Envoy::Server::Configuration::FilterDependenciesPtr; using Envoy::Server::Configuration::NamedHttpFilterConfigFactory; -using testing::Eq; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 3e87f787765d..1b13b76a00bc 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -181,7 +181,7 @@ stat_prefix: router filter_config_provider_manager_); EXPECT_EQ(128, config.tracingConfig()->max_path_tag_length_); - EXPECT_EQ(*context_.local_info_.address_, config.localAddress()); + EXPECT_EQ(*context_.server_factory_context_.local_info_.address_, config.localAddress()); EXPECT_EQ("foo", config.serverName()); EXPECT_EQ(HttpConnectionManagerConfig::HttpConnectionManagerProto::OVERWRITE, config.serverHeaderTransformation()); @@ -212,7 +212,7 @@ stat_prefix: router #ifdef ENVOY_ENABLE_QUIC { - EXPECT_CALL(context_, isQuicListener()).WillOnce(Return(false)); + EXPECT_CALL(listener_info_, isQuic()).WillOnce(Return(false)); EXPECT_THROW_WITH_MESSAGE( HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), @@ -222,7 +222,7 @@ stat_prefix: router EnvoyException, "HTTP/3 codec configured on non-QUIC listener."); } { - EXPECT_CALL(context_, isQuicListener()).WillOnce(Return(true)); + EXPECT_CALL(listener_info_, isQuic()).WillOnce(Return(true)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, scoped_routes_config_provider_manager_, tracer_manager_, @@ -259,7 +259,7 @@ stat_prefix: router "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_, isQuicListener()).WillOnce(Return(true)); + EXPECT_CALL(listener_info_, isQuic()).WillOnce(Return(true)); EXPECT_THROW_WITH_MESSAGE( HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -329,7 +329,7 @@ stat_prefix: router tracing_config.mutable_http()->set_name("zipkin"); tracing_config.mutable_http()->mutable_typed_config()->PackFrom( envoy::config::trace::v3::ZipkinConfig{}); - context_.http_context_.setDefaultTracingConfig(tracing_config); + context_.server_factory_context_.http_context_.setDefaultTracingConfig(tracing_config); // When tracing is not enabled on a given "envoy.filters.network.http_connection_manager" filter, // there is no reason to obtain an actual Tracer. @@ -409,7 +409,7 @@ tracing: {} # notice that tracing is enabled tracing_config.mutable_http()->set_name("zipkin"); tracing_config.mutable_http()->mutable_typed_config()->PackFrom( envoy::config::trace::v3::ZipkinConfig{}); - context_.http_context_.setDefaultTracingConfig(tracing_config); + context_.server_factory_context_.http_context_.setDefaultTracingConfig(tracing_config); // When tracing is enabled on a given "envoy.filters.network.http_connection_manager" filter, // an actual Tracer must be obtained from the HttpTracerManager. @@ -460,7 +460,7 @@ stat_prefix: router bootstrap_tracing_config.mutable_http()->set_name("opencensus"); bootstrap_tracing_config.mutable_http()->mutable_typed_config()->PackFrom( envoy::config::trace::v3::OpenCensusConfig{}); - context_.http_context_.setDefaultTracingConfig(bootstrap_tracing_config); + context_.server_factory_context_.http_context_.setDefaultTracingConfig(bootstrap_tracing_config); // Set up expected tracer provider configuration. envoy::config::trace::v3::Tracing_Http inlined_tracing_config; @@ -807,7 +807,7 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbMaxConfiguredViaRunti "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - ON_CALL(context_.runtime_loader_.snapshot_, + ON_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger("envoy.reloadable_features.max_request_headers_size_kb", 60)) .WillByDefault(Return(9000)); @@ -991,8 +991,9 @@ TEST_F(HttpConnectionManagerConfigTest, ServerOverwrite) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -1014,8 +1015,9 @@ TEST_F(HttpConnectionManagerConfigTest, ServerAppendIfAbsent) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -1037,8 +1039,9 @@ TEST_F(HttpConnectionManagerConfigTest, ServerPassThrough) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -1061,8 +1064,9 @@ TEST_F(HttpConnectionManagerConfigTest, SchemeOverwrite) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -1084,8 +1088,9 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathDefault) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -1110,10 +1115,11 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathRuntime) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("http_connection_manager.normalize_path", An())) .WillOnce(Return(true)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -1136,10 +1142,11 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathTrue) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("http_connection_manager.normalize_path", An())) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -1162,10 +1169,11 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathFalse) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("http_connection_manager.normalize_path", An())) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -1563,10 +1571,10 @@ stat_prefix: router auto proto_config = parseHttpConnectionManagerFromYaml(yaml_string); HttpConnectionManagerFilterConfigFactory factory; // We expect a single slot allocation vs. multiple. - EXPECT_CALL(context_.thread_local_, allocateSlot()); + EXPECT_CALL(context_.server_factory_context_.thread_local_, allocateSlot()); Network::FilterFactoryCb cb1 = factory.createFilterFactoryFromProto(proto_config, context_); Network::FilterFactoryCb cb2 = factory.createFilterFactoryFromProto(proto_config, context_); - EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context_.getServerFactoryContext())); + EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context_.serverFactoryContext())); } TEST_F(HttpConnectionManagerConfigTest, BadHttpConnectionMangerConfig) { @@ -2196,7 +2204,7 @@ class TestRequestIDExtensionFactory : public Server::Configuration::RequestIDExt Http::RequestIDExtensionSharedPtr createExtensionInstance(const Protobuf::Message& config, - Server::Configuration::CommonFactoryContext& context) override { + Server::Configuration::FactoryContext& context) override { const auto& custom_config = MessageUtil::downcastAndValidate< const test::http_connection_manager::CustomRequestIDExtension&>( config, context.messageValidationVisitor()); @@ -2806,11 +2814,12 @@ TEST_F(HttpConnectionManagerConfigTest, PathWithEscapedSlashesActionDefault) { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled(_, An())) .WillRepeatedly(Return(true)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger("http_connection_manager.path_with_escaped_slashes_action", 0)) .WillRepeatedly(Return(0)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -2834,11 +2843,12 @@ TEST_F(HttpConnectionManagerConfigTest, PathWithEscapedSlashesActionDefaultOverr "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled(_, An())) .WillRepeatedly(Return(true)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger("http_connection_manager.path_with_escaped_slashes_action", 0)) .WillRepeatedly(Return(3)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -2850,7 +2860,7 @@ TEST_F(HttpConnectionManagerConfigTest, PathWithEscapedSlashesActionDefaultOverr config.pathWithEscapedSlashesAction()); // Check the UNESCAPE_AND_FORWARD override to mollify coverage check - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger("http_connection_manager.path_with_escaped_slashes_action", 0)) .WillRepeatedly(Return(4)); HttpConnectionManagerConfig config1(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -2876,13 +2886,14 @@ TEST_F(HttpConnectionManagerConfigTest, "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled(_, An())) .WillRepeatedly(Return(true)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); // When configuration value is not the IMPLEMENTATION_SPECIFIC_DEFAULT the runtime override should // not even be considered. - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger("http_connection_manager.path_with_escaped_slashes_action", 0)) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -2907,15 +2918,17 @@ TEST_F(HttpConnectionManagerConfigTest, PathWithEscapedSlashesActionDefaultOverr "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router )EOF"; - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("http_connection_manager.path_with_escaped_slashes_action_enabled", An())) .WillRepeatedly(Return(false)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger("http_connection_manager.path_with_escaped_slashes_action", 0)) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, @@ -3035,10 +3048,12 @@ TEST_F(HttpConnectionManagerConfigTest, HeaderValidatorConfig) { TestHeaderValidatorFactoryConfig factory; Registry::InjectFactory registration(factory); - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); #ifdef ENVOY_ENABLE_UHV HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -3075,10 +3090,12 @@ TEST_F(HttpConnectionManagerConfigTest, HeaderValidatorConfigWithRuntimeDisabled TestHeaderValidatorFactoryConfig factory; Registry::InjectFactory registration(factory); - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); #ifdef ENVOY_ENABLE_UHV HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -3119,10 +3136,12 @@ TEST_F(HttpConnectionManagerConfigTest, DefaultHeaderValidatorConfigWithRuntimeE proto_config; DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory registration(factory); - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); #ifdef ENVOY_ENABLE_UHV HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -3166,10 +3185,12 @@ TEST_F(HttpConnectionManagerConfigTest, DefaultHeaderValidatorConfig) { proto_config; DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory registration(factory); - EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) - .WillRepeatedly(Invoke(&context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, + featureEnabled(_, An())) + .WillRepeatedly(Invoke(&context_.server_factory_context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, scoped_routes_config_provider_manager_, tracer_manager_, @@ -3207,10 +3228,11 @@ TEST_F(HttpConnectionManagerConfigTest, TranslateLegacyConfigToDefaultHeaderVali proto_config; DefaultHeaderValidatorFactoryConfigOverride factory(proto_config); Registry::InjectFactory registration(factory); - EXPECT_CALL(context_.runtime_loader_.snapshot_, + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, featureEnabled(_, An())) .WillRepeatedly(Return(true)); - EXPECT_CALL(context_.runtime_loader_.snapshot_, getInteger(_, _)).Times(AnyNumber()); + EXPECT_CALL(context_.server_factory_context_.runtime_loader_.snapshot_, getInteger(_, _)) + .Times(AnyNumber()); #ifdef ENVOY_ENABLE_UHV HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, date_provider_, route_config_provider_manager_, @@ -3244,9 +3266,10 @@ class HcmUtilityTest : public testing::Test { HcmUtilityTest() { // Although different Listeners will have separate FactoryContexts, // those contexts must share the same SingletonManager. - ON_CALL(context_two_, singletonManager()).WillByDefault([&]() -> Singleton::Manager& { - return *context_one_.singleton_manager_; - }); + ON_CALL(context_two_.server_factory_context_, singletonManager()) + .WillByDefault([this]() -> Singleton::Manager& { + return *context_one_.server_factory_context_.singleton_manager_; + }); } NiceMock context_one_; NiceMock context_two_; diff --git a/test/extensions/filters/network/http_connection_manager/config_test_base.h b/test/extensions/filters/network/http_connection_manager/config_test_base.h index 80ec91140313..0af8d4902e29 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test_base.h +++ b/test/extensions/filters/network/http_connection_manager/config_test_base.h @@ -37,9 +37,12 @@ class HttpConnectionManagerConfigTest : public testing::Test { HttpConnectionManagerConfigTest() { scoped_runtime_.mergeValues( {{"envoy.reloadable_features.no_extension_lookup_by_name", "false"}}); + ON_CALL(context_, listenerInfo()).WillByDefault(testing::ReturnRef(listener_info_)); } + NiceMock listener_info_; NiceMock context_; - Http::SlowDateProviderImpl date_provider_{context_.mainThreadDispatcher().timeSource()}; + Http::SlowDateProviderImpl date_provider_{ + context_.server_factory_context_.mainThreadDispatcher().timeSource()}; NiceMock route_config_provider_manager_; NiceMock scoped_routes_config_provider_manager_; NiceMock tracer_manager_; diff --git a/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc b/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc index 8c9799476365..ee00dbb61b25 100644 --- a/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc +++ b/test/extensions/filters/network/local_ratelimit/local_ratelimit_fuzz_test.cc @@ -35,11 +35,8 @@ DEFINE_PROTO_FUZZER( try { TestUtility::validate(input); - } catch (const ProtoValidationException& e) { - ENVOY_LOG_MISC(debug, "ProtoValidationException: {}", e.what()); - return; - } catch (const ProtobufMessage::DeprecatedProtoFieldException& e) { - ENVOY_LOG_MISC(debug, "DeprecatedProtoFieldException: {}", e.what()); + } catch (const EnvoyException& e) { + ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); return; } try { @@ -47,7 +44,7 @@ DEFINE_PROTO_FUZZER( ENVOY_LOG_MISC(debug, "In fill_interval, msecs must be greater than 50ms!"); return; } - } catch (const DurationUtil::OutOfRangeException& e) { + } catch (const EnvoyException& e) { // TODO: // protoc-gen-validate has an issue on type "Duration" which may generate interval with seconds // > 0 while "nanos" < 0. And negative "nanos" will cause validation inside the filter to fail. diff --git a/test/extensions/filters/network/ratelimit/config_test.cc b/test/extensions/filters/network/ratelimit/config_test.cc index 09c58270a046..7504edc2ccb5 100644 --- a/test/extensions/filters/network/ratelimit/config_test.cc +++ b/test/extensions/filters/network/ratelimit/config_test.cc @@ -48,8 +48,9 @@ TEST(RateLimitFilterConfigTest, CorrectProto) { NiceMock context; - EXPECT_CALL(context.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClient(_, _, _)) - .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { + EXPECT_CALL(context.server_factory_context_.cluster_manager_.async_client_manager_, + getOrCreateRawAsyncClientWithHashKey(_, _, _)) + .WillOnce(Invoke([](const Grpc::GrpcServiceConfigWithHashKey&, Stats::Scope&, bool) { return std::make_unique>(); })); diff --git a/test/extensions/filters/network/ratelimit/ratelimit_test.cc b/test/extensions/filters/network/ratelimit/ratelimit_test.cc index bc8a646419a1..e5c75571f695 100644 --- a/test/extensions/filters/network/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/network/ratelimit/ratelimit_test.cc @@ -518,10 +518,10 @@ domain: foo stat_prefix: name )EOF"; - ON_CALL(factory_context.runtime_loader_.snapshot_, + ON_CALL(factory_context.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.tcp_filter_enabled", 100)) .WillByDefault(Return(true)); - ON_CALL(factory_context.runtime_loader_.snapshot_, + ON_CALL(factory_context.server_factory_context_.runtime_loader_.snapshot_, featureEnabled("ratelimit.tcp_filter_enforcing", 100)) .WillByDefault(Return(true)); @@ -529,20 +529,22 @@ stat_prefix: name TestUtility::loadFromYaml(rl_yaml, proto_config); Extensions::NetworkFilters::RateLimitFilter::ConfigSharedPtr rl_config( - new Extensions::NetworkFilters::RateLimitFilter::Config(proto_config, factory_context.scope_, - factory_context.runtime_loader_)); + new Extensions::NetworkFilters::RateLimitFilter::Config( + proto_config, factory_context.scope_, + factory_context.server_factory_context_.runtime_loader_)); Extensions::Filters::Common::RateLimit::MockClient* rl_client = new Extensions::Filters::Common::RateLimit::MockClient(); manager.addReadFilter(std::make_shared( rl_config, Extensions::Filters::Common::RateLimit::ClientPtr{rl_client})); - factory_context.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy tcp_proxy; tcp_proxy.set_stat_prefix("name"); tcp_proxy.set_cluster("fake_cluster"); TcpProxy::ConfigSharedPtr tcp_proxy_config(new TcpProxy::Config(tcp_proxy, factory_context)); - manager.addReadFilter( - std::make_shared(tcp_proxy_config, factory_context.cluster_manager_)); + manager.addReadFilter(std::make_shared( + tcp_proxy_config, factory_context.server_factory_context_.cluster_manager_)); Extensions::Filters::Common::RateLimit::RequestCallbacks* request_callbacks{}; EXPECT_CALL(*rl_client, limit(_, "foo", @@ -556,7 +558,8 @@ stat_prefix: name EXPECT_EQ(manager.initializeReadFilters(), true); - EXPECT_CALL(factory_context.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(factory_context.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(Upstream::TcpPoolData([]() {}, &conn_pool))); request_callbacks->complete(Extensions::Filters::Common::RateLimit::LimitStatus::OK, nullptr, diff --git a/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc b/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc index 7a53fcba77e4..0e7c2694328c 100644 --- a/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc +++ b/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc @@ -97,11 +97,12 @@ class CommandLookUpSpeedTest { } // namespace Extensions } // namespace Envoy -static void BM_MakeRequests(benchmark::State& state) { +static void bmMakeRequests(benchmark::State& state) { Envoy::Extensions::NetworkFilters::RedisProxy::CommandLookUpSpeedTest context; for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.makeRequests(); } } -BENCHMARK(BM_MakeRequests); +BENCHMARK(bmMakeRequests); diff --git a/test/extensions/filters/network/redis_proxy/command_split_speed_test.cc b/test/extensions/filters/network/redis_proxy/command_split_speed_test.cc index 0feefb51b924..084138716546 100644 --- a/test/extensions/filters/network/redis_proxy/command_split_speed_test.cc +++ b/test/extensions/filters/network/redis_proxy/command_split_speed_test.cc @@ -88,44 +88,48 @@ class CommandSplitSpeedTest { } // namespace Extensions } // namespace Envoy -static void BM_Split_CompositeArray(benchmark::State& state) { +static void bmSplitCompositeArray(benchmark::State& state) { Envoy::Extensions::NetworkFilters::RedisProxy::CommandSplitSpeedTest context; Envoy::Extensions::NetworkFilters::Common::Redis::RespValueSharedPtr request = context.makeSharedBulkStringArray(state.range(0), 36, state.range(1)); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.createLocalCompositeArray(request); } } -BENCHMARK(BM_Split_CompositeArray)->Ranges({{1, 100}, {64, 8 << 14}}); +BENCHMARK(bmSplitCompositeArray)->Ranges({{1, 100}, {64, 8 << 14}}); -static void BM_Split_Copy(benchmark::State& state) { +static void bmSplitCopy(benchmark::State& state) { Envoy::Extensions::NetworkFilters::RedisProxy::CommandSplitSpeedTest context; Envoy::Extensions::NetworkFilters::Common::Redis::RespValueSharedPtr request = context.makeSharedBulkStringArray(state.range(0), 36, state.range(1)); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.copy(request); } } -BENCHMARK(BM_Split_Copy)->Ranges({{1, 100}, {64, 8 << 14}}); +BENCHMARK(bmSplitCopy)->Ranges({{1, 100}, {64, 8 << 14}}); -static void BM_Split_CreateShared(benchmark::State& state) { +static void bmSplitCreateShared(benchmark::State& state) { Envoy::Extensions::NetworkFilters::RedisProxy::CommandSplitSpeedTest context; Envoy::Extensions::NetworkFilters::Common::Redis::RespValueSharedPtr request = context.makeSharedBulkStringArray(state.range(0), 36, state.range(1)); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.createShared(request); } state.counters["use_count"] = request.use_count(); } -BENCHMARK(BM_Split_CreateShared)->Ranges({{1, 100}, {64, 8 << 14}}); +BENCHMARK(bmSplitCreateShared)->Ranges({{1, 100}, {64, 8 << 14}}); -static void BM_Split_CreateVariant(benchmark::State& state) { +static void bmSplitCreateVariant(benchmark::State& state) { Envoy::Extensions::NetworkFilters::RedisProxy::CommandSplitSpeedTest context; Envoy::Extensions::NetworkFilters::Common::Redis::RespValueSharedPtr request = context.makeSharedBulkStringArray(state.range(0), 36, state.range(1)); for (auto _ : state) { + UNREFERENCED_PARAMETER(_); context.createVariant(request); } state.counters["use_count"] = request.use_count(); } -BENCHMARK(BM_Split_CreateVariant)->Ranges({{1, 100}, {64, 8 << 14}}); +BENCHMARK(bmSplitCreateVariant)->Ranges({{1, 100}, {64, 8 << 14}}); diff --git a/test/extensions/filters/network/redis_proxy/config_test.cc b/test/extensions/filters/network/redis_proxy/config_test.cc index f0d220af674c..be97f9e8f539 100644 --- a/test/extensions/filters/network/redis_proxy/config_test.cc +++ b/test/extensions/filters/network/redis_proxy/config_test.cc @@ -80,7 +80,7 @@ stat_prefix: foo NiceMock context; RedisProxyFilterConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); - EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context.getServerFactoryContext())); + EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context.serverFactoryContext())); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); cb(connection); @@ -141,7 +141,7 @@ stat_prefix: foo NiceMock context; RedisProxyFilterConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); - EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context.getServerFactoryContext())); + EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context.serverFactoryContext())); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); cb(connection); @@ -165,7 +165,7 @@ stat_prefix: foo NiceMock context; RedisProxyFilterConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); - EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context.getServerFactoryContext())); + EXPECT_TRUE(factory.isTerminalFilterByProto(proto_config, context.serverFactoryContext())); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); cb(connection); diff --git a/test/extensions/filters/network/redis_proxy/mocks.cc b/test/extensions/filters/network/redis_proxy/mocks.cc index 2b82150e4a28..60c6ac26f9b8 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.cc +++ b/test/extensions/filters/network/redis_proxy/mocks.cc @@ -15,7 +15,7 @@ MockRouter::MockRouter(RouteSharedPtr route) : route_(std::move(route)) { MockRouter::~MockRouter() = default; MockRoute::MockRoute(ConnPool::InstanceSharedPtr conn_pool) : conn_pool_(std::move(conn_pool)) { - ON_CALL(*this, upstream()).WillByDefault(Return(conn_pool_)); + ON_CALL(*this, upstream(_)).WillByDefault(Return(conn_pool_)); ON_CALL(*this, mirrorPolicies()).WillByDefault(ReturnRef(policies_)); } MockRoute::~MockRoute() = default; diff --git a/test/extensions/filters/network/redis_proxy/mocks.h b/test/extensions/filters/network/redis_proxy/mocks.h index 00337512d12c..3d9c96b8f5fe 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.h +++ b/test/extensions/filters/network/redis_proxy/mocks.h @@ -39,7 +39,7 @@ class MockRoute : public Route { MockRoute(ConnPool::InstanceSharedPtr); ~MockRoute() override; - MOCK_METHOD(ConnPool::InstanceSharedPtr, upstream, (), (const)); + MOCK_METHOD(ConnPool::InstanceSharedPtr, upstream, (const std::string&), (const)); MOCK_METHOD(const MirrorPolicies&, mirrorPolicies, (), (const)); ConnPool::InstanceSharedPtr conn_pool_; MirrorPolicies policies_; diff --git a/test/extensions/filters/network/redis_proxy/router_impl_test.cc b/test/extensions/filters/network/redis_proxy/router_impl_test.cc index 8d58b0cea7ce..b3b0deb932e9 100644 --- a/test/extensions/filters/network/redis_proxy/router_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/router_impl_test.cc @@ -20,7 +20,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::InSequence; using testing::Matcher; using testing::NiceMock; using testing::Return; @@ -67,7 +66,21 @@ TEST(PrefixRoutesTest, RoutedToCatchAll) { std::string key("c:bar"); NiceMock stream_info; - EXPECT_EQ(upstream_c, router.upstreamPool(key, stream_info)->upstream()); + EXPECT_EQ(upstream_c, router.upstreamPool(key, stream_info)->upstream("")); +} + +TEST(PrefixRoutesTest, MissingCatchAll) { + Upstreams upstreams; + upstreams.emplace("fake_clusterA", std::make_shared()); + upstreams.emplace("fake_clusterB", std::make_shared()); + + Runtime::MockLoader runtime_; + + PrefixRoutes router(createPrefixRoutes(), std::move(upstreams), runtime_); + + std::string key("c:bar"); + NiceMock stream_info; + EXPECT_EQ(nullptr, router.upstreamPool(key, stream_info)); } TEST(PrefixRoutesTest, RoutedToLongestPrefix) { @@ -83,7 +96,7 @@ TEST(PrefixRoutesTest, RoutedToLongestPrefix) { std::string key("ab:bar"); NiceMock stream_info; - EXPECT_EQ(upstream_a, router.upstreamPool(key, stream_info)->upstream()); + EXPECT_EQ(upstream_a, router.upstreamPool(key, stream_info)->upstream("")); } TEST(PrefixRoutesTest, TestFormatterWithCatchAllRoute) { @@ -120,7 +133,7 @@ TEST(PrefixRoutesTest, TestFormatterWithCatchAllRoute) { PrefixRoutes router(prefix_routes, std::move(upstreams), runtime_); std::string key("removeMe_catchAllKey"); EXPECT_EQ(upstream_catch_all, - router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream()); + router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream("")); EXPECT_EQ("{catchAllKey}-catchAllEnv-subjectCN-{catchAllKey}", key); } @@ -157,7 +170,7 @@ TEST(PrefixRoutesTest, TestFormatterWithPrefixRoute) { PrefixRoutes router(prefix_routes, std::move(upstreams), runtime_); std::string key("abc:bar"); EXPECT_EQ(upstream_c, - router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream()); + router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream("")); EXPECT_EQ("{abc:bar}-test-subjectCN-{abc:bar}", key); } @@ -194,7 +207,7 @@ TEST(PrefixRoutesTest, TestFormatterWithPercentInKey) { PrefixRoutes router(prefix_routes, std::move(upstreams), runtime_); std::string key("%abc:bar%"); EXPECT_EQ(upstream_c, - router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream()); + router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream("")); EXPECT_EQ("{%abc:bar%}-test-subjectCN-{%abc:bar%}", key); } @@ -229,7 +242,7 @@ TEST(PrefixRoutesTest, TestKeyPrefixFormatterWithMissingFilterState) { PrefixRoutes router(prefix_routes, std::move(upstreams), runtime_); std::string key("abc:bar"); EXPECT_EQ(upstream_c, - router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream()); + router.upstreamPool(key, filter_callbacks.connection().streamInfo())->upstream("")); EXPECT_EQ("abc:bar", key); } @@ -249,7 +262,7 @@ TEST(PrefixRoutesTest, CaseUnsensitivePrefix) { std::string key("AB:bar"); NiceMock stream_info; - EXPECT_EQ(upstream_a, router.upstreamPool(key, stream_info)->upstream()); + EXPECT_EQ(upstream_a, router.upstreamPool(key, stream_info)->upstream("")); } TEST(PrefixRoutesTest, RemovePrefix) { @@ -274,7 +287,7 @@ TEST(PrefixRoutesTest, RemovePrefix) { std::string key("abc:bar"); NiceMock stream_info; - EXPECT_EQ(upstream_a, router.upstreamPool(key, stream_info)->upstream()); + EXPECT_EQ(upstream_a, router.upstreamPool(key, stream_info)->upstream("")); EXPECT_EQ(":bar", key); } @@ -291,7 +304,7 @@ TEST(PrefixRoutesTest, RoutedToShortestPrefix) { std::string key("a:bar"); NiceMock stream_info; - EXPECT_EQ(upstream_b, router.upstreamPool(key, stream_info)->upstream()); + EXPECT_EQ(upstream_b, router.upstreamPool(key, stream_info)->upstream("")); EXPECT_EQ("a:bar", key); } @@ -317,10 +330,10 @@ TEST(PrefixRoutesTest, DifferentPrefixesSameUpstream) { NiceMock stream_info; std::string key1("a:bar"); - EXPECT_EQ(upstream_b, router.upstreamPool(key1, stream_info)->upstream()); + EXPECT_EQ(upstream_b, router.upstreamPool(key1, stream_info)->upstream("")); std::string key2("also_route_to_b:bar"); - EXPECT_EQ(upstream_b, router.upstreamPool(key2, stream_info)->upstream()); + EXPECT_EQ(upstream_b, router.upstreamPool(key2, stream_info)->upstream("")); } TEST(PrefixRoutesTest, DuplicatePrefix) { @@ -343,6 +356,40 @@ TEST(PrefixRoutesTest, DuplicatePrefix) { EnvoyException, "prefix `ab` already exists.") } +TEST(PrefixRoutesTest, RouteReadWriteToDiffClusters) { + auto upstream_a = std::make_shared(); + auto upstream_b = std::make_shared(); + auto upstream_c = std::make_shared(); + auto upstream_cr = std::make_shared(); + + Upstreams upstreams; + upstreams.emplace("fake_clusterA", upstream_a); + upstreams.emplace("fake_clusterB", upstream_b); + upstreams.emplace("fake_clusterC", upstream_c); + upstreams.emplace("fake_clusterCR", upstream_cr); + + Runtime::MockLoader runtime_; + + auto prefix_routes = createPrefixRoutes(); + prefix_routes.mutable_catch_all_route()->set_cluster("fake_clusterC"); + auto read_policy = prefix_routes.mutable_catch_all_route()->mutable_read_command_policy(); + read_policy->set_cluster("fake_clusterCR"); + + PrefixRoutes router(prefix_routes, std::move(upstreams), runtime_); + NiceMock stream_info; + + std::string get_command("GET"); + std::string set_command("SET"); + + std::string key1("c:bar"); + EXPECT_EQ(upstream_cr, router.upstreamPool(key1, stream_info)->upstream(get_command)); + EXPECT_EQ(upstream_c, router.upstreamPool(key1, stream_info)->upstream(set_command)); + + std::string key2("ab:bar"); + EXPECT_EQ(upstream_a, router.upstreamPool(key2, stream_info)->upstream(get_command)); + EXPECT_EQ(upstream_a, router.upstreamPool(key2, stream_info)->upstream(set_command)); +} + TEST(MirrorPolicyImplTest, ShouldMirrorDefault) { envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::PrefixRoutes::Route:: RequestMirrorPolicy config; diff --git a/test/extensions/filters/network/set_filter_state/BUILD b/test/extensions/filters/network/set_filter_state/BUILD new file mode 100644 index 000000000000..13a6056f03fd --- /dev/null +++ b/test/extensions/filters/network/set_filter_state/BUILD @@ -0,0 +1,28 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "integration_test", + srcs = [ + "integration_test.cc", + ], + extension_names = ["envoy.filters.network.set_filter_state"], + deps = [ + "//source/common/router:string_accessor_lib", + "//source/extensions/filters/network/echo:config", + "//source/extensions/filters/network/set_filter_state:config", + "//test/integration:integration_lib", + "//test/integration:utility_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/network/set_filter_state/integration_test.cc b/test/extensions/filters/network/set_filter_state/integration_test.cc new file mode 100644 index 000000000000..063a2917a772 --- /dev/null +++ b/test/extensions/filters/network/set_filter_state/integration_test.cc @@ -0,0 +1,68 @@ +#include "source/common/router/string_accessor_impl.h" + +#include "test/integration/integration.h" +#include "test/integration/utility.h" +#include "test/test_common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SetFilterState { + +class ObjectFooFactory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "foo"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +REGISTER_FACTORY(ObjectFooFactory, StreamInfo::FilterState::ObjectFactory); + +class SetFilterStateIntegrationTest : public testing::TestWithParam, + + public BaseIntegrationTest { +public: + SetFilterStateIntegrationTest() : BaseIntegrationTest(GetParam(), config()) {} + + static std::string config() { + return absl::StrCat(ConfigHelper::baseConfig(), R"EOF( + filter_chains: + filters: + - name: direct_response + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.set_filter_state.v3.Config + on_new_connection: + - object_key: foo + format_string: + text_format_source: + inline_string: "bar" + - name: envoy.filters.network.echo + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.echo.v3.Echo + )EOF"); + } + + void SetUp() override { + useListenerAccessLog("%FILTER_STATE(foo)%"); + BaseIntegrationTest::initialize(); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, SetFilterStateIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(SetFilterStateIntegrationTest, OnConnection) { + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + ASSERT_TRUE(tcp_client->write("hello")); + ASSERT_TRUE(tcp_client->connected()); + tcp_client->close(); + EXPECT_THAT(waitForAccessLog(listener_access_log_name_), testing::HasSubstr("bar")); +} + +} // namespace SetFilterState +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/tcp_proxy/config_test.cc b/test/extensions/filters/network/tcp_proxy/config_test.cc index e9a7dfa3e224..cc0444e924ab 100644 --- a/test/extensions/filters/network/tcp_proxy/config_test.cc +++ b/test/extensions/filters/network/tcp_proxy/config_test.cc @@ -59,7 +59,7 @@ TEST(ConfigTest, ConfigTest) { config.set_stat_prefix("prefix"); config.set_cluster("cluster"); - EXPECT_TRUE(factory.isTerminalFilterByProto(config, context.getServerFactoryContext())); + EXPECT_TRUE(factory.isTerminalFilterByProto(config, context.serverFactoryContext())); Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, context); Network::MockConnection connection; diff --git a/test/extensions/filters/network/thrift_proxy/config_test.cc b/test/extensions/filters/network/thrift_proxy/config_test.cc index 14128f7dd4ed..602eb30092b9 100644 --- a/test/extensions/filters/network/thrift_proxy/config_test.cc +++ b/test/extensions/filters/network/thrift_proxy/config_test.cc @@ -63,7 +63,7 @@ class ThriftFilterConfigTestBase { void testConfig(envoy::extensions::filters::network::thrift_proxy::v3::ThriftProxy& config) { Network::FilterFactoryCb cb; EXPECT_NO_THROW({ cb = factory_.createFilterFactoryFromProto(config, context_); }); - EXPECT_TRUE(factory_.isTerminalFilterByProto(config, context_.getServerFactoryContext())); + EXPECT_TRUE(factory_.isTerminalFilterByProto(config, context_.serverFactoryContext())); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); @@ -278,8 +278,8 @@ version_info: "1" EXPECT_TRUE(context_.server_factory_context_.cluster_manager_.subscription_factory_.callbacks_ ->onConfigUpdate(decoded_resources.refvec_, response.version_info()) .ok()); - auto message_ptr = context_.admin_.config_tracker_.config_tracker_callbacks_["trds_routes"]( - universal_name_matcher); + auto message_ptr = context_.server_factory_context_.admin_.config_tracker_ + .config_tracker_callbacks_["trds_routes"](universal_name_matcher); const auto& dump = TestUtility::downcastAndValidate(*message_ptr); EXPECT_EQ(1, dump.dynamic_route_configs().size()); diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index 76c7e887dfef..aaffb72f2be6 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -90,9 +90,9 @@ class ThriftConnectionManagerTest : public testing::Test { public: ThriftConnectionManagerTest() : stats_(ThriftFilterStats::generateStats("test.", *store_.rootScope())) { - route_config_provider_manager_ = - std::make_unique(context_.admin_); - ON_CALL(*context_.access_log_manager_.file_, write(_)) + route_config_provider_manager_ = std::make_unique( + context_.server_factory_context_.admin_); + ON_CALL(*context_.server_factory_context_.access_log_manager_.file_, write(_)) .WillByDefault(SaveArg<0>(&access_log_data_)); } ~ThriftConnectionManagerTest() override { diff --git a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc index 2fd20e7e24da..1c3e9decda59 100644 --- a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc +++ b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc @@ -51,8 +51,9 @@ timeout: "1.337s" NiceMock context; - EXPECT_CALL(context.cluster_manager_.async_client_manager_, getOrCreateRawAsyncClient(_, _, _)) - .WillOnce(Invoke([](const envoy::config::core::v3::GrpcService&, Stats::Scope&, bool) { + EXPECT_CALL(context.server_factory_context_.cluster_manager_.async_client_manager_, + getOrCreateRawAsyncClientWithHashKey(_, _, _)) + .WillOnce(Invoke([](const Grpc::GrpcServiceConfigWithHashKey&, Stats::Scope&, bool) { return std::make_unique>(); })); diff --git a/test/extensions/filters/network/thrift_proxy/integration.cc b/test/extensions/filters/network/thrift_proxy/integration.cc index 236750130871..c2d5239fa274 100644 --- a/test/extensions/filters/network/thrift_proxy/integration.cc +++ b/test/extensions/filters/network/thrift_proxy/integration.cc @@ -102,7 +102,7 @@ void BaseThriftIntegrationTest::preparePayloads(const PayloadOptions& options, void BaseThriftIntegrationTest::readAll(std::string file, Buffer::Instance& buffer) { file = TestEnvironment::substitute(file, version_); - std::string data = api_->fileSystem().fileReadToEnd(file); + std::string data = api_->fileSystem().fileReadToEnd(file).value(); buffer.add(data); } diff --git a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc index b74709369038..5e500d805e37 100644 --- a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc @@ -26,7 +26,7 @@ class ThriftRouteMatcherTest : public testing::Test { protected: RouteMatcher createMatcher( const envoy::extensions::filters::network::thrift_proxy::v3::RouteConfiguration& route) { - return RouteMatcher(route, absl::nullopt); + return {route, absl::nullopt}; } RouteMatcher createMatcher(const std::string& yaml) { diff --git a/test/extensions/filters/network/thrift_proxy/router_ratelimit_test.cc b/test/extensions/filters/network/thrift_proxy/router_ratelimit_test.cc index e5e297c74aa7..50a3d9f3d31b 100644 --- a/test/extensions/filters/network/thrift_proxy/router_ratelimit_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_ratelimit_test.cc @@ -33,8 +33,8 @@ namespace { class ThriftRateLimitConfigurationTest : public testing::Test { public: ThriftRateLimitConfigurationTest() { - route_config_provider_manager_ = - std::make_unique(factory_context_.admin_); + route_config_provider_manager_ = std::make_unique( + factory_context_.server_factory_context_.admin_); } void initializeClusters(const std::vector& cluster_names) { diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index e03bf7b61ee5..3d6957de5adb 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -107,14 +107,15 @@ class ThriftRouterTestBase { return protocol; }), transport_register_(transport_factory_), protocol_register_(protocol_factory_) { - context_.cluster_manager_.initializeThreadLocalClusters({"cluster"}); + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"cluster"}); } void initializeRouter(ShadowWriter& shadow_writer, bool close_downstream_on_error) { route_ = new NiceMock(); route_ptr_.reset(route_); - router_ = std::make_unique(context_.clusterManager(), *stats_, context_.runtime(), + router_ = std::make_unique(context_.server_factory_context_.cluster_manager_, *stats_, + context_.server_factory_context_.runtime_loader_, shadow_writer, close_downstream_on_error); EXPECT_EQ(nullptr, router_->downstreamConnection()); @@ -125,14 +126,17 @@ class ThriftRouterTestBase { } void initializeRouter(bool close_downstream_on_error = true) { - stats_ = std::make_shared("test", context_.scope(), context_.localInfo()); + stats_ = std::make_shared("test", context_.scope(), + context_.server_factory_context_.localInfo()); initializeRouter(shadow_writer_, close_downstream_on_error); } void initializeRouterWithShadowWriter() { - stats_ = std::make_shared("test", context_.scope(), context_.localInfo()); - shadow_writer_impl_ = std::make_shared(context_.clusterManager(), *stats_, - dispatcher_, context_.threadLocal()); + stats_ = std::make_shared("test", context_.scope(), + context_.server_factory_context_.localInfo()); + shadow_writer_impl_ = std::make_shared( + context_.server_factory_context_.clusterManager(), *stats_, dispatcher_, + context_.server_factory_context_.threadLocal()); initializeRouter(*shadow_writer_impl_, true); } @@ -238,7 +242,9 @@ class ThriftRouterTestBase { void initializeUpstreamZone() { upstream_locality_.set_zone("other_zone_name"); - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_, locality()) + ON_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_, + locality()) .WillByDefault(ReturnRef(upstream_locality_)); } @@ -277,18 +283,21 @@ class ThriftRouterTestBase { } void connectUpstream() { - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; })); conn_state_.reset(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, setConnectionState_(_)) .WillOnce(Invoke( [&](Tcp::ConnectionPool::ConnectionStatePtr& cs) -> void { conn_state_.swap(cs); })); @@ -301,7 +310,8 @@ class ThriftRouterTestBase { })); EXPECT_CALL(callbacks_, continueDecoding()); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); EXPECT_NE(nullptr, upstream_callbacks_); } @@ -315,7 +325,8 @@ class ThriftRouterTestBase { initializeMetadata(msg_type, "method", sequence_id); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; @@ -324,7 +335,8 @@ class ThriftRouterTestBase { if (!conn_state_) { conn_state_ = std::make_unique(); } - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); @@ -355,12 +367,15 @@ class ThriftRouterTestBase { })); }; EXPECT_CALL(callbacks_, continueDecoding()).Times(0); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); @@ -541,8 +556,9 @@ class ThriftRouterTestBase { EXPECT_CALL(upstream_connection_, write(_, false)); if (msg_type_ == MessageType::Oneway) { - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, - released(Ref(upstream_connection_))); + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + released(Ref(upstream_connection_))); } EXPECT_EQ(FilterStatus::Continue, router_->messageEnd()); @@ -573,8 +589,9 @@ class ThriftRouterTestBase { EXPECT_CALL(callbacks_, upstreamData(Ref(buffer))) .WillOnce(Return(ThriftFilters::ResponseStatus::Complete)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, - released(Ref(upstream_connection_))); + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + released(Ref(upstream_connection_))); if (is_drain) { EXPECT_CALL(upstream_connection_, close(Network::ConnectionCloseType::NoFlush)) @@ -594,7 +611,8 @@ class ThriftRouterTestBase { void expectStatCalls(Stats::MockStore& cluster_store) { Stats::MockScope& cluster_scope = cluster_store.mockScope(); - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + ON_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + statsScope()) .WillByDefault(ReturnRef(cluster_scope)); EXPECT_CALL(cluster_store, counter("thrift.upstream_rq_call")).Times(AtLeast(1)); @@ -723,9 +741,11 @@ TEST_P(ThriftRouterRainidayTest, PoolRemoteConnectionFailure) { startRequest(MessageType::Call); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) .WillOnce(Invoke([&](const DirectResponse& response, bool end_stream) -> void { @@ -737,20 +757,24 @@ TEST_P(ThriftRouterRainidayTest, PoolRemoteConnectionFailure) { EXPECT_EQ(GetParam(), end_stream); })); EXPECT_CALL(callbacks_, continueDecoding()).Times(GetParam() ? 0 : 1); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); - - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception_local") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception") - .value()); - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception_local") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception") + .value()); + EXPECT_EQ(0UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); } TEST_P(ThriftRouterRainidayTest, PoolLocalConnectionFailure) { @@ -758,17 +782,19 @@ TEST_P(ThriftRouterRainidayTest, PoolLocalConnectionFailure) { startRequest(MessageType::Call); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::LocalConnectionFailure); - - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::LocalConnectionFailure); + + EXPECT_EQ(0UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); } TEST_P(ThriftRouterRainidayTest, PoolTimeout) { @@ -776,9 +802,11 @@ TEST_P(ThriftRouterRainidayTest, PoolTimeout) { startRequest(MessageType::Call); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) .WillOnce(Invoke([&](const DirectResponse& response, bool end_stream) -> void { @@ -788,20 +816,24 @@ TEST_P(ThriftRouterRainidayTest, PoolTimeout) { ContainsRegex(".*connection failure before response start: timeout.*")); EXPECT_EQ(GetParam(), end_stream); })); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginTimeout, _)); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::Timeout); - - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception_local") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception") - .value()); - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginTimeout, _)); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::Timeout); + + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception_local") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception") + .value()); + EXPECT_EQ(0UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); } TEST_P(ThriftRouterRainidayTest, PoolOverflowFailure) { @@ -809,9 +841,11 @@ TEST_P(ThriftRouterRainidayTest, PoolOverflowFailure) { startRequest(MessageType::Call); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) .WillOnce(Invoke([&](const DirectResponse& response, bool end_stream) -> void { @@ -820,36 +854,44 @@ TEST_P(ThriftRouterRainidayTest, PoolOverflowFailure) { EXPECT_THAT(app_ex.what(), ContainsRegex(".*too many connections.*")); EXPECT_FALSE(end_stream); })); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::Overflow, true); - - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception_local") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception") - .value()); - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::Overflow, true); + + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception_local") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception") + .value()); + EXPECT_EQ(0UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); } TEST_P(ThriftRouterRainidayTest, PoolConnectionFailureWithOnewayMessage) { initializeRouter(GetParam()); startRequest(MessageType::Oneway); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_oneway") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_oneway") + .value()); EXPECT_CALL(callbacks_, sendLocalReply(_, Eq(GetParam()))); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception") - .value()); - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception") + .value()); + EXPECT_EQ(0UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); destroyRouter(); } @@ -877,7 +919,8 @@ TEST_P(ThriftRouterRainidayTest, NoCluster) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_, getThreadLocalCluster(Eq(cluster_name_))) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(Eq(cluster_name_))) .WillOnce(Return(nullptr)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) .WillOnce(Invoke([&](const DirectResponse& response, bool end_stream) -> void { @@ -933,7 +976,9 @@ TEST_P(ThriftRouterRainidayTest, ClusterMaintenanceMode) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, maintenanceMode()) + EXPECT_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + maintenanceMode()) .WillOnce(Return(true)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) @@ -945,9 +990,11 @@ TEST_P(ThriftRouterRainidayTest, ClusterMaintenanceMode) { })); EXPECT_EQ(FilterStatus::StopIteration, router_->messageBegin(metadata_)); EXPECT_EQ(1U, context_.scope().counterFromString("test.upstream_rq_maintenance_mode").value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); } TEST_P(ThriftRouterRainidayTest, NoHealthyHosts) { @@ -957,7 +1004,8 @@ TEST_P(ThriftRouterRainidayTest, NoHealthyHosts) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_, tcpConnPool(_, _)) + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + tcpConnPool(_, _)) .WillOnce(Return(absl::nullopt)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) @@ -970,9 +1018,11 @@ TEST_P(ThriftRouterRainidayTest, NoHealthyHosts) { EXPECT_EQ(FilterStatus::StopIteration, router_->messageBegin(metadata_)); EXPECT_EQ(1U, context_.scope().counterFromString("test.no_healthy_upstream").value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); } TEST_F(ThriftRouterTest, TruncatedResponse) { @@ -987,16 +1037,19 @@ TEST_F(ThriftRouterTest, TruncatedResponse) { EXPECT_CALL(callbacks_, startUpstreamResponse(_, _)); EXPECT_CALL(callbacks_, upstreamData(Ref(buffer))) .WillOnce(Return(ThriftFilters::ResponseStatus::MoreData)); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, - released(Ref(upstream_connection_))); + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + released(Ref(upstream_connection_))); EXPECT_CALL(callbacks_, resetDownstreamConnection()); upstream_callbacks_->onUpstreamData(buffer, true); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.downstream_cx_underflow_response_close") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.downstream_cx_underflow_response_close") + .value()); } TEST_F(ThriftRouterTest, UpstreamLocalCloseMidResponse) { @@ -1053,9 +1106,9 @@ TEST_P(ThriftRouterRainidayTest, UnexpectedUpstreamRemoteClose) { EXPECT_EQ(GetParam(), end_stream); })); EXPECT_CALL(callbacks_, onReset()).Times(0); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); router_->onEvent(Network::ConnectionEvent::RemoteClose); } @@ -1097,8 +1150,10 @@ TEST_F(ThriftRouterTest, UnexpectedRouterDestroyBeforeUpstreamConnect) { initializeRouter(); startRequest(MessageType::Call); - EXPECT_EQ(1, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.handles_.size()); - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.handles_.front(), + EXPECT_EQ(1, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.handles_.size()); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .handles_.front(), cancel(Tcp::ConnectionPool::CancelPolicy::Default)); destroyRouter(); } @@ -1114,7 +1169,8 @@ TEST_F(ThriftRouterTest, UnexpectedRouterDestroy) { TEST_F(ThriftRouterTest, ProtocolUpgrade) { Stats::MockStore cluster_store; Stats::MockScope& cluster_scope{cluster_store.mockScope()}; - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + ON_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + statsScope()) .WillByDefault(ReturnRef(cluster_scope)); EXPECT_CALL(cluster_store, counter("thrift.upstream_rq_call")); @@ -1124,17 +1180,20 @@ TEST_F(ThriftRouterTest, ProtocolUpgrade) { initializeRouter(); startRequest(MessageType::Call); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke( [&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; })); conn_state_.reset(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, setConnectionState_(_)) .WillOnce(Invoke( [&](Tcp::ConnectionPool::ConnectionStatePtr& cs) -> void { conn_state_.swap(cs); })); @@ -1169,7 +1228,8 @@ TEST_F(ThriftRouterTest, ProtocolUpgrade) { EXPECT_EQ("upgrade request", buffer.toString()); })); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( + upstream_connection_); EXPECT_NE(nullptr, upstream_callbacks_); Buffer::OwnedImpl buffer; @@ -1199,17 +1259,20 @@ TEST_F(ThriftRouterTest, ProtocolUpgrade) { TEST_F(ThriftRouterTest, ProtocolUpgradeOnExistingUnusedConnection) { initializeRouter(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke( [&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; })); conn_state_.reset(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, setConnectionState_(_)) .WillOnce(Invoke( [&](Tcp::ConnectionPool::ConnectionStatePtr& cs) -> void { conn_state_.swap(cs); })); @@ -1222,10 +1285,13 @@ TEST_F(ThriftRouterTest, ProtocolUpgradeOnExistingUnusedConnection) { })); // Simulate an existing connection that's never been used. - EXPECT_CALL(context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, newConnection(_)) + EXPECT_CALL( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_, + newConnection(_)) .WillOnce( Invoke([&](Tcp::ConnectionPool::Callbacks& cb) -> Tcp::ConnectionPool::Cancellable* { - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.newConnectionImpl(cb); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .newConnectionImpl(cb); EXPECT_CALL(*protocol_, supportsUpgrade()).WillOnce(Return(true)); @@ -1236,8 +1302,8 @@ TEST_F(ThriftRouterTest, ProtocolUpgradeOnExistingUnusedConnection) { return ThriftObjectPtr{upgrade_response}; })); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( - upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolReady(upstream_connection_); return nullptr; })); @@ -1266,28 +1332,36 @@ TEST_F(ThriftRouterTest, ProtocolUpgradeOnExistingUnusedConnection) { returnResponse(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); } TEST_F(ThriftRouterTest, ProtocolUpgradeSkippedOnExistingConnection) { initializeRouter(); startRequest(MessageType::Call); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, addUpstreamCallbacks(_)) .WillOnce(Invoke( [&](Tcp::ConnectionPool::UpstreamCallbacks& cb) -> void { upstream_callbacks_ = &cb; })); conn_state_ = std::make_unique(); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.connection_data_, + EXPECT_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.connection_data_, connectionState()) .WillRepeatedly( Invoke([&]() -> Tcp::ConnectionPool::ConnectionState* { return conn_state_.get(); })); @@ -1307,7 +1381,8 @@ TEST_F(ThriftRouterTest, ProtocolUpgradeSkippedOnExistingConnection) { })); EXPECT_CALL(callbacks_, continueDecoding()); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady(upstream_connection_); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolReady( + upstream_connection_); EXPECT_NE(nullptr, upstream_callbacks_); // Then the actual request... @@ -1316,15 +1391,21 @@ TEST_F(ThriftRouterTest, ProtocolUpgradeSkippedOnExistingConnection) { returnResponse(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); } TEST_F(ThriftRouterTest, PoolTimeoutUpstreamTimeMeasurement) { @@ -1332,7 +1413,8 @@ TEST_F(ThriftRouterTest, PoolTimeoutUpstreamTimeMeasurement) { Stats::MockStore cluster_store; Stats::MockScope& cluster_scope{cluster_store.mockScope()}; - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + ON_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + statsScope()) .WillByDefault(ReturnRef(cluster_scope)); EXPECT_CALL(cluster_store, counter("thrift.upstream_rq_call")); @@ -1356,8 +1438,8 @@ TEST_F(ThriftRouterTest, PoolTimeoutUpstreamTimeMeasurement) { ContainsRegex(".*connection failure before response start: timeout.*")); EXPECT_TRUE(end_stream); })); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::Timeout); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::Timeout); } TEST_P(ThriftRouterFieldTypeTest, OneWay) { @@ -1366,20 +1448,24 @@ TEST_P(ThriftRouterFieldTypeTest, OneWay) { initializeRouter(); startRequest(MessageType::Oneway); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); connectUpstream(); sendTrivialStruct(field_type); completeRequest(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_oneway") - .value()); - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_oneway") + .value()); + EXPECT_EQ(0UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); } TEST_P(ThriftRouterFieldTypeTest, Call) { @@ -1388,28 +1474,34 @@ TEST_P(ThriftRouterFieldTypeTest, Call) { initializeRouter(); startRequest(MessageType::Call); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); connectUpstream(); sendTrivialStruct(field_type); completeRequest(); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::ExtOriginRequestSuccess, _)); returnResponse(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); } TEST_P(ThriftRouterFieldTypeTest, CallWithUpstreamRqTime) { @@ -1419,7 +1511,8 @@ TEST_P(ThriftRouterFieldTypeTest, CallWithUpstreamRqTime) { Stats::MockStore cluster_store; Stats::MockScope& cluster_scope{cluster_store.mockScope()}; - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + ON_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + statsScope()) .WillByDefault(ReturnRef(cluster_scope)); EXPECT_CALL(cluster_store, counter("thrift.upstream_rq_call")); EXPECT_CALL(cluster_store, counter("thrift.upstream_resp_reply")); @@ -1455,31 +1548,39 @@ TEST_P(ThriftRouterFieldTypeTest, Call_Error) { initializeRouter(); startRequest(MessageType::Call); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); connectUpstream(); sendTrivialStruct(field_type); completeRequest(); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); returnResponse(MessageType::Reply, false); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(0UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_error") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(0UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_error") + .value()); } TEST_P(ThriftRouterFieldTypeTest, Exception) { @@ -1488,28 +1589,34 @@ TEST_P(ThriftRouterFieldTypeTest, Exception) { initializeRouter(); startRequest(MessageType::Call); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess, _)); connectUpstream(); sendTrivialStruct(field_type); completeRequest(); - EXPECT_CALL( - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .host_->outlier_detector_, + putResult(Upstream::Outlier::Result::ExtOriginRequestFailed, _)); returnResponse(MessageType::Exception); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_exception_remote") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_exception_remote") + .value()); } TEST_P(ThriftRouterFieldTypeTest, UnknownMessageTypes) { @@ -1523,12 +1630,16 @@ TEST_P(ThriftRouterFieldTypeTest, UnknownMessageTypes) { returnResponse(MessageType::Call); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_invalid_type") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_invalid_type") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_invalid_type") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_invalid_type") + .value()); } // Ensure the service name gets stripped when strip_service_name = true. @@ -1546,15 +1657,21 @@ TEST_P(ThriftRouterFieldTypeTest, StripServiceNameEnabled) { returnResponse(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); } // Ensure the service name prefix isn't stripped when strip_service_name = false. @@ -1572,15 +1689,21 @@ TEST_P(ThriftRouterFieldTypeTest, StripServiceNameDisabled) { returnResponse(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); } TEST_F(ThriftRouterTest, CallWithExistingConnection) { @@ -1598,15 +1721,21 @@ TEST_F(ThriftRouterTest, CallWithExistingConnection) { returnResponse(); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_rq_call") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.upstream_resp_success") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_rq_call") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.upstream_resp_success") + .value()); } TEST_P(ThriftRouterContainerTest, DecoderFilterCallbacks) { @@ -1700,8 +1829,9 @@ TEST_P(ThriftRouterPassthroughTest, PassthroughEnable) { configuration); const auto protocol_option = std::make_shared(configuration); - EXPECT_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, - extensionProtocolOptions(_)) + EXPECT_CALL( + *context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + extensionProtocolOptions(_)) .WillRepeatedly(Return(protocol_option)); initializeRouter(); @@ -1724,8 +1854,8 @@ TEST_P(ThriftRouterPassthroughTest, PassthroughEnable) { EXPECT_THAT(app_ex.what(), ContainsRegex(".*connection failure.*")); EXPECT_TRUE(end_stream); })); - context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( - ConnectionPool::PoolFailureReason::RemoteConnectionFailure); + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_ + .poolFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure); } TEST_F(ThriftRouterTest, RequestResponseSize) { @@ -1779,9 +1909,11 @@ TEST_F(ThriftRouterTest, UpstreamPartialResponse) { upstream_callbacks_->onEvent(Network::ConnectionEvent::LocalClose); destroyRouter(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("thrift.downstream_cx_partial_response_close") - .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("thrift.downstream_cx_partial_response_close") + .value()); } TEST_F(ThriftRouterTest, ShadowRequests) { @@ -1801,7 +1933,8 @@ TEST_F(ThriftRouterTest, ShadowRequests) { auto& upstream_connection = shadow_cluster_info->connection; auto& conn_state = shadow_cluster_info->conn_state; - ON_CALL(context_.cluster_manager_, getThreadLocalCluster(absl::string_view(name))) + ON_CALL(context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster(absl::string_view(name))) .WillByDefault(Return(&shadow_cluster)); EXPECT_CALL(shadow_cluster.tcp_conn_pool_, newConnection(_)) .WillOnce( @@ -1858,15 +1991,18 @@ TEST_F(ThriftRouterTest, UpstreamZoneCallSuccess) { completeRequest(); returnResponse(); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_reply") - .value()); EXPECT_EQ(1UL, - context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_success") .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_success_.value()); + EXPECT_EQ(1UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_success_.value()); } TEST_F(ThriftRouterTest, UpstreamZoneCallError) { @@ -1878,14 +2014,18 @@ TEST_F(ThriftRouterTest, UpstreamZoneCallError) { completeRequest(); returnResponse(MessageType::Reply, false); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_reply") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() - .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_error") - .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_reply") + .value()); + EXPECT_EQ(1UL, + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() + .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_error") + .value()); + EXPECT_EQ(1UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); } TEST_F(ThriftRouterTest, UpstreamZoneCallException) { @@ -1897,17 +2037,19 @@ TEST_F(ThriftRouterTest, UpstreamZoneCallException) { completeRequest(); returnResponse(MessageType::Exception); EXPECT_EQ(1UL, - context_.cluster_manager_.thread_local_cluster_.cluster_.info_->statsScope() + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->statsScope() .counterFromString("zone.zone_name.other_zone_name.thrift.upstream_resp_exception") .value()); - EXPECT_EQ(1UL, context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.host_->stats_ - .rq_error_.value()); + EXPECT_EQ(1UL, context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .tcp_conn_pool_.host_->stats_.rq_error_.value()); } TEST_F(ThriftRouterTest, UpstreamZoneCallWithRqTime) { NiceMock cluster_store; Stats::MockScope& cluster_scope{cluster_store.mockScope()}; - ON_CALL(*context_.cluster_manager_.thread_local_cluster_.cluster_.info_, statsScope()) + ON_CALL(*context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + statsScope()) .WillByDefault(ReturnRef(cluster_scope)); initializeRouter(); diff --git a/test/extensions/filters/network/thrift_proxy/shadow_writer_test.cc b/test/extensions/filters/network/thrift_proxy/shadow_writer_test.cc index 3a5d048eb6f6..ae4cf73cd352 100644 --- a/test/extensions/filters/network/thrift_proxy/shadow_writer_test.cc +++ b/test/extensions/filters/network/thrift_proxy/shadow_writer_test.cc @@ -39,9 +39,10 @@ struct MockNullResponseDecoder : public NullResponseDecoder { class ShadowWriterTest : public testing::Test { public: ShadowWriterTest() { - stats_ = std::make_shared("test", context_.scope(), context_.localInfo()); - shadow_writer_ = - std::make_shared(cm_, *stats_, dispatcher_, context_.threadLocal()); + stats_ = std::make_shared("test", context_.scope(), + context_.server_factory_context_.localInfo()); + shadow_writer_ = std::make_shared( + cm_, *stats_, dispatcher_, context_.server_factory_context_.threadLocal()); metadata_ = std::make_shared(); metadata_->setMethodName("ping"); metadata_->setMessageType(MessageType::Call); diff --git a/test/extensions/filters/network/wasm/config_test.cc b/test/extensions/filters/network/wasm/config_test.cc index 5773041e300f..883e27e65bb3 100644 --- a/test/extensions/filters/network/wasm/config_test.cc +++ b/test/extensions/filters/network/wasm/config_test.cc @@ -27,12 +27,15 @@ class WasmNetworkFilterConfigTest : public testing::TestWithParam> { protected: WasmNetworkFilterConfigTest() : api_(Api::createApiForTest(stats_store_)) { - ON_CALL(context_, api()).WillByDefault(ReturnRef(*api_)); + ON_CALL(context_.server_factory_context_, api()).WillByDefault(ReturnRef(*api_)); ON_CALL(context_, scope()).WillByDefault(ReturnRef(stats_scope_)); - ON_CALL(context_, listenerMetadata()).WillByDefault(ReturnRef(listener_metadata_)); + ON_CALL(context_, listenerInfo()).WillByDefault(ReturnRef(listener_info_)); + ON_CALL(listener_info_, metadata()).WillByDefault(ReturnRef(listener_metadata_)); ON_CALL(context_, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(context_, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(context_, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(context_.server_factory_context_, clusterManager()) + .WillByDefault(ReturnRef(cluster_manager_)); + ON_CALL(context_.server_factory_context_, mainThreadDispatcher()) + .WillByDefault(ReturnRef(dispatcher_)); } void SetUp() override { Envoy::Extensions::Common::Wasm::clearCodeCacheForTesting(); } @@ -46,6 +49,7 @@ class WasmNetworkFilterConfigTest })); } + NiceMock listener_info_; NiceMock context_; Stats::IsolatedStoreImpl stats_store_; Stats::Scope& stats_scope_{*stats_store_.rootScope()}; @@ -397,7 +401,8 @@ TEST_P(WasmNetworkFilterConfigTest, YamlLoadFromRemoteWasmCreateFilter) { return &request; })); NiceMock threadlocal; - EXPECT_CALL(context_, threadLocal()).WillRepeatedly(ReturnRef(threadlocal)); + EXPECT_CALL(context_.server_factory_context_, threadLocal()) + .WillRepeatedly(ReturnRef(threadlocal)); threadlocal.registered_ = false; auto filter_config = std::make_unique(proto_config, context_); EXPECT_EQ(filter_config->createFilter(), nullptr); @@ -436,7 +441,7 @@ TEST_P(WasmNetworkFilterConfigTest, FailedToGetThreadLocalPlugin) { envoy::extensions::filters::network::wasm::v3::Wasm proto_config; TestUtility::loadFromYaml(yaml, proto_config); - EXPECT_CALL(context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); + EXPECT_CALL(context_.server_factory_context_, threadLocal()).WillOnce(ReturnRef(threadlocal)); threadlocal.registered_ = true; auto filter_config = std::make_unique(proto_config, context_); ASSERT_EQ(threadlocal.current_slot_, 1); diff --git a/test/extensions/filters/network/zookeeper_proxy/config_test.cc b/test/extensions/filters/network/zookeeper_proxy/config_test.cc index 14d6ea6ee704..30043aaa11c8 100644 --- a/test/extensions/filters/network/zookeeper_proxy/config_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/config_test.cc @@ -24,6 +24,8 @@ class ZookeeperFilterConfigTest : public testing::Test { std::string yaml = R"EOF( stat_prefix: test_prefix max_packet_bytes: 1048576 +enable_per_opcode_request_bytes_metrics: true +enable_per_opcode_response_bytes_metrics: true enable_latency_threshold_metrics: true default_latency_threshold: "0.1s" latency_threshold_overrides:)EOF"; @@ -179,6 +181,8 @@ stat_prefix: test_prefix TestUtility::loadFromYamlAndValidate(yaml, proto_config_); EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); EXPECT_EQ(proto_config_.max_packet_bytes().value(), 0); + EXPECT_EQ(proto_config_.enable_per_opcode_request_bytes_metrics(), false); + EXPECT_EQ(proto_config_.enable_per_opcode_response_bytes_metrics(), false); EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), false); EXPECT_EQ(proto_config_.default_latency_threshold(), ProtobufWkt::util::TimeUtil::SecondsToDuration(0)); @@ -198,6 +202,8 @@ default_latency_threshold: "0.15s" TestUtility::loadFromYamlAndValidate(yaml, proto_config_); EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); EXPECT_EQ(proto_config_.max_packet_bytes().value(), 0); + EXPECT_EQ(proto_config_.enable_per_opcode_request_bytes_metrics(), false); + EXPECT_EQ(proto_config_.enable_per_opcode_response_bytes_metrics(), false); EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), false); EXPECT_EQ(proto_config_.default_latency_threshold(), ProtobufWkt::util::TimeUtil::MillisecondsToDuration(150)); @@ -219,6 +225,8 @@ stat_prefix: test_prefix TestUtility::loadFromYamlAndValidate(yaml, proto_config_); EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); EXPECT_EQ(proto_config_.max_packet_bytes().value(), 0); + EXPECT_EQ(proto_config_.enable_per_opcode_request_bytes_metrics(), false); + EXPECT_EQ(proto_config_.enable_per_opcode_response_bytes_metrics(), false); EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), false); EXPECT_EQ(proto_config_.default_latency_threshold(), ProtobufWkt::util::TimeUtil::SecondsToDuration(0)); @@ -241,6 +249,8 @@ TEST_F(ZookeeperFilterConfigTest, FullConfig) { EXPECT_EQ(proto_config_.stat_prefix(), "test_prefix"); EXPECT_EQ(proto_config_.max_packet_bytes().value(), 1048576); + EXPECT_EQ(proto_config_.enable_per_opcode_request_bytes_metrics(), true); + EXPECT_EQ(proto_config_.enable_per_opcode_response_bytes_metrics(), true); EXPECT_EQ(proto_config_.enable_latency_threshold_metrics(), true); EXPECT_EQ(proto_config_.default_latency_threshold(), ProtobufWkt::util::TimeUtil::MillisecondsToDuration(100)); diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc index c890a6ed090c..f22f9f14cee4 100644 --- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc @@ -29,13 +29,16 @@ MATCHER_P(MapEq, rhs, "") { return protoMapEq(arg, rhs); } class ZooKeeperFilterTest : public testing::Test { public: void initialize( + const bool enable_per_opcode_request_bytes_metrics = true, + const bool enable_per_opcode_response_bytes_metrics = true, const bool enable_latency_threshold_metrics = true, const std::chrono::milliseconds default_latency_threshold = std::chrono::milliseconds(100), const LatencyThresholdOverrideList& latency_threshold_overrides = LatencyThresholdOverrideList()) { config_ = std::make_shared( - stat_prefix_, 1048576, enable_latency_threshold_metrics, default_latency_threshold, - latency_threshold_overrides, scope_); + stat_prefix_, 1048576, enable_per_opcode_request_bytes_metrics, + enable_per_opcode_response_bytes_metrics, enable_latency_threshold_metrics, + default_latency_threshold, latency_threshold_overrides, scope_); filter_ = std::make_unique(config_, time_system_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } @@ -566,12 +569,15 @@ class ZooKeeperFilterTest : public testing::Test { switch (opcode) { case OpCodes::Create: EXPECT_EQ(1UL, config_->stats().create_rq_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.create_rq_bytes").value()); break; case OpCodes::CreateContainer: EXPECT_EQ(1UL, config_->stats().createcontainer_rq_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.createcontainer_rq_bytes").value()); break; case OpCodes::CreateTtl: EXPECT_EQ(1UL, config_->stats().createttl_rq_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.createttl_rq_bytes").value()); break; default: break; @@ -620,30 +626,86 @@ class ZooKeeperFilterTest : public testing::Test { } EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(32UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } - void testRequest(Buffer::OwnedImpl& data, const std::vector& metadata_values, - const Stats::Counter& stat, const uint64_t request_bytes) { + void testRequest(Buffer::OwnedImpl& data, const std::vector& metadata_values) { expectSetDynamicMetadata(metadata_values); + + std::string opcode = ""; + uint64_t request_bytes = 0; + for (const auto& metadata : metadata_values) { + auto it = metadata.find("opname"); + if (it != metadata.end()) { + opcode = it->second; + } + + it = metadata.find("bytes"); + if (it != metadata.end()) { + request_bytes = std::stoull(it->second); + } + } + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, stat.value()); + EXPECT_EQ(1UL, store_.counter(absl::StrCat("test.zookeeper.", opcode, "_rq")).value()); EXPECT_EQ(request_bytes, config_->stats().request_bytes_.value()); + EXPECT_EQ(request_bytes, + store_.counter(absl::StrCat("test.zookeeper.", opcode, "_rq_bytes")).value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } - void testResponse(const std::vector& metadata_values, - const Stats::Counter& resp_counter, const Stats::Counter& resp_fast_counter, - const Stats::Counter& resp_slow_counter, const uint32_t xid = 1000, + void testControlRequest(Buffer::OwnedImpl& data, const std::vector& metadata_values, + std::string rq_opcode, std::string rq_bytes_opcode) { + expectSetDynamicMetadata(metadata_values); + + uint64_t request_bytes = 0; + for (const auto& metadata : metadata_values) { + auto it = metadata.find("bytes"); + if (it != metadata.end()) { + request_bytes = std::stoull(it->second); + } + } + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, store_.counter(absl::StrCat("test.zookeeper.", rq_opcode, "_rq")).value()); + EXPECT_EQ(request_bytes, config_->stats().request_bytes_.value()); + EXPECT_EQ( + request_bytes, + store_.counter(absl::StrCat("test.zookeeper.", rq_bytes_opcode, "_rq_bytes")).value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + } + + void testResponse(const std::vector& metadata_values, const uint32_t xid = 1000, const uint64_t zxid = 2000, const uint32_t response_count = 1) { + expectSetDynamicMetadata(metadata_values); + Buffer::OwnedImpl data = encodeResponseHeader(xid, zxid, 0); + std::string resp = ""; + for (const auto& metadata : metadata_values) { + auto it = metadata.find("opname"); + if (it != metadata.end()) { + resp = it->second; + } + } + + // Some opcode names in control response metadata have `_response` suffix. + // However, its corresponding metric names have `_resp` suffix. + std::string long_resp_suffix = "_response"; + std::string short_resp_suffix = "_resp"; + size_t pos = resp.rfind(long_resp_suffix); + if (pos != std::string::npos) { + resp.replace(pos, long_resp_suffix.length(), short_resp_suffix); + } - expectSetDynamicMetadata(metadata_values); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); - EXPECT_EQ(1UL * response_count, resp_counter.value()); - EXPECT_EQ(1UL * response_count, resp_fast_counter.value()); - EXPECT_EQ(0UL, resp_slow_counter.value()); + EXPECT_EQ(1UL * response_count, store_.counter(absl::StrCat("test.zookeeper.", resp)).value()); + EXPECT_EQ(1UL * response_count, + store_.counter(absl::StrCat("test.zookeeper.", resp, "_fast")).value()); + EXPECT_EQ(0UL, store_.counter(absl::StrCat("test.zookeeper.", resp, "_slow")).value()); EXPECT_EQ(20UL * response_count, config_->stats().response_bytes_.value()); + EXPECT_EQ(20UL * response_count, + store_.counter(absl::StrCat("test.zookeeper.", resp, "_bytes")).value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); const auto histogram_name = fmt::format("test.zookeeper.{}_latency", metadata_values[0].find("opname")->second); @@ -669,7 +731,7 @@ TEST_F(ZooKeeperFilterTest, DisableErrorBudgetCalculation) { std::chrono::milliseconds default_latency_threshold(200); LatencyThresholdOverrideList latency_threshold_overrides; - initialize(false, default_latency_threshold, latency_threshold_overrides); + initialize(true, true, false, default_latency_threshold, latency_threshold_overrides); EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(50)), ErrorBudgetResponseType::None); @@ -692,7 +754,7 @@ TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithDefaultLatencyThresholdConfig std::chrono::milliseconds default_latency_threshold(200); LatencyThresholdOverrideList latency_threshold_overrides; - initialize(true, default_latency_threshold, latency_threshold_overrides); + initialize(true, true, true, default_latency_threshold, latency_threshold_overrides); EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(50)), ErrorBudgetResponseType::Fast); @@ -719,7 +781,7 @@ TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithMultiLatencyThresholdConfig) threshold_override->set_opcode(LatencyThresholdOverride::Multi); threshold_override->mutable_threshold()->set_nanos(200000000); // 200 milliseconds - initialize(true, default_latency_threshold, latency_threshold_overrides); + initialize(true, true, true, default_latency_threshold, latency_threshold_overrides); EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(50)), ErrorBudgetResponseType::Fast); @@ -749,7 +811,7 @@ TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithDefaultAndOtherLatencyThresho threshold_override->set_opcode(LatencyThresholdOverride::Create); threshold_override->mutable_threshold()->set_nanos(200000000); // 200 milliseconds - initialize(true, default_latency_threshold, latency_threshold_overrides); + initialize(true, true, true, default_latency_threshold, latency_threshold_overrides); EXPECT_EQ(config_->errorBudgetDecision(OpCodes::Connect, std::chrono::milliseconds(150)), ErrorBudgetResponseType::Fast); @@ -774,12 +836,105 @@ TEST_F(ZooKeeperFilterTest, ErrorBudgetDecisionWithDefaultAndOtherLatencyThresho ErrorBudgetResponseType::Slow); } +TEST_F(ZooKeeperFilterTest, DisablePerOpcodeRequestAndResponseBytesMetrics) { + std::chrono::milliseconds default_latency_threshold(100); + LatencyThresholdOverrideList latency_threshold_overrides; + + initialize(false, false, true, default_latency_threshold, latency_threshold_overrides); + + Buffer::OwnedImpl data = encodeConnect(); + expectSetDynamicMetadata({{{"opname", "connect"}}, {{"bytes", "32"}}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, store_.counter("test.zookeeper.connect_rq").value()); + EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.connect_rq_bytes").value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + + data = encodeConnectResponse(); + expectSetDynamicMetadata({{{"opname", "connect_response"}, + {"protocol_version", "0"}, + {"timeout", "10"}, + {"readonly", "0"}}, + {{"bytes", "24"}}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); + EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.connect_resp_bytes").value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); +} + +TEST_F(ZooKeeperFilterTest, DisablePerOpcodeRequestBytesMetrics) { + std::chrono::milliseconds default_latency_threshold(100); + LatencyThresholdOverrideList latency_threshold_overrides; + + initialize(false, true, true, default_latency_threshold, latency_threshold_overrides); + + Buffer::OwnedImpl data = encodeConnect(); + expectSetDynamicMetadata({{{"opname", "connect"}}, {{"bytes", "32"}}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, store_.counter("test.zookeeper.connect_rq").value()); + EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.connect_rq_bytes").value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + + data = encodeConnectResponse(); + expectSetDynamicMetadata({{{"opname", "connect_response"}, + {"protocol_version", "0"}, + {"timeout", "10"}, + {"readonly", "0"}}, + {{"bytes", "24"}}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); + EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.connect_resp_bytes").value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); +} + +TEST_F(ZooKeeperFilterTest, DisablePerOpcodeResponseBytesMetrics) { + std::chrono::milliseconds default_latency_threshold(100); + LatencyThresholdOverrideList latency_threshold_overrides; + + initialize(true, false, true, default_latency_threshold, latency_threshold_overrides); + + Buffer::OwnedImpl data = encodeConnect(); + expectSetDynamicMetadata({{{"opname", "connect"}}, {{"bytes", "32"}}}); + + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, store_.counter("test.zookeeper.connect_rq").value()); + EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(32UL, store_.counter("test.zookeeper.connect_rq_bytes").value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + + data = encodeConnectResponse(); + expectSetDynamicMetadata({{{"opname", "connect_response"}, + {"protocol_version", "0"}, + {"timeout", "10"}, + {"readonly", "0"}}, + {{"bytes", "24"}}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); + EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); + EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.connect_resp_bytes").value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); +} + TEST_F(ZooKeeperFilterTest, Connect) { initialize(); Buffer::OwnedImpl data = encodeConnect(); - testRequest(data, {{{"opname", "connect"}}, {{"bytes", "32"}}}, config_->stats().connect_rq_, 32); + testRequest(data, {{{"opname", "connect"}}, {{"bytes", "32"}}}); data = encodeConnectResponse(); expectSetDynamicMetadata({{{"opname", "connect_response"}, @@ -792,6 +947,7 @@ TEST_F(ZooKeeperFilterTest, Connect) { EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.connect_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); } @@ -801,8 +957,8 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { Buffer::OwnedImpl data = encodeConnect(true); - testRequest(data, {{{"opname", "connect_readonly"}}, {{"bytes", "33"}}}, - config_->stats().connect_readonly_rq_, 33); + testControlRequest(data, {{{"opname", "connect_readonly"}}, {{"bytes", "33"}}}, + "connect_readonly", "connect"); data = encodeConnectResponse(true); expectSetDynamicMetadata({{{"opname", "connect_response"}, @@ -815,6 +971,7 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { EXPECT_EQ(1UL, config_->stats().connect_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().connect_resp_slow_.value()); EXPECT_EQ(25UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(25UL, store_.counter("test.zookeeper.connect_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); } @@ -871,10 +1028,9 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { Buffer::OwnedImpl data = encodePing(); - testRequest(data, {{{"opname", "ping"}}, {{"bytes", "12"}}}, config_->stats().ping_rq_, 12); + testRequest(data, {{{"opname", "ping"}}, {{"bytes", "12"}}}); testResponse({{{"opname", "ping_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().ping_resp_, config_->stats().ping_resp_fast_, - config_->stats().ping_resp_slow_, enumToSignedInt(XidCodes::PingXid)); + enumToSignedInt(XidCodes::PingXid)); } TEST_F(ZooKeeperFilterTest, AuthRequest) { @@ -882,11 +1038,9 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { Buffer::OwnedImpl data = encodeAuth("digest"); - testRequest(data, {{{"opname", "auth"}}, {{"bytes", "36"}}}, - store_.counter("test.zookeeper.auth.digest_rq"), 36); + testControlRequest(data, {{{"opname", "auth"}}, {{"bytes", "36"}}}, "auth.digest", "auth"); testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().auth_resp_, config_->stats().auth_resp_fast_, - config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid)); + enumToSignedInt(XidCodes::AuthXid)); } TEST_F(ZooKeeperFilterTest, GetDataRequest) { @@ -895,11 +1049,8 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { Buffer::OwnedImpl data = encodePathWatch("/foo", true); testRequest(data, - {{{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, {{"bytes", "21"}}}, - config_->stats().getdata_rq_, 21); - testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getdata_resp_, config_->stats().getdata_resp_fast_, - config_->stats().getdata_resp_slow_); + {{{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, {{"bytes", "21"}}}); + testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { @@ -909,70 +1060,51 @@ TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { // by the server. Buffer::OwnedImpl data = encodePathWatch("", true); - testRequest(data, {{{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, {{"bytes", "17"}}}, - config_->stats().getdata_rq_, 17); - testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getdata_resp_, config_->stats().getdata_resp_fast_, - config_->stats().getdata_resp_slow_); + testRequest(data, {{{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, {{"bytes", "17"}}}); + testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistent) { testCreate(CreateFlags::Persistent, 0); - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistentWithNegativeDataLen) { testCreateWithNegativeDataLen(CreateFlags::Persistent, 0); - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistentSequential) { testCreate(CreateFlags::PersistentSequential, 2); - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestEphemeral) { testCreate(CreateFlags::Ephemeral, 1); - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestEphemeralSequential) { testCreate(CreateFlags::EphemeralSequential, 3); - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestContainer) { testCreate(CreateFlags::Container, 4, OpCodes::CreateContainer); testResponse( - {{{"opname", "createcontainer_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().createcontainer_resp_, config_->stats().createcontainer_resp_fast_, - config_->stats().createcontainer_resp_slow_); + {{{"opname", "createcontainer_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestTTL) { testCreate(CreateFlags::PersistentWithTtl, 5, OpCodes::CreateTtl); testResponse( - {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().createttl_resp_, config_->stats().createttl_resp_fast_, - config_->stats().createttl_resp_slow_); + {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequestTTLSequential) { testCreate(CreateFlags::PersistentSequentialWithTtl, 6, OpCodes::CreateTtl); testResponse( - {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().createttl_resp_, config_->stats().createttl_resp_fast_, - config_->stats().createttl_resp_slow_); + {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CreateRequest2) { @@ -981,13 +1113,9 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", 0, false, 1000, enumToSignedInt(OpCodes::Create2)); - testRequest( - data, - {{{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, {{"bytes", "35"}}}, - config_->stats().create2_rq_, 35); - testResponse({{{"opname", "create2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create2_resp_, config_->stats().create2_resp_fast_, - config_->stats().create2_resp_slow_); + testRequest(data, {{{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, + {{"bytes", "35"}}}); + testResponse({{{"opname", "create2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, SetRequest) { @@ -995,11 +1123,8 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { Buffer::OwnedImpl data = encodeSetRequest("/foo", "bar", -1); - testRequest(data, {{{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}}, - config_->stats().setdata_rq_, 31); - testResponse({{{"opname", "setdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setdata_resp_, config_->stats().setdata_resp_fast_, - config_->stats().setdata_resp_slow_); + testRequest(data, {{{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}}); + testResponse({{{"opname", "setdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { @@ -1009,12 +1134,9 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { encodePathWatch("/foo", false, 1000, enumToSignedInt(OpCodes::GetChildren)); testRequest( - data, {{{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, - config_->stats().getchildren_rq_, 21); + data, {{{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}); testResponse( - {{{"opname", "getchildren_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getchildren_resp_, config_->stats().getchildren_resp_fast_, - config_->stats().getchildren_resp_slow_); + {{{"opname", "getchildren_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { @@ -1023,13 +1145,10 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { Buffer::OwnedImpl data = encodePathWatch("/foo", false, 1000, enumToSignedInt(OpCodes::GetChildren2)); - testRequest( - data, {{{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, - config_->stats().getchildren2_rq_, 21); + testRequest(data, {{{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, + {{"bytes", "21"}}}); testResponse( - {{{"opname", "getchildren2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getchildren2_resp_, config_->stats().getchildren2_resp_fast_, - config_->stats().getchildren2_resp_slow_); + {{{"opname", "getchildren2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, DeleteRequest) { @@ -1038,11 +1157,8 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { Buffer::OwnedImpl data = encodeDeleteRequest("/foo", -1); testRequest(data, - {{{"opname", "delete"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "24"}}}, - config_->stats().delete_rq_, 24); - testResponse({{{"opname", "delete_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().delete_resp_, config_->stats().delete_resp_fast_, - config_->stats().delete_resp_slow_); + {{{"opname", "delete"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "24"}}}); + testResponse({{{"opname", "delete_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, ExistsRequest) { @@ -1051,11 +1167,8 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { Buffer::OwnedImpl data = encodePathWatch("/foo", false, 1000, enumToSignedInt(OpCodes::Exists)); testRequest(data, - {{{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, - config_->stats().exists_rq_, 21); - testResponse({{{"opname", "exists_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().exists_resp_, config_->stats().exists_resp_fast_, - config_->stats().exists_resp_slow_); + {{{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}); + testResponse({{{"opname", "exists_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, GetAclRequest) { @@ -1063,11 +1176,8 @@ TEST_F(ZooKeeperFilterTest, GetAclRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GetAcl)); - testRequest(data, {{{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}}, - config_->stats().getacl_rq_, 20); - testResponse({{{"opname", "getacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getacl_resp_, config_->stats().getacl_resp_fast_, - config_->stats().getacl_resp_slow_); + testRequest(data, {{{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}}); + testResponse({{{"opname", "getacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, SetAclRequest) { @@ -1076,11 +1186,8 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { Buffer::OwnedImpl data = encodeSetAclRequest("/foo", "digest", "passwd", -1); testRequest(data, - {{{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "52"}}}, - config_->stats().setacl_rq_, 52); - testResponse({{{"opname", "setacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setacl_resp_, config_->stats().setacl_resp_fast_, - config_->stats().setacl_resp_slow_); + {{{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "52"}}}); + testResponse({{{"opname", "setacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, SyncRequest) { @@ -1088,11 +1195,8 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::Sync)); - testRequest(data, {{{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}}, - config_->stats().sync_rq_, 20); - testResponse({{{"opname", "sync_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().sync_resp_, config_->stats().sync_resp_fast_, - config_->stats().sync_resp_slow_); + testRequest(data, {{{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}}); + testResponse({{{"opname", "sync_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { @@ -1100,12 +1204,9 @@ TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GetEphemerals)); - testRequest(data, {{{"opname", "getephemerals"}, {"path", "/foo"}}, {{"bytes", "20"}}}, - config_->stats().getephemerals_rq_, 20); + testRequest(data, {{{"opname", "getephemerals"}, {"path", "/foo"}}, {{"bytes", "20"}}}); testResponse( - {{{"opname", "getephemerals_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().getephemerals_resp_, config_->stats().getephemerals_resp_fast_, - config_->stats().getephemerals_resp_slow_); + {{{"opname", "getephemerals_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { @@ -1113,13 +1214,9 @@ TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GetAllChildrenNumber)); - testRequest(data, {{{"opname", "getallchildrennumber"}, {"path", "/foo"}}, {{"bytes", "20"}}}, - config_->stats().getallchildrennumber_rq_, 20); + testRequest(data, {{{"opname", "getallchildrennumber"}, {"path", "/foo"}}, {{"bytes", "20"}}}); testResponse({{{"opname", "getallchildrennumber_resp"}, {"zxid", "2000"}, {"error", "0"}}, - {{"bytes", "20"}}}, - config_->stats().getallchildrennumber_resp_, - config_->stats().getallchildrennumber_resp_fast_, - config_->stats().getallchildrennumber_resp_slow_); + {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CheckRequest) { @@ -1132,11 +1229,10 @@ TEST_F(ZooKeeperFilterTest, CheckRequest) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.check_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); - testResponse({{{"opname", "check_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().check_resp_, config_->stats().check_resp_fast_, - config_->stats().check_resp_slow_); + testResponse({{{"opname", "check_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, MultiRequest) { @@ -1164,15 +1260,14 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().multi_rq_.value()); EXPECT_EQ(200UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(200UL, store_.counter("test.zookeeper.multi_rq_bytes").value()); EXPECT_EQ(3UL, config_->stats().create_rq_.value()); EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); EXPECT_EQ(2UL, config_->stats().delete_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); - testResponse({{{"opname", "multi_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().multi_resp_, config_->stats().multi_resp_fast_, - config_->stats().multi_resp_slow_); + testResponse({{{"opname", "multi_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, ReconfigRequest) { @@ -1180,11 +1275,9 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { Buffer::OwnedImpl data = encodeReconfigRequest("s1", "s2", "s3", 1000); - testRequest(data, {{{"opname", "reconfig"}}, {{"bytes", "38"}}}, config_->stats().reconfig_rq_, - 38); - testResponse({{{"opname", "reconfig_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().reconfig_resp_, config_->stats().reconfig_resp_fast_, - config_->stats().reconfig_resp_slow_); + testRequest(data, {{{"opname", "reconfig"}}, {{"bytes", "38"}}}); + testResponse( + {{{"opname", "reconfig_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { @@ -1197,12 +1290,10 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { Buffer::OwnedImpl data = encodeSetWatchesRequest(dataw, existw, childw, enumToSignedInt(XidCodes::SetWatchesXid)); - testRequest(data, {{{"opname", "setwatches"}}, {{"bytes", "84"}}}, - config_->stats().setwatches_rq_, 84); + testRequest(data, {{{"opname", "setwatches"}}, {{"bytes", "84"}}}); testResponse( {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setwatches_resp_, config_->stats().setwatches_resp_fast_, - config_->stats().setwatches_resp_slow_, enumToSignedInt(XidCodes::SetWatchesXid)); + enumToSignedInt(XidCodes::SetWatchesXid)); } TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { @@ -1214,12 +1305,9 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { Buffer::OwnedImpl data = encodeSetWatchesRequest(dataw, existw, childw); - testRequest(data, {{{"opname", "setwatches"}}, {{"bytes", "84"}}}, - config_->stats().setwatches_rq_, 84); + testRequest(data, {{{"opname", "setwatches"}}, {{"bytes", "84"}}}); testResponse( - {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setwatches_resp_, config_->stats().setwatches_resp_fast_, - config_->stats().setwatches_resp_slow_); + {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, SetWatches2Request) { @@ -1234,12 +1322,9 @@ TEST_F(ZooKeeperFilterTest, SetWatches2Request) { Buffer::OwnedImpl data = encodeSetWatches2Request(dataw, existw, childw, persistentw, persistent_recursivew); - testRequest(data, {{{"opname", "setwatches2"}}, {{"bytes", "126"}}}, - config_->stats().setwatches2_rq_, 126); + testRequest(data, {{{"opname", "setwatches2"}}, {{"bytes", "126"}}}); testResponse( - {{{"opname", "setwatches2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().setwatches2_resp_, config_->stats().setwatches2_resp_fast_, - config_->stats().setwatches2_resp_slow_); + {{{"opname", "setwatches2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, AddWatchRequest) { @@ -1249,11 +1334,9 @@ TEST_F(ZooKeeperFilterTest, AddWatchRequest) { encodePathVersion("/foo", enumToSignedInt(AddWatchMode::PersistentRecursive), enumToSignedInt(OpCodes::AddWatch)); - testRequest(data, {{{"opname", "addwatch"}, {"path", "/foo"}, {"mode", "1"}}, {{"bytes", "24"}}}, - config_->stats().addwatch_rq_, 24); - testResponse({{{"opname", "addwatch_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().addwatch_resp_, config_->stats().addwatch_resp_fast_, - config_->stats().addwatch_resp_slow_); + testRequest(data, {{{"opname", "addwatch"}, {"path", "/foo"}, {"mode", "1"}}, {{"bytes", "24"}}}); + testResponse( + {{{"opname", "addwatch_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { @@ -1262,12 +1345,9 @@ TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { Buffer::OwnedImpl data = encodePathVersion("/foo", enumToSignedInt(WatcherType::Children), enumToSignedInt(OpCodes::CheckWatches)); - testRequest(data, {{{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}}, - config_->stats().checkwatches_rq_, 24); + testRequest(data, {{{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}}); testResponse( - {{{"opname", "checkwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().checkwatches_resp_, config_->stats().checkwatches_resp_fast_, - config_->stats().checkwatches_resp_slow_); + {{{"opname", "checkwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { @@ -1276,12 +1356,9 @@ TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { Buffer::OwnedImpl data = encodePathVersion("/foo", enumToSignedInt(WatcherType::Data), enumToSignedInt(OpCodes::RemoveWatches)); - testRequest(data, {{{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}}, - config_->stats().removewatches_rq_, 24); + testRequest(data, {{{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}}); testResponse( - {{{"opname", "removewatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().removewatches_resp_, config_->stats().removewatches_resp_fast_, - config_->stats().removewatches_resp_slow_); + {{{"opname", "removewatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, CloseRequest) { @@ -1289,10 +1366,8 @@ TEST_F(ZooKeeperFilterTest, CloseRequest) { Buffer::OwnedImpl data = encodeCloseRequest(); - testRequest(data, {{{"opname", "close"}}, {{"bytes", "12"}}}, config_->stats().close_rq_, 12); - testResponse({{{"opname", "close_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().close_resp_, config_->stats().close_resp_fast_, - config_->stats().close_resp_slow_); + testRequest(data, {{{"opname", "close"}}, {{"bytes", "12"}}}); + testResponse({{{"opname", "close_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } TEST_F(ZooKeeperFilterTest, WatchEvent) { @@ -1308,6 +1383,10 @@ TEST_F(ZooKeeperFilterTest, WatchEvent) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); EXPECT_EQ(1UL, config_->stats().watch_event_.value()); EXPECT_EQ(36UL, config_->stats().response_bytes_.value()); + // WATCH_XID is generated by the server, it has no corresponding opcode. + // Below expectation makes sure that WATCH_XID does not return the default opcode (which is + // connect). + EXPECT_EQ(0UL, store_.counter("test.zookeeper.connect_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1336,6 +1415,7 @@ TEST_F(ZooKeeperFilterTest, OneRequestWithMultipleOnDataCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(0UL, config_->stats().create_rq_.value()); EXPECT_EQ(0UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1350,12 +1430,11 @@ TEST_F(ZooKeeperFilterTest, OneRequestWithMultipleOnDataCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response. - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); } // |REQ1|REQ2| @@ -1371,15 +1450,13 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithOneOnDataCall) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(2UL, config_->stats().create_rq_.value()); EXPECT_EQ(71UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(71UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. - testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1000, 2000); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1001, 2001, 2); + 1001, 2001, 2); } // |REQ1|REQ2| @@ -1394,15 +1471,14 @@ TEST_F(ZooKeeperFilterTest, MultipleControlRequestsWithOneOnDataCall) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(2UL, store_.counter("test.zookeeper.auth.digest_rq").value()); EXPECT_EQ(72UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(72UL, store_.counter("test.zookeeper.auth_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().auth_resp_, config_->stats().auth_resp_fast_, - config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid), 2000); + enumToSignedInt(XidCodes::AuthXid), 2000); testResponse({{{"opname", "auth_response"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().auth_resp_, config_->stats().auth_resp_fast_, - config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid), 2001, 2); + enumToSignedInt(XidCodes::AuthXid), 2001, 2); } // |REQ1|REQ2| @@ -1418,12 +1494,13 @@ TEST_F(ZooKeeperFilterTest, MixedControlAndDataRequestsWithOneOnDataCall) { EXPECT_EQ(1UL, store_.counter("test.zookeeper.auth.digest_rq").value()); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(71UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(36UL, store_.counter("test.zookeeper.auth_rq_bytes").value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().auth_resp_, config_->stats().auth_resp_fast_, - config_->stats().auth_resp_slow_, enumToSignedInt(XidCodes::AuthXid), 2000); + enumToSignedInt(XidCodes::AuthXid), 2000); Buffer::OwnedImpl resp_data = encodeResponseHeader(1000, 2001, 0); expectSetDynamicMetadata( @@ -1433,6 +1510,8 @@ TEST_F(ZooKeeperFilterTest, MixedControlAndDataRequestsWithOneOnDataCall) { EXPECT_EQ(1UL, config_->stats().create_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().create_resp_slow_.value()); EXPECT_EQ(40UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(20UL, store_.counter("test.zookeeper.auth_resp_bytes").value()); + EXPECT_EQ(20UL, store_.counter("test.zookeeper.create_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.create_resp_latency")); } @@ -1449,6 +1528,7 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(0UL, config_->stats().create_rq_.value()); EXPECT_EQ(0UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1467,6 +1547,7 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1483,15 +1564,14 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(2UL, config_->stats().create_rq_.value()); EXPECT_EQ(71UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(71UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1000, 2000); + 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1001, 2001, 2); + 1001, 2001, 2); } // |REQ1 ------|REQ2|REQ3| @@ -1506,6 +1586,7 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls2) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(0UL, config_->stats().create_rq_.value()); EXPECT_EQ(0UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1523,18 +1604,16 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls2) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(3UL, config_->stats().create_rq_.value()); EXPECT_EQ(108UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(108UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1000, 2000); + 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1001, 2001, 2); + 1001, 2001, 2); testResponse({{{"opname", "create_resp"}, {"zxid", "2002"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1002, 2002, 3); + 1002, 2002, 3); } // |REQ1|REQ2|REQ3 ------| @@ -1552,6 +1631,7 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls3) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(2UL, config_->stats().create_rq_.value()); EXPECT_EQ(71UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(71UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1566,18 +1646,16 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls3) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(3UL, config_->stats().create_rq_.value()); EXPECT_EQ(108UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(108UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1000, 2000); + 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1001, 2001, 2); + 1001, 2001, 2); testResponse({{{"opname", "create_resp"}, {"zxid", "2002"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1002, 2002, 3); + 1002, 2002, 3); } // |REQ1|REQ2 ----------|REQ3| @@ -1595,6 +1673,7 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1607,6 +1686,7 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().create_rq_.value()); EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(35UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. data.drain(data.length()); @@ -1621,18 +1701,16 @@ TEST_F(ZooKeeperFilterTest, MultipleRequestsWithMultipleOnDataCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(3UL, config_->stats().create_rq_.value()); EXPECT_EQ(108UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(108UL, store_.counter("test.zookeeper.create_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Responses. testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1000, 2000); + 1000, 2000); testResponse({{{"opname", "create_resp"}, {"zxid", "2001"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1001, 2001, 2); + 1001, 2001, 2); testResponse({{{"opname", "create_resp"}, {"zxid", "2002"}, {"error", "0"}}, {{"bytes", "20"}}}, - config_->stats().create_resp_, config_->stats().create_resp_fast_, - config_->stats().create_resp_slow_, 1002, 2002, 3); + 1002, 2002, 3); } // |RESP1 ------------| @@ -1646,6 +1724,7 @@ TEST_F(ZooKeeperFilterTest, OneResponseWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false)); EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(21UL, store_.counter("test.zookeeper.getdata_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response (onWrite1). @@ -1656,6 +1735,7 @@ TEST_F(ZooKeeperFilterTest, OneResponseWithMultipleOnWriteCalls) { EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1669,6 +1749,7 @@ TEST_F(ZooKeeperFilterTest, OneResponseWithMultipleOnWriteCalls) { EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1684,6 +1765,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithOneOnWriteCall) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false)); EXPECT_EQ(2UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(42UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(42UL, store_.counter("test.zookeeper.getdata_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response (onWrite1). @@ -1695,6 +1777,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithOneOnWriteCall) { EXPECT_EQ(2UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(48UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(48UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1711,6 +1794,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false)); EXPECT_EQ(2UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(42UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(42UL, store_.counter("test.zookeeper.getdata_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response (onWrite1). @@ -1721,6 +1805,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1738,6 +1823,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1752,6 +1838,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls) { EXPECT_EQ(2UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(50UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(50UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1770,6 +1857,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls2) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false)); EXPECT_EQ(3UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(63UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(63UL, store_.counter("test.zookeeper.getdata_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response (onWrite1). @@ -1780,6 +1868,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls2) { EXPECT_EQ(0UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(0UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(0UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1796,6 +1885,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls2) { EXPECT_EQ(3UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(72UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(72UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1814,6 +1904,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls3) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false)); EXPECT_EQ(3UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(63UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(63UL, store_.counter("test.zookeeper.getdata_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response (onWrite1). @@ -1826,6 +1917,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls3) { EXPECT_EQ(2UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(48UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(48UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1839,6 +1931,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls3) { EXPECT_EQ(3UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(72UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(72UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -1857,6 +1950,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(rq_data, false)); EXPECT_EQ(3UL, config_->stats().getdata_rq_.value()); EXPECT_EQ(63UL, config_->stats().request_bytes_.value()); + EXPECT_EQ(63UL, store_.counter("test.zookeeper.getdata_rq_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Response (onWrite1). @@ -1868,6 +1962,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1881,6 +1976,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(1UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(24UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); // Mock the buffer is drained by the tcp_proxy filter. resp_data.drain(resp_data.length()); @@ -1896,6 +1992,7 @@ TEST_F(ZooKeeperFilterTest, MultipleResponsesWithMultipleOnWriteCalls4) { EXPECT_EQ(3UL, config_->stats().getdata_resp_fast_.value()); EXPECT_EQ(0UL, config_->stats().getdata_resp_slow_.value()); EXPECT_EQ(72UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(72UL, store_.counter("test.zookeeper.getdata_resp_bytes").value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } diff --git a/test/extensions/filters/udp/dns_filter/dns_filter_integration_test.cc b/test/extensions/filters/udp/dns_filter/dns_filter_integration_test.cc index c0f2bfadbdca..01fdf8fd3e48 100644 --- a/test/extensions/filters/udp/dns_filter/dns_filter_integration_test.cc +++ b/test/extensions/filters/udp/dns_filter/dns_filter_integration_test.cc @@ -22,11 +22,6 @@ class DnsFilterIntegrationTest : public testing::TestWithParam(); NiceMock dns_resolver_factory_; @@ -645,7 +646,7 @@ TEST_F(DnsFilterTest, RepeatedTypeAQuerySuccess) { for (size_t i = 0; i < loopCount; i++) { // Generate a changing, non-zero query ID for each lookup - const uint16_t query_id = (random_.random() + i) & 0xFFFF; + const uint16_t query_id = (random_.random() + i) % 0xFFFF + 1; const std::string query = Utils::buildQueryForDomain(domain, DNS_RECORD_TYPE_A, DNS_RECORD_CLASS_IN, query_id); ASSERT_FALSE(query.empty()); @@ -1722,7 +1723,7 @@ TEST_F(DnsFilterTest, InvalidQueryNameTest2) { EXPECT_FALSE(response_ctx_->parse_status_); EXPECT_EQ(DNS_RESPONSE_CODE_FORMAT_ERROR, response_ctx_->getQueryResponseCode()); - // TODO(abaptiste): underflow/overflow stats + // TODO(suniltheta): underflow/overflow stats EXPECT_EQ(1, config_->stats().downstream_rx_invalid_queries_.value()); } diff --git a/test/extensions/filters/udp/dns_filter/dns_filter_test_utils.cc b/test/extensions/filters/udp/dns_filter/dns_filter_test_utils.cc index e22b86b94dd7..ed873d300424 100644 --- a/test/extensions/filters/udp/dns_filter/dns_filter_test_utils.cc +++ b/test/extensions/filters/udp/dns_filter/dns_filter_test_utils.cc @@ -26,7 +26,7 @@ std::string buildQueryForDomain(const std::string& name, uint16_t rec_type, uint const uint16_t query_id) { Random::RandomGeneratorImpl random_; struct DnsHeader query {}; - uint16_t id = (query_id ? query_id : random_.random() & 0xFFFF); + uint16_t id = query_id ? query_id : (random_.random() % 0xFFFF) + 1; // Generate a random query ID query.id = id; @@ -375,7 +375,7 @@ DnsAnswerRecordPtr DnsResponseValidator::parseDnsAnswerRecord(const Buffer::Inst offset += sizeof(uint16_t); available_bytes -= sizeof(uint16_t); - // TODO(abaptiste): Support Extension Mechanisms for DNS (RFC2671) + // TODO(suniltheta): Support Extension Mechanisms for DNS (RFC2671) // // We may see optional records indicating DNS extension support. We need to skip // these records until we have proper support. Encountering one of these records diff --git a/test/extensions/filters/udp/udp_proxy/BUILD b/test/extensions/filters/udp/udp_proxy/BUILD index d0bdab133c6d..5d508de0f039 100644 --- a/test/extensions/filters/udp/udp_proxy/BUILD +++ b/test/extensions/filters/udp/udp_proxy/BUILD @@ -18,6 +18,7 @@ envoy_extension_cc_mock( hdrs = ["mocks.h"], extension_names = ["envoy.filters.udp_listener.udp_proxy"], deps = [ + "//source/extensions/filters/udp/udp_proxy:udp_proxy_filter_lib", "//source/extensions/filters/udp/udp_proxy/session_filters:filter_interface", "//test/mocks/network:network_mocks", "//test/mocks/stream_info:stream_info_mocks", @@ -31,12 +32,18 @@ envoy_extension_cc_test( srcs = ["udp_proxy_filter_test.cc"], extension_names = ["envoy.filters.udp_listener.udp_proxy"], deps = [ + ":mocks", "//source/common/common:hash_lib", + "//source/common/router:string_accessor_lib", + "//source/common/stream_info:uint32_accessor_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/filters/udp/udp_proxy:config", "//source/extensions/filters/udp/udp_proxy:udp_proxy_filter_lib", "//source/extensions/matching/network/common:inputs_lib", + "//test/extensions/filters/udp/udp_proxy/session_filters:drainer_filter_config_lib", + "//test/extensions/filters/udp/udp_proxy/session_filters:drainer_filter_proto_cc_proto", "//test/mocks/api:api_mocks", + "//test/mocks/http:stream_encoder_mock", "//test/mocks/network:socket_mocks", "//test/mocks/server:listener_factory_context_mocks", "//test/mocks/upstream:cluster_manager_mocks", diff --git a/test/extensions/filters/udp/udp_proxy/hash_policy_impl_test.cc b/test/extensions/filters/udp/udp_proxy/hash_policy_impl_test.cc index 147b4aab2578..5453248fdcfd 100644 --- a/test/extensions/filters/udp/udp_proxy/hash_policy_impl_test.cc +++ b/test/extensions/filters/udp/udp_proxy/hash_policy_impl_test.cc @@ -6,6 +6,8 @@ #include "source/common/network/utility.h" #include "source/extensions/filters/udp/udp_proxy/hash_policy_impl.h" +#include "test/test_common/utility.h" + #include "gtest/gtest.h" namespace Envoy { diff --git a/test/extensions/filters/udp/udp_proxy/mocks.cc b/test/extensions/filters/udp/udp_proxy/mocks.cc index 1053c36d0875..7c654662a83e 100644 --- a/test/extensions/filters/udp/udp_proxy/mocks.cc +++ b/test/extensions/filters/udp/udp_proxy/mocks.cc @@ -3,7 +3,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Invoke; using testing::Return; using testing::ReturnRef; @@ -16,6 +15,7 @@ namespace SessionFilters { MockReadFilterCallbacks::MockReadFilterCallbacks() { ON_CALL(*this, sessionId()).WillByDefault(Return(session_id_)); ON_CALL(*this, streamInfo()).WillByDefault(ReturnRef(stream_info_)); + ON_CALL(*this, continueFilterChain()).WillByDefault(Return(true)); } MockReadFilterCallbacks::~MockReadFilterCallbacks() = default; @@ -25,6 +25,21 @@ MockWriteFilterCallbacks::MockWriteFilterCallbacks() { } MockWriteFilterCallbacks::~MockWriteFilterCallbacks() = default; +MockUdpTunnelingConfig::MockUdpTunnelingConfig(Http::HeaderEvaluator& header_evaluator) + : header_evaluator_(header_evaluator) { + ON_CALL(*this, proxyHost(_)).WillByDefault(Return(default_proxy_host_)); + ON_CALL(*this, targetHost(_)).WillByDefault(Return(default_target_host_)); + ON_CALL(*this, proxyPort()).WillByDefault(ReturnRef(default_proxy_port_)); + ON_CALL(*this, defaultTargetPort()).WillByDefault(Return(default_target_port_)); + ON_CALL(*this, postPath()).WillByDefault(ReturnRef(post_path_)); + ON_CALL(*this, headerEvaluator()).WillByDefault(ReturnRef(header_evaluator_)); +} +MockUdpTunnelingConfig::~MockUdpTunnelingConfig() = default; + +MockTunnelCreationCallbacks::~MockTunnelCreationCallbacks() = default; +MockUpstreamTunnelCallbacks::~MockUpstreamTunnelCallbacks() = default; +MockHttpStreamCallbacks::~MockHttpStreamCallbacks() = default; + } // namespace SessionFilters } // namespace UdpProxy } // namespace UdpFilters diff --git a/test/extensions/filters/udp/udp_proxy/mocks.h b/test/extensions/filters/udp/udp_proxy/mocks.h index 089806d5cdff..fb8e913c2821 100644 --- a/test/extensions/filters/udp/udp_proxy/mocks.h +++ b/test/extensions/filters/udp/udp_proxy/mocks.h @@ -1,6 +1,7 @@ #pragma once #include "source/extensions/filters/udp/udp_proxy/session_filters/filter.h" +#include "source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h" #include "test/mocks/stream_info/mocks.h" @@ -21,7 +22,7 @@ class MockReadFilterCallbacks : public ReadFilterCallbacks { MOCK_METHOD(uint64_t, sessionId, (), (const)); MOCK_METHOD(StreamInfo::StreamInfo&, streamInfo, ()); - MOCK_METHOD(void, continueFilterChain, ()); + MOCK_METHOD(bool, continueFilterChain, ()); MOCK_METHOD(void, injectDatagramToFilterChain, (Network::UdpRecvData & data)); uint64_t session_id_{1}; @@ -41,6 +42,71 @@ class MockWriteFilterCallbacks : public WriteFilterCallbacks { NiceMock stream_info_; }; +class MockUdpTunnelingConfig : public UdpTunnelingConfig { +public: + MockUdpTunnelingConfig(Http::HeaderEvaluator& header_evaluator); + ~MockUdpTunnelingConfig() override; + + MOCK_METHOD(const std::string, proxyHost, (const StreamInfo::StreamInfo& stream_info), (const)); + MOCK_METHOD(const std::string, targetHost, (const StreamInfo::StreamInfo& stream_info), (const)); + MOCK_METHOD(const absl::optional&, proxyPort, (), (const)); + MOCK_METHOD(uint32_t, defaultTargetPort, (), (const)); + MOCK_METHOD(bool, usePost, (), (const)); + MOCK_METHOD(const std::string&, postPath, (), (const)); + MOCK_METHOD(Http::HeaderEvaluator&, headerEvaluator, (), (const)); + MOCK_METHOD(uint32_t, maxConnectAttempts, (), (const)); + MOCK_METHOD(bool, bufferEnabled, (), (const)); + MOCK_METHOD(uint32_t, maxBufferedDatagrams, (), (const)); + MOCK_METHOD(uint64_t, maxBufferedBytes, (), (const)); + MOCK_METHOD(void, propagateResponseHeaders, + (Http::ResponseHeaderMapPtr && headers, + const StreamInfo::FilterStateSharedPtr& filter_state), + (const)); + MOCK_METHOD(void, propagateResponseTrailers, + (Http::ResponseTrailerMapPtr && trailers, + const StreamInfo::FilterStateSharedPtr& filter_state), + (const)); + + std::string default_proxy_host_ = "default.host.com"; + std::string default_target_host_ = "default.target.host"; + const absl::optional default_proxy_port_ = 10; + uint32_t default_target_port_{20}; + std::string post_path_ = "/default/post"; + Http::HeaderEvaluator& header_evaluator_; +}; + +class MockTunnelCreationCallbacks : public TunnelCreationCallbacks { +public: + ~MockTunnelCreationCallbacks() override; + + MOCK_METHOD(void, onStreamSuccess, (Http::RequestEncoder & request_encoder)); + MOCK_METHOD(void, onStreamFailure, ()); +}; + +class MockUpstreamTunnelCallbacks : public UpstreamTunnelCallbacks { +public: + ~MockUpstreamTunnelCallbacks() override; + + MOCK_METHOD(void, onUpstreamEvent, (Network::ConnectionEvent event)); + MOCK_METHOD(void, onAboveWriteBufferHighWatermark, ()); + MOCK_METHOD(void, onBelowWriteBufferLowWatermark, ()); + MOCK_METHOD(void, onUpstreamData, (Buffer::Instance & data, bool end_stream)); +}; + +class MockHttpStreamCallbacks : public HttpStreamCallbacks { +public: + ~MockHttpStreamCallbacks() override; + + MOCK_METHOD(void, onStreamReady, + (StreamInfo::StreamInfo * info, std::unique_ptr&& upstream, + Upstream::HostDescriptionConstSharedPtr& host, + const Network::ConnectionInfoProvider& address_provider, + Ssl::ConnectionInfoConstSharedPtr ssl_info)); + MOCK_METHOD(void, onStreamFailure, + (ConnectionPool::PoolFailureReason reason, absl::string_view failure_reason, + Upstream::HostDescriptionConstSharedPtr host)); +}; + } // namespace SessionFilters } // namespace UdpProxy } // namespace UdpFilters diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/BUILD b/test/extensions/filters/udp/udp_proxy/session_filters/BUILD index 066bd6185c2d..a2bb0d9f1527 100644 --- a/test/extensions/filters/udp/udp_proxy/session_filters/BUILD +++ b/test/extensions/filters/udp/udp_proxy/session_filters/BUILD @@ -30,7 +30,7 @@ envoy_proto_library( envoy_cc_test_library( name = "drainer_filter_config_lib", - srcs = ["drainer_filter.h"], + hdrs = ["drainer_filter.h"], deps = [ ":drainer_filter_proto_cc_proto", "//envoy/registry", @@ -48,10 +48,11 @@ envoy_proto_library( envoy_cc_test_library( name = "buffer_filter_config_lib", - srcs = ["buffer_filter.h"], + hdrs = ["buffer_filter.h"], deps = [ ":buffer_filter_proto_cc_proto", "//envoy/registry", + "//source/common/router:string_accessor_lib", "//source/extensions/filters/udp/udp_proxy/session_filters:factory_base_lib", "//source/extensions/filters/udp/udp_proxy/session_filters:filter_interface", "//test/test_common:utility_lib", diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/buffer_filter.h b/test/extensions/filters/udp/udp_proxy/session_filters/buffer_filter.h index e34f431b4e16..8f32cb16b51c 100644 --- a/test/extensions/filters/udp/udp_proxy/session_filters/buffer_filter.h +++ b/test/extensions/filters/udp/udp_proxy/session_filters/buffer_filter.h @@ -5,6 +5,7 @@ #include "envoy/registry/registry.h" #include "source/common/config/utility.h" +#include "source/common/router/string_accessor_impl.h" #include "source/extensions/filters/udp/udp_proxy/session_filters/factory_base.h" #include "source/extensions/filters/udp/udp_proxy/session_filters/filter.h" @@ -33,10 +34,18 @@ class BufferingSessionFilter : public Filter { void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) override { read_callbacks_ = &callbacks; + // Verify that the filter is able to access the stream info. + callbacks.streamInfo().filterState()->setData( + "test.read", std::make_shared("val"), + Envoy::StreamInfo::FilterState::StateType::Mutable); } void initializeWriteFilterCallbacks(WriteFilterCallbacks& callbacks) override { write_callbacks_ = &callbacks; + // Verify that the filter is able to access the stream info. + callbacks.streamInfo().filterState()->setData( + "test.write", std::make_shared("val"), + Envoy::StreamInfo::FilterState::StateType::Mutable); } ReadFilterStatus onNewSession() override { return ReadFilterStatus::Continue; } diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/drainer_filter.h b/test/extensions/filters/udp/udp_proxy/session_filters/drainer_filter.h index 438de329d5a0..b62b0d355414 100644 --- a/test/extensions/filters/udp/udp_proxy/session_filters/drainer_filter.h +++ b/test/extensions/filters/udp/udp_proxy/session_filters/drainer_filter.h @@ -32,6 +32,14 @@ class DrainerUdpSessionReadFilter : public virtual ReadFilter { stop_iteration_on_first_read_(stop_iteration_on_first_read), continue_filter_chain_(continue_filter_chain) {} + void onSessionCompleteInternal() override { + read_callbacks_->streamInfo().filterState()->setData( + "test.udp_session.drainer.on_session_complete", + std::make_shared("session_complete"), + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection, + StreamInfo::StreamSharingMayImpactPooling::SharedWithUpstreamConnection); + } + ReadFilterStatus onNewSession() override { if (stop_iteration_on_new_session_) { // We can count how many times onNewSession was called on a filter chain diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/BUILD b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..3c6b4bd3a4cf --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/BUILD @@ -0,0 +1,69 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", + "envoy_package", + "envoy_proto_library", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_proto_library( + name = "dfp_setter_filter_proto", + srcs = ["dfp_setter.proto"], +) + +envoy_cc_test_library( + name = "dfp_setter_filter_config_lib", + srcs = ["dfp_setter.h"], + deps = [ + ":dfp_setter_filter_proto_cc_proto", + "//envoy/registry", + "//source/common/router:string_accessor_lib", + "//source/common/stream_info:uint32_accessor_lib", + "//source/extensions/filters/udp/udp_proxy/session_filters:factory_base_lib", + "//source/extensions/filters/udp/udp_proxy/session_filters:filter_interface", + "//test/test_common:utility_lib", + ], + alwayslink = 1, +) + +envoy_extension_cc_test( + name = "proxy_filter_test", + srcs = ["proxy_filter_test.cc"], + extension_names = ["envoy.filters.udp.session.dynamic_forward_proxy"], + deps = [ + "//source/common/router:string_accessor_lib", + "//source/common/stream_info:uint32_accessor_lib", + "//source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy:config", + "//test/extensions/common/dynamic_forward_proxy:mocks", + "//test/extensions/filters/udp/udp_proxy:mocks", + "//test/mocks/server:factory_context_mocks", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "dynamic_forward_proxy_filter_integration_test", + srcs = ["proxy_filter_integration_test.cc"], + extension_names = ["envoy.filters.udp.session.dynamic_forward_proxy"], + tags = ["cpu:3"], + deps = [ + ":dfp_setter_filter_config_lib", + ":dfp_setter_filter_proto_cc_proto", + "//envoy/network:filter_interface", + "//envoy/server:filter_config_interface", + "//source/extensions/clusters/dynamic_forward_proxy:cluster", + "//source/extensions/filters/udp/udp_proxy:config", + "//source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy:config", + "//source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy:proxy_filter_lib", + "//test/integration:integration_lib", + "//test/test_common:registry_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.h b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.h new file mode 100644 index 000000000000..28d0e78bb155 --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.h @@ -0,0 +1,77 @@ +#pragma once + +#include "envoy/registry/registry.h" + +#include "source/common/config/utility.h" +#include "source/common/router/string_accessor_impl.h" +#include "source/common/stream_info/uint32_accessor_impl.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/factory_base.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/filter.h" + +#include "test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.pb.h" +#include "test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.pb.validate.h" +#include "test/test_common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { + +using DynamicForwardProxySetterFilterConfig = test::extensions::filters::udp::udp_proxy:: + session_filters::DynamicForwardProxySetterFilterConfig; + +class DynamicForwardProxySetterFilter : public virtual ReadFilter { +public: + DynamicForwardProxySetterFilter(const std::string host, uint32_t port) + : host_(host), port_(port) {} + + ReadFilterStatus onNewSession() override { + read_callbacks_->streamInfo().filterState()->setData( + "envoy.upstream.dynamic_host", std::make_shared(host_), + StreamInfo::FilterState::StateType::Mutable); + read_callbacks_->streamInfo().filterState()->setData( + "envoy.upstream.dynamic_port", std::make_shared(port_), + StreamInfo::FilterState::StateType::Mutable); + return ReadFilterStatus::Continue; + } + + ReadFilterStatus onData(Network::UdpRecvData&) override { return ReadFilterStatus::Continue; } + + void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + +private: + const std::string host_; + uint32_t port_; + ReadFilterCallbacks* read_callbacks_; +}; + +class DynamicForwardProxySetterFilterConfigFactory + : public FactoryBase { +public: + DynamicForwardProxySetterFilterConfigFactory() : FactoryBase("test.udp_session.dfp_setter") {} + +private: + FilterFactoryCb + createFilterFactoryFromProtoTyped(const DynamicForwardProxySetterFilterConfig& config, + Server::Configuration::FactoryContext&) override { + return [config](FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addReadFilter( + std::make_unique(config.host(), config.port())); + }; + } +}; + +static Registry::RegisterFactory + register_dfp_setter_udp_session_read_filter_; + +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.proto b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.proto new file mode 100644 index 000000000000..c42a189735c8 --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package test.extensions.filters.udp.udp_proxy.session_filters; + +message DynamicForwardProxySetterFilterConfig { + string host = 1; + uint32 port = 2; +} diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter_integration_test.cc new file mode 100644 index 000000000000..d3a822c31bab --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -0,0 +1,227 @@ +#include + +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/network/filter.h" +#include "envoy/server/filter_config.h" + +#include "source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/config.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h" + +#include "test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.h" +#include "test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/dfp_setter.pb.h" +#include "test/integration/integration.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/registry.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { +namespace { + +class DynamicForwardProxyIntegrationTest + : public testing::TestWithParam, + public BaseIntegrationTest { +public: + DynamicForwardProxyIntegrationTest() + : BaseIntegrationTest(GetParam(), ConfigHelper::baseUdpListenerConfig()) { + skip_tag_extraction_rule_check_ = true; + } + + struct BufferConfig { + uint32_t max_buffered_datagrams_; + uint32_t max_buffered_bytes_; + }; + + void setup(std::string upsteam_host = "localhost", + absl::optional buffer_config = absl::nullopt, uint32_t max_hosts = 1024, + uint32_t max_pending_requests = 1024) { + setUdpFakeUpstream(FakeUpstreamConfig::UdpConfig()); + + config_helper_.addConfigModifier( + [this, upsteam_host, buffer_config, max_hosts, + max_pending_requests](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + // Switch predefined cluster_0 to CDS filesystem sourcing. + bootstrap.mutable_dynamic_resources()->mutable_cds_config()->set_resource_api_version( + envoy::config::core::v3::ApiVersion::V3); + bootstrap.mutable_dynamic_resources() + ->mutable_cds_config() + ->mutable_path_config_source() + ->set_path(cds_helper_.cdsPath()); + bootstrap.mutable_static_resources()->clear_clusters(); + + std::string filter_config = fmt::format( + R"EOF( +name: udp_proxy +typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig + stat_prefix: foo + matcher: + on_no_match: + action: + name: route + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: cluster_0 + session_filters: + - name: setter + typed_config: + '@type': type.googleapis.com/test.extensions.filters.udp.udp_proxy.session_filters.DynamicForwardProxySetterFilterConfig + host: {} + port: {} + - name: dfp + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.session.dynamic_forward_proxy.v3.FilterConfig + stat_prefix: foo + dns_cache_config: + name: foo + dns_lookup_family: {} + max_hosts: {} + dns_cache_circuit_breaker: + max_pending_requests: {} +)EOF", + upsteam_host, fake_upstreams_[0]->localAddress()->ip()->port(), + Network::Test::ipVersionToDnsFamily(GetParam()), max_hosts, max_pending_requests); + + if (buffer_config.has_value()) { + filter_config += fmt::format(R"EOF( + buffer_options: + max_buffered_datagrams: {} + max_buffered_bytes: {} +)EOF", + buffer_config.value().max_buffered_datagrams_, + buffer_config.value().max_buffered_bytes_); + } + + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter = listener->add_listener_filters(); + TestUtility::loadFromYaml(filter_config, *filter); + }); + + // Setup the initial CDS cluster. + cluster_.mutable_connect_timeout()->CopyFrom( + Protobuf::util::TimeUtil::MillisecondsToDuration(100)); + cluster_.set_name("cluster_0"); + cluster_.set_lb_policy(envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED); + + const std::string cluster_type_config = fmt::format( + R"EOF( +name: envoy.clusters.dynamic_forward_proxy +typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: foo + dns_lookup_family: {} + max_hosts: {} + dns_cache_circuit_breaker: + max_pending_requests: {} +)EOF", + Network::Test::ipVersionToDnsFamily(GetParam()), max_hosts, max_pending_requests); + + TestUtility::loadFromYaml(cluster_type_config, *cluster_.mutable_cluster_type()); + + // Load the CDS cluster and wait for it to initialize. + cds_helper_.setCds({cluster_}); + BaseIntegrationTest::initialize(); + test_server_->waitForCounterEq("cluster_manager.cluster_added", 1); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + } + + CdsHelper cds_helper_; + envoy::config::cluster::v3::Cluster cluster_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, DynamicForwardProxyIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(DynamicForwardProxyIntegrationTest, BasicFlow) { + setup(); + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + Network::Test::UdpSyncPeer client(version_); + + client.write("hello1", *listener_address); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_attempt", 1); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_success", 1); + test_server_->waitForCounterEq("dns_cache.foo.host_added", 1); + + // There is no buffering in this test, so the first message was dropped. Send another message + // to verify that it's able to go through after the DNS resolution completed. + client.write("hello2", *listener_address); + + Network::UdpRecvData request_datagram; + ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram)); + EXPECT_EQ("hello2", request_datagram.buffer_->toString()); +} + +TEST_P(DynamicForwardProxyIntegrationTest, BasicFlowWithBuffering) { + setup("localhost", BufferConfig{1, 1024}); + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + Network::Test::UdpSyncPeer client(version_); + + client.write("hello1", *listener_address); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_attempt", 1); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_success", 1); + test_server_->waitForCounterEq("dns_cache.foo.host_added", 1); + + // Buffering is enabled so check that the first datagram is sent after the resolution completed. + Network::UdpRecvData request_datagram; + ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram)); + EXPECT_EQ("hello1", request_datagram.buffer_->toString()); +} + +TEST_P(DynamicForwardProxyIntegrationTest, BufferOverflowDueToDatagramSize) { + setup("localhost", BufferConfig{1, 2}); + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + Network::Test::UdpSyncPeer client(version_); + + client.write("hello1", *listener_address); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_attempt", 1); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_success", 1); + test_server_->waitForCounterEq("dns_cache.foo.host_added", 1); + test_server_->waitForCounterEq("udp.session.dynamic_forward_proxy.foo.buffer_overflow", 1); + + // The first datagram should be dropped because it exceeds the buffer size. Send another message + // to verify that it's able to go through after the DNS resolution completed. + client.write("hello2", *listener_address); + + Network::UdpRecvData request_datagram; + ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram)); + EXPECT_EQ("hello2", request_datagram.buffer_->toString()); +} + +TEST_P(DynamicForwardProxyIntegrationTest, EmptyDnsResponseDueToDummyHost) { + setup("dummyhost"); + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + Network::Test::UdpSyncPeer client(version_); + + client.write("hello1", *listener_address); + test_server_->waitForCounterEq("dns_cache.foo.dns_query_attempt", 1); + + // The DNS response is empty, so will not be found any valid host and session will be removed. + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_none_healthy", 1); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + + // DNS cache hit but still no host found. + client.write("hello2", *listener_address); + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_none_healthy", 2); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +} // namespace +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter_test.cc new file mode 100644 index 000000000000..6af769197b9d --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter_test.cc @@ -0,0 +1,355 @@ +#include "envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3/dynamic_forward_proxy.pb.h" + +#include "source/common/router/string_accessor_impl.h" +#include "source/common/stream_info/uint32_accessor_impl.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/dynamic_forward_proxy/proxy_filter.h" + +#include "test/extensions/common/dynamic_forward_proxy/mocks.h" +#include "test/extensions/filters/udp/udp_proxy/mocks.h" +#include "test/mocks/server/factory_context.h" + +using testing::Eq; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace DynamicForwardProxy { +namespace { + +using LoadDnsCacheEntryStatus = + Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryStatus; +using MockLoadDnsCacheEntryResult = + Extensions::Common::DynamicForwardProxy::MockDnsCache::MockLoadDnsCacheEntryResult; + +class DynamicProxyFilterTest + : public testing::Test, + public Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory { +public: + DynamicProxyFilterTest() { + recv_data_stub1_.buffer_ = std::make_unique("sometestdata1"); + recv_data_stub2_.buffer_ = std::make_unique("sometestdata2"); + } + + void setFilterStateHost(const std::string& host) { + stream_info_.filterState()->setData( + "envoy.upstream.dynamic_host", std::make_shared(host), + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection); + } + + void setFilterStatePort(uint32_t port) { + stream_info_.filterState()->setData( + "envoy.upstream.dynamic_port", std::make_shared(port), + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection); + } + + void setFilterState(const std::string& host, uint32_t port) { + setFilterStateHost(host); + setFilterStatePort(port); + } + + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { + return dns_cache_manager_; + } + + void setup(absl::optional proto_config = absl::nullopt) { + FilterConfig config; + if (proto_config.has_value()) { + config = proto_config.value(); + } + + EXPECT_CALL(*dns_cache_manager_, getCache(_)); + filter_config_ = std::make_shared(config, *this, server_context_); + filter_ = std::make_unique(filter_config_); + filter_->initializeReadFilterCallbacks(callbacks_); + ON_CALL(callbacks_, streamInfo()).WillByDefault(ReturnRef(stream_info_)); + } + + using MockDnsCacheManager = Extensions::Common::DynamicForwardProxy::MockDnsCacheManager; + std::shared_ptr dns_cache_manager_ = std::make_shared(); + NiceMock server_context_; + ProxyFilterConfigSharedPtr filter_config_; + std::unique_ptr filter_; + NiceMock callbacks_; + NiceMock stream_info_; + NiceMock pending_requests_; + Network::UdpRecvData recv_data_stub1_; + Network::UdpRecvData recv_data_stub2_; +}; + +TEST_F(DynamicProxyFilterTest, DefaultConfig) { + setup(); + EXPECT_FALSE(filter_config_->bufferEnabled()); +} + +TEST_F(DynamicProxyFilterTest, DefaultBufferConfig) { + FilterConfig config; + config.mutable_buffer_options(); + setup(config); + + EXPECT_TRUE(filter_config_->bufferEnabled()); + EXPECT_EQ(1024, filter_config_->maxBufferedDatagrams()); + EXPECT_EQ(16384, filter_config_->maxBufferedBytes()); + filter_config_->disableBuffer(); + EXPECT_FALSE(filter_config_->bufferEnabled()); +} + +TEST_F(DynamicProxyFilterTest, CustomBufferConfig) { + FilterConfig config; + auto* buffer_options = config.mutable_buffer_options(); + buffer_options->mutable_max_buffered_datagrams()->set_value(10); + buffer_options->mutable_max_buffered_bytes()->set_value(20); + + setup(config); + EXPECT_TRUE(filter_config_->bufferEnabled()); + EXPECT_EQ(10, filter_config_->maxBufferedDatagrams()); + EXPECT_EQ(20, filter_config_->maxBufferedBytes()); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnMissingHostAndPort) { + setup(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).Times(0); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnMissingPort) { + setup(); + setFilterStateHost("host"); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).Times(0); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnMissingHost) { + setup(); + setFilterStatePort(50); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).Times(0); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnEmptyHost) { + setup(); + setFilterState("", 50); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).Times(0); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnPortIsZero) { + setup(); + setFilterState("", 0); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).Times(0); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnPortOutOfRange) { + setup(); + setFilterState("", 65536); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).Times(0); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnRequestOverflow) { + setup(); + setFilterState("host", 50); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()).WillOnce(Return(nullptr)); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, StopIterationOnCacheLoadOverflow) { + setup(); + setFilterState("host", 50); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Overflow, nullptr, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, PassesThroughImmediatelyWhenDnsAlreadyInCache) { + setup(); + setFilterState("host", 50); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::InCache, nullptr, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::Continue, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::Continue, filter_->onData(recv_data_stub1_)); +} + +TEST_F(DynamicProxyFilterTest, + RequestWithCacheMissShouldStopIterationBufferPopulateCacheAndFlushWhenReady) { + setup(); + setFilterState("host", 50); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); + + EXPECT_CALL(callbacks_, continueFilterChain()); + + auto host_info = std::make_shared(); + host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 50); + EXPECT_CALL(*host_info, address()); + filter_->onLoadDnsCacheComplete(host_info); + + EXPECT_CALL(*handle, onDestroy()); +} + +TEST_F(DynamicProxyFilterTest, LoadingCacheEntryWithDefaultBufferConfig) { + FilterConfig config; + config.mutable_buffer_options(); + setup(config); + + setFilterState("host", 50); + EXPECT_TRUE(filter_config_->bufferEnabled()); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + // Two datagrams will be buffered. + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub2_)); + + EXPECT_CALL(callbacks_, continueFilterChain()); + EXPECT_CALL(callbacks_, injectDatagramToFilterChain(_)).Times(2); + + auto host_info = std::make_shared(); + host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 50); + EXPECT_CALL(*host_info, address()); + filter_->onLoadDnsCacheComplete(host_info); + + EXPECT_CALL(*handle, onDestroy()); + EXPECT_FALSE(filter_config_->bufferEnabled()); +} + +TEST_F(DynamicProxyFilterTest, LoadingCacheEntryWithBufferSizeOverflow) { + FilterConfig config; + auto* buffer_options = config.mutable_buffer_options(); + buffer_options->mutable_max_buffered_datagrams()->set_value(1); + setup(config); + + setFilterState("host", 50); + EXPECT_TRUE(filter_config_->bufferEnabled()); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + // Buffer size is 1, first datagram will be buffered, second will drop. + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub2_)); + + EXPECT_CALL(callbacks_, continueFilterChain()); + EXPECT_CALL(callbacks_, injectDatagramToFilterChain(_)); + + auto host_info = std::make_shared(); + host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 50); + EXPECT_CALL(*host_info, address()); + filter_->onLoadDnsCacheComplete(host_info); + + EXPECT_CALL(*handle, onDestroy()); + EXPECT_FALSE(filter_config_->bufferEnabled()); +} + +TEST_F(DynamicProxyFilterTest, LoadingCacheEntryWithBufferBytesOverflow) { + FilterConfig config; + auto* buffer_options = config.mutable_buffer_options(); + buffer_options->mutable_max_buffered_bytes()->set_value(strlen("sometestdata1")); + setup(config); + + setFilterState("host", 50); + EXPECT_TRUE(filter_config_->bufferEnabled()); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + // Buffer bytes size is 13, first datagram will be buffered, second will drop. + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub2_)); + + EXPECT_CALL(callbacks_, continueFilterChain()); + EXPECT_CALL(callbacks_, injectDatagramToFilterChain(_)); + + auto host_info = std::make_shared(); + host_info->address_ = Network::Utility::parseInternetAddress("1.2.3.4", 50); + EXPECT_CALL(*host_info, address()); + filter_->onLoadDnsCacheComplete(host_info); + + EXPECT_CALL(*handle, onDestroy()); + EXPECT_FALSE(filter_config_->bufferEnabled()); +} + +TEST_F(DynamicProxyFilterTest, LoadingCacheEntryWithContinueFilterChainFailure) { + FilterConfig config; + config.mutable_buffer_options(); + setup(config); + + setFilterState("host", 50); + EXPECT_TRUE(filter_config_->bufferEnabled()); + Upstream::ResourceAutoIncDec* circuit_breakers_{ + new Upstream::ResourceAutoIncDec(pending_requests_)}; + EXPECT_CALL(*dns_cache_manager_->dns_cache_, canCreateDnsRequest_()) + .WillOnce(Return(circuit_breakers_)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("host"), 50, _, _)) + .WillOnce(Return( + MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle, absl::nullopt})); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onNewSession()); + EXPECT_EQ(ReadFilterStatus::StopIteration, filter_->onData(recv_data_stub1_)); + + // Session is removed and no longer valid, no datagrams will be injected. + EXPECT_CALL(callbacks_, continueFilterChain()).WillOnce(Return(false)); + EXPECT_CALL(callbacks_, injectDatagramToFilterChain(_)).Times(0); + filter_->onLoadDnsCacheComplete(nullptr); + + EXPECT_CALL(*handle, onDestroy()); +} + +} // namespace +} // namespace DynamicForwardProxy +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/BUILD b/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/BUILD new file mode 100644 index 000000000000..eebb7aeeb83c --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/BUILD @@ -0,0 +1,40 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "http_capsule_filter_test", + srcs = ["http_capsule_filter_test.cc"], + extension_names = ["envoy.filters.udp.session.http_capsule"], + deps = [ + "//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:config", + "//test/extensions/filters/udp/udp_proxy:mocks", + "//test/mocks/server:factory_context_mocks", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "http_capsule_integration_test", + srcs = ["http_capsule_integration_test.cc"], + extension_names = ["envoy.filters.udp.session.http_capsule"], + deps = [ + "//envoy/network:filter_interface", + "//envoy/server:filter_config_interface", + "//source/extensions/filters/udp/udp_proxy:config", + "//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:config", + "//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:http_capsule_filter_lib", + "//test/integration:integration_lib", + "//test/test_common:registry_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule_filter_test.cc b/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule_filter_test.cc new file mode 100644 index 000000000000..e11c665d8a99 --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule_filter_test.cc @@ -0,0 +1,212 @@ +#include "envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.pb.h" + +#include "source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h" + +#include "test/extensions/filters/udp/udp_proxy/mocks.h" +#include "test/mocks/server/factory_context.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace HttpCapsule { +namespace { + +class HttpCapsuleFilterTest : public testing::Test { +public: + void setup() { + filter_ = + std::make_unique(server_context_.server_factory_context_.timeSource()); + filter_->initializeReadFilterCallbacks(read_callbacks_); + filter_->initializeWriteFilterCallbacks(write_callbacks_); + } + + NiceMock server_context_; + std::unique_ptr filter_; + NiceMock read_callbacks_; + NiceMock write_callbacks_; + NiceMock stream_info_; +}; + +TEST_F(HttpCapsuleFilterTest, ContinueOnNewSession) { + setup(); + EXPECT_EQ(ReadFilterStatus::Continue, filter_->onNewSession()); +} + +TEST_F(HttpCapsuleFilterTest, EncapsulateEmptyDatagram) { + setup(); + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + EXPECT_EQ(ReadFilterStatus::Continue, filter_->onData(datagram)); + + const std::string expected_data = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "01" // Capsule length + "00" // Context ID + ); + + EXPECT_EQ(expected_data, datagram.buffer_->toString()); +} + +TEST_F(HttpCapsuleFilterTest, EncapsulateDatagram) { + setup(); + + const std::string payload = "payload"; + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(payload); + EXPECT_EQ(ReadFilterStatus::Continue, filter_->onData(datagram)); + + const std::string expected_data = + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "08" // Capsule Length = length(payload) + 1 + "00" // Context ID + ) + + payload; + + EXPECT_EQ(expected_data, datagram.buffer_->toString()); +} + +TEST_F(HttpCapsuleFilterTest, InvalidCapsule) { + setup(); + + const std::string invalid_context_id_fragment = + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "01" // Capsule Length + "c0" // Context ID (Invalid VarInt62) + ); + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(invalid_context_id_fragment); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)).Times(0); + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(datagram)); + EXPECT_EQ(0, datagram.buffer_->length()); +} + +TEST_F(HttpCapsuleFilterTest, IncompatibleCapsule) { + setup(); + + const std::string unexpected_capsule_fragment = + absl::HexStringToBytes("17" // Capsule Type is not DATAGRAM + "01" // Capsule Length + "00" // Unknown capsule payload + ); + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(unexpected_capsule_fragment); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)).Times(0); + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(datagram)); + EXPECT_EQ(0, datagram.buffer_->length()); +} + +TEST_F(HttpCapsuleFilterTest, UnknownContextId) { + setup(); + + const std::string invalid_context_id_fragment = + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "01" // Capsule Length + "01" // Unknown Context ID + ); + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(invalid_context_id_fragment); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)).Times(0); + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(datagram)); + EXPECT_EQ(0, datagram.buffer_->length()); +} + +TEST_F(HttpCapsuleFilterTest, DecapsulateDatagram) { + setup(); + + const std::string payload = "payload"; + const std::string capsule = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "08" // Capsule Length = length(payload) + 1 + "00" // Context ID + ) + + payload; + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(capsule); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)) + .WillOnce(Invoke([payload](Network::UdpRecvData& data) -> void { + EXPECT_EQ(payload, data.buffer_->toString()); + })); + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(datagram)); + EXPECT_EQ(0, datagram.buffer_->length()); +} + +TEST_F(HttpCapsuleFilterTest, DecapsulateSplitPayload) { + setup(); + + const std::string payload = "payload"; + const std::string capsule = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "08" // Capsule Length = length(payload) + 1 + "00" // Context ID + ) + + payload; + + int pivot_index = 4; // Some arbitrary split index of the capsule. + + // Send first part of the capsule and verify that a datagram was not injected. + Network::UdpRecvData payload1; + payload1.buffer_ = std::make_unique(); + payload1.buffer_->add(capsule.substr(0, pivot_index)); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)).Times(0); + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(payload1)); + EXPECT_EQ(0, payload1.buffer_->length()); + + // Send second part of the capsule and verify that a datagram was generated. + Network::UdpRecvData payload2; + payload2.buffer_ = std::make_unique(); + payload2.buffer_->add(capsule.substr(pivot_index)); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)) + .WillOnce(Invoke([payload](Network::UdpRecvData& data) -> void { + EXPECT_EQ(payload, data.buffer_->toString()); + })); + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(payload2)); + EXPECT_EQ(0, payload2.buffer_->length()); +} + +TEST_F(HttpCapsuleFilterTest, DecapsulateMultipleDatagrams) { + setup(); + + const std::string capsule = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "09" // Capsule Length = length(payload1) + 1 + "00" // Context ID + ) + + "payload1" + + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "09" // Capsule Length = length(payload2) + 1 + "00" // Context ID + ) + + "payload2"; + + Network::UdpRecvData datagram; + datagram.buffer_ = std::make_unique(); + datagram.buffer_->add(capsule); + EXPECT_CALL(write_callbacks_, injectDatagramToFilterChain(_)) + .WillOnce(Invoke([](Network::UdpRecvData& data) -> void { + EXPECT_EQ("payload1", data.buffer_->toString()); + })) + .WillOnce(Invoke([](Network::UdpRecvData& data) -> void { + EXPECT_EQ("payload2", data.buffer_->toString()); + })); + + EXPECT_EQ(WriteFilterStatus::StopIteration, filter_->onWrite(datagram)); + EXPECT_EQ(0, datagram.buffer_->length()); +} + +} // namespace +} // namespace HttpCapsule +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule_integration_test.cc b/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule_integration_test.cc new file mode 100644 index 000000000000..9976e03f76cf --- /dev/null +++ b/test/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule_integration_test.cc @@ -0,0 +1,199 @@ +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/network/filter.h" +#include "envoy/server/filter_config.h" + +#include "source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/config.h" +#include "source/extensions/filters/udp/udp_proxy/session_filters/http_capsule/http_capsule.h" + +#include "test/integration/integration.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/registry.h" + +namespace Envoy { +namespace Extensions { +namespace UdpFilters { +namespace UdpProxy { +namespace SessionFilters { +namespace HttpCapsule { +namespace { + +class HttpCapsuleIntegrationTest : public testing::TestWithParam, + public BaseIntegrationTest { +public: + HttpCapsuleIntegrationTest() + : BaseIntegrationTest(GetParam(), ConfigHelper::baseUdpListenerConfig()) {} + + void setup() { + setUdpFakeUpstream(FakeUpstreamConfig::UdpConfig()); + + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + std::string filter_config = fmt::format(R"EOF( +name: udp_proxy +typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig + stat_prefix: foo + matcher: + on_no_match: + action: + name: route + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: cluster_0 + session_filters: + - name: http_capsule + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3.FilterConfig +)EOF"); + + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter = listener->add_listener_filters(); + TestUtility::loadFromYaml(filter_config, *filter); + }); + + BaseIntegrationTest::initialize(); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, HttpCapsuleIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(HttpCapsuleIntegrationTest, BasicFlow) { + setup(); + + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + + const std::string request = "hello"; + const std::string expected_request = + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "06" // Capsule Length = length(hello) + 1 + "00" // Context ID + ) + + request; + + const std::string expected_response = "world"; + const std::string response = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "06" // Capsule Length = length(world) + 1 + "00" // Context ID + ) + + expected_response; + + // Send datagram to be encapsulated. + Network::Test::UdpSyncPeer client(version_, Network::DEFAULT_UDP_MAX_DATAGRAM_SIZE); + client.write(request, *listener_address); + + // Wait for the upstream datagram. + Network::UdpRecvData request_datagram; + ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram)); + EXPECT_EQ(expected_request, request_datagram.buffer_->toString()); + + // Respond from the upstream. + fake_upstreams_[0]->sendUdpDatagram(response, request_datagram.addresses_.peer_); + Network::UdpRecvData response_datagram; + client.recv(response_datagram); + EXPECT_EQ(expected_response, response_datagram.buffer_->toString()); +} + +TEST_P(HttpCapsuleIntegrationTest, SendSplitCapsule) { + setup(); + + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + + const std::string request = "hello"; + const std::string expected_request = + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "06" // Capsule Length = length(hello) + 1 + "00" // Context ID + ) + + request; + + const std::string expected_response = "world"; + const std::string response = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "06" // Capsule Length = length(world) + 1 + "00" // Context ID + ) + + expected_response; + + // Send datagram to be encapsulated. + Network::Test::UdpSyncPeer client(version_, Network::DEFAULT_UDP_MAX_DATAGRAM_SIZE); + client.write(request, *listener_address); + + // Wait for the upstream datagram. + Network::UdpRecvData request_datagram; + ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram)); + EXPECT_EQ(expected_request, request_datagram.buffer_->toString()); + + // Respond from the upstream. + int pivot_index = 4; + fake_upstreams_[0]->sendUdpDatagram(response.substr(0, pivot_index), + request_datagram.addresses_.peer_); + // Make sure that only one payload received, but none sent downstream because it's not a complete + // capsule. + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_rx_datagrams", 1); + EXPECT_EQ(0, test_server_->counter("udp.foo.downstream_sess_tx_datagrams")->value()); + + // Sending the rest of the capsule, so we expect a datagram flushed downstream. + fake_upstreams_[0]->sendUdpDatagram(response.substr(pivot_index), + request_datagram.addresses_.peer_); + Network::UdpRecvData response_datagram; + client.recv(response_datagram); + EXPECT_EQ(expected_response, response_datagram.buffer_->toString()); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_rx_datagrams", 2); + test_server_->waitForCounterEq("udp.foo.downstream_sess_tx_datagrams", 1); +} + +TEST_P(HttpCapsuleIntegrationTest, SendMultipleCapsules) { + setup(); + + const uint32_t port = lookupPort("listener_0"); + const auto listener_address = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + + const std::string request = "hello"; + const std::string expected_request = + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "06" // Capsule Length = length(hello) + 1 + "00" // Context ID + ) + + request; + + std::string response = absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "0A" // Capsule Length = length(response1) + 1 + "00" // Context ID + ) + + "response1" + + absl::HexStringToBytes("00" // DATAGRAM Capsule Type + "0A" // Capsule Length = length(response2) + 1 + "00" // Context ID + ) + + "response2"; + + // Send datagram to be encapsulated. + Network::Test::UdpSyncPeer client(version_, Network::DEFAULT_UDP_MAX_DATAGRAM_SIZE); + client.write(request, *listener_address); + + // Wait for the upstream datagram. + Network::UdpRecvData request_datagram; + ASSERT_TRUE(fake_upstreams_[0]->waitForUdpDatagram(request_datagram)); + EXPECT_EQ(expected_request, request_datagram.buffer_->toString()); + + // // Respond from the upstream. + fake_upstreams_[0]->sendUdpDatagram(response, request_datagram.addresses_.peer_); + Network::UdpRecvData response_datagram; + client.recv(response_datagram); + EXPECT_EQ("response1", response_datagram.buffer_->toString()); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_rx_datagrams", 1); + test_server_->waitForCounterEq("udp.foo.downstream_sess_tx_datagrams", 2); +} + +} // namespace +} // namespace HttpCapsule +} // namespace SessionFilters +} // namespace UdpProxy +} // namespace UdpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc index 59e2d45b09cd..571b4816d4bb 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -7,16 +7,24 @@ #include "source/common/common/hash.h" #include "source/common/network/socket_impl.h" #include "source/common/network/socket_option_impl.h" +#include "source/common/router/string_accessor_impl.h" +#include "source/common/stream_info/uint32_accessor_impl.h" #include "source/extensions/filters/udp/udp_proxy/config.h" #include "source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h" +#include "test/extensions/filters/udp/udp_proxy/mocks.h" +#include "test/extensions/filters/udp/udp_proxy/session_filters/drainer_filter.h" +#include "test/extensions/filters/udp/udp_proxy/session_filters/drainer_filter.pb.h" #include "test/mocks/api/mocks.h" +#include "test/mocks/http/stream_encoder.h" #include "test/mocks/network/socket.h" +#include "test/mocks/server/factory_context.h" #include "test/mocks/server/listener_factory_context.h" #include "test/mocks/upstream/cluster_manager.h" #include "test/mocks/upstream/cluster_update_callbacks.h" #include "test/mocks/upstream/cluster_update_callbacks_handle.h" #include "test/mocks/upstream/host.h" +#include "test/mocks/upstream/load_balancer_context.h" #include "test/mocks/upstream/thread_local_cluster.h" #include "test/test_common/threadsafe_singleton_injector.h" @@ -31,6 +39,7 @@ using testing::InSequence; using testing::InvokeWithoutArgs; using testing::Return; using testing::ReturnNew; +using testing::ReturnRef; using testing::SaveArg; namespace Envoy { @@ -43,7 +52,7 @@ class TestUdpProxyFilter : public UdpProxyFilter { public: using UdpProxyFilter::UdpProxyFilter; - MOCK_METHOD(Network::SocketPtr, createSocket, (const Upstream::HostConstSharedPtr& host)); + MOCK_METHOD(Network::SocketPtr, createUdpSocket, (const Upstream::HostConstSharedPtr& host)); }; Api::IoCallUint64Result makeNoError(uint64_t rc) { @@ -53,8 +62,7 @@ Api::IoCallUint64Result makeNoError(uint64_t rc) { } Api::IoCallUint64Result makeError(int sys_errno) { - return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(sys_errno), - Network::IoSocketError::deleteIoError)); + return {0, Network::IoSocketError::create(sys_errno)}; } class UdpProxyFilterBase : public testing::Test { @@ -173,9 +181,8 @@ class UdpProxyFilterTest : public UdpProxyFilterBase { // Return an EAGAIN result. EXPECT_CALL(*socket_->io_handle_, supportsMmsg()); EXPECT_CALL(*socket_->io_handle_, recvmsg(_, 1, _, _)) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError))))); + .WillOnce(Return(ByMove( + Api::IoCallUint64Result(0, Network::IoSocketError::getIoSocketEagainError())))); } // Kick off the receive. @@ -198,15 +205,19 @@ class UdpProxyFilterTest : public UdpProxyFilterBase { upstream_address_(Network::Utility::parseInternetAddressAndPort(upstream_ip_address_)), peer_address_(std::move(peer_address)) { // Disable strict mock warnings. - ON_CALL(*factory_context_.access_log_manager_.file_, write(_)) + ON_CALL(*factory_context_.server_factory_context_.access_log_manager_.file_, write(_)) .WillByDefault( Invoke([&](absl::string_view data) { output_.push_back(std::string(data)); })); ON_CALL(os_sys_calls_, supportsIpTransparent(_)).WillByDefault(Return(true)); EXPECT_CALL(os_sys_calls_, supportsUdpGro()).Times(AtLeast(0)).WillRepeatedly(Return(true)); EXPECT_CALL(callbacks_, udpListener()).Times(AtLeast(0)); - EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, address()) + EXPECT_CALL( + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, + address()) .WillRepeatedly(Return(upstream_address_)); - EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, coarseHealth()) + EXPECT_CALL( + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, + coarseHealth()) .WillRepeatedly(Return(Upstream::Host::Health::Healthy)); } @@ -215,13 +226,16 @@ class UdpProxyFilterTest : public UdpProxyFilterBase { void setup(const envoy::extensions::filters::udp::udp_proxy::v3::UdpProxyConfig& config, bool has_cluster = true, bool expect_gro = true) { config_ = std::make_shared(factory_context_, config); - EXPECT_CALL(factory_context_.cluster_manager_, addThreadLocalClusterUpdateCallbacks_(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + addThreadLocalClusterUpdateCallbacks_(_)) .WillOnce(DoAll(SaveArgAddress(&cluster_update_callbacks_), ReturnNew())); if (has_cluster) { - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"fake_cluster"}); } - EXPECT_CALL(factory_context_.cluster_manager_, getThreadLocalCluster("fake_cluster")); + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_, + getThreadLocalCluster("fake_cluster")); filter_ = std::make_unique(callbacks_, config_); expect_gro_ = expect_gro; } @@ -240,7 +254,7 @@ class UdpProxyFilterTest : public UdpProxyFilterBase { test_sessions_.emplace_back(*this, address); TestSession& new_session = test_sessions_.back(); new_session.idle_timer_ = new Event::MockTimer(&callbacks_.udp_listener_.dispatcher_); - EXPECT_CALL(*filter_, createSocket(_)) + EXPECT_CALL(*filter_, createUdpSocket(_)) .WillOnce(Return(ByMove(Network::SocketPtr{test_sessions_.back().socket_}))); EXPECT_CALL( *new_session.socket_->io_handle_, @@ -380,7 +394,9 @@ class UdpProxyFilterIpv6Test : public UdpProxyFilterTest { explicit UdpProxyFilterIpv6Test(Network::Address::InstanceConstSharedPtr&& upstream_address_v6) : UdpProxyFilterTest(Network::Utility::parseInternetAddressAndPort(peer_ipv6_address_)), upstream_address_v6_(std::move(upstream_address_v6)) { - EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, address()) + EXPECT_CALL( + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, + address()) .WillRepeatedly(Return(upstream_address_v6_)); } @@ -420,7 +436,12 @@ TEST_F(UdpProxyFilterTest, BasicFlow) { "%DYNAMIC_METADATA(udp.proxy.session:bytes_received)% " "%DYNAMIC_METADATA(udp.proxy.session:datagrams_received)% " "%DYNAMIC_METADATA(udp.proxy.session:bytes_sent)% " - "%DYNAMIC_METADATA(udp.proxy.session:datagrams_sent)%"; + "%DYNAMIC_METADATA(udp.proxy.session:datagrams_sent)% " + "%DOWNSTREAM_REMOTE_ADDRESS% " + "%DOWNSTREAM_LOCAL_ADDRESS% " + "%UPSTREAM_HOST% " + "%STREAM_ID% " + "%ACCESS_LOG_TYPE%"; const std::string proxy_access_log_format = "%DYNAMIC_METADATA(udp.proxy.proxy:bytes_received)% " @@ -470,7 +491,12 @@ stat_prefix: foo filter_.reset(); EXPECT_EQ(output_.size(), 2); EXPECT_EQ(output_.front(), "17 3 17 3 0 1 0"); - EXPECT_EQ(output_.back(), "17 3 17 3"); + const std::string session_access_log_regex = + "17 3 17 3 10.0.0.1:1000 10.0.0.2:80 20.0.0.1:443 " + "[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12} " + + AccessLogType_Name(AccessLog::AccessLogType::UdpSessionEnd); + + EXPECT_TRUE(std::regex_match(output_.back(), std::regex(session_access_log_regex))); } // Route with source IP. @@ -594,35 +620,33 @@ stat_prefix: foo test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); - EXPECT_EQ(5, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_tx_bytes_total_.value()); + EXPECT_EQ(5, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_tx_bytes_total_.value()); test_sessions_[0].recvDataFromUpstream("world2", 0, SOCKET_ERROR_MSG_SIZE); checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); - EXPECT_EQ(6, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_rx_bytes_total_.value()); + EXPECT_EQ(6, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_rx_bytes_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_tx_errors_.value()); test_sessions_[0].recvDataFromUpstream("world2", SOCKET_ERROR_MSG_SIZE, 0); checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); - EXPECT_EQ(6, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_rx_bytes_total_.value()); - EXPECT_EQ( - 1, TestUtility::findCounter( - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_, - "udp.sess_rx_errors") - ->value()); + EXPECT_EQ(6, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_rx_bytes_total_.value()); + EXPECT_EQ(1, TestUtility::findCounter(factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.cluster_.info_->stats_store_, + "udp.sess_rx_errors") + ->value()); test_sessions_[0].expectWriteToUpstream("hello", SOCKET_ERROR_MSG_SIZE); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); checkTransferStats(10 /*rx_bytes*/, 2 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); - EXPECT_EQ(5, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_tx_bytes_total_.value()); - EXPECT_EQ( - 1, TestUtility::findCounter( - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_, - "udp.sess_tx_errors") - ->value()); + EXPECT_EQ(5, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_tx_bytes_total_.value()); + EXPECT_EQ(1, TestUtility::findCounter(factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.cluster_.info_->stats_store_, + "udp.sess_tx_errors") + ->value()); filter_.reset(); EXPECT_EQ(output_.size(), 2); @@ -662,11 +686,10 @@ stat_prefix: foo EXPECT_LOG_CONTAINS("debug", "cannot connect", recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello")); checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); - EXPECT_EQ( - 1, TestUtility::findCounter( - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->stats_store_, - "udp.sess_tx_errors") - ->value()); + EXPECT_EQ(1, TestUtility::findCounter(factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.cluster_.info_->stats_store_, + "udp.sess_tx_errors") + ->value()); filter_.reset(); EXPECT_EQ(output_.size(), 2); @@ -689,11 +712,12 @@ stat_prefix: foo cluster: fake_cluster )EOF")); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Return(nullptr)); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); - EXPECT_EQ(1, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_none_healthy_.value()); + EXPECT_EQ(1, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_none_healthy_.value()); } // No cluster at filter creation. @@ -765,10 +789,12 @@ stat_prefix: foo // Now add the cluster we care about. { Upstream::ThreadLocalClusterCommand command = [this]() -> Upstream::ThreadLocalCluster& { - return factory_context_.cluster_manager_.thread_local_cluster_; + return factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_; }; cluster_update_callbacks_->onClusterAddOrUpdate( - factory_context_.cluster_manager_.thread_local_cluster_.info()->name(), command); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.info() + ->name(), + command); } expectSessionCreate(upstream_address_); test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); @@ -801,8 +827,8 @@ stat_prefix: foo )EOF")); // Allow only a single session. - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( - 1, 0, 0, 0, 0); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->resetResourceManager(1, 0, 0, 0, 0); expectSessionCreate(upstream_address_); test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); @@ -812,8 +838,8 @@ stat_prefix: foo // This should hit the session circuit breaker. recvDataFromDownstream("10.0.0.2:1000", "10.0.0.2:80", "hello"); - EXPECT_EQ(1, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_overflow_.value()); + EXPECT_EQ(1, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_overflow_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); @@ -849,8 +875,10 @@ stat_prefix: foo EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.priority_set_.runUpdateCallbacks( - 0, {}, {factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_}); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_ + .priority_set_.runUpdateCallbacks(0, {}, + {factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.lb_.host_}); EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); @@ -883,7 +911,9 @@ stat_prefix: foo EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); - EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, coarseHealth()) + EXPECT_CALL( + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, + coarseHealth()) .WillRepeatedly(Return(Upstream::Host::Health::Unhealthy)); test_sessions_[0].expectWriteToUpstream("hello"); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); @@ -911,11 +941,14 @@ stat_prefix: foo EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); - EXPECT_CALL(*factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, coarseHealth()) + EXPECT_CALL( + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_, + coarseHealth()) .WillRepeatedly(Return(Upstream::Host::Health::Unhealthy)); auto new_host_address = Network::Utility::parseInternetAddressAndPort("20.0.0.2:443"); auto new_host = createHost(new_host_address); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Return(new_host)); expectSessionCreate(new_host_address); test_sessions_[1].expectWriteToUpstream("hello", 0, nullptr, true); @@ -959,6 +992,28 @@ use_per_packet_load_balancing: true "Only one of use_per_packet_load_balancing or session_filters can be used."); } +TEST_F(UdpProxyFilterTest, MutualExcludePerPacketLoadBalancingAndTunneling) { + auto config = R"EOF( +stat_prefix: foo +matcher: + on_no_match: + action: + name: route + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: fake_cluster +use_per_packet_load_balancing: true +tunneling_config: + proxy_host: host.com + target_host: host.com + default_target_port: 30 + )EOF"; + + EXPECT_THROW_WITH_MESSAGE( + setup(readConfig(config)), EnvoyException, + "Only one of use_per_packet_load_balancing or tunneling_config can be used."); +} + // Verify that on second data packet sent from the client, another upstream host is selected. TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingBasicFlow) { InSequence s; @@ -976,8 +1031,8 @@ use_per_packet_load_balancing: true )EOF")); // Allow for two sessions. - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( - 2, 0, 0, 0, 0); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->resetResourceManager(2, 0, 0, 0, 0); expectSessionCreate(upstream_address_); test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); @@ -990,7 +1045,8 @@ use_per_packet_load_balancing: true auto new_host_address = Network::Utility::parseInternetAddressAndPort("20.0.0.2:443"); auto new_host = createHost(new_host_address); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Return(new_host)); expectSessionCreate(new_host_address); test_sessions_[1].expectWriteToUpstream("hello2", 0, nullptr, true); @@ -1000,7 +1056,8 @@ use_per_packet_load_balancing: true checkTransferStats(11 /*rx_bytes*/, 2 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); // On next datagram, first session should be used - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillRepeatedly(DoDefault()); test_sessions_[0].expectWriteToUpstream("hello3"); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello3"); @@ -1025,13 +1082,14 @@ stat_prefix: foo use_per_packet_load_balancing: true )EOF")); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Return(nullptr)); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); EXPECT_EQ(0, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); - EXPECT_EQ(1, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_none_healthy_.value()); + EXPECT_EQ(1, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_none_healthy_.value()); } // Verify that when on second packet no host is available, message is dropped. @@ -1055,16 +1113,17 @@ use_per_packet_load_balancing: true recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); - EXPECT_EQ(0, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_none_healthy_.value()); + EXPECT_EQ(0, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_none_healthy_.value()); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Return(nullptr)); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello2"); EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); - EXPECT_EQ(1, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_none_healthy_.value()); + EXPECT_EQ(1, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_none_healthy_.value()); } // Verify that all sessions for a host are removed when a host is removed. @@ -1089,8 +1148,10 @@ use_per_packet_load_balancing: true EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.priority_set_.runUpdateCallbacks( - 0, {}, {factory_context_.cluster_manager_.thread_local_cluster_.lb_.host_}); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_ + .priority_set_.runUpdateCallbacks(0, {}, + {factory_context_.server_factory_context_.cluster_manager_ + .thread_local_cluster_.lb_.host_}); EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); @@ -1118,8 +1179,8 @@ use_per_packet_load_balancing: true )EOF")); // Allow for two sessions. - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( - 2, 0, 0, 0, 0); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->resetResourceManager(2, 0, 0, 0, 0); expectSessionCreate(upstream_address_); test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); @@ -1129,7 +1190,8 @@ use_per_packet_load_balancing: true auto new_host_address = Network::Utility::parseInternetAddressAndPort("20.0.0.2:443"); auto new_host = createHost(new_host_address); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Return(new_host)); expectSessionCreate(new_host_address); test_sessions_[1].expectWriteToUpstream("hello2", 0, nullptr, true); @@ -1163,12 +1225,12 @@ use_per_packet_load_balancing: true )EOF")); // Don't allow for any session. - factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager( - 0, 0, 0, 0, 0); + factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ + ->resetResourceManager(0, 0, 0, 0, 0); recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); - EXPECT_EQ(1, factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_ - ->traffic_stats_->upstream_cx_overflow_.value()); + EXPECT_EQ(1, factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_ + .cluster_.info_->traffic_stats_->upstream_cx_overflow_.value()); } // Make sure socket option is set correctly if use_original_src_ip is set in case of ipv6. @@ -1333,7 +1395,8 @@ stat_prefix: foo auto host = createHost(upstream_address_); auto generated_hash = HashUtil::xxHash64("10.0.0.1"); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Invoke([host, generated_hash]( Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { auto hash = context->computeHashKey(); @@ -1363,7 +1426,8 @@ stat_prefix: foo )EOF")); auto host = createHost(upstream_address_); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce( Invoke([host](Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { auto hash = context->computeHashKey(); @@ -1435,7 +1499,8 @@ stat_prefix: foo auto host = createHost(upstream_address_); auto generated_hash = HashUtil::xxHash64("key"); - EXPECT_CALL(factory_context_.cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + EXPECT_CALL(factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.lb_, + chooseHost(_)) .WillOnce(Invoke([host, generated_hash]( Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { auto hash = context->computeHashKey(); @@ -1449,6 +1514,624 @@ stat_prefix: foo test_sessions_[0].recvDataFromUpstream("world"); } +TEST_F(UdpProxyFilterTest, EnrichAccessLogOnSessionComplete) { + InSequence s; + + const std::string session_access_log_format = + "%FILTER_STATE(test.udp_session.drainer.on_session_complete)%"; + + setup(accessLogConfig(R"EOF( +stat_prefix: foo +matcher: + on_no_match: + action: + name: route + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: fake_cluster +session_filters: +- name: foo + typed_config: + '@type': type.googleapis.com/test.extensions.filters.udp.udp_proxy.session_filters.DrainerUdpSessionReadFilterConfig + downstream_bytes_to_drain: 0 + stop_iteration_on_new_session: false + stop_iteration_on_first_read: false + continue_filter_chain: false + )EOF", + session_access_log_format, "")); + + expectSessionCreate(upstream_address_); + test_sessions_[0].expectWriteToUpstream("hello", 0, nullptr, true); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + + test_sessions_[0].idle_timer_->invokeCallback(); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); + + filter_.reset(); + EXPECT_EQ(output_.size(), 1); + EXPECT_THAT(output_.front(), testing::HasSubstr("session_complete")); +} + +class HttpUpstreamImplTest : public testing::Test { +public: + struct HeaderToAdd { + std::string key_; + std::string value_; + }; + + Http::TestRequestHeaderMapImpl + expectedHeaders(bool is_ssl = false, bool use_post = false, + absl::optional opt_authority = absl::nullopt, + absl::optional opt_path = absl::nullopt, + absl::optional header_to_add = absl::nullopt) { + // In case connect-udp is used, Envoy expect the H2 headers to be normalized with H1, + // so expect that the request headers here match H1 headers, even though + // eventually H2 headers will be sent. When the headers are normalized to H1, the method + // is replaced with GET, a header with the 'upgrade' key is added with 'connect-udp' + // value and a header with the 'connection' key is added with 'Upgrade' value. + std::string scheme = is_ssl ? "https" : "http"; + std::string authority = + opt_authority.has_value() ? opt_authority.value() : "default.host.com:10"; + std::string method = use_post ? "POST" : "GET"; + std::string path = + opt_path.has_value() ? opt_path.value() : "/.well-known/masque/udp/default.target.host/20/"; + + auto headers = Http::TestRequestHeaderMapImpl{ + {":scheme", scheme}, {":authority", authority}, {":method", method}, {":path", path}}; + + if (!use_post) { + headers.addCopy("capsule-protocol", "?1"); + headers.addCopy("upgrade", "connect-udp"); + headers.addCopy("connection", "Upgrade"); + } + + if (header_to_add) { + headers.addCopy(header_to_add.value().key_, header_to_add.value().value_); + } + + return headers; + } + + void setAndExpectRequestEncoder(Http::TestRequestHeaderMapImpl expected_headers, + bool is_ssl = false) { + EXPECT_CALL(request_encoder_.stream_, addCallbacks(_)); + EXPECT_CALL(request_encoder_, encodeHeaders(_, _)) + .WillOnce(Invoke([expected_headers](const Http::RequestHeaderMap& headers, + bool end_stream) -> Http::Status { + EXPECT_THAT(&headers, HeaderMapEqualIgnoreOrder(&expected_headers)); + EXPECT_FALSE(end_stream); + return absl::OkStatus(); + })); + + upstream_->setRequestEncoder(request_encoder_, is_ssl); + + // Verify that resetEncoder is called only once. + EXPECT_CALL(request_encoder_.stream_, removeCallbacks(_)); + } + + void filterStateOverride(absl::optional proxy_port = absl::nullopt, + absl::optional target_port = absl::nullopt) { + if (proxy_port) { + stream_info_.filterState()->setData( + "udp.connect.proxy_port", + std::make_shared(proxy_port.value()), + Envoy::StreamInfo::FilterState::StateType::Mutable); + } + + if (target_port) { + stream_info_.filterState()->setData( + "udp.connect.target_port", + std::make_shared(target_port.value()), + Envoy::StreamInfo::FilterState::StateType::Mutable); + } + } + + void setup(absl::optional header_to_add = absl::nullopt) { + Protobuf::RepeatedPtrField headers_to_add; + if (header_to_add) { + envoy::config::core::v3::HeaderValueOption* header = headers_to_add.Add(); + header->mutable_header()->set_key(header_to_add.value().key_); + header->mutable_header()->set_value(header_to_add.value().value_); + } + + header_evaluator_ = Envoy::Router::HeaderParser::configure(headers_to_add); + config_ = std::make_unique>(*header_evaluator_); + upstream_ = std::make_unique(callbacks_, *config_, stream_info_); + upstream_->setTunnelCreationCallbacks(creation_callbacks_); + } + + NiceMock callbacks_; + NiceMock stream_info_; + NiceMock request_encoder_; + NiceMock creation_callbacks_; + std::unique_ptr> config_; + std::unique_ptr header_evaluator_; + std::unique_ptr upstream_; +}; + +TEST_F(HttpUpstreamImplTest, EncodeData) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + EXPECT_CALL(request_encoder_, encodeData(_, false)); + Buffer::OwnedImpl data; + upstream_->encodeData(data); +} + +TEST_F(HttpUpstreamImplTest, WatermarksCallBack) { + setup(); + + EXPECT_CALL(callbacks_, onAboveWriteBufferHighWatermark()); + upstream_->onAboveWriteBufferHighWatermark(); + EXPECT_CALL(callbacks_, onBelowWriteBufferLowWatermark()); + upstream_->onBelowWriteBufferLowWatermark(); +} + +TEST_F(HttpUpstreamImplTest, OnResetStream) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + // If the creation callbacks are active will resetting the stream, it means that response + // headers were not received, so it's expected to be a failure. + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + upstream_->onResetStream(Http::StreamResetReason::ConnectionTimeout, "reason"); +} + +TEST_F(HttpUpstreamImplTest, LocalCloseByDownstreamResetsStream) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + // If the creation callbacks are active will resetting the stream, it means that response + // headers were not received, so it's expected to be a failure. + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + EXPECT_CALL(request_encoder_.stream_, resetStream(Http::StreamResetReason::LocalReset)); + upstream_->onDownstreamEvent(Network::ConnectionEvent::LocalClose); +} + +TEST_F(HttpUpstreamImplTest, RemoteCloseByDownstreamResetsStream) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + // If the creation callbacks are active will resetting the stream, it means that response + // headers were not received, so it's expected to be a failure. + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + EXPECT_CALL(request_encoder_.stream_, resetStream(Http::StreamResetReason::LocalReset)); + upstream_->onDownstreamEvent(Network::ConnectionEvent::RemoteClose); +} + +TEST_F(HttpUpstreamImplTest, NullOperationFunctions) { + setup(); + + // Calling functions that implemented by no-op for coverage. + upstream_->responseDecoder().decodeMetadata(nullptr); + upstream_->responseDecoder().decode1xxHeaders(nullptr); + std::array buffer; + OutputBufferStream ostream{buffer.data(), buffer.size()}; + upstream_->responseDecoder().dumpState(ostream, 1); +} + +TEST_F(HttpUpstreamImplTest, FailureResponseHeadersNot200Status) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "500"}, {"Capsule-Protocol", "?1"}}); + upstream_->responseDecoder().decodeHeaders(std::move(response_headers), /*end_stream=*/false); +} + +TEST_F(HttpUpstreamImplTest, FailureResponseHeadersEndStream) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "101"}, {"upgrade", "connect-udp"}}); + upstream_->responseDecoder().decodeHeaders(std::move(response_headers), /*end_stream=*/true); +} + +TEST_F(HttpUpstreamImplTest, SuccessResponseHeaders) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + EXPECT_CALL(creation_callbacks_, onStreamFailure()).Times(0); + EXPECT_CALL(creation_callbacks_, onStreamSuccess(_)); + + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "101"}, {"upgrade", "connect-udp"}}); + upstream_->responseDecoder().decodeHeaders(std::move(response_headers), /*end_stream=*/false); +} + +TEST_F(HttpUpstreamImplTest, DecodeData) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + Buffer::OwnedImpl data; + + // Second decodeData call will reset the stream, because it is end of stream. + // Since response headers were not received it's considered as a failure. + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + EXPECT_CALL(callbacks_, onUpstreamData(_, _)) + .WillOnce(Invoke([](Buffer::Instance&, bool end_stream) { EXPECT_FALSE(end_stream); })) + .WillOnce(Invoke([](Buffer::Instance&, bool end_stream) { EXPECT_TRUE(end_stream); })); + + upstream_->responseDecoder().decodeData(data, /*end_stream=*/false); + upstream_->responseDecoder().decodeData(data, /*end_stream=*/true); +} + +TEST_F(HttpUpstreamImplTest, DecodeDataAfterSuccessHeaders) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + Buffer::OwnedImpl data; + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "101"}, {"upgrade", "connect-udp"}}); + + EXPECT_CALL(creation_callbacks_, onStreamFailure()).Times(0); + EXPECT_CALL(creation_callbacks_, onStreamSuccess(_)); + EXPECT_CALL(callbacks_, onUpstreamData(_, _)) + .WillOnce(Invoke([](Buffer::Instance&, bool end_stream) { EXPECT_FALSE(end_stream); })) + .WillOnce(Invoke([](Buffer::Instance&, bool end_stream) { EXPECT_TRUE(end_stream); })); + + upstream_->responseDecoder().decodeHeaders(std::move(response_headers), /*end_stream=*/false); + upstream_->responseDecoder().decodeData(data, /*end_stream=*/false); + upstream_->responseDecoder().decodeData(data, /*end_stream=*/true); +} + +TEST_F(HttpUpstreamImplTest, DecodeTrailers) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + // Decode trailers should reset the stream. + EXPECT_CALL(creation_callbacks_, onStreamFailure()); + upstream_->responseDecoder().decodeTrailers(nullptr); +} + +TEST_F(HttpUpstreamImplTest, DecodeTrailersAfterSuccessHeaders) { + setup(); + setAndExpectRequestEncoder(expectedHeaders()); + + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "101"}, {"upgrade", "connect-udp"}}); + + EXPECT_CALL(creation_callbacks_, onStreamFailure()).Times(0); + EXPECT_CALL(creation_callbacks_, onStreamSuccess(_)); + + upstream_->responseDecoder().decodeHeaders(std::move(response_headers), /*end_stream=*/false); + upstream_->responseDecoder().decodeTrailers(nullptr); +} + +TEST_F(HttpUpstreamImplTest, EncodeHeaders) { + HeaderToAdd header{"test_key", "test_val"}; + absl::optional port; + bool is_ssl = false; + + setup(header); + + EXPECT_CALL(*config_, proxyHost(_)).WillRepeatedly(Return("proxy.host")); + EXPECT_CALL(*config_, proxyPort()).WillRepeatedly(ReturnRef(port)); + EXPECT_CALL(*config_, targetHost(_)).WillRepeatedly(Return("target.host")); + EXPECT_CALL(*config_, defaultTargetPort()).WillRepeatedly(Return(200)); + + auto expected_headers = expectedHeaders( + is_ssl, /*use_post=*/false, /*opt_authority=*/"proxy.host", + /*opt_path=*/"/.well-known/masque/udp/target.host/200/", /*header_to_add=*/header); + + setAndExpectRequestEncoder(expected_headers, is_ssl); +} + +TEST_F(HttpUpstreamImplTest, EncodeHeadersWithPost) { + std::string post_path = "/post/path"; + absl::optional port = 100; + bool is_ssl = true; + + setup(); + + EXPECT_CALL(*config_, proxyHost(_)).WillRepeatedly(Return("proxy.host")); + EXPECT_CALL(*config_, proxyPort()).WillRepeatedly(ReturnRef(port)); + EXPECT_CALL(*config_, usePost()).WillRepeatedly(Return(true)); + EXPECT_CALL(*config_, postPath()).WillRepeatedly(ReturnRef(post_path)); + + auto expected_headers = + expectedHeaders(is_ssl, /*use_post=*/true, /*opt_authority=*/"proxy.host:100", + /*opt_path=*/post_path, /*header_to_add=*/absl::nullopt); + + setAndExpectRequestEncoder(expected_headers, is_ssl); +} + +TEST_F(HttpUpstreamImplTest, EncodeHeadersWithFilterStateOverrides) { + HeaderToAdd header{"test_key", "test_val"}; + bool is_ssl = false; + + setup(header); + filterStateOverride(30, 60); + + EXPECT_CALL(*config_, proxyHost(_)).WillRepeatedly(Return("proxy.host")); + EXPECT_CALL(*config_, targetHost(_)).WillRepeatedly(Return("target.host")); + + // Filter state override has precedence, so they are not called. + EXPECT_CALL(*config_, proxyPort()).Times(0); + EXPECT_CALL(*config_, defaultTargetPort()).Times(0); + + auto expected_headers = expectedHeaders( + is_ssl, /*use_post=*/false, /*opt_authority=*/"proxy.host:30", + /*opt_path=*/"/.well-known/masque/udp/target.host/60/", /*header_to_add=*/header); + + setAndExpectRequestEncoder(expected_headers, is_ssl); +} + +TEST_F(HttpUpstreamImplTest, TargetHostPercentEncoding) { + bool is_ssl = false; + setup(); + + EXPECT_CALL(*config_, proxyHost(_)).WillRepeatedly(Return("proxy.host")); + EXPECT_CALL(*config_, targetHost(_)).WillRepeatedly(Return("2001:db8::42")); + + auto expected_headers = + expectedHeaders(is_ssl, /*use_post=*/false, /*opt_authority=*/"proxy.host:10", + /*opt_path=*/"/.well-known/masque/udp/2001%3Adb8%3A%3A42/20/"); + + setAndExpectRequestEncoder(expected_headers, is_ssl); +} + +class TunnelingConnectionPoolImplTest : public testing::Test { +public: + void setup() { + Protobuf::RepeatedPtrField headers_to_add; + header_evaluator_ = Envoy::Router::HeaderParser::configure(headers_to_add); + config_ = std::make_unique>(*header_evaluator_); + stream_info_.downstream_connection_info_provider_->setConnectionID(0); + session_access_logs_ = std::make_unique>(); + pool_ = std::make_unique( + cluster_, &context_, *config_, callbacks_, stream_info_, false, *session_access_logs_); + } + + void createNewStream() { pool_->newStream(stream_callbacks_); } + + NiceMock cluster_; + NiceMock context_; + std::unique_ptr header_evaluator_; + std::unique_ptr> config_; + NiceMock callbacks_; + NiceMock stream_info_; + std::unique_ptr> session_access_logs_; + NiceMock stream_callbacks_; + NiceMock request_encoder_; + std::shared_ptr> upstream_host_{ + new NiceMock()}; + std::unique_ptr pool_; +}; + +TEST_F(TunnelingConnectionPoolImplTest, ValidPool) { + setup(); + EXPECT_TRUE(pool_->valid()); +} + +TEST_F(TunnelingConnectionPoolImplTest, InvalidPool) { + EXPECT_CALL(cluster_, httpConnPool(_, _, _)).WillOnce(Return(absl::nullopt)); + setup(); + EXPECT_FALSE(pool_->valid()); +} + +TEST_F(TunnelingConnectionPoolImplTest, OnNewStream) { + setup(); + EXPECT_CALL(cluster_.conn_pool_, newStream(_, _, _)); + createNewStream(); +} + +TEST_F(TunnelingConnectionPoolImplTest, PoolFailure) { + setup(); + createNewStream(); + EXPECT_CALL(stream_callbacks_, onStreamFailure(_, _, _)); + + std::string upstream_host_name = "upstream_host_test"; + EXPECT_CALL(*upstream_host_, hostname()).WillOnce(ReturnRef(upstream_host_name)); + pool_->onPoolFailure(Http::ConnectionPool::PoolFailureReason::Timeout, "reason", upstream_host_); + EXPECT_EQ(stream_info_.upstreamInfo()->upstreamHost()->hostname(), upstream_host_name); + EXPECT_EQ(stream_info_.upstreamInfo()->upstreamTransportFailureReason(), "reason"); +} + +TEST_F(TunnelingConnectionPoolImplTest, PoolReady) { + setup(); + createNewStream(); + EXPECT_CALL(request_encoder_.stream_, addCallbacks(_)); + + std::string upstream_host_name = "upstream_host_test"; + EXPECT_CALL(*upstream_host_, hostname()).WillOnce(ReturnRef(upstream_host_name)); + pool_->onPoolReady(request_encoder_, upstream_host_, stream_info_, absl::nullopt); + EXPECT_EQ(stream_info_.upstreamInfo()->upstreamHost()->hostname(), upstream_host_name); +} + +TEST_F(TunnelingConnectionPoolImplTest, OnStreamFailure) { + setup(); + createNewStream(); + EXPECT_CALL(stream_callbacks_, + onStreamFailure(ConnectionPool::PoolFailureReason::RemoteConnectionFailure, "", _)); + pool_->onStreamFailure(); +} + +TEST_F(TunnelingConnectionPoolImplTest, OnStreamSuccess) { + setup(); + createNewStream(); + EXPECT_CALL(stream_callbacks_, onStreamReady(_, _, _, _, _)); + pool_->onStreamSuccess(request_encoder_); +} + +TEST_F(TunnelingConnectionPoolImplTest, FactoryTest) { + setup(); + + TunnelingConnectionPoolFactory factory; + auto valid_pool = factory.createConnPool(cluster_, &context_, *config_, callbacks_, stream_info_, + false, *session_access_logs_); + EXPECT_FALSE(valid_pool == nullptr); + + EXPECT_CALL(cluster_, httpConnPool(_, _, _)).WillOnce(Return(absl::nullopt)); + auto invalid_pool = factory.createConnPool(cluster_, &context_, *config_, callbacks_, + stream_info_, false, *session_access_logs_); + EXPECT_TRUE(invalid_pool == nullptr); +} + +TEST(TunnelingConfigImplTest, DefaultConfigs) { + NiceMock context; + TunnelingConfig proto_config; + proto_config.set_use_post(true); + TunnelingConfigImpl config(proto_config, context); + + EXPECT_EQ(1, config.maxConnectAttempts()); + EXPECT_EQ(1024, config.maxBufferedDatagrams()); + EXPECT_EQ(16384, config.maxBufferedBytes()); +} + +TEST(TunnelingConfigImplTest, DefaultPostPath) { + NiceMock context; + TunnelingConfig proto_config; + proto_config.set_use_post(true); + TunnelingConfigImpl config(proto_config, context); + + EXPECT_EQ("/", config.postPath()); +} + +TEST(TunnelingConfigImplTest, PostPathWithoutPostMethod) { + NiceMock context; + TunnelingConfig proto_config; + proto_config.set_post_path("/path"); + EXPECT_THROW_WITH_MESSAGE(TunnelingConfigImpl(proto_config, context), EnvoyException, + "Can't set a post path when POST method isn't used"); +} + +TEST(TunnelingConfigImplTest, PostWithInvalidPath) { + NiceMock context; + TunnelingConfig proto_config; + proto_config.set_use_post(true); + proto_config.set_post_path("path"); + EXPECT_THROW_WITH_MESSAGE(TunnelingConfigImpl(proto_config, context), EnvoyException, + "Path must start with '/'"); +} + +TEST(TunnelingConfigImplTest, ValidProxyPort) { + NiceMock context; + TunnelingConfig proto_config; + proto_config.mutable_proxy_port()->set_value(443); + TunnelingConfigImpl config(proto_config, context); + EXPECT_EQ(443, config.proxyPort()); +} + +TEST(TunnelingConfigImplTest, ProxyPortOutOfRange) { + NiceMock context; + + { + TunnelingConfig proto_config; + proto_config.mutable_proxy_port()->set_value(0); + EXPECT_THROW_WITH_MESSAGE(TunnelingConfigImpl(proto_config, context), EnvoyException, + "Port value not in range"); + } + { + TunnelingConfig proto_config; + proto_config.mutable_proxy_port()->set_value(65536); + EXPECT_THROW_WITH_MESSAGE(TunnelingConfigImpl(proto_config, context), EnvoyException, + "Port value not in range"); + } +} + +TEST(TunnelingConfigImplTest, InvalidHeadersToAdd) { + NiceMock context; + + { + TunnelingConfig proto_config; + auto* header_to_add = proto_config.add_headers_to_add(); + auto* header = header_to_add->mutable_header(); + // Can't add pseudo header. + header->set_key(":method"); + header->set_value("GET"); + EXPECT_THROW_WITH_MESSAGE(TunnelingConfigImpl(proto_config, context), EnvoyException, + ":-prefixed or host headers may not be modified"); + } + + { + TunnelingConfig proto_config; + auto* header_to_add = proto_config.add_headers_to_add(); + auto* header = header_to_add->mutable_header(); + // Can't modify host. + header->set_key("host"); + header->set_value("example.net:80"); + EXPECT_THROW_WITH_MESSAGE(TunnelingConfigImpl(proto_config, context), EnvoyException, + ":-prefixed or host headers may not be modified"); + } +} + +TEST(TunnelingConfigImplTest, HeadersToAdd) { + NiceMock context; + NiceMock stream_info; + + stream_info.filterState()->setData( + "test_key", std::make_shared("test_val"), + Envoy::StreamInfo::FilterState::StateType::Mutable); + + TunnelingConfig proto_config; + auto* header_to_add = proto_config.add_headers_to_add(); + auto* header = header_to_add->mutable_header(); + header->set_key("test_key"); + header->set_value("%FILTER_STATE(test_key:PLAIN)%"); + TunnelingConfigImpl config(proto_config, context); + + auto headers = Http::TestRequestHeaderMapImpl{{":scheme", "http"}, {":authority", "host.com"}}; + config.headerEvaluator().evaluateHeaders(headers, {}, stream_info); + EXPECT_EQ("test_val", headers.get_("test_key")); +} + +TEST(TunnelingConfigImplTest, ProxyHostFromFilterState) { + NiceMock context; + NiceMock stream_info; + + stream_info.filterState()->setData( + "test-proxy-host", std::make_shared("test.host.com"), + Envoy::StreamInfo::FilterState::StateType::Mutable); + + TunnelingConfig proto_config; + proto_config.set_proxy_host("%FILTER_STATE(test-proxy-host:PLAIN)%"); + TunnelingConfigImpl config(proto_config, context); + + EXPECT_EQ("test.host.com", config.proxyHost(stream_info)); +} + +TEST(TunnelingConfigImplTest, TargetHostFromFilterState) { + NiceMock context; + NiceMock stream_info; + + stream_info.filterState()->setData( + "test-proxy-host", std::make_shared("test.host.com"), + Envoy::StreamInfo::FilterState::StateType::Mutable); + + TunnelingConfig proto_config; + proto_config.set_target_host("%FILTER_STATE(test-proxy-host:PLAIN)%"); + TunnelingConfigImpl config(proto_config, context); + + EXPECT_EQ("test.host.com", config.targetHost(stream_info)); +} + +TEST(TunnelingConfigImplTest, BufferingState) { + NiceMock context; + + { + TunnelingConfig proto_config; + TunnelingConfigImpl config(proto_config, context); + // Buffering should be disabled by default. + EXPECT_FALSE(config.bufferEnabled()); + } + + { + TunnelingConfig proto_config; + proto_config.mutable_buffer_options(); // Will set buffering. + TunnelingConfigImpl config(proto_config, context); + EXPECT_TRUE(config.bufferEnabled()); + } +} + } // namespace } // namespace UdpProxy } // namespace UdpFilters diff --git a/test/extensions/formatter/cel/cel_test.cc b/test/extensions/formatter/cel/cel_test.cc index c90b71daed67..2ada1036d0b5 100644 --- a/test/extensions/formatter/cel/cel_test.cc +++ b/test/extensions/formatter/cel/cel_test.cc @@ -34,8 +34,16 @@ class CELFormatterTest : public ::testing::Test { }; #ifdef USE_CEL_PARSER +TEST_F(CELFormatterTest, TestNodeId) { + auto cel_parser = std::make_unique(context_.server_factory_context_); + absl::optional max_length = absl::nullopt; + auto formatter = cel_parser->parse("CEL", "xds.node.id", max_length); + EXPECT_THAT(formatter->formatValueWithContext(formatter_context_, stream_info_), + ProtoEq(ValueUtil::stringValue("node_name"))); +} + TEST_F(CELFormatterTest, TestFormatValue) { - auto cel_parser = std::make_unique(context_); + auto cel_parser = std::make_unique(context_.server_factory_context_); absl::optional max_length = absl::nullopt; auto formatter = cel_parser->parse("CEL", "request.headers[':method']", max_length); EXPECT_THAT(formatter->formatValueWithContext(formatter_context_, stream_info_), @@ -43,14 +51,14 @@ TEST_F(CELFormatterTest, TestFormatValue) { } TEST_F(CELFormatterTest, TestParseFail) { - auto cel_parser = std::make_unique(context_); + auto cel_parser = std::make_unique(context_.server_factory_context_); absl::optional max_length = absl::nullopt; EXPECT_EQ(nullptr, cel_parser->parse("INVALID_CMD", "requests.headers['missing_headers']", max_length)); } TEST_F(CELFormatterTest, TestNullFormatValue) { - auto cel_parser = std::make_unique(context_); + auto cel_parser = std::make_unique(context_.server_factory_context_); absl::optional max_length = absl::nullopt; auto formatter = cel_parser->parse("CEL", "requests.headers['missing_headers']", max_length); EXPECT_THAT(formatter->formatValueWithContext(formatter_context_, stream_info_), diff --git a/test/extensions/formatter/metadata/metadata_test.cc b/test/extensions/formatter/metadata/metadata_test.cc index d16492dca24b..a265a5ac9d26 100644 --- a/test/extensions/formatter/metadata/metadata_test.cc +++ b/test/extensions/formatter/metadata/metadata_test.cc @@ -129,6 +129,23 @@ TEST_F(MetadataFormatterTest, NonExistentRouteMetadata) { getTestMetadataFormatter("ROUTE")->formatWithContext(formatter_context_, stream_info_)); } +// Test that METADATA(LISTENER accesses stream_info listener metadata. +TEST_F(MetadataFormatterTest, ListenerMetadata) { + auto listener_info = std::make_shared>(); + EXPECT_CALL(*listener_info, metadata()).WillRepeatedly(testing::ReturnRef(*metadata_)); + stream_info_.downstream_connection_info_provider_->setListenerInfo(listener_info); + EXPECT_EQ( + "test_value", + getTestMetadataFormatter("LISTENER")->formatWithContext(formatter_context_, stream_info_)); +} + +// Test that METADATA(LISTENER handles no listener info. +TEST_F(MetadataFormatterTest, NoListenerMetadata) { + EXPECT_EQ( + "-", + getTestMetadataFormatter("LISTENER")->formatWithContext(formatter_context_, stream_info_)); +} + } // namespace Formatter } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/geoip_providers/maxmind/BUILD b/test/extensions/geoip_providers/maxmind/BUILD new file mode 100644 index 000000000000..1757c2c53c8b --- /dev/null +++ b/test/extensions/geoip_providers/maxmind/BUILD @@ -0,0 +1,53 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + size = "small", + srcs = select({ + "//bazel:linux": ["config_test.cc"], + "//conditions:default": [], + }), + data = [ + "//test/extensions/geoip_providers/maxmind/test_data:geolocation_databases", + ], + extension_names = ["envoy.geoip_providers.maxmind"], + tags = ["skip_on_windows"], + deps = [ + "//source/extensions/geoip_providers/maxmind:config", + "//source/extensions/geoip_providers/maxmind:provider_impl", + "//test/mocks/server:factory_context_mocks", + "@envoy_api//envoy/extensions/geoip_providers/maxmind/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "geoip_provider_test", + size = "small", + srcs = select({ + "//bazel:linux": ["geoip_provider_test.cc"], + "//conditions:default": [], + }), + data = [ + "//test/extensions/geoip_providers/maxmind/test_data:geolocation_databases", + ], + extension_names = ["envoy.geoip_providers.maxmind"], + tags = ["skip_on_windows"], + deps = [ + "//envoy/registry", + "//source/extensions/geoip_providers/maxmind:config", + "//source/extensions/geoip_providers/maxmind:provider_impl", + "//test/mocks/server:factory_context_mocks", + "@envoy_api//envoy/extensions/geoip_providers/maxmind/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/geoip_providers/maxmind/config_test.cc b/test/extensions/geoip_providers/maxmind/config_test.cc new file mode 100644 index 000000000000..53b40ec948f3 --- /dev/null +++ b/test/extensions/geoip_providers/maxmind/config_test.cc @@ -0,0 +1,359 @@ +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.h" +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.validate.h" + +#include "source/extensions/geoip_providers/maxmind/config.h" +#include "source/extensions/geoip_providers/maxmind/geoip_provider.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/environment.h" + +#include "absl/strings/str_format.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::AllOf; + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Maxmind { + +using MaxmindProviderConfig = envoy::extensions::geoip_providers::maxmind::v3::MaxMindConfig; + +class GeoipProviderPeer { +public: + static const absl::optional& cityDbPath(const GeoipProvider& provider) { + return provider.config_->cityDbPath(); + } + static const absl::optional& ispDbPath(const GeoipProvider& provider) { + return provider.config_->ispDbPath(); + } + static const absl::optional& anonDbPath(const GeoipProvider& provider) { + return provider.config_->anonDbPath(); + } + static const absl::optional& countryHeader(const GeoipProvider& provider) { + return provider.config_->countryHeader(); + } + static const absl::optional& cityHeader(const GeoipProvider& provider) { + return provider.config_->cityHeader(); + } + static const absl::optional& regionHeader(const GeoipProvider& provider) { + return provider.config_->regionHeader(); + } + static const absl::optional& asnHeader(const GeoipProvider& provider) { + return provider.config_->asnHeader(); + } + static const absl::optional& anonVpnHeader(const GeoipProvider& provider) { + return provider.config_->anonVpnHeader(); + } + static const absl::optional& anonTorHeader(const GeoipProvider& provider) { + return provider.config_->anonTorHeader(); + } + static const absl::optional& anonProxyHeader(const GeoipProvider& provider) { + return provider.config_->anonProxyHeader(); + } + static const absl::optional& anonHostingHeader(const GeoipProvider& provider) { + return provider.config_->anonHostingHeader(); + } +}; + +MATCHER_P(HasCityDbPath, expected_db_path, "") { + auto provider = std::static_pointer_cast(arg); + auto city_db_path = GeoipProviderPeer::cityDbPath(*provider); + if (city_db_path && testing::Matches(expected_db_path)(city_db_path.value())) { + return true; + } + *result_listener << "expected city_db_path=" << expected_db_path + << " but city_db_path was not found in provider config"; + return false; +} + +MATCHER_P(HasIspDbPath, expected_db_path, "") { + auto provider = std::static_pointer_cast(arg); + auto isp_db_path = GeoipProviderPeer::ispDbPath(*provider); + if (isp_db_path && testing::Matches(expected_db_path)(isp_db_path.value())) { + return true; + } + *result_listener << "expected isp_db_path=" << expected_db_path + << " but isp_db_path was not found in provider config"; + return false; +} + +MATCHER_P(HasAnonDbPath, expected_db_path, "") { + auto provider = std::static_pointer_cast(arg); + auto anon_db_path = GeoipProviderPeer::anonDbPath(*provider); + if (anon_db_path && testing::Matches(expected_db_path)(anon_db_path.value())) { + return true; + } + *result_listener << "expected anon_db_path=" << expected_db_path + << " but anon_db_path was not found in provider config"; + return false; +} + +MATCHER_P(HasCountryHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto country_header = GeoipProviderPeer::countryHeader(*provider); + if (country_header && testing::Matches(expected_header)(country_header.value())) { + return true; + } + *result_listener << "expected country header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasCityHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto city_header = GeoipProviderPeer::cityHeader(*provider); + if (city_header && testing::Matches(expected_header)(city_header.value())) { + return true; + } + *result_listener << "expected city header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasRegionHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto region_header = GeoipProviderPeer::regionHeader(*provider); + if (region_header && testing::Matches(expected_header)(region_header.value())) { + return true; + } + *result_listener << "expected region header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasAsnHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto asn_header = GeoipProviderPeer::asnHeader(*provider); + if (asn_header && testing::Matches(expected_header)(asn_header.value())) { + return true; + } + *result_listener << "expected asn header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasAnonVpnHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto anon_vpn_header = GeoipProviderPeer::anonVpnHeader(*provider); + if (anon_vpn_header && testing::Matches(expected_header)(anon_vpn_header.value())) { + return true; + } + *result_listener << "expected anon_vpn header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasAnonTorHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto anon_tor_header = GeoipProviderPeer::anonTorHeader(*provider); + if (anon_tor_header && testing::Matches(expected_header)(anon_tor_header.value())) { + return true; + } + *result_listener << "expected anon_tor header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasAnonProxyHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto anon_proxy_header = GeoipProviderPeer::anonProxyHeader(*provider); + if (anon_proxy_header && testing::Matches(expected_header)(anon_proxy_header.value())) { + return true; + } + *result_listener << "expected anon_proxy header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +MATCHER_P(HasAnonHostingHeader, expected_header, "") { + auto provider = std::static_pointer_cast(arg); + auto anon_hosting_header = GeoipProviderPeer::anonHostingHeader(*provider); + if (anon_hosting_header && testing::Matches(expected_header)(anon_hosting_header.value())) { + return true; + } + *result_listener << "expected anon_hosting_header header=" << expected_header + << " but header was not found in provider config with expected value"; + return false; +} + +std::string genGeoDbFilePath(std::string db_name) { + return TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/" + db_name); +} + +TEST(MaxmindProviderConfigTest, EmptyProto) { + MaxmindProviderFactory factory; + EXPECT_TRUE(factory.createEmptyConfigProto() != nullptr); +} + +TEST(MaxmindProviderConfigTest, ProviderConfigWithCorrectProto) { + const auto provider_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + anon_vpn: "x-anon-vpn" + asn: "x-geo-asn" + is_anon: "x-geo-anon" + anon_vpn: "x-anon-vpn" + anon_tor: "x-anon-tor" + anon_proxy: "x-anon-proxy" + anon_hosting: "x-anon-hosting" + city_db_path: %s + isp_db_path: %s + anon_db_path: %s + )EOF"; + MaxmindProviderConfig provider_config; + auto city_db_path = genGeoDbFilePath("GeoLite2-City-Test.mmdb"); + auto asn_db_path = genGeoDbFilePath("GeoLite2-ASN-Test.mmdb"); + auto anon_db_path = genGeoDbFilePath("GeoIP2-Anonymous-IP-Test.mmdb"); + auto processed_provider_config_yaml = + absl::StrFormat(provider_config_yaml, city_db_path, asn_db_path, anon_db_path); + TestUtility::loadFromYaml(processed_provider_config_yaml, provider_config); + NiceMock context; + EXPECT_CALL(context, messageValidationVisitor()); + MaxmindProviderFactory factory; + Geolocation::DriverSharedPtr driver = + factory.createGeoipProviderDriver(provider_config, "maxmind", context); + EXPECT_THAT(driver, AllOf(HasCityDbPath(city_db_path), HasIspDbPath(asn_db_path), + HasAnonDbPath(anon_db_path), HasCountryHeader("x-geo-country"), + HasCityHeader("x-geo-city"), HasRegionHeader("x-geo-region"), + HasAsnHeader("x-geo-asn"), HasAnonVpnHeader("x-anon-vpn"), + HasAnonTorHeader("x-anon-tor"), HasAnonProxyHeader("x-anon-proxy"), + HasAnonHostingHeader("x-anon-hosting"))); +} + +TEST(MaxmindProviderConfigTest, ProviderConfigWithNoDbPaths) { + std::string provider_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + )EOF"; + MaxmindProviderConfig provider_config; + TestUtility::loadFromYaml(provider_config_yaml, provider_config); + NiceMock context; + EXPECT_CALL(context, messageValidationVisitor()); + MaxmindProviderFactory factory; + EXPECT_THROW_WITH_MESSAGE(factory.createGeoipProviderDriver(provider_config, "maxmind", context), + Envoy::EnvoyException, + "At least one geolocation database path needs to be configured: " + "city_db_path, isp_db_path or anon_db_path"); +} + +TEST(MaxmindProviderConfigTest, ProviderConfigWithNoGeoHeaders) { + std::string provider_config_yaml = R"EOF( + isp_db_path: "/geoip2/Isp.mmdb" + )EOF"; + MaxmindProviderConfig provider_config; + TestUtility::loadFromYaml(provider_config_yaml, provider_config); + NiceMock context; + EXPECT_CALL(context, messageValidationVisitor()); + MaxmindProviderFactory factory; + EXPECT_THROW_WITH_REGEX(factory.createGeoipProviderDriver(provider_config, "maxmind", context), + ProtoValidationException, + "Proto constraint validation failed.*value is required.*"); +} + +TEST(MaxmindProviderConfigTest, DbPathFormatValidatedWhenNonEmptyValue) { + std::string provider_config_yaml = R"EOF( + isp_db_path: "/geoip2/Isp.exe" + )EOF"; + MaxmindProviderConfig provider_config; + TestUtility::loadFromYaml(provider_config_yaml, provider_config); + NiceMock context; + EXPECT_CALL(context, messageValidationVisitor()); + MaxmindProviderFactory factory; + EXPECT_THROW_WITH_REGEX( + factory.createGeoipProviderDriver(provider_config, "maxmind", context), + ProtoValidationException, + "Proto constraint validation failed.*value does not match regex pattern.*"); +} + +TEST(MaxmindProviderConfigTest, ReusesProviderInstanceForSameProtoConfig) { + const auto provider_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + anon_vpn: "x-anon-vpn" + asn: "x-geo-asn" + anon_tor: "x-anon-tor" + anon_proxy: "x-anon-proxy" + anon_hosting: "x-anon-hosting" + city_db_path: %s + isp_db_path: %s + anon_db_path: %s + )EOF"; + MaxmindProviderConfig provider_config; + auto city_db_path = genGeoDbFilePath("GeoLite2-City-Test.mmdb"); + auto asn_db_path = genGeoDbFilePath("GeoLite2-ASN-Test.mmdb"); + auto anon_db_path = genGeoDbFilePath("GeoIP2-Anonymous-IP-Test.mmdb"); + auto processed_provider_config_yaml = + absl::StrFormat(provider_config_yaml, city_db_path, asn_db_path, anon_db_path); + TestUtility::loadFromYaml(processed_provider_config_yaml, provider_config); + NiceMock context; + EXPECT_CALL(context, messageValidationVisitor()).Times(2); + MaxmindProviderFactory factory; + Geolocation::DriverSharedPtr driver1 = + factory.createGeoipProviderDriver(provider_config, "maxmind", context); + Geolocation::DriverSharedPtr driver2 = + factory.createGeoipProviderDriver(provider_config, "maxmind", context); + EXPECT_EQ(driver1.get(), driver2.get()); +} + +TEST(MaxmindProviderConfigTest, DifferentProviderInstancesForDifferentProtoConfig) { + const auto provider_config_yaml1 = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + anon_vpn: "x-anon-vpn" + asn: "x-geo-asn" + anon_tor: "x-anon-tor" + anon_proxy: "x-anon-proxy" + anon_hosting: "x-anon-hosting" + city_db_path: %s + isp_db_path: %s + anon_db_path: %s + )EOF"; + const auto provider_config_yaml2 = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + anon_vpn: "x-anon-vpn" + anon_tor: "x-anon-tor" + anon_proxy: "x-anon-proxy" + anon_hosting: "x-anon-hosting" + city_db_path: %s + anon_db_path: %s + )EOF"; + MaxmindProviderConfig provider_config1; + MaxmindProviderConfig provider_config2; + auto city_db_path = genGeoDbFilePath("GeoLite2-City-Test.mmdb"); + auto asn_db_path = genGeoDbFilePath("GeoLite2-ASN-Test.mmdb"); + auto anon_db_path = genGeoDbFilePath("GeoIP2-Anonymous-IP-Test.mmdb"); + auto processed_provider_config_yaml1 = + absl::StrFormat(provider_config_yaml1, city_db_path, asn_db_path, anon_db_path); + auto processed_provider_config_yaml2 = + absl::StrFormat(provider_config_yaml2, city_db_path, anon_db_path); + TestUtility::loadFromYaml(processed_provider_config_yaml1, provider_config1); + TestUtility::loadFromYaml(processed_provider_config_yaml2, provider_config2); + NiceMock context; + EXPECT_CALL(context, messageValidationVisitor()).Times(2); + MaxmindProviderFactory factory; + Geolocation::DriverSharedPtr driver1 = + factory.createGeoipProviderDriver(provider_config1, "maxmind", context); + Geolocation::DriverSharedPtr driver2 = + factory.createGeoipProviderDriver(provider_config2, "maxmind", context); + EXPECT_NE(driver1.get(), driver2.get()); +} + +} // namespace Maxmind +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc b/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc new file mode 100644 index 000000000000..214acf4babff --- /dev/null +++ b/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc @@ -0,0 +1,303 @@ +#include "envoy/extensions/geoip_providers/maxmind/v3/maxmind.pb.h" +#include "envoy/registry/registry.h" + +#include "source/common/network/address_impl.h" +#include "source/common/network/utility.h" +#include "source/extensions/geoip_providers/maxmind/config.h" +#include "source/extensions/geoip_providers/maxmind/geoip_provider.h" + +#include "test/mocks/server/factory_context.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::NiceMock; +using testing::ReturnRef; +using testing::SaveArg; + +namespace Envoy { +namespace Extensions { +namespace GeoipProviders { +namespace Maxmind { + +class GeoipProviderPeer { +public: + static Stats::Scope& providerScope(const DriverSharedPtr& driver) { + auto provider = std::static_pointer_cast(driver); + return provider->config_->getStatsScopeForTest(); + } +}; + +class GeoipProviderTest : public testing::Test { +public: + GeoipProviderTest() { + provider_factory_ = dynamic_cast( + Registry::FactoryRegistry::getFactory( + "envoy.geoip_providers.maxmind")); + ASSERT(provider_factory_); + } + + void initializeProvider(const std::string& yaml) { + EXPECT_CALL(context_, scope()).WillRepeatedly(ReturnRef(*scope_)); + envoy::extensions::geoip_providers::maxmind::v3::MaxMindConfig config; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); + provider_ = provider_factory_->createGeoipProviderDriver(config, "prefix.", context_); + } + + void expectStats(const std::string& db_type, const uint32_t total_count = 1, + const uint32_t hit_count = 1, const uint32_t error_count = 0) { + auto& provider_scope = GeoipProviderPeer::providerScope(provider_); + EXPECT_EQ(provider_scope.counterFromString(absl::StrCat(db_type, ".total")).value(), + total_count); + EXPECT_EQ(provider_scope.counterFromString(absl::StrCat(db_type, ".hit")).value(), hit_count); + EXPECT_EQ(provider_scope.counterFromString(absl::StrCat(db_type, ".lookup_error")).value(), + error_count); + } + + Stats::IsolatedStoreImpl stats_store_; + Stats::ScopeSharedPtr scope_{stats_store_.createScope("")}; + NiceMock context_; + DriverSharedPtr provider_; + MaxmindProviderFactory* provider_factory_; + absl::flat_hash_map captured_lookup_response_; +}; + +TEST_F(GeoipProviderTest, ValidConfigCityAndIspDbsSuccessfulLookup) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + asn: "x-geo-asn" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + isp_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-ASN-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("78.26.243.166"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(4, captured_lookup_response_.size()); + const auto& city_it = captured_lookup_response_.find("x-geo-city"); + EXPECT_EQ("Boxford", city_it->second); + const auto& region_it = captured_lookup_response_.find("x-geo-region"); + EXPECT_EQ("ENG", region_it->second); + const auto& country_it = captured_lookup_response_.find("x-geo-country"); + EXPECT_EQ("GB", country_it->second); + const auto& asn_it = captured_lookup_response_.find("x-geo-asn"); + EXPECT_EQ("15169", asn_it->second); + expectStats("city_db"); + expectStats("isp_db"); +} + +TEST_F(GeoipProviderTest, ValidConfigCityLookupError) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/MaxMind-DB-test-ipv4-24.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("2345:0425:2CA1:0:0:0567:5673:23b5"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + expectStats("city_db", 1, 0, 1); + EXPECT_EQ(0, captured_lookup_response_.size()); +} + +// Tests for anonymous database replicate expectations from corresponding Maxmind tests: +// https://github.com/maxmind/GeoIP2-perl/blob/main/t/GeoIP2/Database/Reader-Anonymous-IP.t +TEST_F(GeoipProviderTest, ValidConfigAnonVpnSuccessfulLookup) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + is_anon: "x-geo-anon" + anon_vpn: "x-geo-anon-vpn" + anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("1.2.0.0"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(2, captured_lookup_response_.size()); + const auto& anon_it = captured_lookup_response_.find("x-geo-anon"); + EXPECT_EQ("true", anon_it->second); + const auto& anon_vpn_it = captured_lookup_response_.find("x-geo-anon-vpn"); + EXPECT_EQ("true", anon_vpn_it->second); + expectStats("anon_db"); +} + +TEST_F(GeoipProviderTest, ValidConfigAnonHostingSuccessfulLookup) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + is_anon: "x-geo-anon" + anon_hosting: "x-geo-anon-hosting" + anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("71.160.223.45"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(2, captured_lookup_response_.size()); + const auto& anon_it = captured_lookup_response_.find("x-geo-anon"); + EXPECT_EQ("true", anon_it->second); + const auto& anon_hosting_it = captured_lookup_response_.find("x-geo-anon-hosting"); + EXPECT_EQ("true", anon_hosting_it->second); + expectStats("anon_db"); +} + +TEST_F(GeoipProviderTest, ValidConfigAnonTorNodeSuccessfulLookup) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + is_anon: "x-geo-anon" + anon_tor: "x-geo-anon-tor" + anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("65.4.3.2"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(2, captured_lookup_response_.size()); + const auto& anon_it = captured_lookup_response_.find("x-geo-anon"); + EXPECT_EQ("true", anon_it->second); + const auto& anon_tor_it = captured_lookup_response_.find("x-geo-anon-tor"); + EXPECT_EQ("true", anon_tor_it->second); + expectStats("anon_db"); +} + +TEST_F(GeoipProviderTest, ValidConfigAnonProxySuccessfulLookup) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + is_anon: "x-geo-anon" + anon_proxy: "x-geo-anon-proxy" + anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("abcd:1000::1"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(2, captured_lookup_response_.size()); + const auto& anon_it = captured_lookup_response_.find("x-geo-anon"); + EXPECT_EQ("true", anon_it->second); + const auto& anon_tor_it = captured_lookup_response_.find("x-geo-anon-proxy"); + EXPECT_EQ("true", anon_tor_it->second); + expectStats("anon_db"); +} + +TEST_F(GeoipProviderTest, ValidConfigEmptyLookupResult) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + is_anon: "x-geo-anon" + anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("10.10.10.10"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(0, captured_lookup_response_.size()); + expectStats("anon_db", 1, 0); +} + +TEST_F(GeoipProviderTest, ValidConfigCityMultipleLookups) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address1 = + Network::Utility::parseInternetAddress("78.26.243.166"); + Geolocation::LookupRequest lookup_rq1{std::move(remote_address1)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq1), std::move(lookup_cb_std)); + EXPECT_EQ(3, captured_lookup_response_.size()); + // Another lookup request. + Network::Address::InstanceConstSharedPtr remote_address2 = + Network::Utility::parseInternetAddress("63.25.243.11"); + Geolocation::LookupRequest lookup_rq2{std::move(remote_address2)}; + testing::MockFunction lookup_cb2; + auto lookup_cb_std2 = lookup_cb2.AsStdFunction(); + EXPECT_CALL(lookup_cb2, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq2), std::move(lookup_cb_std2)); + EXPECT_EQ(3, captured_lookup_response_.size()); + expectStats("city_db", 2, 2); +} + +using GeoipProviderDeathTest = GeoipProviderTest; + +TEST_F(GeoipProviderDeathTest, GeoDbNotSetForConfiguredHeader) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + city: "x-geo-city" + asn: "x-geo-asn" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + initializeProvider(config_yaml); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddress("78.26.243.166"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + EXPECT_DEATH(provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)), + "assert failure: isp_db_. Details: Maxmind asn database is not initialized for " + "performing lookups"); +} + +TEST_F(GeoipProviderDeathTest, GeoDbPathDoesNotExist) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + city: "x-geo-city" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data_atc/GeoLite2-City-Test.mmdb" + )EOF"; + EXPECT_DEATH(initializeProvider(config_yaml), ".*Unable to open Maxmind database file.*"); +} + +} // namespace Maxmind +} // namespace GeoipProviders +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/geoip_providers/maxmind/test_data/BUILD b/test/extensions/geoip_providers/maxmind/test_data/BUILD new file mode 100644 index 000000000000..7323b7eab2ac --- /dev/null +++ b/test/extensions/geoip_providers/maxmind/test_data/BUILD @@ -0,0 +1,15 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +# Copies of certain lighweight testing Maxmind databases are included into this filegroup: +# https://github.com/maxmind/MaxMind-DB/tree/main/test-data +filegroup( + name = "geolocation_databases", + srcs = glob(["*.mmdb"]), +) diff --git a/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb b/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb new file mode 100644 index 000000000000..ad9895c20de9 Binary files /dev/null and b/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb differ diff --git a/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-ASN-Test.mmdb b/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-ASN-Test.mmdb new file mode 100644 index 000000000000..bc31924dc151 Binary files /dev/null and b/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-ASN-Test.mmdb differ diff --git a/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb b/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb new file mode 100644 index 000000000000..f391123b82e1 Binary files /dev/null and b/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb differ diff --git a/test/extensions/geoip_providers/maxmind/test_data/MaxMind-DB-test-ipv4-24.mmdb b/test/extensions/geoip_providers/maxmind/test_data/MaxMind-DB-test-ipv4-24.mmdb new file mode 100644 index 000000000000..71ff41e91dd3 Binary files /dev/null and b/test/extensions/geoip_providers/maxmind/test_data/MaxMind-DB-test-ipv4-24.mmdb differ diff --git a/test/extensions/http/cache/file_system_http_cache/BUILD b/test/extensions/http/cache/file_system_http_cache/BUILD index ce137ae24c27..a5bf24f6ef25 100644 --- a/test/extensions/http/cache/file_system_http_cache/BUILD +++ b/test/extensions/http/cache/file_system_http_cache/BUILD @@ -18,7 +18,6 @@ envoy_extension_cc_test( "//source/extensions/filters/http/cache:cache_entry_utils_lib", "//source/extensions/http/cache/file_system_http_cache:config", "//test/extensions/common/async_files:mocks", - "//test/extensions/filters/http/cache:common", "//test/extensions/filters/http/cache:http_cache_implementation_test_common_lib", "//test/mocks/server:factory_context_mocks", "//test/test_common:simulated_time_system_lib", diff --git a/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc b/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc index 0a5e19a0c9ae..004eb996d33e 100644 --- a/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc +++ b/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc @@ -12,7 +12,6 @@ #include "source/extensions/http/cache/file_system_http_cache/file_system_http_cache.h" #include "test/extensions/common/async_files/mocks.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/extensions/filters/http/cache/http_cache_implementation_test_common.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/environment.h" @@ -63,9 +62,8 @@ class FileSystemCacheTestContext { throw EnvoyException( fmt::format("Didn't find a registered implementation for type: '{}'", type)); } - ON_CALL(context_.api_, threadFactory()).WillByDefault([]() -> Thread::ThreadFactory& { - return Thread::threadFactoryForTest(); - }); + ON_CALL(context_.server_factory_context_.api_, threadFactory()) + .WillByDefault([]() -> Thread::ThreadFactory& { return Thread::threadFactoryForTest(); }); } void initCache() { @@ -266,23 +264,25 @@ class MockSingletonManager : public Singleton::ManagerImpl { public: MockSingletonManager() : Singleton::ManagerImpl(Thread::threadFactoryForTest()) { // By default just act like a real SingletonManager, but allow overrides. - ON_CALL(*this, get(_, _)) + ON_CALL(*this, get(_, _, _)) .WillByDefault(std::bind(&MockSingletonManager::realGet, this, std::placeholders::_1, - std::placeholders::_2)); + std::placeholders::_2, std::placeholders::_3)); } MOCK_METHOD(Singleton::InstanceSharedPtr, get, - (const std::string& name, Singleton::SingletonFactoryCb cb)); - Singleton::InstanceSharedPtr realGet(const std::string& name, Singleton::SingletonFactoryCb cb) { - return Singleton::ManagerImpl::get(name, cb); + (const std::string& name, Singleton::SingletonFactoryCb cb, bool pin)); + Singleton::InstanceSharedPtr realGet(const std::string& name, Singleton::SingletonFactoryCb cb, + bool pin) { + return Singleton::ManagerImpl::get(name, cb, pin); } }; class FileSystemHttpCacheTestWithMockFiles : public FileSystemHttpCacheTest { public: FileSystemHttpCacheTestWithMockFiles() { - ON_CALL(context_, singletonManager()).WillByDefault(ReturnRef(mock_singleton_manager_)); - ON_CALL(mock_singleton_manager_, get(HasSubstr("async_file_manager_factory_singleton"), _)) + ON_CALL(context_.server_factory_context_, singletonManager()) + .WillByDefault(ReturnRef(mock_singleton_manager_)); + ON_CALL(mock_singleton_manager_, get(HasSubstr("async_file_manager_factory_singleton"), _, _)) .WillByDefault(Return(mock_async_file_manager_factory_)); ON_CALL(*mock_async_file_manager_factory_, getAsyncFileManager(_, _)) .WillByDefault(Return(mock_async_file_manager_)); @@ -1252,9 +1252,8 @@ TEST(Registration, GetCacheFromFactory) { ASSERT_NE(factory, nullptr); envoy::extensions::filters::http::cache::v3::CacheConfig cache_config; NiceMock factory_context; - ON_CALL(factory_context.api_, threadFactory()).WillByDefault([]() -> Thread::ThreadFactory& { - return Thread::threadFactoryForTest(); - }); + ON_CALL(factory_context.server_factory_context_.api_, threadFactory()) + .WillByDefault([]() -> Thread::ThreadFactory& { return Thread::threadFactoryForTest(); }); TestUtility::loadFromYaml(std::string(yaml_config), cache_config); EXPECT_EQ(factory->getCache(cache_config, factory_context)->cacheInfo().name_, "envoy.extensions.http.cache.file_system_http_cache"); diff --git a/test/extensions/http/cache/simple_http_cache/BUILD b/test/extensions/http/cache/simple_http_cache/BUILD index 744f4bd8e289..a349aa162993 100644 --- a/test/extensions/http/cache/simple_http_cache/BUILD +++ b/test/extensions/http/cache/simple_http_cache/BUILD @@ -15,7 +15,6 @@ envoy_extension_cc_test( deps = [ "//source/extensions/filters/http/cache:cache_entry_utils_lib", "//source/extensions/http/cache/simple_http_cache:config", - "//test/extensions/filters/http/cache:common", "//test/extensions/filters/http/cache:http_cache_implementation_test_common_lib", "//test/mocks/server:factory_context_mocks", "//test/test_common:simulated_time_system_lib", diff --git a/test/extensions/http/cache/simple_http_cache/simple_http_cache_test.cc b/test/extensions/http/cache/simple_http_cache/simple_http_cache_test.cc index 718681770e73..3bb376ac41ea 100644 --- a/test/extensions/http/cache/simple_http_cache/simple_http_cache_test.cc +++ b/test/extensions/http/cache/simple_http_cache/simple_http_cache_test.cc @@ -6,7 +6,6 @@ #include "source/extensions/filters/http/cache/cache_headers_utils.h" #include "source/extensions/http/cache/simple_http_cache/simple_http_cache.h" -#include "test/extensions/filters/http/cache/common.h" #include "test/extensions/filters/http/cache/http_cache_implementation_test_common.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/simulated_time_system.h" diff --git a/test/extensions/http/header_validators/envoy_default/header_validator_utils.cc b/test/extensions/http/header_validators/envoy_default/header_validator_utils.cc index 7dc65c0277bc..8c3dc2a16c16 100644 --- a/test/extensions/http/header_validators/envoy_default/header_validator_utils.cc +++ b/test/extensions/http/header_validators/envoy_default/header_validator_utils.cc @@ -11,7 +11,6 @@ namespace HeaderValidators { namespace EnvoyDefault { using ::Envoy::Http::HeaderString; -using ::Envoy::Http::testCharInTable; void HeaderValidatorUtils::validateAllCharactersInUrlPath( ::Envoy::Http::ServerHeaderValidator& validator, absl::string_view path, diff --git a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc index 33a881681479..8556ec7d1173 100644 --- a/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http1_header_validator_test.cc @@ -610,7 +610,7 @@ TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapNormalizePath) { EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); // The transform method should normalize path EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir2"); } TEST_F(Http1HeaderValidatorTest, ValidateRequestHeaderMapRejectPath) { @@ -781,7 +781,7 @@ TEST_F(Http1HeaderValidatorTest, AdditionalCharactersInPathAllowed) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Note that \ is translated to / and [] remain unencoded - EXPECT_EQ(headers.path(), "/path/with%22%3C%3E[]%5E%60%7B%7D/%7C"); + EXPECT_EQ(headers.getPathValue(), "/path/with%22%3C%3E[]%5E%60%7B%7D/%7C"); } // Validate that without path normalization additional characters remain untouched @@ -796,7 +796,7 @@ TEST_F(Http1HeaderValidatorTest, AdditionalCharactersInPathAllowedWithoutPathNor {":path", absl::StrCat("/path/with", AdditionallyAllowedCharacters)}}; EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), R"str(/path/with"<>[]^`{}\|)str"); + EXPECT_EQ(headers.getPathValue(), R"str(/path/with"<>[]^`{}\|)str"); } TEST_F(Http1HeaderValidatorTest, AdditionalCharactersInQueryAllowed) { @@ -814,7 +814,7 @@ TEST_F(Http1HeaderValidatorTest, AdditionalCharactersInQueryAllowed) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Additional characters in query always remain untouched - EXPECT_EQ(headers.path(), R"str(/path/with?value="<>[]^`{}\|)str"); + EXPECT_EQ(headers.getPathValue(), R"str(/path/with?value="<>[]^`{}\|)str"); } TEST_F(Http1HeaderValidatorTest, AdditionalCharactersInFragmentAllowed) { @@ -832,7 +832,7 @@ TEST_F(Http1HeaderValidatorTest, AdditionalCharactersInFragmentAllowed) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // UHV strips fragment from URL path - EXPECT_EQ(headers.path(), "/path/with?value=aaa"); + EXPECT_EQ(headers.getPathValue(), "/path/with?value=aaa"); } } // namespace diff --git a/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc b/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc index 83ada3244c69..831f7c9ca529 100644 --- a/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http2_header_validator_test.cc @@ -280,10 +280,10 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnect) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Extended CONNECT is transformed to H/1 upgrade - EXPECT_EQ(headers.method(), "GET"); + EXPECT_EQ(headers.getMethodValue(), "GET"); EXPECT_EQ(headers.getUpgradeValue(), "websocket"); EXPECT_EQ(headers.getConnectionValue(), "upgrade"); - EXPECT_EQ(headers.protocol(), ""); + EXPECT_EQ(headers.getProtocolValue(), ""); } TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoScheme) { @@ -295,7 +295,7 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoScheme) { EXPECT_REJECT_WITH_DETAILS(uhv->validateRequestHeaders(headers), UhvResponseCodeDetail::get().InvalidScheme); // If validation failed, the extended CONNECT request should not be transformed to H/1 upgrade - EXPECT_EQ(headers.method(), "CONNECT"); + EXPECT_EQ(headers.getMethodValue(), "CONNECT"); } TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoPath) { @@ -330,12 +330,12 @@ TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectPathNormalization) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir2"); // Expect transformation to H/1 upgrade as well - EXPECT_EQ(headers.method(), "GET"); + EXPECT_EQ(headers.getMethodValue(), "GET"); EXPECT_EQ(headers.getUpgradeValue(), "websocket"); EXPECT_EQ(headers.getConnectionValue(), "upgrade"); - EXPECT_EQ(headers.protocol(), ""); + EXPECT_EQ(headers.getProtocolValue(), ""); } TEST_F(Http2HeaderValidatorTest, RequestExtendedConnectNoAuthorityIsOk) { @@ -369,13 +369,13 @@ TEST_F(Http2HeaderValidatorTest, UpstreamUpgradeRequestTransformedToExtendedConn auto result = uhv->transformRequestHeaders(headers); EXPECT_TRUE(result.status.ok()); // Expect the extended CONNECT request - EXPECT_EQ(result.new_headers->method(), "CONNECT"); - EXPECT_EQ(result.new_headers->protocol(), "websocket"); + EXPECT_EQ(result.new_headers->getMethodValue(), "CONNECT"); + EXPECT_EQ(result.new_headers->getProtocolValue(), "websocket"); EXPECT_EQ(result.new_headers->Upgrade(), nullptr); EXPECT_EQ(result.new_headers->Connection(), nullptr); // No path normalization is expected at the client codec - EXPECT_EQ(result.new_headers->path(), "/./dir1/../dir2"); - EXPECT_EQ(result.new_headers->host(), "envoy.com"); + EXPECT_EQ(result.new_headers->getPathValue(), "/./dir1/../dir2"); + EXPECT_EQ(result.new_headers->getHostValue(), "envoy.com"); EXPECT_EQ(result.new_headers->getSchemeValue(), "https"); // New headers must be valid H/2 extended CONNECT headers @@ -672,7 +672,7 @@ TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapNormalizePath) { EXPECT_TRUE(uhv->validateRequestHeaders(headers).ok()); EXPECT_TRUE(uhv->transformRequestHeaders(headers).ok()); - EXPECT_EQ(headers.path(), "/dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir2"); } TEST_F(Http2HeaderValidatorTest, ValidateRequestHeaderMapRejectPath) { @@ -832,8 +832,8 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInPathAllowedHttp2) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Note that \ is translated to / and [] remain unencoded - EXPECT_EQ(headers.path(), absl::StrCat("/path/with%09%20%22%3C%3E[]%5E%60%7B%7D/%7C", - generatePercentEncodedExtendedAscii())); + EXPECT_EQ(headers.getPathValue(), absl::StrCat("/path/with%09%20%22%3C%3E[]%5E%60%7B%7D/%7C", + generatePercentEncodedExtendedAscii())); } // Validate that H/3 UHV allows additional characters "<>[]^`{}\| space TAB @@ -853,7 +853,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInPathAllowedHttp3) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Note that \ is translated to / and [] remain unencoded - EXPECT_EQ(headers.path(), "/path/with%09%20%22%3C%3E[]%5E%60%7B%7D/%7C"); + EXPECT_EQ(headers.getPathValue(), "/path/with%09%20%22%3C%3E[]%5E%60%7B%7D/%7C"); } // Validate that without path normalization additional characters remain untouched @@ -868,7 +868,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInPathAllowedWithoutPathNor {":path", absl::StrCat("/path/with", AdditionallyAllowedCharactersHttp2)}}; EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), + EXPECT_EQ(headers.getPathValue(), absl::StrCat("/path/with\t \"<>[]^`{}\\|", generateExtendedAsciiString())); } @@ -884,7 +884,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInPathAllowedWithoutPathNor {":path", absl::StrCat("/path/with", AdditionallyAllowedCharactersHttp3)}}; EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/path/with\t \"<>[]^`{}\\|"); + EXPECT_EQ(headers.getPathValue(), "/path/with\t \"<>[]^`{}\\|"); } TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInQueryAllowedHttp2) { @@ -902,7 +902,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInQueryAllowedHttp2) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Additional characters in query always remain untouched - EXPECT_EQ(headers.path(), + EXPECT_EQ(headers.getPathValue(), absl::StrCat("/path/with?value=\t \"<>[]^`{}\\|", generateExtendedAsciiString())); } @@ -922,7 +922,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInQueryAllowedHttp3) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // Additional characters in query always remain untouched - EXPECT_EQ(headers.path(), "/path/with?value=\t \"<>[]^`{}\\|"); + EXPECT_EQ(headers.getPathValue(), "/path/with?value=\t \"<>[]^`{}\\|"); } TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInFragmentAllowedHttp2) { @@ -940,7 +940,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInFragmentAllowedHttp2) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // UHV strips fragment from URL path - EXPECT_EQ(headers.path(), "/path/with?value=aaa"); + EXPECT_EQ(headers.getPathValue(), "/path/with?value=aaa"); } TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInFragmentAllowedHttp3) { @@ -959,7 +959,7 @@ TEST_F(Http2HeaderValidatorTest, AdditionalCharactersInFragmentAllowedHttp3) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); // UHV strips fragment from URL path - EXPECT_EQ(headers.path(), "/path/with?value=aaa"); + EXPECT_EQ(headers.getPathValue(), "/path/with?value=aaa"); } } // namespace diff --git a/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc b/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc index 059772d6c316..08de2d95443e 100644 --- a/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc +++ b/test/extensions/http/header_validators/envoy_default/http_common_validation_test.cc @@ -16,7 +16,6 @@ namespace HeaderValidators { namespace EnvoyDefault { namespace { -using ::Envoy::Http::HeaderString; using ::Envoy::Http::Protocol; using ::Envoy::Http::TestRequestHeaderMapImpl; using ::Envoy::Http::UhvResponseCodeDetail; @@ -64,7 +63,7 @@ TEST_P(HttpCommonValidationTest, MalformedUrlEncodingAllowed) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/path%Z0with%xYbad%7Jencoding%"); + EXPECT_EQ(headers.getPathValue(), "/path%Z0with%xYbad%7Jencoding%"); } TEST_P(HttpCommonValidationTest, MalformedUrlEncodingRejectedWithOverride) { @@ -90,7 +89,7 @@ TEST_P(HttpCommonValidationTest, BackslashInPathIsTranslatedToSlash) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/path/with/back/slash%5C"); + EXPECT_EQ(headers.getPathValue(), "/path/with/back/slash%5C"); } TEST_P(HttpCommonValidationTest, BackslashInPathIsRejectedWithOverride) { @@ -148,7 +147,7 @@ TEST_P(HttpCommonValidationTest, FragmentStrippedFromPathWhenConfigured) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/path/with?query=and"); + EXPECT_EQ(headers.getPathValue(), "/path/with?query=and"); } TEST_P(HttpCommonValidationTest, ValidateRequestHeaderMapRedirectPath) { @@ -165,7 +164,7 @@ TEST_P(HttpCommonValidationTest, ValidateRequestHeaderMapRedirectPath) { ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect); EXPECT_EQ(result.details(), "uhv.path_normalization_redirect"); // By default decoded backslash (%5C) is converted to forward slash. - EXPECT_EQ(headers.path(), "/dir1/dir2/dir4"); + EXPECT_EQ(headers.getPathValue(), "/dir1/dir2/dir4"); } TEST_P(HttpCommonValidationTest, ValidateRequestHeadersNoPathNormalization) { @@ -181,7 +180,7 @@ TEST_P(HttpCommonValidationTest, ValidateRequestHeadersNoPathNormalization) { EXPECT_EQ( result.action(), ::Envoy::Http::ServerHeaderValidator::RequestHeadersTransformationResult::Action::Redirect); - EXPECT_EQ(headers.path(), "/dir1/dir2\\.."); + EXPECT_EQ(headers.getPathValue(), "/dir1/dir2\\.."); } TEST_P(HttpCommonValidationTest, NoPathNormalizationNoSlashDecoding) { @@ -193,7 +192,7 @@ TEST_P(HttpCommonValidationTest, NoPathNormalizationNoSlashDecoding) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); auto result = uhv->transformRequestHeaders(headers); EXPECT_ACCEPT(result); - EXPECT_EQ(headers.path(), "/dir1%2Fdir2%5cdir3"); + EXPECT_EQ(headers.getPathValue(), "/dir1%2Fdir2%5cdir3"); } TEST_P(HttpCommonValidationTest, NoPathNormalizationDecodeSlashesAndForward) { @@ -205,7 +204,7 @@ TEST_P(HttpCommonValidationTest, NoPathNormalizationDecodeSlashesAndForward) { EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); auto result = uhv->transformRequestHeaders(headers); EXPECT_ACCEPT(result); - EXPECT_EQ(headers.path(), "/dir1/dir2\\../dir3"); + EXPECT_EQ(headers.getPathValue(), "/dir1/dir2\\../dir3"); } TEST_P(HttpCommonValidationTest, RejectEncodedSlashes) { @@ -238,7 +237,7 @@ TEST_P(HttpCommonValidationTest, AllowPercent00WithOverride) { auto uhv = createUhv(empty_config); EXPECT_ACCEPT(uhv->validateRequestHeaders(headers)); EXPECT_ACCEPT(uhv->transformRequestHeaders(headers)); - EXPECT_EQ(headers.path(), "/dir1%00dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir1%00dir2"); } } // namespace diff --git a/test/extensions/http/header_validators/envoy_default/path_normalizer_fuzz_test.cc b/test/extensions/http/header_validators/envoy_default/path_normalizer_fuzz_test.cc index 501646fb1be0..b86ac9d3a217 100644 --- a/test/extensions/http/header_validators/envoy_default/path_normalizer_fuzz_test.cc +++ b/test/extensions/http/header_validators/envoy_default/path_normalizer_fuzz_test.cc @@ -49,11 +49,11 @@ DEFINE_PROTO_FUZZER( Extensions::Http::HeaderValidators::EnvoyDefault::PathNormalizer:: PathNormalizationResult::Action::Redirect) { // Additional sanity checks on the normalized path - RELEASE_ASSERT(header_map->path().size() <= input.path().size(), + RELEASE_ASSERT(header_map->getPathValue().size() <= input.path().size(), "Normalized path is always shorter or the same length."); - RELEASE_ASSERT(header_map->method() == input.method(), ":method should not change."); + RELEASE_ASSERT(header_map->getMethodValue() == input.method(), ":method should not change."); auto original_query_or_fragment = input.path().find_first_of("?#"); - auto normalized_query_or_fragment = header_map->path().find_first_of("?#"); + auto normalized_query_or_fragment = header_map->getPathValue().find_first_of("?#"); RELEASE_ASSERT( (original_query_or_fragment != std::string::npos && normalized_query_or_fragment != absl::string_view::npos) || @@ -62,23 +62,26 @@ DEFINE_PROTO_FUZZER( "Query/fragment must be present or absent in both original and normalized paths"); if (original_query_or_fragment != std::string::npos) { RELEASE_ASSERT(input.path().substr(original_query_or_fragment) == - header_map->path().substr(normalized_query_or_fragment), + header_map->getPathValue().substr(normalized_query_or_fragment), "Original and normalized query/path should be the same"); } if (!input.options().skip_merging_slashes()) { - RELEASE_ASSERT(header_map->path().substr(0, normalized_query_or_fragment).find("//") == - absl::string_view::npos, - ":path must not contain adjacent slashes."); + RELEASE_ASSERT( + header_map->getPathValue().substr(0, normalized_query_or_fragment).find("//") == + absl::string_view::npos, + ":path must not contain adjacent slashes."); } if (!input.options().skip_path_normalization()) { - RELEASE_ASSERT(header_map->path().substr(0, normalized_query_or_fragment).find("/./") == - absl::string_view::npos, - ":path must not contain /./"); - RELEASE_ASSERT(header_map->path().substr(0, normalized_query_or_fragment).find("/../") == - absl::string_view::npos, - ":path must not contain /../"); + RELEASE_ASSERT( + header_map->getPathValue().substr(0, normalized_query_or_fragment).find("/./") == + absl::string_view::npos, + ":path must not contain /./"); + RELEASE_ASSERT( + header_map->getPathValue().substr(0, normalized_query_or_fragment).find("/../") == + absl::string_view::npos, + ":path must not contain /../"); } } } diff --git a/test/extensions/http/header_validators/envoy_default/path_normalizer_test.cc b/test/extensions/http/header_validators/envoy_default/path_normalizer_test.cc index e2cafabed358..30a956b4e039 100644 --- a/test/extensions/http/header_validators/envoy_default/path_normalizer_test.cc +++ b/test/extensions/http/header_validators/envoy_default/path_normalizer_test.cc @@ -172,7 +172,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriRoot) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/"); + EXPECT_EQ(headers.getPathValue(), "/"); EXPECT_TRUE(result.ok()); } @@ -182,7 +182,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriRootPreserveQueryFragment) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/root/child?x=1#anchor"); + EXPECT_EQ(headers.getPathValue(), "/root/child?x=1#anchor"); EXPECT_TRUE(result.ok()); } @@ -192,7 +192,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriRootPreserveFragment) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/root/child#anchor"); + EXPECT_EQ(headers.getPathValue(), "/root/child#anchor"); EXPECT_TRUE(result.ok()); } @@ -202,7 +202,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriDotDot) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir2"); EXPECT_TRUE(result.ok()); } @@ -212,7 +212,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriDot) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/dir1/dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir1/dir2"); EXPECT_TRUE(result.ok()); } @@ -222,7 +222,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriTrailingDotDot) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/"); + EXPECT_EQ(headers.getPathValue(), "/"); EXPECT_TRUE(result.ok()); } @@ -232,7 +232,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriEncodedDotDot) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/dir2"); + EXPECT_EQ(headers.getPathValue(), "/dir2"); EXPECT_TRUE(result.ok()); } @@ -242,7 +242,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriTrailingDot) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/dir1/"); + EXPECT_EQ(headers.getPathValue(), "/dir1/"); EXPECT_TRUE(result.ok()); } @@ -252,7 +252,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriDotInSegments) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/dir1/.dir2/..dir3/dir.4/dir..5"); + EXPECT_EQ(headers.getPathValue(), "/dir1/.dir2/..dir3/dir.4/dir..5"); EXPECT_TRUE(result.ok()); } @@ -262,7 +262,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriMergeSlashes) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/root/child/"); + EXPECT_EQ(headers.getPathValue(), "/root/child/"); EXPECT_TRUE(result.ok()); } @@ -273,7 +273,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriPercentDecodeNormalized) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/%FF"); + EXPECT_EQ(headers.getPathValue(), "/%FF"); EXPECT_TRUE(result.ok()); } @@ -283,7 +283,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriPercentDecoded) { auto normalizer = create(empty_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/~/dir1"); + EXPECT_EQ(headers.getPathValue(), "/~/dir1"); EXPECT_TRUE(result.ok()); } @@ -293,7 +293,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriSkipMergingSlashes) { auto normalizer = create(skip_merging_slashes_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "//root//child//"); + EXPECT_EQ(headers.getPathValue(), "//root//child//"); EXPECT_TRUE(result.ok()); } @@ -303,7 +303,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriSkipMergingSlashesWithDecodeSlashes) auto normalizer = create(skip_merging_slashes_with_decode_slashes_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/root//child//"); + EXPECT_EQ(headers.getPathValue(), "/root//child//"); EXPECT_TRUE(result.ok()); } @@ -314,7 +314,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriDecodeSlashes) { auto normalizer = create(decode_encoded_slash_config); auto result = normalizer->normalizePathUri(headers); - EXPECT_EQ(headers.path(), "/dir1/dir2/dir3/dir4"); + EXPECT_EQ(headers.getPathValue(), "/dir1/dir2/dir3/dir4"); EXPECT_TRUE(result.ok()); } @@ -346,7 +346,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriRedirectEncodedSlashes) { EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Redirect); EXPECT_EQ(result.details(), "uhv.path_normalization_redirect"); - EXPECT_EQ(headers.path(), "/dir1/dir2/dir3"); + EXPECT_EQ(headers.getPathValue(), "/dir1/dir2/dir3"); } TEST_F(PathNormalizerTest, NormalizePathUriNormalizeEncodedSlashesDefault) { @@ -358,7 +358,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriNormalizeEncodedSlashesDefault) { EXPECT_TRUE(result.ok()); // By default slashes are not decoded - EXPECT_EQ(headers.path(), "/dir1%2Fdir2%5Cdir3"); + EXPECT_EQ(headers.getPathValue(), "/dir1%2Fdir2%5Cdir3"); } TEST_F(PathNormalizerTest, NormalizePathUriNormalizeEncodedSlashesKept) { @@ -369,7 +369,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriNormalizeEncodedSlashesKept) { auto result = normalizer->normalizePathUri(headers); EXPECT_TRUE(result.ok()); - EXPECT_EQ(headers.path(), "/dir1%2Fdir2%5Cdir3"); + EXPECT_EQ(headers.getPathValue(), "/dir1%2Fdir2%5Cdir3"); } TEST_F(PathNormalizerTest, NormalizePathUriNormalizeEncodedSlashesImplDefault) { @@ -380,7 +380,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriNormalizeEncodedSlashesImplDefault) { auto result = normalizer->normalizePathUri(headers); EXPECT_TRUE(result.ok()); - EXPECT_EQ(headers.path(), "/dir1%2Fdir2"); + EXPECT_EQ(headers.getPathValue(), "/dir1%2Fdir2"); } TEST_F(PathNormalizerTest, NormalizePathUriInvalidBeyondRoot) { @@ -424,7 +424,7 @@ TEST_F(PathNormalizerTest, MalformedUrlEncodingAllowed) { auto result = normalizer->normalizePathUri(headers); EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); - EXPECT_EQ(headers.path(), "/path%Z0with%xYbad%7Jencoding%A"); + EXPECT_EQ(headers.getPathValue(), "/path%Z0with%xYbad%7Jencoding%A"); } TEST_F(PathNormalizerTest, NormalizePathUriAuthorityFormConnect) { @@ -435,7 +435,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriAuthorityFormConnect) { auto result = normalizer->normalizePathUri(headers); EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); - EXPECT_EQ(headers.path(), ""); + EXPECT_EQ(headers.getPathValue(), ""); } TEST_F(PathNormalizerTest, NormalizePathUriAuthorityFormWithPathConnect) { @@ -468,7 +468,7 @@ TEST_F(PathNormalizerTest, NormalizePathUriAsteriskFormOptions) { auto result = normalizer->normalizePathUri(headers); EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); - EXPECT_EQ(headers.path(), "*"); + EXPECT_EQ(headers.getPathValue(), "*"); } TEST_F(PathNormalizerTest, NormalizePathUriAsteriskFormNotOptions) { @@ -491,7 +491,7 @@ TEST_F(PathNormalizerTest, BackslashTranslatedToSlashByDefault) { EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); // In query backslash is untouched - EXPECT_EQ(headers.path(), "/path/with/back/slash%5c?key=val\\ue"); + EXPECT_EQ(headers.getPathValue(), "/path/with/back/slash%5c?key=val\\ue"); } TEST_F(PathNormalizerTest, BackslashPreservedWithOverride) { @@ -502,7 +502,7 @@ TEST_F(PathNormalizerTest, BackslashPreservedWithOverride) { auto result = normalizer->normalizePathUri(headers); EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); - EXPECT_EQ(headers.path(), "/path\\with/back\\/slash%5C"); + EXPECT_EQ(headers.getPathValue(), "/path\\with/back\\/slash%5C"); } TEST_F(PathNormalizerTest, PreservePercentEncodedCaseByDefault) { @@ -512,7 +512,7 @@ TEST_F(PathNormalizerTest, PreservePercentEncodedCaseByDefault) { auto result = normalizer->normalizePathUri(headers); EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); - EXPECT_EQ(headers.path(), "/dir1%Abdir2%3a%fFZ"); + EXPECT_EQ(headers.getPathValue(), "/dir1%Abdir2%3a%fFZ"); } TEST_F(PathNormalizerTest, NormalizePercentEncodedCase) { @@ -523,7 +523,7 @@ TEST_F(PathNormalizerTest, NormalizePercentEncodedCase) { auto result = normalizer->normalizePathUri(headers); EXPECT_EQ(result.action(), PathNormalizer::PathNormalizationResult::Action::Accept); - EXPECT_EQ(headers.path(), "/dir1%ABdir2%3A%FFZ"); + EXPECT_EQ(headers.getPathValue(), "/dir1%ABdir2%3A%FFZ"); } } // namespace EnvoyDefault diff --git a/test/extensions/http/stateful_session/cookie/config_test.cc b/test/extensions/http/stateful_session/cookie/config_test.cc index 889bc4633160..47a3383aaff4 100644 --- a/test/extensions/http/stateful_session/cookie/config_test.cc +++ b/test/extensions/http/stateful_session/cookie/config_test.cc @@ -26,7 +26,7 @@ TEST(CookieBasedSessionStateFactoryConfigTest, Basic) { )EOF"; TestUtility::loadFromYaml(yaml, proto_config); - NiceMock context; + NiceMock context; EXPECT_NE(factory->createSessionStateFactory(proto_config, context), nullptr); } @@ -44,7 +44,7 @@ TEST(CookieBasedSessionStateFactoryConfigTest, NegativeTTL) { )EOF"; TestUtility::loadFromYaml(yaml, proto_config); - NiceMock context; + NiceMock context; EXPECT_THROW(factory->createSessionStateFactory(proto_config, context), EnvoyException); } diff --git a/test/extensions/http/stateful_session/header/config_test.cc b/test/extensions/http/stateful_session/header/config_test.cc index ac8008993f82..2a763669158b 100644 --- a/test/extensions/http/stateful_session/header/config_test.cc +++ b/test/extensions/http/stateful_session/header/config_test.cc @@ -23,7 +23,7 @@ TEST(HeaderBasedSessionStateFactoryConfigTest, Basic) { )EOF"; TestUtility::loadFromYaml(yaml, proto_config); - NiceMock context; + NiceMock context; EXPECT_NE(factory->createSessionStateFactory(proto_config, context), nullptr); } @@ -38,7 +38,7 @@ TEST(HeaderBasedSessionStateFactoryConfigTest, MissingHeaderName) { )EOF"; TestUtility::loadFromYaml(yaml, proto_config); - NiceMock context; + NiceMock context; EXPECT_THROW(factory->createSessionStateFactory(proto_config, context), EnvoyException); } diff --git a/test/extensions/io_socket/user_space/io_handle_impl_test.cc b/test/extensions/io_socket/user_space/io_handle_impl_test.cc index 17aeb8b7a99c..b3bf35512ef8 100644 --- a/test/extensions/io_socket/user_space/io_handle_impl_test.cc +++ b/test/extensions/io_socket/user_space/io_handle_impl_test.cc @@ -412,6 +412,26 @@ TEST_F(IoHandleImplTest, ShutDownOptionsNotSupported) { ASSERT_DEBUG_DEATH(io_handle_peer_->shutdown(ENVOY_SHUT_RDWR), ""); } +// This test is ensure the memory created by BufferFragment won't be released +// after the write. +TEST_F(IoHandleImplTest, WriteBufferFragement) { + Buffer::OwnedImpl buf("a"); + bool released = false; + auto buf_frag = Buffer::OwnedBufferFragmentImpl::create( + std::string(255, 'b'), [&released](const Buffer::OwnedBufferFragmentImpl* fragment) { + released = true; + delete fragment; + }); + buf.addBufferFragment(*buf_frag.release()); + + auto result = io_handle_->write(buf); + EXPECT_FALSE(released); + EXPECT_EQ(0, buf.length()); + io_handle_peer_->read(buf, absl::nullopt); + buf.drain(buf.length()); + EXPECT_TRUE(released); +} + TEST_F(IoHandleImplTest, WriteByMove) { Buffer::OwnedImpl buf("0123456789"); auto result = io_handle_peer_->write(buf); diff --git a/test/extensions/key_value/file_based/alternate_protocols_cache_impl_test.cc b/test/extensions/key_value/file_based/alternate_protocols_cache_impl_test.cc index 3e1040e33076..9363f6487253 100644 --- a/test/extensions/key_value/file_based/alternate_protocols_cache_impl_test.cc +++ b/test/extensions/key_value/file_based/alternate_protocols_cache_impl_test.cc @@ -20,7 +20,8 @@ class HttpServerPropertiesCacheManagerTest : public testing::Test, options_.mutable_max_entries()->set_value(10); } void initialize() { - Http::AlternateProtocolsData data = {context_}; + Http::AlternateProtocolsData data{context_.server_factory_context_, + context_.messageValidationVisitor()}; factory_ = std::make_unique( singleton_manager_, tls_, data); manager_ = factory_->get(); diff --git a/test/extensions/load_balancing_policies/cluster_provided/config_test.cc b/test/extensions/load_balancing_policies/cluster_provided/config_test.cc index 3bec7e1bbbd8..b760611342db 100644 --- a/test/extensions/load_balancing_policies/cluster_provided/config_test.cc +++ b/test/extensions/load_balancing_policies/cluster_provided/config_test.cc @@ -15,7 +15,7 @@ namespace ClusterProvided { namespace { TEST(ClusterProvidedConfigTest, ClusterProvidedConfigTest) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; diff --git a/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc b/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc index c190be34bc8d..0ccfed539376 100644 --- a/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc +++ b/test/extensions/load_balancing_policies/cluster_provided/integration_test.cc @@ -24,76 +24,108 @@ class ClusterProvidedIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); - std::string cluster_yaml = R"EOF( + std::string cluster_yaml = R"EOF( name: cluster_0 connect_timeout: 1.250s type: ORIGINAL_DST lb_policy: CLUSTER_PROVIDED original_dst_lb_config: use_http_header: true - )EOF"; + )EOF"; - TestUtility::loadFromYaml(cluster_yaml, *cluster_0); + TestUtility::loadFromYaml(cluster_yaml, *cluster_0); - auto* policy = cluster_0->mutable_load_balancing_policy(); + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED); + return; + } - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.cluster_provided - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.cluster_provided.v3.ClusterProvided - )EOF"; + auto* policy = cluster_0->mutable_load_balancing_policy(); - TestUtility::loadFromYaml(policy_yaml, *policy); - }); - } -}; + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.cluster_provided + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.cluster_provided.v3.ClusterProvided + )EOF"; -INSTANTIATE_TEST_SUITE_P(IpVersions, ClusterProvidedIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + TestUtility::loadFromYaml(policy_yaml, *policy); + }); -// Test the case where the cluster provided load balancer is configured by the load balancing -// policy API and it works as expected. -TEST_P(ClusterProvidedIntegrationTest, NormalLoadBalancing) { - initialize(); + HttpIntegrationTest::initialize(); + } - for (uint64_t i = 0; i < 4; i++) { - for (size_t upstream_index = 0; upstream_index < fake_upstreams_.size(); upstream_index++) { - codec_client_ = makeHttpConnection(lookupPort("http")); + void runNormalLoadBalancing() { + for (uint64_t i = 0; i < 4; i++) { + for (size_t upstream_index = 0; upstream_index < fake_upstreams_.size(); upstream_index++) { + codec_client_ = makeHttpConnection(lookupPort("http")); - const auto& upstream_target_address = - fake_upstreams_[upstream_index]->localAddress()->asString(); + const auto& upstream_target_address = + fake_upstreams_[upstream_index]->localAddress()->asString(); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, - {":path", "/"}, - {":scheme", "http"}, - {":authority", "example.com"}, - {"x-envoy-original-dst-host", upstream_target_address}}; + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "example.com"}, + {"x-envoy-original-dst-host", upstream_target_address}}; - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto upstream = waitForNextUpstreamRequest({0, 1, 2}); - EXPECT_EQ(upstream.value(), upstream_index); + auto upstream = waitForNextUpstreamRequest({0, 1, 2}); + EXPECT_EQ(upstream.value(), upstream_index); - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - cleanupUpstreamAndDownstream(); + cleanupUpstreamAndDownstream(); + } } } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, ClusterProvidedIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Test the case where the cluster provided load balancer is configured by the load balancing +// policy API and it works as expected. +TEST_P(ClusterProvidedIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} + +// Test the case where the cluster provided load balancer is configured by the legacy cluster LB +// policy API and it works as expected. +TEST_P(ClusterProvidedIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} + +// Test the case where the cluster provided load balancer is configured by the legacy cluster LB +// policy API (but disable the API conversion) and it works as expected. +TEST_P(ClusterProvidedIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/common/BUILD b/test/extensions/load_balancing_policies/common/BUILD new file mode 100644 index 000000000000..eec0c0100900 --- /dev/null +++ b/test/extensions/load_balancing_policies/common/BUILD @@ -0,0 +1,25 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test_library( + name = "benchmark_base_tester_lib", + srcs = ["benchmark_base_tester.cc"], + hdrs = ["benchmark_base_tester.h"], + deps = [ + "//source/common/common:random_generator_lib", + "//source/common/memory:stats_lib", + "//source/common/upstream:upstream_lib", + "//test/common/upstream:utility_lib", + "//test/mocks/upstream:cluster_info_mocks", + "//test/test_common:printers_lib", + "//test/test_common:simulated_time_system_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/load_balancing_policies/common/benchmark_base_tester.cc b/test/extensions/load_balancing_policies/common/benchmark_base_tester.cc new file mode 100644 index 000000000000..3cc30ae2eb69 --- /dev/null +++ b/test/extensions/load_balancing_policies/common/benchmark_base_tester.cc @@ -0,0 +1,44 @@ +#include "test/extensions/load_balancing_policies/common/benchmark_base_tester.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Common { + +BaseTester::BaseTester(uint64_t num_hosts, uint32_t weighted_subset_percent, uint32_t weight, + bool attach_metadata) { + Upstream::HostVector hosts; + ASSERT(num_hosts < 65536); + for (uint64_t i = 0; i < num_hosts; i++) { + const bool should_weight = i < num_hosts * (weighted_subset_percent / 100.0); + const std::string url = fmt::format("tcp://10.0.{}.{}:6379", i / 256, i % 256); + const auto effective_weight = should_weight ? weight : 1; + if (attach_metadata) { + envoy::config::core::v3::Metadata metadata; + ProtobufWkt::Value value; + value.set_number_value(i); + ProtobufWkt::Struct& map = + (*metadata.mutable_filter_metadata())[Config::MetadataFilters::get().ENVOY_LB]; + (*map.mutable_fields())[std::string(metadata_key)] = value; + + hosts.push_back(Upstream::makeTestHost(info_, url, metadata, simTime(), effective_weight)); + } else { + hosts.push_back(Upstream::makeTestHost(info_, url, simTime(), effective_weight)); + } + } + + Upstream::HostVectorConstSharedPtr updated_hosts = std::make_shared(hosts); + Upstream::HostsPerLocalityConstSharedPtr hosts_per_locality = + Upstream::makeHostsPerLocality({hosts}); + priority_set_.updateHosts( + 0, Upstream::HostSetImpl::partitionHosts(updated_hosts, hosts_per_locality), {}, hosts, {}, + absl::nullopt); + local_priority_set_.updateHosts( + 0, Upstream::HostSetImpl::partitionHosts(updated_hosts, hosts_per_locality), {}, hosts, {}, + absl::nullopt); +} + +} // namespace Common +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/common/benchmark_base_tester.h b/test/extensions/load_balancing_policies/common/benchmark_base_tester.h new file mode 100644 index 000000000000..229f4832aec2 --- /dev/null +++ b/test/extensions/load_balancing_policies/common/benchmark_base_tester.h @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include "envoy/config/cluster/v3/cluster.pb.h" + +#include "source/common/common/random_generator.h" +#include "source/common/memory/stats.h" +#include "source/common/upstream/upstream_impl.h" + +#include "test/common/upstream/utility.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/test_common/simulated_time_system.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Common { + +class BaseTester : public Event::TestUsingSimulatedTime { +public: + static constexpr absl::string_view metadata_key = "key"; + // We weight the first weighted_subset_percent of hosts with weight. + BaseTester(uint64_t num_hosts, uint32_t weighted_subset_percent = 0, uint32_t weight = 0, + bool attach_metadata = false); + + Envoy::Thread::MutexBasicLockable lock_; + // Reduce default log level to warn while running this benchmark to avoid problems due to + // excessive debug logging in upstream_impl.cc + Envoy::Logger::Context logging_context_{spdlog::level::warn, + Envoy::Logger::Logger::DEFAULT_LOG_FORMAT, lock_, false}; + + Upstream::PrioritySetImpl priority_set_; + Upstream::PrioritySetImpl local_priority_set_; + + // The following are needed to create a load balancer by the load balancer factory. + Upstream::LoadBalancerParams lb_params_{priority_set_, &local_priority_set_}; + + Stats::IsolatedStoreImpl stats_store_; + Stats::Scope& stats_scope_{*stats_store_.rootScope()}; + Upstream::ClusterLbStatNames stat_names_{stats_store_.symbolTable()}; + Upstream::ClusterLbStats stats_{stat_names_, stats_scope_}; + NiceMock runtime_; + Random::RandomGeneratorImpl random_; + envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_; + envoy::config::cluster::v3::Cluster::RoundRobinLbConfig round_robin_lb_config_; + std::shared_ptr info_{new NiceMock()}; +}; + +} // namespace Common +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/least_request/config_test.cc b/test/extensions/load_balancing_policies/least_request/config_test.cc index 0f05f3887a9e..d4869da78e53 100644 --- a/test/extensions/load_balancing_policies/least_request/config_test.cc +++ b/test/extensions/load_balancing_policies/least_request/config_test.cc @@ -15,7 +15,7 @@ namespace LeastRequest { namespace { TEST(LeastRequestConfigTest, ValidateFail) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; @@ -29,7 +29,7 @@ TEST(LeastRequestConfigTest, ValidateFail) { EXPECT_EQ("envoy.load_balancing_policies.least_request", factory.name()); auto lb_config = - factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor()); + factory.loadConfig(*factory.createEmptyConfigProto(), context.messageValidationVisitor()); auto thread_aware_lb = factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, context.api_.random_, context.time_system_); diff --git a/test/extensions/load_balancing_policies/least_request/integration_test.cc b/test/extensions/load_balancing_policies/least_request/integration_test.cc index 8c66ac355a57..6176802b97b0 100644 --- a/test/extensions/load_balancing_policies/least_request/integration_test.cc +++ b/test/extensions/load_balancing_policies/least_request/integration_test.cc @@ -24,78 +24,106 @@ class LeastRequestIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); - auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); - - constexpr absl::string_view endpoints_yaml = R"EOF( - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - )EOF"; - - const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); - TestUtility::loadFromYaml( - fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); - - auto* policy = cluster_0->mutable_load_balancing_policy(); - - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.least_request - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest - )EOF"; - - TestUtility::loadFromYaml(policy_yaml, *policy); - }); + void initializeConfig(bool legacy_api = false, bool disable_lagacy_api_conversion = false) { + if (disable_lagacy_api_conversion) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.convert_legacy_lb_config", + "false"); + } + + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + constexpr absl::string_view endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + )EOF"; + + const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); + TestUtility::loadFromYaml( + fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); + + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::LEAST_REQUEST); + return; + } + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.least_request + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + + HttpIntegrationTest::initialize(); } -}; -INSTANTIATE_TEST_SUITE_P(IpVersions, LeastRequestIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + void runNormalLoadBalancing() { + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); -TEST_P(LeastRequestIntegrationTest, NormalLoadBalancing) { - initialize(); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; - for (uint64_t i = 0; i < 8; i++) { - codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + upstream_request_->encodeHeaders(default_response_headers_, true); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); - ASSERT(upstream_index.has_value()); + ASSERT_TRUE(response->waitForEndStream()); - upstream_request_->encodeHeaders(default_response_headers_, true); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - ASSERT_TRUE(response->waitForEndStream()); + cleanupUpstreamAndDownstream(); + } + } +}; - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); +INSTANTIATE_TEST_SUITE_P(IpVersions, LeastRequestIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); - cleanupUpstreamAndDownstream(); - } +TEST_P(LeastRequestIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} + +TEST_P(LeastRequestIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} + +TEST_P(LeastRequestIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/maglev/BUILD b/test/extensions/load_balancing_policies/maglev/BUILD index 87d543682990..c8618de817f8 100644 --- a/test/extensions/load_balancing_policies/maglev/BUILD +++ b/test/extensions/load_balancing_policies/maglev/BUILD @@ -30,6 +30,27 @@ envoy_extension_cc_test( ], ) +# Runs the same test suite as :maglev_lb_test with the forced original implementation +# to ensure coverage on x86. +envoy_extension_cc_test( + name = "maglev_lb_force_original_impl_test", + srcs = ["maglev_lb_test.cc"], + extension_names = ["envoy.load_balancing_policies.maglev"], + deps = [ + "//source/extensions/load_balancing_policies/maglev:maglev_lb_force_original_impl_lib", + "//test/common/upstream:utility_lib", + "//test/mocks:common_lib", + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:host_mocks", + "//test/mocks/upstream:host_set_mocks", + "//test/mocks/upstream:load_balancer_context_mock", + "//test/mocks/upstream:priority_set_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + ], +) + envoy_extension_cc_test( name = "config_test", srcs = ["config_test.cc"], diff --git a/test/extensions/load_balancing_policies/maglev/config_test.cc b/test/extensions/load_balancing_policies/maglev/config_test.cc index 636fd4c77794..65d2c8402fe6 100644 --- a/test/extensions/load_balancing_policies/maglev/config_test.cc +++ b/test/extensions/load_balancing_policies/maglev/config_test.cc @@ -16,7 +16,7 @@ namespace Maglev { namespace { TEST(MaglevConfigTest, Validate) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; @@ -31,7 +31,7 @@ TEST(MaglevConfigTest, Validate) { EXPECT_EQ("envoy.load_balancing_policies.maglev", factory.name()); auto lb_config = - factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor()); + factory.loadConfig(*factory.createEmptyConfigProto(), context.messageValidationVisitor()); auto thread_aware_lb = factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, context.api_.random_, context.time_system_); @@ -59,7 +59,7 @@ TEST(MaglevConfigTest, Validate) { auto message_ptr = factory.createEmptyConfigProto(); message_ptr->MergeFrom(config_msg); - auto lb_config = factory.loadConfig(std::move(message_ptr), context.messageValidationVisitor()); + auto lb_config = factory.loadConfig(*message_ptr, context.messageValidationVisitor()); EXPECT_THROW_WITH_MESSAGE(factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, context.api_.random_, diff --git a/test/extensions/load_balancing_policies/maglev/integration_test.cc b/test/extensions/load_balancing_policies/maglev/integration_test.cc index 6ed7437a9c91..2f3266767565 100644 --- a/test/extensions/load_balancing_policies/maglev/integration_test.cc +++ b/test/extensions/load_balancing_policies/maglev/integration_test.cc @@ -25,48 +25,6 @@ class MaglevIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); - auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); - - constexpr absl::string_view endpoints_yaml = R"EOF( - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - )EOF"; - - const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); - TestUtility::loadFromYaml( - fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); - - auto* policy = cluster_0->mutable_load_balancing_policy(); - - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.maglev - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev - )EOF"; - - TestUtility::loadFromYaml(policy_yaml, *policy); - }); - config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { @@ -79,45 +37,116 @@ class MaglevIntegrationTest : public testing::TestWithParamset_header_name("x-hash"); }); } -}; -INSTANTIATE_TEST_SUITE_P(IpVersions, MaglevIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + void initializeConfig(bool legacy_api = false, bool disable_lagacy_api_conversion = false) { + if (disable_lagacy_api_conversion) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.convert_legacy_lb_config", + "false"); + } -TEST_P(MaglevIntegrationTest, NormalLoadBalancing) { - initialize(); + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + constexpr absl::string_view endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + )EOF"; + + const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); + TestUtility::loadFromYaml( + fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); + + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::MAGLEV); + cluster_0->mutable_maglev_lb_config(); + return; + } + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.maglev + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); - absl::optional unique_upstream_index; + HttpIntegrationTest::initialize(); + } - for (uint64_t i = 0; i < 8; i++) { - codec_client_ = makeHttpConnection(lookupPort("http")); + void runNormalLoadBalancing() { + absl::optional unique_upstream_index; - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}, - {"x-hash", "hash"}, - }; + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}, + {"x-hash", "hash"}, + }; - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); - ASSERT(upstream_index.has_value()); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - if (unique_upstream_index.has_value()) { - EXPECT_EQ(unique_upstream_index.value(), upstream_index.value()); - } else { - unique_upstream_index = upstream_index.value(); - } + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); + + if (unique_upstream_index.has_value()) { + EXPECT_EQ(unique_upstream_index.value(), upstream_index.value()); + } else { + unique_upstream_index = upstream_index.value(); + } - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - cleanupUpstreamAndDownstream(); + cleanupUpstreamAndDownstream(); + } } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, MaglevIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(MaglevIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} + +TEST_P(MaglevIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} + +TEST_P(MaglevIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc index e0cf7e3f06c8..e36cd294c016 100644 --- a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc +++ b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc @@ -45,14 +45,10 @@ class TestLoadBalancerContext : public LoadBalancerContextBase { // Note: ThreadAwareLoadBalancer base is heavily tested by RingHashLoadBalancerTest. Only basic // functionality is covered here. -class MaglevLoadBalancerTest : public Event::TestUsingSimulatedTime, - public testing::TestWithParam { +class MaglevLoadBalancerTest : public Event::TestUsingSimulatedTime, public testing::Test { public: MaglevLoadBalancerTest() - : stat_names_(stats_store_.symbolTable()), stats_(stat_names_, *stats_store_.rootScope()) { - scoped_runtime_.mergeValues( - {{"envoy.reloadable_features.allow_compact_maglev", GetParam() ? "true" : "false"}}); - } + : stat_names_(stats_store_.symbolTable()), stats_(stat_names_, *stats_store_.rootScope()) {} void createLb() { lb_ = std::make_unique( @@ -91,13 +87,10 @@ class MaglevLoadBalancerTest : public Event::TestUsingSimulatedTime, NiceMock runtime_; NiceMock random_; std::unique_ptr lb_; - TestScopedRuntime scoped_runtime_; }; -INSTANTIATE_TEST_SUITE_P(MaglevTests, MaglevLoadBalancerTest, ::testing::Bool()); - // Works correctly without any hosts. -TEST_P(MaglevLoadBalancerTest, NoHost) { +TEST_F(MaglevLoadBalancerTest, NoHost) { init(7); EXPECT_EQ(nullptr, lb_->factory()->create(lb_params_)->chooseHost(nullptr)); }; @@ -106,7 +99,7 @@ TEST_P(MaglevLoadBalancerTest, NoHost) { // cluster, the operation does not immediately reach the worker thread. There may be cases where the // thread aware load balancer is destructed, but the load balancer factory is still used in the // worker thread. -TEST_P(MaglevLoadBalancerTest, LbDestructedBeforeFactory) { +TEST_F(MaglevLoadBalancerTest, LbDestructedBeforeFactory) { init(7); auto factory = lb_->factory(); @@ -116,13 +109,13 @@ TEST_P(MaglevLoadBalancerTest, LbDestructedBeforeFactory) { } // Throws an exception if table size is not a prime number. -TEST_P(MaglevLoadBalancerTest, NoPrimeNumber) { +TEST_F(MaglevLoadBalancerTest, NoPrimeNumber) { EXPECT_THROW_WITH_MESSAGE(init(8), EnvoyException, "The table size of maglev must be prime number"); }; // Check it has default table size if config is null or table size has invalid value. -TEST_P(MaglevLoadBalancerTest, DefaultMaglevTableSize) { +TEST_F(MaglevLoadBalancerTest, DefaultMaglevTableSize) { const uint64_t defaultValue = MaglevTable::DefaultTableSize; config_ = envoy::config::cluster::v3::Cluster::MaglevLbConfig(); @@ -135,7 +128,7 @@ TEST_P(MaglevLoadBalancerTest, DefaultMaglevTableSize) { }; // Basic sanity tests. -TEST_P(MaglevLoadBalancerTest, Basic) { +TEST_F(MaglevLoadBalancerTest, Basic) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()), makeTestHost(info_, "tcp://127.0.0.1:91", simTime()), makeTestHost(info_, "tcp://127.0.0.1:92", simTime()), @@ -168,7 +161,7 @@ TEST_P(MaglevLoadBalancerTest, Basic) { } // Basic with hostname. -TEST_P(MaglevLoadBalancerTest, BasicWithHostName) { +TEST_F(MaglevLoadBalancerTest, BasicWithHostName) { host_set_.hosts_ = {makeTestHost(info_, "90", "tcp://127.0.0.1:90", simTime()), makeTestHost(info_, "91", "tcp://127.0.0.1:91", simTime()), makeTestHost(info_, "92", "tcp://127.0.0.1:92", simTime()), @@ -203,7 +196,7 @@ TEST_P(MaglevLoadBalancerTest, BasicWithHostName) { } // Basic with metadata hash_key. -TEST_P(MaglevLoadBalancerTest, BasicWithMetadataHashKey) { +TEST_F(MaglevLoadBalancerTest, BasicWithMetadataHashKey) { host_set_.hosts_ = {makeTestHostWithHashKey(info_, "90", "tcp://127.0.0.1:90", simTime()), makeTestHostWithHashKey(info_, "91", "tcp://127.0.0.1:91", simTime()), makeTestHostWithHashKey(info_, "92", "tcp://127.0.0.1:92", simTime()), @@ -238,7 +231,7 @@ TEST_P(MaglevLoadBalancerTest, BasicWithMetadataHashKey) { } // Same ring as the Basic test, but exercise retry host predicate behavior. -TEST_P(MaglevLoadBalancerTest, BasicWithRetryHostPredicate) { +TEST_F(MaglevLoadBalancerTest, BasicWithRetryHostPredicate) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()), makeTestHost(info_, "tcp://127.0.0.1:91", simTime()), makeTestHost(info_, "tcp://127.0.0.1:92", simTime()), @@ -286,7 +279,7 @@ TEST_P(MaglevLoadBalancerTest, BasicWithRetryHostPredicate) { } // Basic stability test. -TEST_P(MaglevLoadBalancerTest, BasicStability) { +TEST_F(MaglevLoadBalancerTest, BasicStability) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()), makeTestHost(info_, "tcp://127.0.0.1:91", simTime()), makeTestHost(info_, "tcp://127.0.0.1:92", simTime()), @@ -331,7 +324,7 @@ TEST_P(MaglevLoadBalancerTest, BasicStability) { } // Weighted sanity test. -TEST_P(MaglevLoadBalancerTest, Weighted) { +TEST_F(MaglevLoadBalancerTest, Weighted) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1), makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), 2)}; host_set_.healthy_hosts_ = host_set_.hosts_; @@ -369,7 +362,7 @@ TEST_P(MaglevLoadBalancerTest, Weighted) { // Locality weighted sanity test when localities have the same weights. Host weights for hosts in // different localities shouldn't matter. -TEST_P(MaglevLoadBalancerTest, LocalityWeightedSameLocalityWeights) { +TEST_F(MaglevLoadBalancerTest, LocalityWeightedSameLocalityWeights) { envoy::config::core::v3::Locality zone_a; zone_a.set_zone("A"); envoy::config::core::v3::Locality zone_b; @@ -417,7 +410,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedSameLocalityWeights) { // Locality weighted sanity test when localities have different weights. Host weights for hosts in // different localities shouldn't matter. -TEST_P(MaglevLoadBalancerTest, LocalityWeightedDifferentLocalityWeights) { +TEST_F(MaglevLoadBalancerTest, LocalityWeightedDifferentLocalityWeights) { envoy::config::core::v3::Locality zone_a; zone_a.set_zone("A"); envoy::config::core::v3::Locality zone_b; @@ -467,7 +460,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedDifferentLocalityWeights) { } // Locality weighted with all localities zero weighted. -TEST_P(MaglevLoadBalancerTest, LocalityWeightedAllZeroLocalityWeights) { +TEST_F(MaglevLoadBalancerTest, LocalityWeightedAllZeroLocalityWeights) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1)}; host_set_.healthy_hosts_ = host_set_.hosts_; host_set_.hosts_per_locality_ = makeHostsPerLocality({{host_set_.hosts_[0]}}); @@ -483,7 +476,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedAllZeroLocalityWeights) { // Validate that when we are in global panic and have localities, we get sane // results (fall back to non-healthy hosts). -TEST_P(MaglevLoadBalancerTest, LocalityWeightedGlobalPanic) { +TEST_F(MaglevLoadBalancerTest, LocalityWeightedGlobalPanic) { envoy::config::core::v3::Locality zone_a; zone_a.set_zone("A"); envoy::config::core::v3::Locality zone_b; @@ -531,7 +524,7 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedGlobalPanic) { // Given extremely lopsided locality weights, and a table that isn't large enough to fit all hosts, // expect that the least-weighted hosts appear once, and the most-weighted host fills the remainder. -TEST_P(MaglevLoadBalancerTest, LocalityWeightedLopsided) { +TEST_F(MaglevLoadBalancerTest, LocalityWeightedLopsided) { envoy::config::core::v3::Locality zone_a; zone_a.set_zone("A"); envoy::config::core::v3::Locality zone_b; diff --git a/test/extensions/load_balancing_policies/random/config_test.cc b/test/extensions/load_balancing_policies/random/config_test.cc index 4e1acac4f811..b115469b9ed0 100644 --- a/test/extensions/load_balancing_policies/random/config_test.cc +++ b/test/extensions/load_balancing_policies/random/config_test.cc @@ -15,7 +15,7 @@ namespace Random { namespace { TEST(RandomConfigTest, ValidateFail) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; @@ -29,7 +29,7 @@ TEST(RandomConfigTest, ValidateFail) { EXPECT_EQ("envoy.load_balancing_policies.random", factory.name()); auto lb_config = - factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor()); + factory.loadConfig(*factory.createEmptyConfigProto(), context.messageValidationVisitor()); auto thread_aware_lb = factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, context.api_.random_, context.time_system_); diff --git a/test/extensions/load_balancing_policies/random/integration_test.cc b/test/extensions/load_balancing_policies/random/integration_test.cc index ec2389b3e36a..f62bec18af36 100644 --- a/test/extensions/load_balancing_policies/random/integration_test.cc +++ b/test/extensions/load_balancing_policies/random/integration_test.cc @@ -24,78 +24,107 @@ class RandomIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); - auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); - - constexpr absl::string_view endpoints_yaml = R"EOF( - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - )EOF"; - - const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); - TestUtility::loadFromYaml( - fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); - - auto* policy = cluster_0->mutable_load_balancing_policy(); - - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.random - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random - )EOF"; - - TestUtility::loadFromYaml(policy_yaml, *policy); - }); + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + constexpr absl::string_view endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + )EOF"; + + const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); + TestUtility::loadFromYaml( + fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); + + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::RANDOM); + return; + } + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.random + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + + HttpIntegrationTest::initialize(); } -}; -INSTANTIATE_TEST_SUITE_P(IpVersions, RandomIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + void runNormalLoadBalancing() { + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); -TEST_P(RandomIntegrationTest, NormalLoadBalancing) { - initialize(); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + + auto response = codec_client_->makeRequestWithBody(request_headers, 0); + + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); - for (uint64_t i = 0; i < 8; i++) { - codec_client_ = makeHttpConnection(lookupPort("http")); + upstream_request_->encodeHeaders(default_response_headers_, true); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + ASSERT_TRUE(response->waitForEndStream()); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); - ASSERT(upstream_index.has_value()); + cleanupUpstreamAndDownstream(); + } + } +}; - upstream_request_->encodeHeaders(default_response_headers_, true); +INSTANTIATE_TEST_SUITE_P(IpVersions, RandomIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); - ASSERT_TRUE(response->waitForEndStream()); +TEST_P(RandomIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); +TEST_P(RandomIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} - cleanupUpstreamAndDownstream(); - } +TEST_P(RandomIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/ring_hash/config_test.cc b/test/extensions/load_balancing_policies/ring_hash/config_test.cc index 24a59c6627d9..19eca2c14a56 100644 --- a/test/extensions/load_balancing_policies/ring_hash/config_test.cc +++ b/test/extensions/load_balancing_policies/ring_hash/config_test.cc @@ -16,7 +16,7 @@ namespace RingHash { namespace { TEST(RingHashConfigTest, Validate) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; @@ -31,7 +31,7 @@ TEST(RingHashConfigTest, Validate) { EXPECT_EQ("envoy.load_balancing_policies.ring_hash", factory.name()); auto lb_config = - factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor()); + factory.loadConfig(*factory.createEmptyConfigProto(), context.messageValidationVisitor()); auto thread_aware_lb = factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, context.api_.random_, context.time_system_); @@ -63,7 +63,7 @@ TEST(RingHashConfigTest, Validate) { auto message_ptr = factory.createEmptyConfigProto(); message_ptr->MergeFrom(config_msg); - auto lb_config = factory.loadConfig(std::move(message_ptr), context.messageValidationVisitor()); + auto lb_config = factory.loadConfig(*message_ptr, context.messageValidationVisitor()); EXPECT_THROW_WITH_MESSAGE( factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, diff --git a/test/extensions/load_balancing_policies/ring_hash/integration_test.cc b/test/extensions/load_balancing_policies/ring_hash/integration_test.cc index 52ae84e92057..c92abc4514d9 100644 --- a/test/extensions/load_balancing_policies/ring_hash/integration_test.cc +++ b/test/extensions/load_balancing_policies/ring_hash/integration_test.cc @@ -25,48 +25,6 @@ class RingHashIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); - auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); - - constexpr absl::string_view endpoints_yaml = R"EOF( - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - )EOF"; - - const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); - TestUtility::loadFromYaml( - fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); - - auto* policy = cluster_0->mutable_load_balancing_policy(); - - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.ring_hash - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.ring_hash.v3.RingHash - )EOF"; - - TestUtility::loadFromYaml(policy_yaml, *policy); - }); - config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { @@ -79,45 +37,115 @@ class RingHashIntegrationTest : public testing::TestWithParamset_header_name("x-hash"); }); } -}; -INSTANTIATE_TEST_SUITE_P(IpVersions, RingHashIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + void initializeConfig(bool legacy_api = false, bool disable_lagacy_api_conversion = false) { + if (disable_lagacy_api_conversion) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.convert_legacy_lb_config", + "false"); + } -TEST_P(RingHashIntegrationTest, NormalLoadBalancing) { - initialize(); + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + constexpr absl::string_view endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + )EOF"; + + const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); + TestUtility::loadFromYaml( + fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); + + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::RING_HASH); + return; + } + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.maglev + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); - absl::optional unique_upstream_index; + HttpIntegrationTest::initialize(); + } - for (uint64_t i = 0; i < 8; i++) { - codec_client_ = makeHttpConnection(lookupPort("http")); + void runNormalLoadBalancing() { + absl::optional unique_upstream_index; - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}, - {"x-hash", "hash"}, - }; + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}, + {"x-hash", "hash"}, + }; - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); - ASSERT(upstream_index.has_value()); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - if (unique_upstream_index.has_value()) { - EXPECT_EQ(unique_upstream_index.value(), upstream_index.value()); - } else { - unique_upstream_index = upstream_index.value(); - } + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); + + if (unique_upstream_index.has_value()) { + EXPECT_EQ(unique_upstream_index.value(), upstream_index.value()); + } else { + unique_upstream_index = upstream_index.value(); + } - upstream_request_->encodeHeaders(default_response_headers_, true); + upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->waitForEndStream()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - cleanupUpstreamAndDownstream(); + cleanupUpstreamAndDownstream(); + } } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, RingHashIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(RingHashIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} + +TEST_P(RingHashIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} + +TEST_P(RingHashIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/round_robin/config_test.cc b/test/extensions/load_balancing_policies/round_robin/config_test.cc index 2e460ec81e34..6b2263b9771c 100644 --- a/test/extensions/load_balancing_policies/round_robin/config_test.cc +++ b/test/extensions/load_balancing_policies/round_robin/config_test.cc @@ -15,7 +15,7 @@ namespace RoundRobin { namespace { TEST(RoundRobinConfigTest, ValidateFail) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; @@ -29,7 +29,7 @@ TEST(RoundRobinConfigTest, ValidateFail) { EXPECT_EQ("envoy.load_balancing_policies.round_robin", factory.name()); auto lb_config = - factory.loadConfig(factory.createEmptyConfigProto(), context.messageValidationVisitor()); + factory.loadConfig(*factory.createEmptyConfigProto(), context.messageValidationVisitor()); auto thread_aware_lb = factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, diff --git a/test/extensions/load_balancing_policies/round_robin/integration_test.cc b/test/extensions/load_balancing_policies/round_robin/integration_test.cc index b3635f5d0bae..8e13ec35470f 100644 --- a/test/extensions/load_balancing_policies/round_robin/integration_test.cc +++ b/test/extensions/load_balancing_policies/round_robin/integration_test.cc @@ -24,86 +24,114 @@ class RoundRobinIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); - auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); - - constexpr absl::string_view endpoints_yaml = R"EOF( - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - )EOF"; - - const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); - TestUtility::loadFromYaml( - fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); - - auto* policy = cluster_0->mutable_load_balancing_policy(); - - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.round_robin - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin - )EOF"; - - TestUtility::loadFromYaml(policy_yaml, *policy); - }); } -}; -INSTANTIATE_TEST_SUITE_P(IpVersions, RoundRobinIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + void initializeConfig(bool legacy_api = false, bool disable_lagacy_api_conversion = false) { + if (disable_lagacy_api_conversion) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.convert_legacy_lb_config", + "false"); + } + + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + constexpr absl::string_view endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + )EOF"; + + const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); + TestUtility::loadFromYaml( + fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); + + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::ROUND_ROBIN); + return; + } + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.round_robin + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + + HttpIntegrationTest::initialize(); + } -TEST_P(RoundRobinIntegrationTest, NormalLoadBalancing) { - initialize(); + void runNormalLoadBalancing() { + std::vector indexs; - std::vector indexs; + for (uint64_t i = 0; i < 8; i++) { + codec_client_ = makeHttpConnection(lookupPort("http")); - for (uint64_t i = 0; i < 8; i++) { - codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "example.com"}}; + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); + ASSERT(upstream_index.has_value()); + indexs.push_back(upstream_index.value()); - auto upstream_index = waitForNextUpstreamRequest({0, 1, 2}); - ASSERT(upstream_index.has_value()); - indexs.push_back(upstream_index.value()); + upstream_request_->encodeHeaders(default_response_headers_, true); - upstream_request_->encodeHeaders(default_response_headers_, true); + ASSERT_TRUE(response->waitForEndStream()); - ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); + cleanupUpstreamAndDownstream(); + } - cleanupUpstreamAndDownstream(); + for (uint64_t i = 2; i < 8; i++) { + EXPECT_NE(indexs[i], indexs[i - 1]); + EXPECT_NE(indexs[i - 1], indexs[i - 2]); + } } +}; - for (uint64_t i = 2; i < 8; i++) { - EXPECT_NE(indexs[i], indexs[i - 1]); - EXPECT_NE(indexs[i - 1], indexs[i - 2]); - } +INSTANTIATE_TEST_SUITE_P(IpVersions, RoundRobinIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(RoundRobinIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} + +TEST_P(RoundRobinIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} + +TEST_P(RoundRobinIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/subset/BUILD b/test/extensions/load_balancing_policies/subset/BUILD index 417a9affab11..22a1130b3faf 100644 --- a/test/extensions/load_balancing_policies/subset/BUILD +++ b/test/extensions/load_balancing_policies/subset/BUILD @@ -4,6 +4,8 @@ load( ) load( "//test/extensions:extensions_build_system.bzl", + "envoy_extension_benchmark_test", + "envoy_extension_cc_benchmark_binary", "envoy_extension_cc_test", ) @@ -47,13 +49,46 @@ envoy_extension_cc_test( srcs = ["subset_test.cc"], extension_names = ["envoy.load_balancing_policies.subset"], deps = [ - "//source/extensions/load_balancing_policies/random:config", + "//source/common/common:minimal_logger_lib", + "//source/common/network:utility_lib", + "//source/common/upstream:load_balancer_lib", + "//source/common/upstream:upstream_includes", + "//source/common/upstream:upstream_lib", "//source/extensions/load_balancing_policies/subset:config", - "//test/mocks/server:factory_context_mocks", + "//test/common/upstream:utility_lib", + "//test/mocks:common_lib", + "//test/mocks/access_log:access_log_mocks", + "//test/mocks/filesystem:filesystem_mocks", + "//test/mocks/runtime:runtime_mocks", "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:host_mocks", + "//test/mocks/upstream:host_set_mocks", "//test/mocks/upstream:load_balancer_context_mock", + "//test/mocks/upstream:load_balancer_mocks", "//test/mocks/upstream:priority_set_mocks", + "//test/test_common:simulated_time_system_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", ], ) + +envoy_extension_cc_benchmark_binary( + name = "subset_benchmark", + srcs = ["subset_benchmark.cc"], + extension_names = ["envoy.load_balancing_policies.subset"], + external_deps = [ + "benchmark", + ], + deps = [ + "//source/extensions/load_balancing_policies/subset:config", + "//test/extensions/load_balancing_policies/common:benchmark_base_tester_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + ], +) + +envoy_extension_benchmark_test( + name = "subset_benchmark_test", + timeout = "long", + benchmark_binary = "subset_benchmark", + extension_names = ["envoy.load_balancing_policies.subset"], +) diff --git a/test/extensions/load_balancing_policies/subset/config_test.cc b/test/extensions/load_balancing_policies/subset/config_test.cc index f4bdd68267d5..fdc45f97b19d 100644 --- a/test/extensions/load_balancing_policies/subset/config_test.cc +++ b/test/extensions/load_balancing_policies/subset/config_test.cc @@ -16,7 +16,7 @@ namespace Subset { namespace { TEST(SubsetConfigTest, SubsetConfigTest) { - NiceMock context; + NiceMock context; NiceMock cluster_info; NiceMock main_thread_priority_set; NiceMock thread_local_priority_set; @@ -50,7 +50,7 @@ TEST(SubsetConfigTest, SubsetConfigTest) { auto& factory = Config::Utility::getAndCheckFactory(config); EXPECT_EQ("envoy.load_balancing_policies.subset", factory.name()); - auto lb_config = factory.loadConfig(std::move(config_msg), context.messageValidationVisitor()); + auto lb_config = factory.loadConfig(*config_msg, context.messageValidationVisitor()); auto thread_aware_lb = factory.create(*lb_config, cluster_info, main_thread_priority_set, context.runtime_loader_, @@ -100,7 +100,7 @@ TEST(SubsetConfigTest, SubsetConfigTestWithUnknownSubsetLoadBalancingPolicy) { EXPECT_EQ("envoy.load_balancing_policies.subset", factory.name()); EXPECT_THROW_WITH_MESSAGE( - factory.loadConfig(std::move(config_msg), context.messageValidationVisitor()), EnvoyException, + factory.loadConfig(*config_msg, context.messageValidationVisitor()), EnvoyException, "cluster: didn't find a registered load balancer factory implementation for subset lb with " "names from [envoy.load_balancing_policies.unknown]"); } diff --git a/test/extensions/load_balancing_policies/subset/integration_test.cc b/test/extensions/load_balancing_policies/subset/integration_test.cc index e995d26ee6c8..2e2fd010a1e9 100644 --- a/test/extensions/load_balancing_policies/subset/integration_test.cc +++ b/test/extensions/load_balancing_policies/subset/integration_test.cc @@ -35,113 +35,158 @@ class SubsetIntegrationTest : public testing::TestWithParammutable_clusters()->Mutable(0); - ASSERT(cluster_0->name() == "cluster_0"); - auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); - - constexpr absl::string_view endpoints_yaml = R"EOF( - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - metadata: - filter_metadata: - envoy.lb: - version: v1 - stage: canary - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - metadata: - filter_metadata: - envoy.lb: - version: v2 - stage: canary - - endpoint: - address: - socket_address: - address: {} - port_value: 0 - metadata: - filter_metadata: - envoy.lb: - version: v3 - )EOF"; - - const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); - TestUtility::loadFromYaml( - fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); - - auto* policy = cluster_0->mutable_load_balancing_policy(); - - const std::string policy_yaml = R"EOF( - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.subset - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.subset.v3.Subset - fallback_policy: ANY_ENDPOINT - subset_selectors: - - keys: - - "version" - - "stage" - fallback_policy: NO_FALLBACK - - keys: - - "version" + config_helper_.addConfigModifier( + [legacy_api](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters()->Mutable(0); + ASSERT(cluster_0->name() == "cluster_0"); + auto* endpoint = cluster_0->mutable_load_assignment()->mutable_endpoints()->Mutable(0); + + constexpr absl::string_view endpoints_yaml = R"EOF( + lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + metadata: + filter_metadata: + envoy.lb: + version: v1 + stage: canary + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + metadata: + filter_metadata: + envoy.lb: + version: v2 + stage: canary + - endpoint: + address: + socket_address: + address: {} + port_value: 0 + metadata: + filter_metadata: + envoy.lb: + version: v3 + )EOF"; + + const std::string local_address = Network::Test::getLoopbackAddressString(GetParam()); + TestUtility::loadFromYaml( + fmt::format(endpoints_yaml, local_address, local_address, local_address), *endpoint); + + // If legacy API is used, set the LB policy by the old way. + if (legacy_api) { + // Set the inner LB policy of the subset LB policy to RANDOM. + cluster_0->set_lb_policy(envoy::config::cluster::v3::Cluster::RANDOM); + + auto* mutable_subset_lb_config = cluster_0->mutable_lb_subset_config(); + + const std::string subset_lb_config_yaml = R"EOF( + fallback_policy: ANY_ENDPOINT + subset_selectors: + - keys: + - "version" + - "stage" + fallback_policy: NO_FALLBACK + - keys: + - "version" + fallback_policy: ANY_ENDPOINT + list_as_any: true + )EOF"; + + TestUtility::loadFromYaml(subset_lb_config_yaml, *mutable_subset_lb_config); + return; + } + + auto* policy = cluster_0->mutable_load_balancing_policy(); + + const std::string policy_yaml = R"EOF( + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.subset + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.subset.v3.Subset fallback_policy: ANY_ENDPOINT - list_as_any: true - subset_lb_policy: - policies: - - typed_extension_config: - name: envoy.load_balancing_policies.random - typed_config: - "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random - )EOF"; - - TestUtility::loadFromYaml(policy_yaml, *policy); - }); + subset_selectors: + - keys: + - "version" + - "stage" + fallback_policy: NO_FALLBACK + - keys: + - "version" + fallback_policy: ANY_ENDPOINT + list_as_any: true + subset_lb_policy: + policies: + - typed_extension_config: + name: envoy.load_balancing_policies.random + typed_config: + "@type": type.googleapis.com/envoy.extensions.load_balancing_policies.random.v3.Random + )EOF"; + + TestUtility::loadFromYaml(policy_yaml, *policy); + }); + + HttpIntegrationTest::initialize(); } -}; -INSTANTIATE_TEST_SUITE_P(IpVersions, SubsetIntegrationTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); + void runNormalLoadBalancing() { + for (uint64_t i = 1; i <= 3; i++) { -// Test the case where the subset load balancer is configured by the load balancing -// policy API and it works as expected. -TEST_P(SubsetIntegrationTest, NormalLoadBalancing) { - initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); - for (uint64_t i = 1; i <= 3; i++) { + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "example.com"}, + {"version", fmt::format("v{}", i)}}; - codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeRequestWithBody(request_headers, 0); - Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, - {":path", "/"}, - {":scheme", "http"}, - {":authority", "example.com"}, - {"version", fmt::format("v{}", i)}}; + waitForNextUpstreamRequest(i - 1); - auto response = codec_client_->makeRequestWithBody(request_headers, 0); + upstream_request_->encodeHeaders(default_response_headers_, true); - waitForNextUpstreamRequest(i - 1); + ASSERT_TRUE(response->waitForEndStream()); - upstream_request_->encodeHeaders(default_response_headers_, true); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); - ASSERT_TRUE(response->waitForEndStream()); + cleanupUpstreamAndDownstream(); + } + } +}; - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_TRUE(response->complete()); +INSTANTIATE_TEST_SUITE_P(IpVersions, SubsetIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); - cleanupUpstreamAndDownstream(); - } +TEST_P(SubsetIntegrationTest, NormalLoadBalancing) { + initializeConfig(); + runNormalLoadBalancing(); +} + +TEST_P(SubsetIntegrationTest, NormalLoadBalancingWithLegacyAPI) { + initializeConfig(true); + runNormalLoadBalancing(); +} + +TEST_P(SubsetIntegrationTest, NormalLoadBalancingWithLegacyAPIAndDisableAPIConversion) { + initializeConfig(true, true); + runNormalLoadBalancing(); } } // namespace diff --git a/test/extensions/load_balancing_policies/subset/subset_benchmark.cc b/test/extensions/load_balancing_policies/subset/subset_benchmark.cc new file mode 100644 index 000000000000..d775c8851e33 --- /dev/null +++ b/test/extensions/load_balancing_policies/subset/subset_benchmark.cc @@ -0,0 +1,116 @@ +// Usage: bazel run //test/common/upstream:load_balancer_benchmark + +#include + +#include "envoy/config/cluster/v3/cluster.pb.h" + +#include "source/common/common/random_generator.h" +#include "source/common/memory/stats.h" +#include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" +#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb.h" + +#include "test/benchmark/main.h" +#include "test/common/upstream/utility.h" +#include "test/extensions/load_balancing_policies/common/benchmark_base_tester.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/test_common/simulated_time_system.h" + +#include "absl/types/optional.h" +#include "benchmark/benchmark.h" + +namespace Envoy { +namespace Extensions { +namespace LoadBalancingPolices { +namespace Subset { +namespace { + +class SubsetLbTester : public LoadBalancingPolices::Common::BaseTester { +public: + SubsetLbTester(uint64_t num_hosts, bool single_host_per_subset) + : BaseTester(num_hosts, 0, 0, true /* attach metadata */) { + envoy::config::cluster::v3::Cluster::LbSubsetConfig subset_config; + subset_config.set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT); + auto* selector = subset_config.mutable_subset_selectors()->Add(); + selector->set_single_host_per_subset(single_host_per_subset); + *selector->mutable_keys()->Add() = std::string(metadata_key); + + subset_info_ = std::make_unique(subset_config); + auto child_lb_creator = std::make_unique( + Upstream::LoadBalancerType::Random, absl::nullopt, absl::nullopt, absl::nullopt, + absl::nullopt, common_config_); + lb_ = std::make_unique( + *subset_info_, std::move(child_lb_creator), priority_set_, &local_priority_set_, stats_, + stats_scope_, runtime_, random_, simTime()); + + const Upstream::HostVector& hosts = priority_set_.getOrCreateHostSet(0).hosts(); + ASSERT(hosts.size() == num_hosts); + orig_hosts_ = std::make_shared(hosts); + smaller_hosts_ = std::make_shared(hosts.begin() + 1, hosts.end()); + ASSERT(smaller_hosts_->size() + 1 == orig_hosts_->size()); + orig_locality_hosts_ = Upstream::makeHostsPerLocality({*orig_hosts_}); + smaller_locality_hosts_ = Upstream::makeHostsPerLocality({*smaller_hosts_}); + } + + // Remove a host and add it back. + void update() { + priority_set_.updateHosts( + 0, Upstream::HostSetImpl::partitionHosts(smaller_hosts_, smaller_locality_hosts_), nullptr, + {}, host_moved_, absl::nullopt); + priority_set_.updateHosts( + 0, Upstream::HostSetImpl::partitionHosts(orig_hosts_, orig_locality_hosts_), nullptr, + host_moved_, {}, absl::nullopt); + } + + std::unique_ptr subset_info_; + std::unique_ptr lb_; + Upstream::HostVectorConstSharedPtr orig_hosts_; + Upstream::HostVectorConstSharedPtr smaller_hosts_; + Upstream::HostsPerLocalitySharedPtr orig_locality_hosts_; + Upstream::HostsPerLocalitySharedPtr smaller_locality_hosts_; + Upstream::HostVector host_moved_; +}; + +void benchmarkSubsetLoadBalancerCreate(::benchmark::State& state) { + const bool single_host_per_subset = state.range(0); + const uint64_t num_hosts = state.range(1); + + if (benchmark::skipExpensiveBenchmarks() && num_hosts > 100) { + state.SkipWithError("Skipping expensive benchmark"); + return; + } + + for (auto _ : state) { // NOLINT: Silences warning about dead store + SubsetLbTester tester(num_hosts, single_host_per_subset); + } +} + +BENCHMARK(benchmarkSubsetLoadBalancerCreate) + ->Ranges({{false, true}, {50, 2500}}) + ->Unit(::benchmark::kMillisecond); + +void benchmarkSubsetLoadBalancerUpdate(::benchmark::State& state) { + const bool single_host_per_subset = state.range(0); + const uint64_t num_hosts = state.range(1); + if (benchmark::skipExpensiveBenchmarks() && num_hosts > 100) { + state.SkipWithError("Skipping expensive benchmark"); + return; + } + + SubsetLbTester tester(num_hosts, single_host_per_subset); + for (auto _ : state) { // NOLINT: Silences warning about dead store + tester.update(); + } +} + +BENCHMARK(benchmarkSubsetLoadBalancerUpdate) + ->Ranges({{false, true}, {50, 2500}}) + ->Unit(::benchmark::kMillisecond); + +} // namespace +} // namespace Subset +} // namespace LoadBalancingPolices +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/load_balancing_policies/subset/subset_test.cc b/test/extensions/load_balancing_policies/subset/subset_test.cc index c91df65bb5ca..032fc45b5662 100644 --- a/test/extensions/load_balancing_policies/subset/subset_test.cc +++ b/test/extensions/load_balancing_policies/subset/subset_test.cc @@ -1,21 +1,3198 @@ -#include "envoy/config/core/v3/extension.pb.h" -#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.h" +#include +#include +#include +#include +#include +#include +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/config/core/v3/base.pb.h" + +#include "source/common/common/logger.h" +#include "source/common/config/metadata.h" #include "source/common/router/metadatamatchcriteria_impl.h" +#include "source/common/upstream/upstream_impl.h" #include "source/extensions/load_balancing_policies/subset/config.h" -#include "test/mocks/server/factory_context.h" +#include "test/common/upstream/utility.h" +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/common.h" +#include "test/mocks/filesystem/mocks.h" +#include "test/mocks/runtime/mocks.h" #include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/host.h" +#include "test/mocks/upstream/host_set.h" +#include "test/mocks/upstream/load_balancer.h" #include "test/mocks/upstream/load_balancer_context.h" #include "test/mocks/upstream/priority_set.h" +#include "test/test_common/simulated_time_system.h" +#include "absl/types/optional.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + namespace Envoy { -namespace Extensions { -namespace LoadBalancingPolices { -namespace Subset { -namespace { +namespace Upstream { + +class SubsetLoadBalancerInternalStateTester { +public: + SubsetLoadBalancerInternalStateTester(std::shared_ptr lb) : lb_(lb) {} + + using MetadataVector = std::vector>; + + void testDescribeMetadata(std::string expected, const MetadataVector& metadata) { + const SubsetLoadBalancer::SubsetMetadata& subset_metadata(metadata); + EXPECT_EQ(expected, lb_.get()->describeMetadata(subset_metadata)); + } + + void validateLbTypeConfigs() const { + const auto* legacy_child_lb_creator = + dynamic_cast(lb_->child_lb_creator_.get()); + + if (legacy_child_lb_creator == nullptr) { + return; + } + + // Each of these expects that an lb_type is set to that type iff the + // returned config for that type exists. + EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RingHash, + legacy_child_lb_creator->lbRingHashConfig() != absl::nullopt); + EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::Maglev, + legacy_child_lb_creator->lbMaglevConfig() != absl::nullopt); + EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RoundRobin, + legacy_child_lb_creator->lbRoundRobinConfig() != absl::nullopt); + EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::LeastRequest, + legacy_child_lb_creator->lbLeastRequestConfig() != absl::nullopt); + } + +private: + std::shared_ptr lb_; +}; + +class TestMetadataMatchCriterion : public Router::MetadataMatchCriterion { +public: + TestMetadataMatchCriterion(const std::string& name, const HashedValue& value) + : name_(name), value_(value) {} + + const std::string& name() const override { return name_; } + const HashedValue& value() const override { return value_; } + +private: + std::string name_; + HashedValue value_; +}; + +class TestMetadataMatchCriteria : public Router::MetadataMatchCriteria { +public: + TestMetadataMatchCriteria(const std::map matches) { + for (const auto& it : matches) { + ProtobufWkt::Value v; + v.set_string_value(it.second); + + matches_.emplace_back( + std::make_shared(it.first, HashedValue(v))); + } + } + + TestMetadataMatchCriteria(const std::map matches) { + for (const auto& it : matches) { + matches_.emplace_back( + std::make_shared(it.first, HashedValue(it.second))); + } + } + + const std::vector& + metadataMatchCriteria() const override { + return matches_; + } + + Router::MetadataMatchCriteriaConstPtr + mergeMatchCriteria(const ProtobufWkt::Struct& override) const override { + auto new_criteria = std::make_unique(*this); + + // TODO: this is copied from MetadataMatchCriteriaImpl::extractMetadataMatchCriteria. + // should we start using real impl? + std::vector v; + absl::node_hash_map existing; + + for (const auto& it : matches_) { + existing.emplace(it->name(), v.size()); + v.emplace_back(it); + } + + // Add values from matches, replacing name/values copied from parent. + for (const auto& it : override.fields()) { + const auto index_it = existing.find(it.first); + if (index_it != existing.end()) { + v[index_it->second] = std::make_shared(it.first, it.second); + } else { + v.emplace_back(std::make_shared(it.first, it.second)); + } + } + std::sort(v.begin(), v.end(), + [](const Router::MetadataMatchCriterionConstSharedPtr& a, + const Router::MetadataMatchCriterionConstSharedPtr& b) -> bool { + return a->name() < b->name(); + }); + + new_criteria->matches_ = v; + return new_criteria; + } + + Router::MetadataMatchCriteriaConstPtr + filterMatchCriteria(const std::set& names) const override { + auto new_criteria = std::make_unique(*this); + for (auto it = new_criteria->matches_.begin(); it != new_criteria->matches_.end();) { + if (names.count(it->get()->name()) == 0) { + it = new_criteria->matches_.erase(it); + } else { + it++; + } + } + return new_criteria; + } + +private: + std::vector matches_; +}; + +namespace SubsetLoadBalancerTest { + +class TestLoadBalancerContext : public LoadBalancerContextBase { +public: + TestLoadBalancerContext( + std::initializer_list::value_type> metadata_matches) + : matches_( + new TestMetadataMatchCriteria(std::map(metadata_matches))) {} + + TestLoadBalancerContext( + std::initializer_list::value_type> metadata_matches) + : matches_(new TestMetadataMatchCriteria( + std::map(metadata_matches))) {} + + // Upstream::LoadBalancerContext + absl::optional computeHashKey() override { return {}; } + const Network::Connection* downstreamConnection() const override { return nullptr; } + const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return matches_.get(); } + const Http::RequestHeaderMap* downstreamHeaders() const override { return nullptr; } + + std::shared_ptr matches_; +}; + +enum class UpdateOrder { RemovesFirst, Simultaneous }; + +class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, + public testing::TestWithParam { +public: + SubsetLoadBalancerTest() + : scope_(stats_store_.createScope("testprefix")), stat_names_(stats_store_.symbolTable()), + stats_(stat_names_, *stats_store_.rootScope()) { + least_request_lb_config_.mutable_choice_count()->set_value(2); + } + + using HostMetadata = std::map; + using HostListMetadata = std::map>; + using HostURLMetadataMap = std::map; + + void init() { + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + }); + } + + void configureHostSet(const HostURLMetadataMap& host_metadata, MockHostSet& host_set) { + HostVector hosts; + for (const auto& it : host_metadata) { + hosts.emplace_back(makeHost(it.first, it.second)); + } + + host_set.hosts_ = hosts; + host_set.hosts_per_locality_ = makeHostsPerLocality({hosts}); + host_set.healthy_hosts_ = host_set.hosts_; + host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; + + host_set.runCallbacks({}, {}); + } + + void configureWeightedHostSet(const HostURLMetadataMap& first_locality_host_metadata, + const HostURLMetadataMap& second_locality_host_metadata, + MockHostSet& host_set, LocalityWeights locality_weights) { + HostVector all_hosts; + HostVector first_locality_hosts; + envoy::config::core::v3::Locality first_locality; + first_locality.set_zone("0"); + for (const auto& it : first_locality_host_metadata) { + auto host = makeHost(it.first, it.second, first_locality); + first_locality_hosts.emplace_back(host); + all_hosts.emplace_back(host); + } + + envoy::config::core::v3::Locality second_locality; + second_locality.set_zone("1"); + HostVector second_locality_hosts; + for (const auto& it : second_locality_host_metadata) { + auto host = makeHost(it.first, it.second, second_locality); + second_locality_hosts.emplace_back(host); + all_hosts.emplace_back(host); + } + + host_set.hosts_ = all_hosts; + host_set.hosts_per_locality_ = + makeHostsPerLocality({first_locality_hosts, second_locality_hosts}); + host_set.healthy_hosts_ = host_set.hosts_; + host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; + host_set.locality_weights_ = std::make_shared(locality_weights); + } + + void init(const HostURLMetadataMap& host_metadata) { + HostURLMetadataMap failover; + init(host_metadata, failover); + } + + void init(const HostURLMetadataMap& host_metadata, + const HostURLMetadataMap& failover_host_metadata, bool use_actual_subset_info = false) { + + if (!use_actual_subset_info) { + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + } + + configureHostSet(host_metadata, host_set_); + if (!failover_host_metadata.empty()) { + configureHostSet(failover_host_metadata, *priority_set_.getMockHostSet(1)); + } + + auto child_lb_creator = std::make_unique( + lb_type_, + lb_type_ == LoadBalancerType::RingHash + ? makeOptRef( + ring_hash_lb_config_) + : absl::nullopt, + lb_type_ == LoadBalancerType::Maglev + ? makeOptRef( + maglev_lb_config_) + : absl::nullopt, + lb_type_ == LoadBalancerType::RoundRobin + ? makeOptRef( + round_robin_lb_config_) + : absl::nullopt, + lb_type_ == LoadBalancerType::LeastRequest + ? makeOptRef( + least_request_lb_config_) + : absl::nullopt, + common_config_); + + lb_ = std::make_shared( + use_actual_subset_info ? static_cast(*actual_subset_info_) + : static_cast(subset_info_), + std::move(child_lb_creator), priority_set_, nullptr, stats_, *scope_, runtime_, random_, + simTime()); + } + + void zoneAwareInit(const std::vector& host_metadata_per_locality, + const std::vector& local_host_metadata_per_locality) { + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + + std::vector> localities; + for (uint32_t i = 0; i < 10; ++i) { + envoy::config::core::v3::Locality locality; + locality.set_zone(std::to_string(i)); + localities.emplace_back(std::make_shared(locality)); + } + ASSERT(host_metadata_per_locality.size() <= localities.size()); + ASSERT(local_host_metadata_per_locality.size() <= localities.size()); + + HostVector hosts; + std::vector hosts_per_locality; + for (uint32_t i = 0; i < host_metadata_per_locality.size(); ++i) { + const auto& host_metadata = host_metadata_per_locality[i]; + HostVector locality_hosts; + for (const auto& host_entry : host_metadata) { + HostSharedPtr host = makeHost(host_entry.first, host_entry.second, *localities[i]); + hosts.emplace_back(host); + locality_hosts.emplace_back(host); + } + hosts_per_locality.emplace_back(locality_hosts); + } + + host_set_.hosts_ = hosts; + host_set_.hosts_per_locality_ = makeHostsPerLocality(std::move(hosts_per_locality)); + + host_set_.healthy_hosts_ = host_set_.hosts_; + host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; + + local_hosts_ = std::make_shared(); + std::vector local_hosts_per_locality_vector; + for (uint32_t i = 0; i < local_host_metadata_per_locality.size(); ++i) { + const auto& local_host_metadata = local_host_metadata_per_locality[i]; + HostVector local_locality_hosts; + for (const auto& host_entry : local_host_metadata) { + HostSharedPtr host = makeHost(host_entry.first, host_entry.second, *localities[i]); + local_hosts_->emplace_back(host); + local_locality_hosts.emplace_back(host); + } + local_hosts_per_locality_vector.emplace_back(local_locality_hosts); + } + local_hosts_per_locality_ = makeHostsPerLocality(std::move(local_hosts_per_locality_vector)); + + local_priority_set_.updateHosts( + 0, + HostSetImpl::updateHostsParams( + local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), local_hosts_per_locality_, + std::make_shared(), HostsPerLocalityImpl::empty(), + std::make_shared(), HostsPerLocalityImpl::empty()), + {}, {}, {}, absl::nullopt); + + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + + lb_ = std::make_shared(subset_info_, std::move(child_lb_creator), + priority_set_, &local_priority_set_, stats_, *scope_, + runtime_, random_, simTime()); + } + + HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata) { + envoy::config::core::v3::Metadata m; + for (const auto& m_it : metadata) { + Config::Metadata::mutableMetadataValue(m, Config::MetadataFilters::get().ENVOY_LB, m_it.first) + .set_string_value(m_it.second); + } + + return makeTestHost(info_, url, m, simTime()); + } + + HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata, + const envoy::config::core::v3::Locality& locality) { + envoy::config::core::v3::Metadata m; + for (const auto& m_it : metadata) { + Config::Metadata::mutableMetadataValue(m, Config::MetadataFilters::get().ENVOY_LB, m_it.first) + .set_string_value(m_it.second); + } + + return makeTestHost(info_, url, m, locality, simTime()); + } + + HostSharedPtr makeHost(const std::string& url, const HostListMetadata& metadata) { + envoy::config::core::v3::Metadata m; + for (const auto& m_it : metadata) { + auto& metadata = Config::Metadata::mutableMetadataValue( + m, Config::MetadataFilters::get().ENVOY_LB, m_it.first); + for (const auto& value : m_it.second) { + metadata.mutable_list_value()->add_values()->set_string_value(value); + } + } + + return makeTestHost(info_, url, m, simTime()); + } + + ProtobufWkt::Struct makeDefaultSubset(HostMetadata metadata) { + ProtobufWkt::Struct default_subset; + + auto* fields = default_subset.mutable_fields(); + for (const auto& it : metadata) { + ProtobufWkt::Value v; + v.set_string_value(it.second); + fields->insert({it.first, v}); + } + + return default_subset; + } + + SubsetSelectorPtr + makeSelector(const std::set& selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy fallback_policy, + const std::set& fallback_keys_subset, + bool single_host_per_subset = false) { + Protobuf::RepeatedPtrField selector_keys_mapped; + for (const auto& it : selector_keys) { + selector_keys_mapped.Add(std::string(it)); + } + + Protobuf::RepeatedPtrField fallback_keys_subset_mapped; + for (const auto& it : fallback_keys_subset) { + fallback_keys_subset_mapped.Add(std::string(it)); + } + + return std::make_shared( + selector_keys_mapped, fallback_policy, fallback_keys_subset_mapped, single_host_per_subset); + } + + SubsetSelectorPtr makeSelector( + const std::set& selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy fallback_policy = + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED) { + return makeSelector(selector_keys, fallback_policy, {}); + } + + void modifyHosts(HostVector add, HostVector remove, absl::optional add_in_locality = {}, + uint32_t priority = 0) { + MockHostSet& host_set = *priority_set_.getMockHostSet(priority); + for (const auto& host : remove) { + auto it = std::find(host_set.hosts_.begin(), host_set.hosts_.end(), host); + if (it != host_set.hosts_.end()) { + host_set.hosts_.erase(it); + } + host_set.healthy_hosts_ = host_set.hosts_; + + std::vector locality_hosts_copy = host_set.hosts_per_locality_->get(); + for (auto& locality_hosts : locality_hosts_copy) { + auto it = std::find(locality_hosts.begin(), locality_hosts.end(), host); + if (it != locality_hosts.end()) { + locality_hosts.erase(it); + } + } + host_set.hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); + host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; + } + + if (GetParam() == UpdateOrder::RemovesFirst && !remove.empty()) { + host_set.runCallbacks({}, remove); + } + + for (const auto& host : add) { + host_set.hosts_.emplace_back(host); + host_set.healthy_hosts_ = host_set.hosts_; + + if (add_in_locality) { + std::vector locality_hosts_copy = host_set.hosts_per_locality_->get(); + locality_hosts_copy[add_in_locality.value()].emplace_back(host); + host_set.hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); + host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; + } + } + + if (GetParam() == UpdateOrder::RemovesFirst) { + if (!add.empty()) { + host_set.runCallbacks(add, {}); + } + } else if (!add.empty() || !remove.empty()) { + host_set.runCallbacks(add, remove); + } + } + + void modifyLocalHosts(HostVector add, HostVector remove, uint32_t add_in_locality) { + for (const auto& host : remove) { + auto it = std::find(local_hosts_->begin(), local_hosts_->end(), host); + if (it != local_hosts_->end()) { + local_hosts_->erase(it); + } + + std::vector locality_hosts_copy = local_hosts_per_locality_->get(); + for (auto& locality_hosts : locality_hosts_copy) { + auto it = std::find(locality_hosts.begin(), locality_hosts.end(), host); + if (it != locality_hosts.end()) { + locality_hosts.erase(it); + } + } + local_hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); + } + + if (GetParam() == UpdateOrder::RemovesFirst && !remove.empty()) { + local_priority_set_.updateHosts( + 0, + updateHostsParams(local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), + local_hosts_per_locality_), + {}, {}, remove, absl::nullopt); + } + + for (const auto& host : add) { + local_hosts_->emplace_back(host); + std::vector locality_hosts_copy = local_hosts_per_locality_->get(); + locality_hosts_copy[add_in_locality].emplace_back(host); + local_hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); + } + + if (GetParam() == UpdateOrder::RemovesFirst) { + if (!add.empty()) { + local_priority_set_.updateHosts( + 0, + updateHostsParams(local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), + local_hosts_per_locality_), + {}, add, {}, absl::nullopt); + } + } else if (!add.empty() || !remove.empty()) { + local_priority_set_.updateHosts( + 0, + updateHostsParams(local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), + local_hosts_per_locality_), + {}, add, remove, absl::nullopt); + } + } + + void doLbTypeTest(LoadBalancerType type) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + lb_type_ = type; + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}}}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + + HostSharedPtr added_host = makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}); + modifyHosts({added_host}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); + } + + MetadataConstSharedPtr buildMetadata(const std::string& version, bool is_default = false) const { + envoy::config::core::v3::Metadata metadata; + + if (!version.empty()) { + Envoy::Config::Metadata::mutableMetadataValue( + metadata, Config::MetadataFilters::get().ENVOY_LB, "version") + .set_string_value(version); + } + + if (is_default) { + Envoy::Config::Metadata::mutableMetadataValue( + metadata, Config::MetadataFilters::get().ENVOY_LB, "default") + .set_string_value("true"); + } + + return std::make_shared(metadata); + } + + MetadataConstSharedPtr buildMetadataWithStage(const std::string& version, + const std::string& stage = "") const { + envoy::config::core::v3::Metadata metadata; + + if (!version.empty()) { + Envoy::Config::Metadata::mutableMetadataValue( + metadata, Config::MetadataFilters::get().ENVOY_LB, "version") + .set_string_value(version); + } + + if (!stage.empty()) { + Envoy::Config::Metadata::mutableMetadataValue( + metadata, Config::MetadataFilters::get().ENVOY_LB, "stage") + .set_string_value(stage); + } + + return std::make_shared(metadata); + } + + ProtobufWkt::Value valueFromJson(std::string json) { + ProtobufWkt::Value v; + TestUtility::loadFromJson(json, v); + return v; + } + + LoadBalancerType lb_type_{LoadBalancerType::RoundRobin}; + NiceMock priority_set_; + MockHostSet& host_set_ = *priority_set_.getMockHostSet(0); + // Mock subset info is used for testing most logic. + NiceMock subset_info_; + // Actual subset info is used for testing actual subset config parsing and behavior. + std::unique_ptr actual_subset_info_; + std::shared_ptr info_{new NiceMock()}; + envoy::config::cluster::v3::Cluster::RingHashLbConfig ring_hash_lb_config_; + envoy::config::cluster::v3::Cluster::MaglevLbConfig maglev_lb_config_; + envoy::config::cluster::v3::Cluster::LeastRequestLbConfig least_request_lb_config_; + envoy::config::cluster::v3::Cluster::RoundRobinLbConfig round_robin_lb_config_; + envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_; + NiceMock runtime_; + NiceMock random_; + Stats::IsolatedStoreImpl stats_store_; + Stats::ScopeSharedPtr scope_; + ClusterLbStatNames stat_names_; + ClusterLbStats stats_; + PrioritySetImpl local_priority_set_; + HostVectorSharedPtr local_hosts_; + HostsPerLocalitySharedPtr local_hosts_per_locality_; + std::shared_ptr lb_; +}; + +TEST_F(SubsetLoadBalancerTest, NoFallback) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + init(); + + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); + + EXPECT_EQ(nullptr, lb_->peekAnotherHost(nullptr)); + EXPECT_FALSE(lb_->lifetimeCallbacks().has_value()); + std::vector hash_key; + auto mock_host = std::make_shared>(); + EXPECT_FALSE(lb_->selectExistingConnection(nullptr, *mock_host, hash_key).has_value()); +} + +// Validate that SubsetLoadBalancer unregisters its priority set member update +// callback. Regression for heap-use-after-free. +TEST_F(SubsetLoadBalancerTest, DeregisterCallbacks) { + init(); + lb_.reset(); + host_set_.runCallbacks({}, {}); +} + +TEST_P(SubsetLoadBalancerTest, NoFallbackAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + init(); + + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + + modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}})}, {host_set_.hosts_.back()}); + + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); +} + +TEST_F(SubsetLoadBalancerTest, FallbackAnyEndpoint) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + init(); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); +} + +TEST_P(SubsetLoadBalancerTest, FallbackAnyEndpointAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + init(); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + + HostSharedPtr added_host = makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}); + modifyHosts({added_host}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); +} + +TEST_F(SubsetLoadBalancerTest, FallbackDefaultSubset) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); + EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); +} + +TEST_F(SubsetLoadBalancerTest, FallbackPanicMode) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + EXPECT_CALL(subset_info_, panicModeAny()).WillRepeatedly(Return(true)); + + // The default subset will be empty. + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "none"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }); + + EXPECT_TRUE(lb_->chooseHost(nullptr) != nullptr); + EXPECT_EQ(1U, stats_.lb_subsets_fallback_panic_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); +} + +TEST_P(SubsetLoadBalancerTest, FallbackPanicModeWithUpdates) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + EXPECT_CALL(subset_info_, panicModeAny()).WillRepeatedly(Return(true)); + + // The default subset will be empty. + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "none"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + init({{"tcp://127.0.0.1:80", {{"version", "default"}}}}); + EXPECT_TRUE(lb_->chooseHost(nullptr) != nullptr); + + // Removing current host, adding a new one. + HostSharedPtr added_host = makeHost("tcp://127.0.0.2:8000", {{"version", "new"}}); + modifyHosts({added_host}, {host_set_.hosts_[0]}); + + EXPECT_EQ(1, host_set_.hosts_.size()); + EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); +} + +TEST_P(SubsetLoadBalancerTest, FallbackDefaultSubsetAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); + + HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:8000", {{"version", "new"}}); + HostSharedPtr added_host2 = makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}); + + modifyHosts({added_host1, added_host2}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host2, lb_->chooseHost(nullptr)); +} + +TEST_F(SubsetLoadBalancerTest, FallbackEmptyDefaultSubsetConvertsToAnyEndpoint) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + EXPECT_CALL(subset_info_, defaultSubset()) + .WillRepeatedly(ReturnRef(ProtobufWkt::Struct::default_instance())); + + init(); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); + EXPECT_EQ(2U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_selected_.value()); +} + +TEST_F(SubsetLoadBalancerTest, FallbackOnUnknownMetadata) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + init(); + + TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); + TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_unknown_value)); +} + +TEST_F(SubsetLoadBalancerTest, BalancesSubset) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_11({{"version", "1.1"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); + EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_selected_.value()); +} + +TEST_P(SubsetLoadBalancerTest, BalancesSubsetAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_11({{"version", "1.1"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); + EXPECT_EQ(2U, stats_.lb_subsets_created_.value()); + + modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {host_set_.hosts_[1], host_set_.hosts_[2]}); + + TestLoadBalancerContext context_12({{"version", "1.2"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_12)); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabled) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + { + TestLoadBalancerContext context({{"version", "1.2"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + } + TestLoadBalancerContext context({{"version", "1.2.1"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabledMultipleLists) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), + makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.2", "1.2"}}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_TRUE(host_set_.hosts()[2] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[2] == lb_->chooseHost(&context)); + } + { + // This should LB between both hosts marked with version 1.2. + TestLoadBalancerContext context({{"version", "1.2"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + { + // Choose a host multiple times to ensure that hosts()[0] is the *only* + // thing selected for this subset. + TestLoadBalancerContext context({{"version", "1.2.1"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + } + + TestLoadBalancerContext context({{"version", "1.2.2"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabledMultipleListsForSingleHost) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version", "hardware"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}, + {"hardware", std::vector{"a", "b"}}}), + makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.1", "1.1.1"}}, + {"hardware", std::vector{"b", "c"}}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.2"}, {"hardware", "a"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + } + + { + TestLoadBalancerContext context({{"version", "1.1"}, {"hardware", "b"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + + { + TestLoadBalancerContext context({{"version", "1.1"}, {"hardware", "a"}}); + EXPECT_TRUE(nullptr == lb_->chooseHost(&context)); + } + + TestLoadBalancerContext context({{"version", "1.2.1"}, {"hardware", "b"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyDisable) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + TestLoadBalancerContext context({{"version", "1.2"}}); + EXPECT_TRUE(nullptr == lb_->chooseHost(&context)); +} + +// Test that adding backends to a failover group causes no problems. +TEST_P(SubsetLoadBalancerTest, UpdateFailover) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + + // Start with an empty lb. Choosing a host should result in failure. + init({}); + EXPECT_TRUE(nullptr == lb_->chooseHost(&context_10).get()); + + // Add hosts to the group at priority 1. + // These hosts should be selected as there are no healthy hosts with priority 0 + modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 1); + EXPECT_FALSE(nullptr == lb_->chooseHost(&context_10).get()); + + // Finally update the priority 0 hosts. The LB should now select hosts. + modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + EXPECT_FALSE(nullptr == lb_->chooseHost(&context_10).get()); +} + +TEST_P(SubsetLoadBalancerTest, OnlyMetadataChanged) { + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_12({{"version", "1.2"}}); + TestLoadBalancerContext context_13({{"version", "1.3"}}); + TestLoadBalancerContext context_default({{"default", "true"}}); + + std::vector subset_selectors = { + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"default"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"default", "true"}}); + + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, + {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"default", "true"}}}}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); + + // Swap the default version. + host_set_.hosts_[0]->metadata(buildMetadata("1.2", true)); + host_set_.hosts_[1]->metadata(buildMetadata("1.0")); + + host_set_.runCallbacks({}, {}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_13)); + + // Bump 1.0 to 1.3, one subset should be removed. + host_set_.hosts_[1]->metadata(buildMetadata("1.3")); + + // No hosts added nor removed, so we bypass modifyHosts(). + host_set_.runCallbacks({}, {}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + + // Rollback from 1.3 to 1.0. + host_set_.hosts_[1]->metadata(buildMetadata("1.0")); + + host_set_.runCallbacks({}, {}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_13)); + + // Make 1.0 default again. + host_set_.hosts_[1]->metadata(buildMetadata("1.0", true)); + host_set_.hosts_[0]->metadata(buildMetadata("1.2")); + + host_set_.runCallbacks({}, {}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); +} + +TEST_P(SubsetLoadBalancerTest, EmptySubsetsPurged) { + std::vector subset_selectors = {makeSelector({"version"}), + makeSelector({"version", "stage"})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + // Simple add and remove. + init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, + {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"stage", "prod"}}}}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + + host_set_.hosts_[0]->metadata(buildMetadataWithStage("1.3")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); + + // Move host that was in the version + stage subset into a new version only subset. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.4")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_removed_.value()); + + // Create a new version + stage subset. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.5", "devel")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(7U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_removed_.value()); + + // Now move it back to its original version + stage subset. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.0", "prod")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(9U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(6U, stats_.lb_subsets_removed_.value()); + + // Finally, remove the original version + stage subset again. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.6")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(10U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(8U, stats_.lb_subsets_removed_.value()); +} + +TEST_P(SubsetLoadBalancerTest, EmptySubsetsPurgedCollapsed) { + std::vector subset_selectors = {makeSelector({"version"}), + makeSelector({"version", "stage"})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + // Init subsets. + init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, + {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"stage", "prod"}}}}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + + // Get rid of 1.0. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2", "prod")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); + + // Get rid of stage prod. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(1U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_removed_.value()); + + // Add stage prod back. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2", "prod")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_removed_.value()); +} + +TEST_P(SubsetLoadBalancerTest, EmptySubsetsPurgedVersionChanged) { + std::vector subset_selectors = {makeSelector({"version"}), + makeSelector({"version", "stage"})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + // Init subsets. + init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, + {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"stage", "prod"}}}}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + + // Get rid of 1.0. + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.2", "prod")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); + + // Change versions. + host_set_.hosts_[0]->metadata(buildMetadataWithStage("1.3")); + host_set_.hosts_[1]->metadata(buildMetadataWithStage("1.4", "prod")); + host_set_.runCallbacks({}, {}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(7U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_removed_.value()); +} + +TEST_P(SubsetLoadBalancerTest, MetadataChangedHostsAddedRemoved) { + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_12({{"version", "1.2"}}); + TestLoadBalancerContext context_13({{"version", "1.3"}}); + TestLoadBalancerContext context_14({{"version", "1.4"}}); + TestLoadBalancerContext context_default({{"default", "true"}}); + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"default", "true"}}); + + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + std::vector subset_selectors = { + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"default"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, + {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"default", "true"}}}}); + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); + + // Swap the default version. + host_set_.hosts_[0]->metadata(buildMetadata("1.2", true)); + host_set_.hosts_[1]->metadata(buildMetadata("1.0")); + + // Add a new host. + modifyHosts({makeHost("tcp://127.0.0.1:8002", {{"version", "1.3"}})}, {}); + + EXPECT_EQ(4U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_13)); + + // Swap default again and remove the previous one. + host_set_.hosts_[0]->metadata(buildMetadata("1.2")); + host_set_.hosts_[1]->metadata(buildMetadata("1.0", true)); + + modifyHosts({}, {host_set_.hosts_[2]}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); + + // Swap the default version once more, this time adding a new host and removing + // the current default version. + host_set_.hosts_[0]->metadata(buildMetadata("1.2", true)); + host_set_.hosts_[1]->metadata(buildMetadata("1.0")); + + modifyHosts({makeHost("tcp://127.0.0.1:8003", {{"version", "1.4"}})}, {host_set_.hosts_[1]}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_13)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_14)); + + // Make 1.4 default, without hosts being added/removed. + host_set_.hosts_[0]->metadata(buildMetadata("1.2")); + host_set_.hosts_[1]->metadata(buildMetadata("1.4", true)); + + host_set_.runCallbacks({}, {}); + + EXPECT_EQ(3U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(5U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_removed_.value()); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_12)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_default)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_13)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_14)); +} + +TEST_P(SubsetLoadBalancerTest, UpdateRemovingLastSubsetHost) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }); + + HostSharedPtr host_v10 = host_set_.hosts_[0]; + HostSharedPtr host_v11 = host_set_.hosts_[1]; + + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_EQ(host_v10, lb_->chooseHost(&context)); + EXPECT_EQ(1U, stats_.lb_subsets_selected_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_created_.value()); + + modifyHosts({}, {host_v10}); + + // fallback to any endpoint + EXPECT_EQ(host_v11, lb_->chooseHost(&context)); + EXPECT_EQ(1U, stats_.lb_subsets_selected_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(2U, stats_.lb_subsets_created_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); +} + +TEST_P(SubsetLoadBalancerTest, UpdateRemovingUnknownHost) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"stage", "version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"stage", "prod"}, {"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"stage", "prod"}, {"version", "1.1"}}}, + }); + + TestLoadBalancerContext context({{"stage", "prod"}, {"version", "1.0"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context)); + + modifyHosts({}, {makeHost("tcp://127.0.0.1:8000", {{"version", "1.2"}}), + makeHost("tcp://127.0.0.1:8001", {{"stage", "prod"}, {"version", "1.2"}})}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context)); +} + +TEST_F(SubsetLoadBalancerTest, UpdateModifyingOnlyHostHealth) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"hardware"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_11({{"version", "1.1"}}); + + // All hosts are healthy. + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); + + host_set_.hosts_[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + host_set_.hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_OUTLIER_CHECK); + host_set_.healthy_hosts_ = {host_set_.hosts_[1], host_set_.hosts_[3]}; + host_set_.runCallbacks({}, {}); + + // Unhealthy hosts are excluded. + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); +} + +TEST_F(SubsetLoadBalancerTest, BalancesDisjointSubsets) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"hardware"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "std"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"hardware", "bigmem"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}, {"hardware", "std"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}, {"hardware", "bigmem"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_bigmem({{"hardware", "bigmem"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_bigmem)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_bigmem)); +} + +TEST_F(SubsetLoadBalancerTest, BalancesOverlappingSubsets) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"stage", "version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.0"}, {"stage", "off"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:84", {{"version", "999"}, {"stage", "dev"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_10_prod({{"version", "1.0"}, {"stage", "prod"}}); + TestLoadBalancerContext context_dev({{"version", "999"}, {"stage", "dev"}}); + TestLoadBalancerContext context_unknown({{"version", "2.0"}, {"stage", "prod"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_10)); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10_prod)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10_prod)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10_prod)); + + EXPECT_EQ(host_set_.hosts_[4], lb_->chooseHost(&context_dev)); + EXPECT_EQ(host_set_.hosts_[4], lb_->chooseHost(&context_dev)); + + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown)); +} + +TEST_F(SubsetLoadBalancerTest, BalancesNestedSubsets) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"stage", "version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"stage"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.0"}, {"stage", "off"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:84", {{"version", "999"}, {"stage", "dev"}}}, + }); + + TestLoadBalancerContext context_prod({{"stage", "prod"}}); + TestLoadBalancerContext context_prod_10({{"version", "1.0"}, {"stage", "prod"}}); + TestLoadBalancerContext context_unknown_stage({{"stage", "larval"}}); + TestLoadBalancerContext context_unknown_version({{"version", "2.0"}, {"stage", "prod"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_prod)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_prod)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_prod)); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_prod_10)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_prod_10)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_prod_10)); + + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_stage)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_version)); +} + +TEST_F(SubsetLoadBalancerTest, IgnoresUnselectedMetadata) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "ignored"}}}, + {"tcp://127.0.0.1:81", {{"ignore", "value"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + }); + + TestLoadBalancerContext context_ignore({{"ignore", "value"}}); + TestLoadBalancerContext context_version({{"version", "1.0"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_version)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_version)); + + EXPECT_EQ(nullptr, lb_->chooseHost(&context_ignore)); +} + +TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) { + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + HostVector hosts; + hosts.emplace_back(makeTestHost(info_, "tcp://127.0.0.1:80", simTime())); + hosts.emplace_back(makeHost("tcp://127.0.0.1:81", {{"version", "1.0"}})); + + host_set_.hosts_ = hosts; + host_set_.hosts_per_locality_ = makeHostsPerLocality({hosts}); + + host_set_.healthy_hosts_ = host_set_.hosts_; + host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; + + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); + + TestLoadBalancerContext context_version({{"version", "1.0"}}); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_version)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_version)); +} + +// TODO(mattklein123): The following 4 tests verify basic functionality with all sub-LB tests. +// Optimally these would also be some type of TEST_P, but that is a little bit complicated as +// modifyHosts() also needs params. Clean this up. +TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRoundRobin) { + doLbTypeTest(LoadBalancerType::RoundRobin); + auto tester = SubsetLoadBalancerInternalStateTester(lb_); + tester.validateLbTypeConfigs(); +} + +TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesLeastRequest) { + doLbTypeTest(LoadBalancerType::LeastRequest); + auto tester = SubsetLoadBalancerInternalStateTester(lb_); + tester.validateLbTypeConfigs(); +} + +TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRandom) { + doLbTypeTest(LoadBalancerType::Random); + auto tester = SubsetLoadBalancerInternalStateTester(lb_); + tester.validateLbTypeConfigs(); +} + +TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRingHash) { + doLbTypeTest(LoadBalancerType::RingHash); + auto tester = SubsetLoadBalancerInternalStateTester(lb_); + tester.validateLbTypeConfigs(); +} + +TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesMaglev) { + doLbTypeTest(LoadBalancerType::Maglev); + auto tester = SubsetLoadBalancerInternalStateTester(lb_); + tester.validateLbTypeConfigs(); +} + +TEST_F(SubsetLoadBalancerTest, ZoneAwareFallback) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = {makeSelector( + {"x"}, envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + common_config_.mutable_healthy_panic_threshold()->set_value(40); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 40)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + }, + { + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.1"}}}, + }}, + {{ + {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, + }, + { + {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, + }}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(nullptr)); +} + +TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = {makeSelector( + {"x"}, envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + }, + { + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.1"}}}, + }}, + {{ + {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, + }, + { + {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, + }}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(nullptr)); + + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + + modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}, local_locality)}, + {host_set_.hosts_[0]}, absl::optional(0)); + + modifyLocalHosts({makeHost("tcp://127.0.0.1:9000", {{"version", "1.0"}}, local_locality)}, + {local_hosts_->at(0)}, 0); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); +} + +TEST_F(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubset) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "new"}}}, + {"tcp://127.0.0.1:83", {{"version", "default"}}}, + {"tcp://127.0.0.1:84", {{"version", "new"}}}, + {"tcp://127.0.0.1:85", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:86", {{"version", "new"}}}, + {"tcp://127.0.0.1:87", {{"version", "default"}}}, + {"tcp://127.0.0.1:88", {{"version", "new"}}}, + {"tcp://127.0.0.1:89", {{"version", "default"}}}, + }}, + {{ + {"tcp://127.0.0.1:90", {{"version", "new"}}}, + {"tcp://127.0.0.1:91", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:92", {{"version", "new"}}}, + {"tcp://127.0.0.1:93", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:94", {{"version", "new"}}}, + {"tcp://127.0.0.1:95", {{"version", "default"}}}, + }}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); +} + +TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubsetAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "new"}}}, + {"tcp://127.0.0.1:83", {{"version", "default"}}}, + {"tcp://127.0.0.1:84", {{"version", "new"}}}, + {"tcp://127.0.0.1:85", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:86", {{"version", "new"}}}, + {"tcp://127.0.0.1:87", {{"version", "default"}}}, + {"tcp://127.0.0.1:88", {{"version", "new"}}}, + {"tcp://127.0.0.1:89", {{"version", "default"}}}, + }}, + {{ + {"tcp://127.0.0.1:90", {{"version", "new"}}}, + {"tcp://127.0.0.1:91", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:92", {{"version", "new"}}}, + {"tcp://127.0.0.1:93", {{"version", "default"}}}, + }, + { + {"tcp://127.0.0.1:94", {{"version", "new"}}}, + {"tcp://127.0.0.1:95", {{"version", "default"}}}, + }}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); + + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + + modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}, local_locality)}, + {host_set_.hosts_[1]}, absl::optional(0)); + + modifyLocalHosts({local_hosts_->at(1)}, + {makeHost("tcp://127.0.0.1:9001", {{"version", "default"}}, local_locality)}, 0); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(nullptr)); +} + +TEST_F(SubsetLoadBalancerTest, ZoneAwareBalancesSubsets) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, + }}, + {{ + {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:93", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:94", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:95", {{"version", "1.1"}}}, + }}); + + TestLoadBalancerContext context({{"version", "1.1"}}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ZoneAwareBalancesSubsetsAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, + }}, + {{ + {"tcp://127.0.0.1:90", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:91", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:92", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:93", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:94", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:95", {{"version", "1.1"}}}, + }}); + + TestLoadBalancerContext context({{"version", "1.1"}}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); + + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + + modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "1.1"}}, local_locality)}, + {host_set_.hosts_[1]}, absl::optional(0)); + + modifyLocalHosts({local_hosts_->at(1)}, + {makeHost("tcp://127.0.0.1:9001", {{"version", "1.1"}}, local_locality)}, 0); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + + // Force request out of small zone. + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); +} + +TEST_F(SubsetLoadBalancerTest, ZoneAwareComplicatedBalancesSubsets) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + // L=local cluster host + // U=upstream host + // + // residuals + // A: 2L 0U 0.00% + // B: 2L 2U 6.67% + // C: 2L 2U 6.67% + // D: 0L 1U 20.00% + // total: 6L 5U 33.33% + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:90", {{"version", "1.1"}}}, + }}, + {{ + {"tcp://127.0.0.1:91", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:92", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:93", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:94", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:95", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:96", {{"version", "1.1"}}}, + }}); + + TestLoadBalancerContext context({{"version", "1.1"}}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(666)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(667)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[2][1], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(1334)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[3][0], lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ZoneAwareComplicatedBalancesSubsetsAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillRepeatedly(Return(2)); + + // Before update: + // + // L=local cluster host + // U=upstream host + // + // residuals + // A: 2L 0U 0.00% + // B: 2L 2U 6.67% + // C: 2L 2U 6.67% + // D: 0L 1U 20.00% + // total: 6L 5U 33.33% + + zoneAwareInit({{ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:86", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:87", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:88", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:89", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:90", {{"version", "1.1"}}}, + }}, + {{ + {"tcp://127.0.0.1:91", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:92", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:93", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:94", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:95", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:96", {{"version", "1.1"}}}, + }}); + + TestLoadBalancerContext context({{"version", "1.1"}}); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(666)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(667)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[2][1], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(0)).WillOnce(Return(1334)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[3][0], lb_->chooseHost(&context)); + + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + envoy::config::core::v3::Locality locality_2; + locality_2.set_zone("2"); + + modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "1.1"}}, local_locality)}, {}, + absl::optional(0)); + + modifyLocalHosts({makeHost("tcp://127.0.0.1:9001", {{"version", "1.1"}}, locality_2)}, {}, 2); + + // After update: + // + // L=local cluster host + // U=upstream host + // + // residuals + // A: 2L 1U 0.00% + // B: 2L 2U 4.76% + // C: 3L 2U 0.00% + // D: 0L 1U 16.67% + // total: 7L 6U 21.42% + // + // Chance of sampling local host in zone 0: 58.34% + + EXPECT_CALL(random_, random()) + .WillOnce(Return(0)) + .WillOnce(Return(5830)); // 58.31% local routing chance due to rounding error + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(5831)).WillOnce(Return(475)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(476)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[3][0], lb_->chooseHost(&context)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2143)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); +} + +TEST_F(SubsetLoadBalancerTest, DescribeMetadata) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + init(); + + ProtobufWkt::Value str_value; + str_value.set_string_value("abc"); + + ProtobufWkt::Value num_value; + num_value.set_number_value(100); + + auto tester = SubsetLoadBalancerInternalStateTester(lb_); + tester.testDescribeMetadata("version=\"abc\"", {{"version", str_value}}); + tester.testDescribeMetadata("number=100", {{"number", num_value}}); + tester.testDescribeMetadata("x=\"abc\", y=100", {{"x", str_value}, {"y", num_value}}); + tester.testDescribeMetadata("y=100, x=\"abc\"", {{"y", num_value}, {"x", str_value}}); + tester.testDescribeMetadata("", {}); +} + +TEST_F(SubsetLoadBalancerTest, DisabledLocalityWeightAwareness) { + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + + // We configure a weighted host set that heavily favors the second locality. + configureWeightedHostSet( + { + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + host_set_, {1, 100}); + + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); + + TestLoadBalancerContext context({{"version", "1.1"}}); + + // Since we don't respect locality weights, the first locality is selected. + EXPECT_CALL(random_, random()).WillOnce(Return(0)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(&context)); +} + +// Verifies that we do *not* invoke coarseHealth() on hosts when constructing the load balancer. +// Since health is modified concurrently from multiple threads, it is not safe to call on the worker +// threads. +TEST_F(SubsetLoadBalancerTest, DoesNotCheckHostHealth) { + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + + auto mock_host = std::make_shared(); + HostVector hosts{mock_host}; + host_set_.hosts_ = hosts; + + EXPECT_CALL(*mock_host, weight()).WillRepeatedly(Return(1)); + + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); +} + +TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) { + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); + + // We configure a weighted host set that heavily favors the second locality. + configureWeightedHostSet( + { + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + host_set_, {1, 100}); + + common_config_.mutable_locality_weighted_lb_config(); + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); + + TestLoadBalancerContext context({{"version", "1.1"}}); + + // Since we respect locality weights, the second locality is selected. + EXPECT_CALL(random_, random()).WillOnce(Return(0)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(&context)); +} + +TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeights) { + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); + + // We configure a weighted host set is weighted equally between each locality. + configureWeightedHostSet( + { + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + host_set_, {50, 50}); + + common_config_.mutable_locality_weighted_lb_config(); + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); + TestLoadBalancerContext context({{"version", "1.1"}}); + + // Since we scale the locality weights by number of hosts removed, we expect to see the second + // locality to be selected less because we've excluded more hosts in that locality than in the + // first. + // The localities are split 50/50, but because of the scaling we expect to see 66/33 instead. + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][3], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); +} + +TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeightsRounding) { + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); + + // We configure a weighted host set where the locality weights are very low to test + // that we are rounding computation instead of flooring it. + configureWeightedHostSet( + { + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + { + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:84", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.1"}}}, + }, + host_set_, {2, 2}); + + common_config_.mutable_locality_weighted_lb_config(); + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); + TestLoadBalancerContext context({{"version", "1.0"}}); + + // We expect to see a 33/66 split because 2 * 1 / 2 = 1 and 2 * 3 / 4 = 1.5 -> 2 + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][2], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); +} + +// Regression for bug where missing locality weights crashed scaling and locality aware subset LBs. +TEST_F(SubsetLoadBalancerTest, ScaleLocalityWeightsWithNoLocalityWeights) { + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); + EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); + + configureHostSet( + { + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.1"}}}, + }, + host_set_); + + auto child_lb_creator = std::make_unique( + lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, + least_request_lb_config_, common_config_); + lb_ = std::make_shared( + subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, + *stats_store_.rootScope(), runtime_, random_, simTime()); +} + +TEST_P(SubsetLoadBalancerTest, GaugesUpdatedOnDestroy) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + }); + + EXPECT_EQ(1U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(0U, stats_.lb_subsets_removed_.value()); + + lb_ = nullptr; + + EXPECT_EQ(0U, stats_.lb_subsets_active_.value()); + EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorNoFallbackPerSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_11({{"version", "1.1"}}); + TestLoadBalancerContext context_12({{"version", "1.2"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_12)); + EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_selected_.value()); +} + +TEST_P(SubsetLoadBalancerTest, FallbackNotDefinedForIntermediateSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = { + makeSelector( + {"stage"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"stage", "version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"stage", "canary"}}}}); + + TestLoadBalancerContext context_match_host0({{"stage", "dev"}}); + TestLoadBalancerContext context_stage_nx({{"stage", "test"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_stage_nx)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_stage_nx)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorFallbackOverridesTopLevelOne) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init(); + + TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); + TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorNoFallbackMatchesTopLevelOne) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init(); + + TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); + TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); + + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_key)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); +} + +TEST_F(SubsetLoadBalancerTest, AllowRedundantKeysForSubset) { + // Yaml config for subset load balancer. + const std::string yaml = R"EOF( + subset_selectors: + - keys: + - A + fallback_policy: NO_FALLBACK + - keys: + - A + - B + fallback_policy: NO_FALLBACK + - keys: + - A + - B + - C + fallback_policy: NO_FALLBACK + - keys: + - A + - D + fallback_policy: NO_FALLBACK + - keys: + - version + - stage + fallback_policy: NO_FALLBACK + fallback_policy: NO_FALLBACK + allow_redundant_keys: true + )EOF"; + + envoy::extensions::load_balancing_policies::subset::v3::Subset subset_proto_config; + TestUtility::loadFromYaml(yaml, subset_proto_config); + + actual_subset_info_ = std::make_unique(subset_proto_config); + // Always be true for the LoadBalancerSubsetInfoImpl. + EXPECT_TRUE(actual_subset_info_->isEnabled()); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:80", {{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-0"}, {"D", "D-V-0"}}}, + {"tcp://127.0.0.1:81", {{"A", "A-V-1"}, {"B", "B-V-1"}, {"C", "C-V-1"}, {"D", "D-V-1"}}}, + {"tcp://127.0.0.1:82", {{"A", "A-V-2"}, {"B", "B-V-2"}, {"C", "C-V-2"}, {"D", "D-V-2"}}}, + {"tcp://127.0.0.1:83", {{"A", "A-V-3"}, {"B", "B-V-3"}, {"C", "C-V-3"}, {"D", "D-V-3"}}}, + {"tcp://127.0.0.1:84", {{"A", "A-V-4"}, {"B", "B-V-4"}, {"C", "C-V-4"}, {"D", "D-V-4"}}}, + {"tcp://127.0.0.1:85", {{"version", "1.0"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:86", {{"version", "1.0"}, {"stage", "canary"}}}}, + {}, true); + + TestLoadBalancerContext context_empty( + std::initializer_list::value_type>{}); + context_empty.matches_.reset(); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_empty)); + + // Request metadata is same with {version, stage}. + // version, stage will be kept and host 6 will be selected. + TestLoadBalancerContext context_v_s_0({{"version", "1.0"}, {"stage", "canary"}}); + EXPECT_EQ(host_set_.hosts_[6], lb_->chooseHost(&context_v_s_0)); + + // Request metadata is superset of {version, stage}. The redundant key will be ignored. + // version, stage will be kept and host 5 will be selected. + TestLoadBalancerContext context_v_s_1({{"version", "1.0"}, {"stage", "dev"}, {"redundant", "X"}}); + EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&context_v_s_1)); + + // Request metadata is superset of {version, stage}. The redundant key will be ignored. + // But one of value not match, so no host will be selected. + TestLoadBalancerContext context_v_s_2( + {{"version", "1.0"}, {"stage", "prod"}, {"redundant", "X"}}); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_v_s_2)); + + // Request metadata is same with {A, B, C} and is superset of selectors {A}, {A, B}. + // All A, B, C will be kept and host 0 will be selected. + TestLoadBalancerContext context_0({{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-0"}}); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_0)); + + // Request metadata is same with {A, B, C} and is superset of selectors {A}, {A, B}. + // All A, B, C will be kept But one of value not match, so no host will be selected. + TestLoadBalancerContext context_1({{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-X"}}); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_1)); + + // Request metadata is superset of selectors {A}, {A, B} {A, B, C}, {A, D}, the longest win. + // A, B, C will be kept and D will be ignored, so host 1 will be selected. + TestLoadBalancerContext context_2( + {{"A", "A-V-1"}, {"B", "B-V-1"}, {"C", "C-V-1"}, {"D", "D-V-X"}}); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_2)); + + // Request metadata is superset of selectors {A}, {A, B} {A, B, C}, {A, D}, the longest win. + // A, B, C will be kept and D will be ignored, but one of value not match, so no host will be + // selected. + TestLoadBalancerContext context_3( + {{"A", "A-V-1"}, {"B", "B-V-1"}, {"C", "C-V-X"}, {"D", "D-V-X"}}); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_3)); + + // Request metadata is superset of selectors {A}, {A, B}, {A, D}, the longest and first win. + // Only A, B will be kept and D will be ignored, so host 2 will be selected. + TestLoadBalancerContext context_4({{"A", "A-V-2"}, {"B", "B-V-2"}, {"D", "D-V-X"}}); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_4)); + + // Request metadata is superset of selectors {A}, {A, B}, {A, D}, the longest and first win. + // Only A, B will be kept and D will be ignored, but one of value not match, so no host will be + // selected. + TestLoadBalancerContext context_5({{"A", "A-V-3"}, {"B", "B-V-X"}, {"D", "D-V-3"}}); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_5)); + + // Request metadata is superset of selectors {A}, {A, D}, the longest win. + // Only A, D will be kept and C will be ignored, so host 3 will be selected. + TestLoadBalancerContext context_6({{"A", "A-V-3"}, {"C", "C-V-X"}, {"D", "D-V-3"}}); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_6)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorDefaultAnyFallbackPerSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET), + makeSelector( + {"app"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT), + makeSelector( + {"foo"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"bar", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:81", {{"version", "0.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"app", "envoy"}}}, + {"tcp://127.0.0.1:84", {{"foo", "abc"}, {"bar", "default"}}}}); + + TestLoadBalancerContext context_ver_10({{"version", "1.0"}}); + TestLoadBalancerContext context_ver_nx({{"version", "x"}}); + TestLoadBalancerContext context_app({{"app", "envoy"}}); + TestLoadBalancerContext context_app_nx({{"app", "ngnix"}}); + TestLoadBalancerContext context_foo({{"foo", "abc"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_app_nx)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_app_nx)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_app)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_ver_nx)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_foo)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorDefaultAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); + + HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:8000", {{"version", "new"}}); + HostSharedPtr added_host2 = makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}); + + TestLoadBalancerContext context_ver_nx({{"version", "x"}}); + + modifyHosts({added_host1, added_host2}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host2, lb_->chooseHost(&context_ver_nx)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorAnyAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT)}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:81", {{"version", "1"}}}, + {"tcp://127.0.0.1:82", {{"version", "2"}}}, + }); + + TestLoadBalancerContext context_ver_nx({{"version", "x"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_ver_nx)); + + HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:83", {{"version", "3"}}); + + modifyHosts({added_host1}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host1, lb_->chooseHost(&context_ver_nx)); +} + +TEST_P(SubsetLoadBalancerTest, FallbackForCompoundSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"foo", "bar"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + std::vector subset_selectors = { + makeSelector( + {"version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED), + makeSelector( + {"version", "hardware", "stage"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK), + makeSelector( + {"version", "hardware"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET), + makeSelector( + {"version", "stage"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, + {"version"})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "c32"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"hardware", "c32"}, {"foo", "bar"}}}, + {"tcp://127.0.0.1:82", {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:83", {{"version", "2.0"}}}}); + + TestLoadBalancerContext context_match_host0({{"version", "1.0"}, {"hardware", "c32"}}); + TestLoadBalancerContext context_ver_nx({{"version", "x"}, {"hardware", "c32"}}); + TestLoadBalancerContext context_stage_nx( + {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "x"}}); + TestLoadBalancerContext context_hardware_nx( + {{"version", "2.0"}, {"hardware", "zzz"}, {"stage", "dev"}}); + TestLoadBalancerContext context_match_host2( + {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "dev"}}); + TestLoadBalancerContext context_ver_20({{"version", "2.0"}}); + TestLoadBalancerContext context_ver_stage_match_host2({{"version", "2.0"}, {"stage", "dev"}}); + TestLoadBalancerContext context_ver_stage_nx({{"version", "2.0"}, {"stage", "canary"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_ver_nx)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_hardware_nx)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_stage_nx)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_match_host2)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_match_host2)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_20)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_stage_match_host2)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_stage_match_host2)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_ver_stage_nx)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_stage_nx)); +} + +TEST_P(SubsetLoadBalancerTest, KeysSubsetFallbackChained) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + makeSelector( + {"stage"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK), + makeSelector( + {"stage", "version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, + {"stage"}), + makeSelector( + {"stage", "version", "hardware"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, + {"version", "stage"})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "c32"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:81", {{"version", "2.0"}, {"hardware", "c64"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.0"}, {"hardware", "c32"}, {"stage", "test"}}}}); + + TestLoadBalancerContext context_match_host0( + {{"version", "1.0"}, {"hardware", "c32"}, {"stage", "dev"}}); + TestLoadBalancerContext context_hw_nx( + {{"version", "2.0"}, {"hardware", "arm"}, {"stage", "dev"}}); + TestLoadBalancerContext context_ver_hw_nx( + {{"version", "1.2"}, {"hardware", "arm"}, {"stage", "dev"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_hw_nx)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_hw_nx)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_ver_hw_nx)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_ver_hw_nx)); +} + +TEST_P(SubsetLoadBalancerTest, KeysSubsetFallbackToNotExistingSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = {makeSelector( + {"stage", "version"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, + {"stage"})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "dev"}}}}); + + TestLoadBalancerContext context_nx({{"version", "1.0"}, {"stage", "test"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_nx)); + EXPECT_EQ(1U, stats_.lb_subsets_fallback_.value()); +} + +TEST_P(SubsetLoadBalancerTest, MetadataFallbackList) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + EXPECT_CALL(subset_info_, metadataFallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::FALLBACK_LIST)); + + std::vector subset_selectors = {makeSelector({"version"})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "2.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "3.0"}}}}); + + const auto version1_host = host_set_.hosts_[0]; + const auto version2_host = host_set_.hosts_[1]; + const auto version3_host = host_set_.hosts_[2]; + + // No context. + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + + TestLoadBalancerContext context_without_metadata({{"key", "value"}}); + context_without_metadata.matches_ = nullptr; + + // No metadata in context. + EXPECT_EQ(nullptr, lb_->chooseHost(&context_without_metadata)); + + TestLoadBalancerContext context_with_fallback({{"fallback_list", valueFromJson(R""""( + [ + {"version": "2.0"}, + {"version": "1.0"} + ] + )"""")}}); + + // version 2.0 is preferred, should be selected + EXPECT_EQ(version2_host, lb_->chooseHost(&context_with_fallback)); + EXPECT_EQ(version2_host, lb_->chooseHost(&context_with_fallback)); + EXPECT_EQ(version2_host, lb_->chooseHost(&context_with_fallback)); + + modifyHosts({}, {version2_host}); + + // version 1.0 is a fallback, should be used when host with version 2.0 is removed + EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_fallback)); + EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_fallback)); + EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_fallback)); + + // if fallback_list is not a list, it should be ignored + // regular metadata is in effect + ProtobufWkt::Value null_value; + null_value.set_null_value(ProtobufWkt::NullValue::NULL_VALUE); + TestLoadBalancerContext context_with_invalid_fallback_list_null( + {{"version", valueFromJson("\"3.0\"")}, {"fallback_list", null_value}}); + + EXPECT_EQ(version3_host, lb_->chooseHost(&context_with_invalid_fallback_list_null)); + EXPECT_EQ(version3_host, lb_->chooseHost(&context_with_invalid_fallback_list_null)); + + // should ignore fallback list entry which is not a struct + TestLoadBalancerContext context_with_invalid_fallback_list_entry( + {{"fallback_list", valueFromJson(R""""( + [ + "invalid string entry", + {"version": "1.0"} + ] + )"""")}}); + + EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_invalid_fallback_list_entry)); + EXPECT_EQ(version1_host, lb_->chooseHost(&context_with_invalid_fallback_list_entry)); + + // simple metadata with no fallback should work as usual + TestLoadBalancerContext context_no_fallback({{"version", "1.0"}}); + EXPECT_EQ(version1_host, lb_->chooseHost(&context_no_fallback)); + EXPECT_EQ(version1_host, lb_->chooseHost(&context_no_fallback)); + + // fallback metadata overrides regular metadata value + TestLoadBalancerContext context_fallback_overrides_metadata_value( + {{"version", valueFromJson("\"1.0\"")}, {"fallback_list", valueFromJson(R""""( + [ + {"hardware": "arm"}, + {"version": "5.0"}, + {"version": "3.0"} + ] + )"""")}}); + EXPECT_EQ(version3_host, lb_->chooseHost(&context_fallback_overrides_metadata_value)); + EXPECT_EQ(version3_host, lb_->chooseHost(&context_fallback_overrides_metadata_value)); +} + +TEST_P(SubsetLoadBalancerTest, MetadataFallbackDisabled) { + + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + EXPECT_CALL(subset_info_, metadataFallbackPolicy()) + .WillRepeatedly( + Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::METADATA_NO_FALLBACK)); + + std::vector subset_selectors = {makeSelector({"fallback_list"})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({{"tcp://127.0.0.1:80", {{"fallback_list", "lorem"}}}, + {"tcp://127.0.0.1:81", {{"fallback_list", "ipsum"}}}}); + + // should treat 'fallback_list' as a regular metadata key + TestLoadBalancerContext context({{"fallback_list", "ipsum"}}); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, MetadataFallbackAndSubsetFallback) { + + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + EXPECT_CALL(subset_info_, metadataFallbackPolicy()) + .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::FALLBACK_LIST)); + + std::vector subset_selectors = { + makeSelector( + {"hardware"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK), + makeSelector( + {"hardware", "stage"}, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET, + {"hardware"})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({{"tcp://127.0.0.1:80", {{"hardware", "c32"}, {"stage", "production"}}}, + {"tcp://127.0.0.1:81", {{"hardware", "c64"}, {"stage", "canary"}}}, + {"tcp://127.0.0.1:82", {{"hardware", "c64"}, {"stage", "production"}}}}); + + const auto c32_production_host = host_set_.hosts_[0]; + const auto c64_canary_host = host_set_.hosts_[1]; + const auto c64_production_host = host_set_.hosts_[2]; + + TestLoadBalancerContext context_canary_c32_preffered( + {{"stage", valueFromJson("\"canary\"")}, {"fallback_list", valueFromJson(R""""( + [ + {"hardware": "c32"}, + {"hardware": "c64"} + ] + )"""")}}); + + // Should select c32_production_host using first fallback entry, even + // when it doesn't match on requested 'stage' - because of the subset fallback policy. + // There is the c64_canary_host which exactly matches second fallback entry, but + // that entry is not used. + EXPECT_EQ(c32_production_host, lb_->chooseHost(&context_canary_c32_preffered)); + EXPECT_EQ(c32_production_host, lb_->chooseHost(&context_canary_c32_preffered)); + + TestLoadBalancerContext context_canary_c16_preffered( + {{"stage", valueFromJson("\"canary\"")}, {"fallback_list", valueFromJson(R""""( + [ + {"hardware": "c16"}, + {"hardware": "c64"} + ] + )"""")}}); + + // Should select c64_canary_host using second fallback entry. First fallback + // entry doesn't match anything even considering subset fallback policy. + EXPECT_EQ(c64_canary_host, lb_->chooseHost(&context_canary_c16_preffered)); + EXPECT_EQ(c64_canary_host, lb_->chooseHost(&context_canary_c16_preffered)); + + TestLoadBalancerContext context_unknown_or_c64({{"fallback_list", valueFromJson(R""""( + [ + {"unknown": "ipsum"}, + {"hardware": "c64"} + ] + )"""")}}); + + // should select any host using first fallback entry, because of ANY_ENDPOINT + // subset fallback policy + EXPECT_EQ(c32_production_host, lb_->chooseHost(&context_unknown_or_c64)); + EXPECT_EQ(c64_canary_host, lb_->chooseHost(&context_unknown_or_c64)); + EXPECT_EQ(c64_production_host, lb_->chooseHost(&context_unknown_or_c64)); +} + +INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerTest, + testing::ValuesIn({UpdateOrder::RemovesFirst, UpdateOrder::Simultaneous})); + +class SubsetLoadBalancerSingleHostPerSubsetTest : public SubsetLoadBalancerTest { +public: + SubsetLoadBalancerSingleHostPerSubsetTest() + : default_subset_selectors_({ + makeSelector({"key"}, true), + }) { + ON_CALL(subset_info_, subsetSelectors()).WillByDefault(ReturnRef(default_subset_selectors_)); + ON_CALL(subset_info_, fallbackPolicy()) + .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + } + + using SubsetLoadBalancerTest::init; + void init() { + init({ + {"tcp://127.0.0.1:80", {}}, + {"tcp://127.0.0.1:81", {{"key", "a"}}}, + {"tcp://127.0.0.1:82", {{"key", "b"}}}, + + }); + } + + using SubsetLoadBalancerTest::makeSelector; + SubsetSelectorPtr makeSelector(const std::set& selector_keys, + bool single_host_per_subset) { + return makeSelector( + selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED, {}, + single_host_per_subset); + } + + std::vector default_subset_selectors_; +}; + +TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, AcceptMultipleSelectors) { + std::vector subset_selectors = { + makeSelector({"version"}, false), + makeSelector({"stage"}, true), + }; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + ON_CALL(subset_info_, fallbackPolicy()) + .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + init({ + {"tcp://127.0.0.1:80", {}}, + {"tcp://127.0.0.1:81", {{"version", "v1"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:82", {{"version", "v1"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:83", {{"version", "v1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:84", {{"version", "v1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:85", {{"version", "v2"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:86", {{"version", "v2"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:87", {{"version", "v2"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:88", {{"version", "v2"}, {"stage", "prod"}}}, + }); + + TestLoadBalancerContext version_v1({{"version", "v1"}}); + TestLoadBalancerContext version_v2({{"version", "v2"}}); + TestLoadBalancerContext stage_dev({{"stage", "dev"}}); + TestLoadBalancerContext stage_prod({{"stage", "prod"}}); + TestLoadBalancerContext stage_test({{"stage", "test"}}); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&version_v1)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&version_v1)); + + EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&version_v2)); + EXPECT_EQ(host_set_.hosts_[6], lb_->chooseHost(&version_v2)); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&stage_dev)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&stage_dev)); + + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&stage_prod)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&stage_prod)); + + EXPECT_EQ(nullptr, lb_->chooseHost(&stage_test)); +} + +TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, AcceptMultipleKeys) { + std::vector subset_selectors = { + makeSelector({"version", "stage"}, true), + }; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + ON_CALL(subset_info_, fallbackPolicy()) + .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + init({ + {"tcp://127.0.0.1:80", {}}, + {"tcp://127.0.0.1:81", {{"version", "v1"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:82", {{"version", "v1"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:83", {{"version", "v1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:84", {{"version", "v1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:85", {{"version", "v2"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:86", {{"version", "v2"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:87", {{"version", "v2"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:88", {{"version", "v2"}, {"stage", "prod"}}}, + }); + + TestLoadBalancerContext v1_dev({{"version", "v1"}, {"stage", "dev"}}); + TestLoadBalancerContext v1_prod({{"version", "v1"}, {"stage", "prod"}}); + TestLoadBalancerContext v2_dev({{"version", "v2"}, {"stage", "dev"}}); + TestLoadBalancerContext v2_prod({{"version", "v2"}, {"stage", "prod"}}); + TestLoadBalancerContext v2_test({{"version", "v2"}, {"stage", "test"}}); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&v1_dev)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&v1_prod)); + EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&v2_dev)); + EXPECT_EQ(host_set_.hosts_[7], lb_->chooseHost(&v2_prod)); + EXPECT_EQ(nullptr, lb_->chooseHost(&v2_test)); +} + +TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, HybridMultipleSelectorsAndKeys) { + std::vector subset_selectors = { + makeSelector({"version", "stage"}, true), + makeSelector({"stage"}, false), + }; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + ON_CALL(subset_info_, fallbackPolicy()) + .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK)); + + init({ + {"tcp://127.0.0.1:80", {}}, + {"tcp://127.0.0.1:81", {{"version", "v1"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:82", {{"version", "v1"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:83", {{"version", "v1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:84", {{"version", "v1"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:85", {{"version", "v2"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:86", {{"version", "v2"}, {"stage", "dev"}}}, + {"tcp://127.0.0.1:87", {{"version", "v2"}, {"stage", "prod"}}}, + {"tcp://127.0.0.1:88", {{"version", "v2"}, {"stage", "prod"}}}, + }); + + TestLoadBalancerContext v1_dev({{"version", "v1"}, {"stage", "dev"}}); + TestLoadBalancerContext v1_prod({{"version", "v1"}, {"stage", "prod"}}); + TestLoadBalancerContext v2_dev({{"version", "v2"}, {"stage", "dev"}}); + TestLoadBalancerContext v2_prod({{"version", "v2"}, {"stage", "prod"}}); + TestLoadBalancerContext v2_test({{"version", "v2"}, {"stage", "test"}}); + TestLoadBalancerContext stage_dev({{"stage", "dev"}}); + TestLoadBalancerContext stage_prod({{"stage", "prod"}}); + TestLoadBalancerContext stage_test({{"stage", "test"}}); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&v1_dev)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&v1_prod)); + EXPECT_EQ(host_set_.hosts_[5], lb_->chooseHost(&v2_dev)); + EXPECT_EQ(host_set_.hosts_[7], lb_->chooseHost(&v2_prod)); + EXPECT_EQ(nullptr, lb_->chooseHost(&v2_test)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&stage_dev)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&stage_dev)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&stage_prod)); + EXPECT_EQ(host_set_.hosts_[4], lb_->chooseHost(&stage_prod)); + EXPECT_EQ(nullptr, lb_->chooseHost(&stage_test)); +} + +TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, DuplicateMetadataStat) { + init({ + {"tcp://127.0.0.1:80", {{"key", "a"}}}, + {"tcp://127.0.0.1:81", {{"key", "a"}}}, + {"tcp://127.0.0.1:82", {{"key", "a"}}}, + {"tcp://127.0.0.1:83", {{"key", "b"}}}, + }); + // The first 'a' is the original, the next 2 instances of 'a' are duplicates (counted + // in stat), and 'b' is another non-duplicate. + for (auto& gauge : stats_store_.gauges()) { + ENVOY_LOG_MISC(debug, "name {} value {}", gauge->name(), gauge->value()); + } + EXPECT_EQ(2, TestUtility::findGauge(stats_store_, + "testprefix.lb_subsets_single_host_per_subset_duplicate") + ->value()); +} + +TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, Match) { + init(); + + TestLoadBalancerContext host_1({{"key", "a"}}); + TestLoadBalancerContext host_2({{"key", "b"}}); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_1)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_1)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_2)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_2)); +} + +TEST_F(SubsetLoadBalancerSingleHostPerSubsetTest, FallbackOnUnknownMetadata) { + init(); + + TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); + TestLoadBalancerContext context_unknown_value({{"key", "unknown"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_unknown_value)); +} + +TEST_P(SubsetLoadBalancerSingleHostPerSubsetTest, Update) { + init(); + + TestLoadBalancerContext host_a({{"key", "a"}}); + TestLoadBalancerContext host_b({{"key", "b"}}); + TestLoadBalancerContext host_c({{"key", "c"}}); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_b)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_b)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&host_c)); // fallback + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_c)); // fallback + + HostSharedPtr added_host = makeHost("tcp://127.0.0.1:8000", {{"key", "c"}}); + + // Remove b, add c + modifyHosts({added_host}, {host_set_.hosts_.back()}); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_a)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_c)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_c)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&host_b)); // fallback + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&host_b)); // fallback + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&host_b)); // fallback +} + +INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerSingleHostPerSubsetTest, + testing::ValuesIn({UpdateOrder::RemovesFirst, UpdateOrder::Simultaneous})); // Test to improve coverage of the SubsetLoadBalancerFactory. TEST(LoadBalancerContextWrapperTest, LoadBalancingContextWrapperTest) { @@ -51,8 +3228,6 @@ TEST(LoadBalancerContextWrapperTest, LoadBalancingContextWrapperTest) { wrapper.overrideHostToSelect(); } -} // namespace -} // namespace Subset -} // namespace LoadBalancingPolices -} // namespace Extensions +} // namespace SubsetLoadBalancerTest +} // namespace Upstream } // namespace Envoy diff --git a/test/extensions/matching/input_matchers/cel_matcher/BUILD b/test/extensions/matching/input_matchers/cel_matcher/BUILD index bd175a3dea4b..a5a2f50ca4fa 100644 --- a/test/extensions/matching/input_matchers/cel_matcher/BUILD +++ b/test/extensions/matching/input_matchers/cel_matcher/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", "envoy_package", ) load( @@ -11,14 +12,17 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_test_library( + name = "cel_matcher_test_lib", + hdrs = ["cel_matcher_test.h"], +) + envoy_extension_cc_test( name = "cel_matcher_test", - srcs = [ - "cel_matcher_test.cc", - "cel_matcher_test.h", - ], + srcs = ["cel_matcher_test.cc"], extension_names = ["envoy.matching.matchers.cel_matcher"], deps = [ + ":cel_matcher_test_lib", "//source/common/matcher:matcher_lib", "//source/extensions/matching/http/cel_input:cel_input_lib", "//source/extensions/matching/input_matchers/cel_matcher:cel_matcher_lib", @@ -29,7 +33,7 @@ envoy_extension_cc_test( "//test/mocks/server:factory_context_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:registry_lib", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc index b4959988c4cb..eb77fdff8e41 100644 --- a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc +++ b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc @@ -19,6 +19,7 @@ #include "source/common/network/address_impl.h" #include "source/common/network/filter_impl.h" #include "source/common/network/listen_socket_impl.h" +#include "source/common/network/tcp_listener_impl.h" #include "source/common/network/utility.h" #include "source/common/stream_info/stream_info_impl.h" #include "source/extensions/network/dns_resolver/cares/dns_impl.h" @@ -728,7 +729,11 @@ class DnsImplTest : public testing::TestWithParam { socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); NiceMock listener_config; - listener_ = dispatcher_->createListener(socket_, *server_, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + listener_ = std::make_unique( + *dispatcher_, api_->randomGenerator(), runtime_, socket_, *server_, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); updateDnsResolverOptions(); // Create a resolver options on stack here to emulate what actually happens in envoy bootstrap. diff --git a/test/extensions/path/match/uri_template/library_test.cc b/test/extensions/path/match/uri_template/library_test.cc index ca10ecf37e81..e154986aa0e2 100644 --- a/test/extensions/path/match/uri_template/library_test.cc +++ b/test/extensions/path/match/uri_template/library_test.cc @@ -43,6 +43,36 @@ TEST(MatchTest, BasicUsage) { EXPECT_TRUE(matcher->match("/bar/en/us")); } +TEST(MatchTest, MatchDoubleEqualsInWildcard) { + const std::string yaml_string = R"EOF( + name: envoy.path.match.uri_template.uri_template_matcher + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + path_template: "/bar/{lang}/{country=**}" +)EOF"; + + Router::PathMatcherSharedPtr matcher = createMatcherFromYaml(yaml_string); + EXPECT_EQ(matcher->uriTemplate(), "/bar/{lang}/{country=**}"); + EXPECT_EQ(matcher->name(), "envoy.path.match.uri_template.uri_template_matcher"); + + EXPECT_TRUE(matcher->match("/bar/en/us==")); +} + +TEST(MatchTest, MatchDoubleEquals) { + const std::string yaml_string = R"EOF( + name: envoy.path.match.uri_template.uri_template_matcher + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.match.uri_template.v3.UriTemplateMatchConfig + path_template: "/bar/{lang}/{country=**}" +)EOF"; + + Router::PathMatcherSharedPtr matcher = createMatcherFromYaml(yaml_string); + EXPECT_EQ(matcher->uriTemplate(), "/bar/{lang}/{country=**}"); + EXPECT_EQ(matcher->name(), "envoy.path.match.uri_template.uri_template_matcher"); + + EXPECT_TRUE(matcher->match("/bar/en==/us")); +} + } // namespace Match } // namespace UriTemplate } // namespace Extensions diff --git a/test/extensions/path/rewrite/uri_template/library_test.cc b/test/extensions/path/rewrite/uri_template/library_test.cc index bddbea1f3989..6d34771872e8 100644 --- a/test/extensions/path/rewrite/uri_template/library_test.cc +++ b/test/extensions/path/rewrite/uri_template/library_test.cc @@ -71,6 +71,34 @@ TEST(RewriteTest, BasicUsage) { EXPECT_EQ(rewriter->name(), "envoy.path.rewrite.uri_template.uri_template_rewriter"); } +TEST(RewriteTest, DoubleEqualAtEndOfPath) { + const std::string yaml_string = R"EOF( + name: envoy.path.rewrite.uri_template.uri_template_rewriter + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + path_template_rewrite: "/bar/{country}/{final}" +)EOF"; + + Router::PathRewriterSharedPtr rewriter = createRewriterFromYaml(yaml_string); + EXPECT_EQ(rewriter->rewritePath("/bar/usa/final==1", "/bar/{final}/{country}").value(), + "/bar/final==1/usa"); + EXPECT_EQ(rewriter->name(), "envoy.path.rewrite.uri_template.uri_template_rewriter"); +} + +TEST(RewriteTest, DoubleEqual) { + const std::string yaml_string = R"EOF( + name: envoy.path.rewrite.uri_template.uri_template_rewriter + typed_config: + "@type": type.googleapis.com/envoy.extensions.path.rewrite.uri_template.v3.UriTemplateRewriteConfig + path_template_rewrite: "/bar/{country}/final" +)EOF"; + + Router::PathRewriterSharedPtr rewriter = createRewriterFromYaml(yaml_string); + EXPECT_EQ(rewriter->rewritePath("/bar/usa==/final", "/bar/{country}/final").value(), + "/bar/usa==/final"); + EXPECT_EQ(rewriter->name(), "envoy.path.rewrite.uri_template.uri_template_rewriter"); +} + TEST(RewriteTest, PatternNotMatched) { const std::string yaml_string = R"EOF( name: envoy.path.rewrite.uri_template.uri_template_rewriter diff --git a/test/extensions/path/uri_template_lib/uri_template_internal_test.cc b/test/extensions/path/uri_template_lib/uri_template_internal_test.cc index ab03e28c6f1c..254eef55eea4 100644 --- a/test/extensions/path/uri_template_lib/uri_template_internal_test.cc +++ b/test/extensions/path/uri_template_lib/uri_template_internal_test.cc @@ -49,7 +49,7 @@ TEST(InternalParsing, ParsedPathDebugString) { EXPECT_EQ(patt2.debugString(), "/{var}"); } -TEST(InternalParsing, isValidLiteralWorks) { +TEST(InternalParsing, IsValidLiteralWorks) { EXPECT_TRUE(isValidLiteral("123abcABC")); EXPECT_TRUE(isValidLiteral("._~-")); EXPECT_TRUE(isValidLiteral("-._~%20!$&'()+,;:@")); @@ -57,7 +57,7 @@ TEST(InternalParsing, isValidLiteralWorks) { EXPECT_FALSE(isValidLiteral("abc/")); EXPECT_FALSE(isValidLiteral("ab*c")); EXPECT_FALSE(isValidLiteral("a**c")); - EXPECT_FALSE(isValidLiteral("a=c")); + EXPECT_TRUE(isValidLiteral("a=c")); EXPECT_FALSE(isValidLiteral("?abc")); EXPECT_FALSE(isValidLiteral("?a=c")); EXPECT_FALSE(isValidLiteral("{abc")); @@ -75,7 +75,7 @@ TEST(InternalParsing, IsValidRewriteLiteralWorks) { EXPECT_FALSE(isValidRewriteLiteral("`~!@#$%^&()-_+;:,<.>'\"| ")); EXPECT_FALSE(isValidRewriteLiteral("ab}c")); EXPECT_FALSE(isValidRewriteLiteral("ab{c")); - EXPECT_FALSE(isValidRewriteLiteral("a=c")); + EXPECT_TRUE(isValidRewriteLiteral("a=c")); EXPECT_FALSE(isValidRewriteLiteral("?a=c")); } @@ -133,7 +133,8 @@ class ParseVariableSuccess : public testing::TestWithParam {}; INSTANTIATE_TEST_SUITE_P(ParseVariableSuccessTestSuite, ParseVariableSuccess, testing::Values("{var=*}", "{Var}", "{v1=**}", "{v_1=*/abc/**}", - "{v3=abc}", "{v=123/*/*}", "{var=abc/*/def}")); + "{v3=abc}", "{v=123/*/*}", "{var=abc/*/def}", + "{var=abc=def}")); TEST_P(ParseVariableSuccess, ParseVariableSuccessTest) { std::string pattern = GetParam(); @@ -151,8 +152,7 @@ class ParseVariableFailure : public testing::TestWithParam {}; INSTANTIATE_TEST_SUITE_P(ParseVariableFailureTestSuite, ParseVariableFailure, testing::Values("{var", "{=abc}", "{_var=*}", "{1v}", "{1v=abc}", "{var=***}", "{v-a-r}", "{var=*/abc?q=1}", "{var=abc/a*}", - "{var=*def/abc}", "{var=}", "{var=abc=def}", - "{rc=||||(A+yl/}", "/")); + "{var=*def/abc}", "{var=}", "{rc=||||(A+yl/}", "/")); TEST_P(ParseVariableFailure, ParseVariableFailureTest) { std::string pattern = GetParam(); @@ -268,8 +268,8 @@ TEST(InternalRegexGen, DollarSignMatchesIfself) { } TEST(InternalRegexGen, OperatorRegexPattern) { - EXPECT_EQ(toRegexPattern(Operator::PathGlob), "[a-zA-Z0-9-._~%!$&'()+,;:@]+"); - EXPECT_EQ(toRegexPattern(Operator::TextGlob), "[a-zA-Z0-9-._~%!$&'()+,;:@/]*"); + EXPECT_EQ(toRegexPattern(Operator::PathGlob), "[a-zA-Z0-9-._~%!$&'()+,;:@=]+"); + EXPECT_EQ(toRegexPattern(Operator::TextGlob), "[a-zA-Z0-9-._~%!$&'()+,;:@=/]*"); } TEST(InternalRegexGen, PathGlobRegex) { @@ -291,10 +291,10 @@ TEST(InternalRegexGen, TextGlobRegex) { } TEST(InternalRegexGen, VariableRegexPattern) { - EXPECT_EQ(toRegexPattern(Variable("var1", {})), "(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+)"); + EXPECT_EQ(toRegexPattern(Variable("var1", {})), "(?P[a-zA-Z0-9-._~%!$&'()+,;:@=]+)"); EXPECT_EQ(toRegexPattern(Variable("var2", {Operator::PathGlob, "abc", Operator::TextGlob})), - "(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+/abc/" - "[a-zA-Z0-9-._~%!$&'()+,;:@/]*)"); + "(?P[a-zA-Z0-9-._~%!$&'()+,;:@=]+/abc/" + "[a-zA-Z0-9-._~%!$&'()+,;:@=/]*)"); } TEST(InternalRegexGen, VariableRegexDefaultMatch) { diff --git a/test/extensions/path/uri_template_lib/uri_template_test.cc b/test/extensions/path/uri_template_lib/uri_template_test.cc index abebf4833a01..14d44b2b8603 100644 --- a/test/extensions/path/uri_template_lib/uri_template_test.cc +++ b/test/extensions/path/uri_template_lib/uri_template_test.cc @@ -21,6 +21,7 @@ namespace { using ::Envoy::StatusHelpers::IsOkAndHolds; using ::Envoy::StatusHelpers::StatusIs; +using testing::HasSubstr; // Capture regex for /{var1}/{var2}/{var3}/{var4}/{var5} static constexpr absl::string_view kCaptureRegex = "/(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+)/" @@ -32,15 +33,15 @@ static constexpr absl::string_view kCaptureRegex = "/(?P[a-zA-Z0-9-._~%!$& TEST(ConvertPathPattern, ValidPattern) { EXPECT_THAT(convertPathPatternSyntaxToRegex("/abc"), IsOkAndHolds("/abc")); EXPECT_THAT(convertPathPatternSyntaxToRegex("/**.mpd"), - IsOkAndHolds("/[a-zA-Z0-9-._~%!$&'()+,;:@/]*\\.mpd")); + IsOkAndHolds("/[a-zA-Z0-9-._~%!$&'()+,;:@=/]*\\.mpd")); EXPECT_THAT(convertPathPatternSyntaxToRegex("/api/*/{resource=*}/{method=**}"), - IsOkAndHolds("/api/[a-zA-Z0-9-._~%!$&'()+,;:@]+/" - "(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+)/" - "(?P[a-zA-Z0-9-._~%!$&'()+,;:@/]*)")); + IsOkAndHolds("/api/[a-zA-Z0-9-._~%!$&'()+,;:@=]+/" + "(?P[a-zA-Z0-9-._~%!$&'()+,;:@=]+)/" + "(?P[a-zA-Z0-9-._~%!$&'()+,;:@=/]*)")); EXPECT_THAT(convertPathPatternSyntaxToRegex("/api/{VERSION}/{version}/{verSION}"), - IsOkAndHolds("/api/(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+)/" - "(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+)/" - "(?P[a-zA-Z0-9-._~%!$&'()+,;:@]+)")); + IsOkAndHolds("/api/(?P[a-zA-Z0-9-._~%!$&'()+,;:@=]+)/" + "(?P[a-zA-Z0-9-._~%!$&'()+,;:@=]+)/" + "(?P[a-zA-Z0-9-._~%!$&'()+,;:@=]+)")); } TEST(ConvertPathPattern, InvalidPattern) { @@ -67,7 +68,17 @@ TEST_P(ParseRewriteHelperSuccess, ParseRewriteHelperSuccessTest) { std::string pattern = GetParam(); SCOPED_TRACE(pattern); - EXPECT_OK(parseRewritePattern(pattern)); + const auto result = parseRewritePattern(pattern); + EXPECT_OK(result); + + // The following is to exercise operator<< in ParseSegments. + const auto& parsed_segments_vec = result.value(); + std::stringstream all_segments_str; + for (const auto& parsed_segment : parsed_segments_vec) { + all_segments_str << parsed_segment; + } + EXPECT_THAT(all_segments_str.str(), HasSubstr("kind = Literal, value =")); + EXPECT_THAT(all_segments_str.str(), HasSubstr("kind = Variable, value =")); } class ParseRewriteHelperFailure : public testing::TestWithParam {}; diff --git a/test/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator_test.cc b/test/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator_test.cc index 5e8571d2420e..023512ffb18e 100644 --- a/test/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator_test.cc +++ b/test/extensions/quic/connection_id_generator/envoy_deterministic_connection_id_generator_test.cc @@ -16,16 +16,14 @@ using ::quic::QuicConnectionId; using ::quic::test::QuicTest; using ::quic::test::TestConnectionId; using ::quic::test::TestConnectionIdNineBytesLong; -using ::testing::ElementsAre; class EnvoyDeterministicConnectionIdGeneratorTest : public QuicTest { public: EnvoyDeterministicConnectionIdGeneratorTest() - : connection_id_length_(12), - generator_(EnvoyDeterministicConnectionIdGenerator(connection_id_length_)) {} + : generator_(EnvoyDeterministicConnectionIdGenerator(connection_id_length_)) {} protected: - int connection_id_length_; + int connection_id_length_{12}; EnvoyDeterministicConnectionIdGenerator generator_; }; diff --git a/test/extensions/quic/proof_source/BUILD b/test/extensions/quic/proof_source/BUILD new file mode 100644 index 000000000000..43dcae18636e --- /dev/null +++ b/test/extensions/quic/proof_source/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test_library( + name = "pending_proof_source_factory_impl_lib", + srcs = ["pending_proof_source_factory_impl.cc"], + hdrs = ["pending_proof_source_factory_impl.h"], + tags = ["nofips"], + deps = [ + "//envoy/registry", + "//source/common/quic:envoy_quic_proof_source_factory_interface", + "//source/common/quic:envoy_quic_proof_source_lib", + ], +) diff --git a/test/extensions/quic/proof_source/pending_proof_source_factory_impl.cc b/test/extensions/quic/proof_source/pending_proof_source_factory_impl.cc new file mode 100644 index 000000000000..9dd472518eb3 --- /dev/null +++ b/test/extensions/quic/proof_source/pending_proof_source_factory_impl.cc @@ -0,0 +1,39 @@ +#include "test/extensions/quic/proof_source/pending_proof_source_factory_impl.h" + +#include "source/common/quic/envoy_quic_proof_source.h" + +namespace Envoy { +namespace Quic { + +class PendingProofSource : public EnvoyQuicProofSource { +public: + PendingProofSource(Network::Socket& listen_socket, + Network::FilterChainManager& filter_chain_manager, + Server::ListenerStats& listener_stats, TimeSource& time_source) + : EnvoyQuicProofSource(listen_socket, filter_chain_manager, listener_stats, time_source) {} + +protected: + void signPayload(const quic::QuicSocketAddress& /*server_address*/, + const quic::QuicSocketAddress& /*client_address*/, + const std::string& /*hostname*/, uint16_t /*signature_algorithm*/, + absl::string_view /*in*/, + std::unique_ptr callback) override { + // Make the callback pending. + pending_callbacks_.push_back(std::move(callback)); + } + +private: + std::vector> pending_callbacks_; +}; + +std::unique_ptr PendingProofSourceFactoryImpl::createQuicProofSource( + Network::Socket& listen_socket, Network::FilterChainManager& filter_chain_manager, + Server::ListenerStats& listener_stats, TimeSource& time_source) { + return std::make_unique(listen_socket, filter_chain_manager, listener_stats, + time_source); +} + +REGISTER_FACTORY(PendingProofSourceFactoryImpl, EnvoyQuicProofSourceFactoryInterface); + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic/proof_source/pending_proof_source_factory_impl.h b/test/extensions/quic/proof_source/pending_proof_source_factory_impl.h new file mode 100644 index 000000000000..35b77043d385 --- /dev/null +++ b/test/extensions/quic/proof_source/pending_proof_source_factory_impl.h @@ -0,0 +1,30 @@ +#pragma once + +#include "envoy/registry/registry.h" + +#include "source/common/protobuf/protobuf.h" +#include "source/common/quic/envoy_quic_proof_source_factory_interface.h" + +namespace Envoy { +namespace Quic { + +// Provides a ProofSource implementation which makes signing pending. +class PendingProofSourceFactoryImpl : public EnvoyQuicProofSourceFactoryInterface { +public: + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + // Using Struct instead of a custom config proto. This is only allowed in tests. + return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Struct()}; + } + + std::string name() const override { return "envoy.quic.proof_source.pending_signing"; } + + std::unique_ptr + createQuicProofSource(Network::Socket& listen_socket, + Network::FilterChainManager& filter_chain_manager, + Server::ListenerStats& listener_stats, TimeSource& time_source) override; +}; + +DECLARE_FACTORY(PendingProofSourceFactoryImpl); + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/rate_limit_descriptors/expr/config_test.cc b/test/extensions/rate_limit_descriptors/expr/config_test.cc index fcfd6c3cb24c..b7f0f5f8d758 100644 --- a/test/extensions/rate_limit_descriptors/expr/config_test.cc +++ b/test/extensions/rate_limit_descriptors/expr/config_test.cc @@ -32,7 +32,7 @@ class RateLimitPolicyEntryTest : public testing::Test { rate_limit_entry_ = std::make_unique(rate_limit, context_); } - NiceMock context_; + NiceMock context_; std::unique_ptr rate_limit_entry_; Http::TestRequestHeaderMapImpl header_; std::vector descriptors_; diff --git a/test/extensions/resource_monitors/downstream_connections/BUILD b/test/extensions/resource_monitors/downstream_connections/BUILD index 58f2b7505b56..057f29049d57 100644 --- a/test/extensions/resource_monitors/downstream_connections/BUILD +++ b/test/extensions/resource_monitors/downstream_connections/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_test", "envoy_package", ) load( @@ -36,3 +37,24 @@ envoy_extension_cc_test( "@envoy_api//envoy/extensions/resource_monitors/downstream_connections/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "cx_limit_overload_integration_test", + size = "large", + srcs = ["cx_limit_overload_integration_test.cc"], + tags = [ + "cpu:3", + ], + deps = [ + "//envoy/network:filter_interface", + "//envoy/registry", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/resource_monitors/downstream_connections:config", + "//test/config:utility_lib", + "//test/integration:http_protocol_integration_lib", + "//test/test_common:logging_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/resource_monitors/downstream_connections/cx_limit_overload_integration_test.cc b/test/extensions/resource_monitors/downstream_connections/cx_limit_overload_integration_test.cc new file mode 100644 index 000000000000..7242c9f89cd6 --- /dev/null +++ b/test/extensions/resource_monitors/downstream_connections/cx_limit_overload_integration_test.cc @@ -0,0 +1,182 @@ +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/overload/v3/overload.pb.h" + +#include "test/integration/integration.h" +#include "test/test_common/test_runtime.h" + +namespace Envoy { + +namespace { + +envoy::config::overload::v3::OverloadManager +generateMaxDownstreamConnectionsOverloadConfig(uint32_t max_cx) { + return TestUtility::parseYaml( + fmt::format(R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: {} + )EOF", + std::to_string(max_cx))); +} + +class GlobalDownstreamCxLimitIntegrationTest : public testing::Test, public BaseIntegrationTest { +protected: + GlobalDownstreamCxLimitIntegrationTest() + : BaseIntegrationTest(Network::Address::IpVersion::v4, ConfigHelper::tcpProxyConfig()) {} + + void initializeOverloadManager(uint32_t max_cx) { + overload_manager_config_ = generateMaxDownstreamConnectionsOverloadConfig(max_cx); + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + *bootstrap.mutable_overload_manager() = this->overload_manager_config_; + }); + initialize(); + } + + AssertionResult waitForConnections(uint32_t expected_connections) { + absl::Notification num_downstream_conns_reached; + + test_server_->server().dispatcher().post([this, &num_downstream_conns_reached, + expected_connections]() { + auto& overload_state = test_server_->server().overloadManager().getThreadLocalOverloadState(); + const auto& monitor = overload_state.getProactiveResourceMonitorForTest( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections); + ASSERT_TRUE(monitor.has_value()); + test_server_->waitForProactiveOverloadResourceUsageEq( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, + expected_connections, test_server_->server().dispatcher(), + std::chrono::milliseconds(500)); + num_downstream_conns_reached.Notify(); + }); + if (num_downstream_conns_reached.WaitForNotificationWithTimeout(absl::Milliseconds(550))) { + return AssertionSuccess(); + } else { + return AssertionFailure(); + } + } + +private: + envoy::config::overload::v3::OverloadManager overload_manager_config_; +}; + +TEST_F(GlobalDownstreamCxLimitIntegrationTest, GlobalLimitInOverloadManager) { + initializeOverloadManager(6); + std::vector tcp_clients; + std::vector raw_conns; + for (int i = 0; i < 6; ++i) { + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(conn)); + raw_conns.push_back(std::move(conn)); + ASSERT_TRUE(tcp_clients.back()->connected()); + } + test_server_->waitForCounterEq("listener.127.0.0.1_0.downstream_global_cx_overflow", 0); + // 7th connection should fail because we have hit the configured limit for + // `max_active_downstream_connections`. + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn; + ASSERT_FALSE(fake_upstreams_[0]->waitForRawConnection(conn, std::chrono::milliseconds(500))); + tcp_clients.back()->waitForDisconnect(); + test_server_->waitForCounterEq("listener.127.0.0.1_0.downstream_global_cx_overflow", 1); + // Get rid of the client that failed to connect. + tcp_clients.back()->close(); + tcp_clients.pop_back(); + // Get rid of the first successfully connected client to be able to establish new connection. + tcp_clients.front()->close(); + ASSERT_TRUE(raw_conns.front()->waitForDisconnect()); + // test_server_->waitForProactiveOverloadResourceUsageEq(Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, + // 5, test_server_->server().dispatcher(), std::chrono::milliseconds(5000)); + ASSERT_TRUE(waitForConnections(5)); + // As 6th client disconnected, we can again establish a connection. + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn1; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(conn1)); + raw_conns.push_back(std::move(conn1)); + ASSERT_TRUE(tcp_clients.back()->connected()); + for (auto& tcp_client : tcp_clients) { + tcp_client->close(); + } +} + +TEST_F(GlobalDownstreamCxLimitIntegrationTest, GlobalLimitSetViaRuntimeKeyAndOverloadManager) { + // Configure global connections limit via deprecated runtime key. + config_helper_.addRuntimeOverride("overload.global_downstream_max_connections", "3"); + initializeOverloadManager(2); + const std::string log_line = + "Global downstream connections limits is configured via runtime key " + "overload.global_downstream_max_connections and in " + "envoy.resource_monitors.global_downstream_max_connections. Using overload manager " + "config."; + std::vector tcp_clients; + std::vector raw_conns; + std::function establish_conns = [this, &tcp_clients, &raw_conns]() { + for (int i = 0; i < 2; ++i) { + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(conn)); + raw_conns.push_back(std::move(conn)); + ASSERT_TRUE(tcp_clients.back()->connected()); + } + }; + EXPECT_LOG_CONTAINS_N_TIMES("warn", log_line, 1, { establish_conns(); }); + // Third connection should fail because we have hit the configured limit in overload manager. + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn1; + ASSERT_FALSE(fake_upstreams_[0]->waitForRawConnection(conn1, std::chrono::milliseconds(500))); + tcp_clients.back()->waitForDisconnect(); + test_server_->waitForCounterEq("listener.127.0.0.1_0.downstream_global_cx_overflow", 1); + for (auto& tcp_client : tcp_clients) { + tcp_client->close(); + } +} + +TEST_F(GlobalDownstreamCxLimitIntegrationTest, GlobalLimitOptOutRespected) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + listener->set_ignore_global_conn_limit(true); + }); + initializeOverloadManager(2); + std::vector tcp_clients; + std::vector raw_conns; + // All clients succeed to connect due to global conn limit opt out set. + for (int i = 0; i <= 5; ++i) { + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(conn)); + raw_conns.push_back(std::move(conn)); + ASSERT_TRUE(tcp_clients.back()->connected()); + } + test_server_->waitForCounterEq("listener.127.0.0.1_0.downstream_global_cx_overflow", 0); + for (auto& tcp_client : tcp_clients) { + tcp_client->close(); + } +} + +TEST_F(GlobalDownstreamCxLimitIntegrationTest, PerListenerLimitAndGlobalLimitInOverloadManager) { + config_helper_.addRuntimeOverride("envoy.resource_limits.listener.listener_0.connection_limit", + "2"); + initializeOverloadManager(5); + std::vector tcp_clients; + std::vector raw_conns; + for (int i = 0; i < 2; ++i) { + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(conn)); + raw_conns.push_back(std::move(conn)); + ASSERT_TRUE(tcp_clients.back()->connected()); + } + // Third connection should fail because we have hit the configured limit per listener. + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + FakeRawConnectionPtr conn1; + ASSERT_FALSE(fake_upstreams_[0]->waitForRawConnection(conn1, std::chrono::milliseconds(500))); + tcp_clients.back()->waitForDisconnect(); + test_server_->waitForCounterEq("listener.127.0.0.1_0.downstream_cx_overflow", 1); + test_server_->waitForCounterEq("listener.127.0.0.1_0.downstream_global_cx_overflow", 0); + for (auto& tcp_client : tcp_clients) { + tcp_client->close(); + } +} + +} // namespace +} // namespace Envoy diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc index 88cbd4b8a5fd..d9be99300106 100644 --- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc +++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_integration_test.cc @@ -42,14 +42,6 @@ class OverloadIntegrationTest : public testing::TestWithParam http2_options) override { - IntegrationCodecClientPtr codec = - HttpIntegrationTest::makeRawHttpConnection(std::move(conn), http2_options); - return codec; - } - void initialize() override { config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { const std::string overload_config = diff --git a/test/extensions/router/cluster_specifiers/lua/BUILD b/test/extensions/router/cluster_specifiers/lua/BUILD new file mode 100644 index 000000000000..f086f868f56c --- /dev/null +++ b/test/extensions/router/cluster_specifiers/lua/BUILD @@ -0,0 +1,35 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "lua_cluster_specifier_test", + srcs = ["lua_cluster_specifier_test.cc"], + extension_names = ["envoy.router.cluster_specifier_plugin.lua"], + deps = [ + "//source/extensions/router/cluster_specifiers/lua:lua_cluster_specifier_lib", + "//test/mocks/router:router_mocks", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.router.cluster_specifier_plugin.lua"], + deps = [ + "//source/extensions/router/cluster_specifiers/lua:config", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/router/cluster_specifiers/lua/config_test.cc b/test/extensions/router/cluster_specifiers/lua/config_test.cc new file mode 100644 index 000000000000..ff23f0a556b6 --- /dev/null +++ b/test/extensions/router/cluster_specifiers/lua/config_test.cc @@ -0,0 +1,69 @@ +#include "source/extensions/router/cluster_specifiers/lua/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Router { +namespace Lua { + +TEST(LuaClusterSpecifierPluginConfigTest, EmptyConfig) { + LuaClusterSpecifierPluginFactoryConfig factory; + + ProtobufTypes::MessagePtr empty_config = factory.createEmptyConfigProto(); + EXPECT_NE(nullptr, empty_config); +} + +TEST(LuaClusterSpecifierPluginConfigTest, NormalConfig) { + const std::string normal_lua_config_yaml = R"EOF( + source_code: + inline_string: | + function envoy_on_route(route_handle) + local header_value = route_handle:headers():get("header_key") + if header_value == "fake" then + return "fake_service" + end + return "web_service" + end + default_cluster: default_service + )EOF"; + + LuaClusterSpecifierConfigProto proto_config{}; + TestUtility::loadFromYaml(normal_lua_config_yaml, proto_config); + NiceMock context; + LuaClusterSpecifierPluginFactoryConfig factory; + Envoy::Router::ClusterSpecifierPluginSharedPtr plugin = + factory.createClusterSpecifierPlugin(proto_config, context); + EXPECT_NE(nullptr, plugin); +} + +TEST(LuaClusterSpecifierPluginConfigTest, NoOnRouteConfig) { + const std::string normal_lua_config_yaml = R"EOF( + source_code: + inline_string: | + function envoy_on_no_route(route_handle) + local header_value = route_handle:headers():get("header_key") + if header_value == "fake" then + return "fake_service" + end + return "web_service" + end + default_cluster: default_service + )EOF"; + + LuaClusterSpecifierConfigProto proto_config{}; + TestUtility::loadFromYaml(normal_lua_config_yaml, proto_config); + NiceMock context; + LuaClusterSpecifierPluginFactoryConfig factory; + EXPECT_THROW_WITH_MESSAGE( + factory.createClusterSpecifierPlugin(proto_config, context), Envoy::EnvoyException, + "envoy_on_route() function not found. Lua will not hook cluster specifier."); +} + +} // namespace Lua +} // namespace Router +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/router/cluster_specifiers/lua/lua_cluster_specifier_test.cc b/test/extensions/router/cluster_specifiers/lua/lua_cluster_specifier_test.cc new file mode 100644 index 000000000000..08e9480b769a --- /dev/null +++ b/test/extensions/router/cluster_specifiers/lua/lua_cluster_specifier_test.cc @@ -0,0 +1,144 @@ +#include "source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.h" + +#include "test/mocks/router/mocks.h" +#include "test/mocks/server/factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Router { +namespace Lua { + +using testing::InSequence; +using testing::NiceMock; +using testing::Return; + +class LuaClusterSpecifierPluginTest : public testing::Test { +public: + void setUpTest(const std::string& yaml) { + LuaClusterSpecifierConfigProto proto_config{}; + TestUtility::loadFromYaml(yaml, proto_config); + + config_ = std::make_shared(proto_config, server_factory_context_); + + plugin_ = std::make_unique(config_); + } + + const std::string normal_lua_config_yaml_ = R"EOF( + source_code: + inline_string: | + function envoy_on_route(route_handle) + local header_value = route_handle:headers():get("header_key") + if header_value == "fake" then + return "fake_service" + end + return "web_service" + end + default_cluster: default_service + )EOF"; + + const std::string error_lua_config_yaml_ = R"EOF( + source_code: + inline_string: | + function envoy_on_route(route_handle) + local header_value = route_handle:headers():get({}) + if header_value == "fake" then + return "fake_service" + end + return "web_service" + end + default_cluster: default_service + )EOF"; + + const std::string return_type_not_string_lua_config_yaml_ = R"EOF( + source_code: + inline_string: | + function envoy_on_route(route_handle) + local header_value = route_handle:headers():get("header_key") + if header_value == "fake" then + return "fake_service" + end + return {} + end + default_cluster: default_service + )EOF"; + + NiceMock server_factory_context_; + std::unique_ptr plugin_; + LuaClusterSpecifierConfigSharedPtr config_; +}; + +// Normal lua code test. +TEST_F(LuaClusterSpecifierPluginTest, NormalLuaCode) { + setUpTest(normal_lua_config_yaml_); + + auto mock_route = std::make_shared>(); + { + Http::TestRequestHeaderMapImpl headers{{":path", "/"}, {"header_key", "fake"}}; + auto route = plugin_->route(mock_route, headers); + EXPECT_EQ("fake_service", route->routeEntry()->clusterName()); + } + + { + Http::TestRequestHeaderMapImpl headers{{":path", "/"}, {"header_key", "header_value"}}; + auto route = plugin_->route(mock_route, headers); + EXPECT_EQ("web_service", route->routeEntry()->clusterName()); + } +} + +// Error lua code test. +TEST_F(LuaClusterSpecifierPluginTest, ErrorLuaCode) { + setUpTest(error_lua_config_yaml_); + + auto mock_route = std::make_shared>(); + { + Http::TestRequestHeaderMapImpl headers{{":path", "/"}, {"header_key", "fake"}}; + auto route = plugin_->route(mock_route, headers); + EXPECT_EQ("default_service", route->routeEntry()->clusterName()); + } + + { + Http::TestRequestHeaderMapImpl headers{{":path", "/"}, {"header_key", "header_value"}}; + auto route = plugin_->route(mock_route, headers); + EXPECT_EQ("default_service", route->routeEntry()->clusterName()); + } +} + +// Return type not string lua code test. +TEST_F(LuaClusterSpecifierPluginTest, ReturnTypeNotStringLuaCode) { + setUpTest(return_type_not_string_lua_config_yaml_); + + auto mock_route = std::make_shared>(); + { + Http::TestRequestHeaderMapImpl headers{{":path", "/"}, {"header_key", "fake"}}; + auto route = plugin_->route(mock_route, headers); + EXPECT_EQ("fake_service", route->routeEntry()->clusterName()); + } + + { + Http::TestRequestHeaderMapImpl headers{{":path", "/"}, {"header_key", "header_value"}}; + auto route = plugin_->route(mock_route, headers); + EXPECT_EQ("default_service", route->routeEntry()->clusterName()); + } +} + +TEST_F(LuaClusterSpecifierPluginTest, DestructLuaClusterSpecifierConfig) { + setUpTest(normal_lua_config_yaml_); + InSequence s; + EXPECT_CALL(server_factory_context_.dispatcher_, isThreadSafe()).WillOnce(Return(false)); + EXPECT_CALL(server_factory_context_.dispatcher_, post(_)); + EXPECT_CALL(server_factory_context_.dispatcher_, isThreadSafe()).WillOnce(Return(true)); + EXPECT_CALL(server_factory_context_.dispatcher_, post(_)).Times(0); + + LuaClusterSpecifierConfigProto proto_config{}; + TestUtility::loadFromYaml(normal_lua_config_yaml_, proto_config); + config_ = std::make_shared(proto_config, server_factory_context_); + config_.reset(); +} + +} // namespace Lua +} // namespace Router +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/stats_sinks/common/statsd/statsd_test.cc b/test/extensions/stats_sinks/common/statsd/statsd_test.cc index 59aa1b6eefab..f7d08b6c859f 100644 --- a/test/extensions/stats_sinks/common/statsd/statsd_test.cc +++ b/test/extensions/stats_sinks/common/statsd/statsd_test.cc @@ -83,9 +83,24 @@ TEST_F(TcpStatsdSinkTest, BasicFlow) { gauge.used_ = true; snapshot_.gauges_.push_back(gauge); + Stats::PrimitiveCounter host_counter; + host_counter.add(3); + Stats::PrimitiveCounterSnapshot host_counter_snap(host_counter); + host_counter_snap.setName("test_host_counter"); + snapshot_.host_counters_.push_back(host_counter_snap); + + Stats::PrimitiveGauge host_gauge; + host_gauge.add(4); + Stats::PrimitiveGaugeSnapshot host_gauge_snap(host_gauge); + host_gauge_snap.setName("test_host_gauge"); + snapshot_.host_gauges_.push_back(host_gauge_snap); + expectCreateConnection(); - EXPECT_CALL(*connection_, - write(BufferStringEqual("envoy.test_counter:1|c\nenvoy.test_gauge:2|g\n"), _)); + EXPECT_CALL(*connection_, write(BufferStringEqual("envoy.test_counter:1|c\n" + "envoy.test_host_counter:3|c\n" + "envoy.test_gauge:2|g\n" + "envoy.test_host_gauge:4|g\n"), + _)); sink_->flush(snapshot_); connection_->runHighWatermarkCallbacks(); diff --git a/test/extensions/stats_sinks/common/statsd/udp_statsd_test.cc b/test/extensions/stats_sinks/common/statsd/udp_statsd_test.cc index 57bed7796eff..a240abc68ebd 100644 --- a/test/extensions/stats_sinks/common/statsd/udp_statsd_test.cc +++ b/test/extensions/stats_sinks/common/statsd/udp_statsd_test.cc @@ -98,13 +98,39 @@ TEST_P(UdpStatsdSinkTest, InitWithIpAddress) { gauge.used_ = true; snapshot.gauges_.push_back(gauge); + Stats::PrimitiveCounter host_counter; + host_counter.add(3); + Stats::PrimitiveCounterSnapshot host_counter_snap(host_counter); + host_counter_snap.setName("test_host_counter"); + snapshot.host_counters_.push_back(host_counter_snap); + + Stats::PrimitiveGauge host_gauge; + host_gauge.add(4); + Stats::PrimitiveGaugeSnapshot host_gauge_snap(host_gauge); + host_gauge_snap.setName("test_host_gauge"); + snapshot.host_gauges_.push_back(host_gauge_snap); + sink.flush(snapshot); - Network::UdpRecvData data; - server.recv(data); - EXPECT_EQ("envoy.test_counter:1|c", data.buffer_->toString()); - Network::UdpRecvData data2; - server.recv(data2); - EXPECT_EQ("envoy.test_gauge:1|g", data2.buffer_->toString()); + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_counter:1|c", data.buffer_->toString()); + } + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_host_counter:3|c", data.buffer_->toString()); + } + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_gauge:1|g", data.buffer_->toString()); + } + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_host_gauge:4|g", data.buffer_->toString()); + } NiceMock timer; timer.name_ = "test_timer"; @@ -142,13 +168,41 @@ TEST_P(UdpStatsdSinkWithTagsTest, InitWithIpAddress) { gauge.setTags(tags); snapshot.gauges_.push_back(gauge); + Stats::PrimitiveCounter host_counter; + host_counter.add(3); + Stats::PrimitiveCounterSnapshot host_counter_snap(host_counter); + host_counter_snap.setTagExtractedName("test_host_counter"); + host_counter_snap.setTags(tags); + snapshot.host_counters_.push_back(host_counter_snap); + + Stats::PrimitiveGauge host_gauge; + host_gauge.add(4); + Stats::PrimitiveGaugeSnapshot host_gauge_snap(host_gauge); + host_gauge_snap.setTagExtractedName("test_host_gauge"); + host_gauge_snap.setTags(tags); + snapshot.host_gauges_.push_back(host_gauge_snap); + sink.flush(snapshot); - Network::UdpRecvData data; - server.recv(data); - EXPECT_EQ("envoy.test_counter:1|c|#node:test", data.buffer_->toString()); - Network::UdpRecvData data2; - server.recv(data2); - EXPECT_EQ("envoy.test_gauge:1|g|#node:test", data2.buffer_->toString()); + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_counter:1|c|#node:test", data.buffer_->toString()); + } + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_host_counter:3|c|#node:test", data.buffer_->toString()); + } + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_gauge:1|g|#node:test", data.buffer_->toString()); + } + { + Network::UdpRecvData data; + server.recv(data); + EXPECT_EQ("envoy.test_host_gauge:4|g|#node:test", data.buffer_->toString()); + } NiceMock timer; timer.name_ = "test_timer"; diff --git a/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc b/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc index 80de5836b1d1..6e56cf4f8c28 100644 --- a/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc +++ b/test/extensions/stats_sinks/open_telemetry/open_telemetry_impl_test.cc @@ -64,6 +64,18 @@ class OpenTelemetryStatsSinkTests : public testing::Test { snapshot_.counters_.push_back({delta, *counter_storage_.back()}); } + void addHostCounterToSnapshot(const std::string& name, uint64_t delta, uint64_t value) { + Stats::PrimitiveCounter counter; + counter.add(value - delta); + counter.latch(); + counter.add(delta); + Stats::PrimitiveCounterSnapshot s(counter); + s.setName(std::string(name)); + s.setTagExtractedName(getTagExtractedName(name)); + s.setTags({{"counter_key", "counter_val"}}); + snapshot_.host_counters_.push_back(s); + } + void addGaugeToSnapshot(const std::string& name, uint64_t value, bool used = true) { gauge_storage_.emplace_back(std::make_unique>()); gauge_storage_.back()->name_ = name; @@ -75,12 +87,35 @@ class OpenTelemetryStatsSinkTests : public testing::Test { snapshot_.gauges_.push_back(*gauge_storage_.back()); } + void addHostGaugeToSnapshot(const std::string& name, uint64_t value) { + Stats::PrimitiveGauge gauge; + gauge.add(value); + Stats::PrimitiveGaugeSnapshot s(gauge); + s.setName(std::string(name)); + s.setTagExtractedName(getTagExtractedName(name)); + s.setTags({{"gauge_key", "gauge_val"}}); + snapshot_.host_gauges_.push_back(s); + } + void addHistogramToSnapshot(const std::string& name, bool is_delta = false, bool used = true) { auto histogram = std::make_unique>(); histogram_t* hist = hist_alloc(); - for (uint64_t value = 1; value <= 10; value++) { - hist_insert_intscale(hist, is_delta ? value + 10 : value, 0, 1); + + // Using the default histogram boundaries for testing. For delta histogram, it's expected that + // even indexes will have count of 1, and 0 otherwise. For cumulative histogram, it's expected + // that odd indexes will have count of 1, and 0 otherwise. + std::vector values; + if (is_delta) { + values = {0.2, 3, 16, 75, 400, 2000, 7500, 50000, 500000, 3000000}; + } else { + // The last value will be used to test a scenario of a bucket which is outside the defined + // histogram bounds. + values = {0.7, 7, 35, 200, 750, 4000, 20000, 200000, 1500000, 4000000}; + } + + for (auto value : values) { + hist_insert(hist, value, 1); } histogram_ptrs_.push_back(hist); @@ -183,7 +218,8 @@ class OtlpMetricsFlusherTests : public OpenTelemetryStatsSinkTests { EXPECT_EQ(10, data_point.count()); const int default_buckets_count = 19; - EXPECT_EQ(default_buckets_count, data_point.bucket_counts().size()); + // The buckets count needs to be one element bigger than the bounds size. + EXPECT_EQ(default_buckets_count + 1, data_point.bucket_counts().size()); EXPECT_EQ(default_buckets_count, data_point.explicit_bounds().size()); EXPECT_EQ(expected_time_ns_, data_point.time_unix_nano()); @@ -191,12 +227,17 @@ class OtlpMetricsFlusherTests : public OpenTelemetryStatsSinkTests { EXPECT_EQ(AggregationTemporality::AGGREGATION_TEMPORALITY_DELTA, metric.histogram().aggregation_temporality()); - EXPECT_GE(data_point.sum(), 160); + EXPECT_GE(data_point.sum(), 3559994.2); } else { EXPECT_EQ(AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE, metric.histogram().aggregation_temporality()); - EXPECT_GE(data_point.sum(), 55); + EXPECT_GE(data_point.sum(), 5724992.7); + } + + for (int idx = 0; idx < data_point.bucket_counts().size(); idx++) { + int expected_value = (is_delta) ? (1 - (idx % 2)) : (idx % 2); + EXPECT_EQ(expected_value, data_point.bucket_counts()[idx]); } } @@ -216,11 +257,13 @@ TEST_F(OtlpMetricsFlusherTests, MetricsWithDefaultOptions) { OtlpMetricsFlusherImpl flusher(otlpOptions()); addCounterToSnapshot("test_counter", 1, 1); + addHostCounterToSnapshot("test_host_counter", 2, 3); addGaugeToSnapshot("test_gauge", 1); + addHostGaugeToSnapshot("test_host_gauge", 4); addHistogramToSnapshot("test_histogram"); MetricsExportRequestSharedPtr metrics = flusher.flush(snapshot_); - expectMetricsCount(metrics, 3); + expectMetricsCount(metrics, 5); { const auto metric = metricAt(0, metrics); @@ -230,33 +273,49 @@ TEST_F(OtlpMetricsFlusherTests, MetricsWithDefaultOptions) { { const auto metric = metricAt(1, metrics); + expectGauge(metric, getTagExtractedName("test_host_gauge"), 4); + expectAttributes(metric.gauge().data_points()[0].attributes(), "gauge_key", "gauge_val"); + } + + { + const auto metric = metricAt(2, metrics); expectSum(metric, getTagExtractedName("test_counter"), 1, false); expectAttributes(metric.sum().data_points()[0].attributes(), "counter_key", "counter_val"); } { - const auto metric = metricAt(2, metrics); + const auto metric = metricAt(3, metrics); + expectSum(metric, getTagExtractedName("test_host_counter"), 3, false); + expectAttributes(metric.sum().data_points()[0].attributes(), "counter_key", "counter_val"); + } + + { + const auto metric = metricAt(4, metrics); expectHistogram(metric, getTagExtractedName("test_histogram"), false); expectAttributes(metric.histogram().data_points()[0].attributes(), "hist_key", "hist_val"); } gauge_storage_.back()->used_ = false; metrics = flusher.flush(snapshot_); - expectMetricsCount(metrics, 2); + expectMetricsCount(metrics, 4); } TEST_F(OtlpMetricsFlusherTests, MetricsWithStatsPrefix) { OtlpMetricsFlusherImpl flusher(otlpOptions(false, false, true, true, "prefix")); addCounterToSnapshot("test_counter", 1, 1); + addHostCounterToSnapshot("test_host_counter", 1, 1); addGaugeToSnapshot("test_gauge", 1); + addGaugeToSnapshot("test_host_gauge", 1); addHistogramToSnapshot("test_histogram"); MetricsExportRequestSharedPtr metrics = flusher.flush(snapshot_); - expectMetricsCount(metrics, 3); + expectMetricsCount(metrics, 5); expectGauge(metricAt(0, metrics), getTagExtractedName("prefix.test_gauge"), 1); - expectSum(metricAt(1, metrics), getTagExtractedName("prefix.test_counter"), 1, false); - expectHistogram(metricAt(2, metrics), getTagExtractedName("prefix.test_histogram"), false); + expectGauge(metricAt(1, metrics), getTagExtractedName("prefix.test_host_gauge"), 1); + expectSum(metricAt(2, metrics), getTagExtractedName("prefix.test_counter"), 1, false); + expectSum(metricAt(3, metrics), getTagExtractedName("prefix.test_host_counter"), 1, false); + expectHistogram(metricAt(4, metrics), getTagExtractedName("prefix.test_histogram"), false); } TEST_F(OtlpMetricsFlusherTests, MetricsWithNoTaggedName) { @@ -307,11 +366,15 @@ TEST_F(OtlpMetricsFlusherTests, GaugeMetric) { addGaugeToSnapshot("test_gauge1", 1); addGaugeToSnapshot("test_gauge2", 2); + addHostGaugeToSnapshot("test_host_gauge1", 3); + addHostGaugeToSnapshot("test_host_gauge2", 4); MetricsExportRequestSharedPtr metrics = flusher.flush(snapshot_); - expectMetricsCount(metrics, 2); + expectMetricsCount(metrics, 4); expectGauge(metricAt(0, metrics), getTagExtractedName("test_gauge1"), 1); expectGauge(metricAt(1, metrics), getTagExtractedName("test_gauge2"), 2); + expectGauge(metricAt(2, metrics), getTagExtractedName("test_host_gauge1"), 3); + expectGauge(metricAt(3, metrics), getTagExtractedName("test_host_gauge2"), 4); } TEST_F(OtlpMetricsFlusherTests, CumulativeCounterMetric) { @@ -319,11 +382,15 @@ TEST_F(OtlpMetricsFlusherTests, CumulativeCounterMetric) { addCounterToSnapshot("test_counter1", 1, 1); addCounterToSnapshot("test_counter2", 2, 3); + addHostCounterToSnapshot("test_host_counter1", 2, 4); + addHostCounterToSnapshot("test_host_counter2", 5, 10); MetricsExportRequestSharedPtr metrics = flusher.flush(snapshot_); - expectMetricsCount(metrics, 2); + expectMetricsCount(metrics, 4); expectSum(metricAt(0, metrics), getTagExtractedName("test_counter1"), 1, false); expectSum(metricAt(1, metrics), getTagExtractedName("test_counter2"), 3, false); + expectSum(metricAt(2, metrics), getTagExtractedName("test_host_counter1"), 4, false); + expectSum(metricAt(3, metrics), getTagExtractedName("test_host_counter2"), 10, false); } TEST_F(OtlpMetricsFlusherTests, DeltaCounterMetric) { @@ -331,11 +398,15 @@ TEST_F(OtlpMetricsFlusherTests, DeltaCounterMetric) { addCounterToSnapshot("test_counter1", 1, 1); addCounterToSnapshot("test_counter2", 2, 3); + addHostCounterToSnapshot("test_host_counter1", 2, 4); + addHostCounterToSnapshot("test_host_counter2", 5, 10); MetricsExportRequestSharedPtr metrics = flusher.flush(snapshot_); - expectMetricsCount(metrics, 2); + expectMetricsCount(metrics, 4); expectSum(metricAt(0, metrics), getTagExtractedName("test_counter1"), 1, true); expectSum(metricAt(1, metrics), getTagExtractedName("test_counter2"), 2, true); + expectSum(metricAt(2, metrics), getTagExtractedName("test_host_counter1"), 2, true); + expectSum(metricAt(3, metrics), getTagExtractedName("test_host_counter2"), 5, true); } TEST_F(OtlpMetricsFlusherTests, CumulativeHistogramMetric) { diff --git a/test/extensions/stats_sinks/open_telemetry/open_telemetry_integration_test.cc b/test/extensions/stats_sinks/open_telemetry/open_telemetry_integration_test.cc index f170834fdbdb..6661a52a9445 100644 --- a/test/extensions/stats_sinks/open_telemetry/open_telemetry_integration_test.cc +++ b/test/extensions/stats_sinks/open_telemetry/open_telemetry_integration_test.cc @@ -38,7 +38,7 @@ class OpenTelemetryGrpcIntegrationTest : public Grpc::GrpcClientIntegrationParam void setStatPrefix(const std::string& stat_prefix) { stat_prefix_ = stat_prefix; } const std::string getFullStatName(const std::string& stat_name) { - if (stat_prefix_ == "") { + if (stat_prefix_.empty()) { return stat_name; } @@ -61,7 +61,7 @@ class OpenTelemetryGrpcIntegrationTest : public Grpc::GrpcClientIntegrationParam metrics_sink->mutable_typed_config()->PackFrom(sink_config); bootstrap.mutable_stats_flush_interval()->CopyFrom( - Protobuf::util::TimeUtil::MillisecondsToDuration(300)); + Protobuf::util::TimeUtil::MillisecondsToDuration(500)); }); HttpIntegrationTest::initialize(); @@ -136,7 +136,7 @@ class OpenTelemetryGrpcIntegrationTest : public Grpc::GrpcClientIntegrationParam known_histogram_exists = true; EXPECT_EQ(1, metric.histogram().data_points().size()); EXPECT_EQ(metric.histogram().data_points()[0].bucket_counts().size(), - Stats::HistogramSettingsImpl::defaultBuckets().size()); + Stats::HistogramSettingsImpl::defaultBuckets().size() + 1); EXPECT_TRUE(metric.histogram().data_points()[0].time_unix_nano() > 0); if (previous_time_stamp > 0) { diff --git a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc index 07bd22ec9955..09c337108c1b 100644 --- a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc @@ -183,9 +183,8 @@ class OpenTracingDriverTest : public testing::Test { } const std::string operation_name_{"test"}; - Http::TestRequestHeaderMapImpl request_headers_{ + Tracing::TestTraceContextImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; - const Http::TestResponseHeaderMapImpl response_headers_{{":status", "500"}}; NiceMock stream_info_; std::unique_ptr driver_; @@ -317,7 +316,8 @@ TEST_F(OpenTracingDriverTest, InjectFailure) { const auto span_context_injection_error_count = stats_.counter("tracing.opentracing.span_context_injection_error").value(); - EXPECT_FALSE(request_headers_.has(Http::CustomHeaders::get().OtSpanContext)); + EXPECT_FALSE( + request_headers_.context_map_.contains(Http::CustomHeaders::get().OtSpanContext.get())); span->injectContext(request_headers_, nullptr); EXPECT_EQ(span_context_injection_error_count + 1, @@ -365,7 +365,7 @@ TEST_F(OpenTracingDriverTest, ExtractUsingForeach) { {Tracing::Reason::Sampling, true}); for (const auto& [key, value] : extracted_headers) { - EXPECT_EQ(value, request_headers_.getByKey(key)); + EXPECT_EQ(value, request_headers_.get(key)); } } diff --git a/test/extensions/tracers/datadog/BUILD b/test/extensions/tracers/datadog/BUILD index d292aa9c0f39..4eba5b353436 100644 --- a/test/extensions/tracers/datadog/BUILD +++ b/test/extensions/tracers/datadog/BUILD @@ -19,6 +19,7 @@ envoy_extension_cc_test( "dict_util_test.cc", "event_scheduler_test.cc", "logger_test.cc", + "naming_test.cc", "span_test.cc", "time_util_test.cc", "tracer_stats_test.cc", diff --git a/test/extensions/tracers/datadog/agent_http_client_test.cc b/test/extensions/tracers/datadog/agent_http_client_test.cc index 98e60c971578..08a461b90894 100644 --- a/test/extensions/tracers/datadog/agent_http_client_test.cc +++ b/test/extensions/tracers/datadog/agent_http_client_test.cc @@ -78,7 +78,7 @@ TEST_F(DatadogAgentHttpClientTest, PathFromURL) { .WillOnce( Invoke([this](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks&, const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { - EXPECT_EQ(url_.path, message->headers().path()); + EXPECT_EQ(url_.path, message->headers().getPathValue()); return &request_; })); @@ -121,17 +121,23 @@ TEST_F(DatadogAgentHttpClientTest, RequestHeaders) { }; EXPECT_CALL(cluster_manager_.instance_.thread_local_cluster_.async_client_, send_(_, _, _)) - .WillOnce( - Invoke([this](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks&, - const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { - EXPECT_EQ("test_host", message->headers().getHostValue()); - - EXPECT_EQ("bar", message->headers().getByKey("foo")); - EXPECT_EQ("boing boing", message->headers().getByKey("baz-boing")); - EXPECT_EQ("boing boing boing", message->headers().getByKey("boing-boing")); - - return &request_; - })); + .WillOnce(Invoke([this](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks&, + const Http::AsyncClient::RequestOptions&) + -> Http::AsyncClient::Request* { + EXPECT_EQ("test_host", message->headers().getHostValue()); + + EXPECT_EQ("bar", + message->headers().get(Http::LowerCaseString("foo"))[0]->value().getStringView()); + EXPECT_EQ( + "boing boing", + message->headers().get(Http::LowerCaseString("baz-boing"))[0]->value().getStringView()); + EXPECT_EQ("boing boing boing", message->headers() + .get(Http::LowerCaseString("boing-boing"))[0] + ->value() + .getStringView()); + + return &request_; + })); // `~AgentHTTPClient()` will cancel the request since we don't finish it here. EXPECT_CALL(request_, cancel()); diff --git a/test/extensions/tracers/datadog/config_test.cc b/test/extensions/tracers/datadog/config_test.cc index 1a412108253e..acb8de730e19 100644 --- a/test/extensions/tracers/datadog/config_test.cc +++ b/test/extensions/tracers/datadog/config_test.cc @@ -80,7 +80,7 @@ class DatadogConfigTest : public testing::Test { } const std::string operation_name_{"test"}; - Http::TestRequestHeaderMapImpl request_headers_{ + Tracing::TestTraceContextImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; NiceMock tls_; diff --git a/test/extensions/tracers/datadog/naming_test.cc b/test/extensions/tracers/datadog/naming_test.cc new file mode 100644 index 000000000000..0d45cde32b1a --- /dev/null +++ b/test/extensions/tracers/datadog/naming_test.cc @@ -0,0 +1,223 @@ +/** + * The tests in this file aren't specific to a class, but instead test all + * behavior related to spans' "operation name" (a.k.a. "span name"), + * "resource name," and "service name." + * + * Datadog's model of a span is different from Envoy's. Each Datadog span, + * in addition to having a "service name" and an "operation name," also has a + * "resource name." The operation name indicates the _kind_ of operation + * that is being performed by the service, whereas the resource name contains + * more specifics about what is being operated upon, or about what is doing the + * operating. Envoy has no notion of "resource name," and instead uses + * operation name and tags for this purpose. + * + * When Envoy's tracing interface indicates an operation name, the Datadog + * tracer translates it into a resource name instead. The actual Datadog + * operation name is always hard-coded to the value "envoy.proxy". + * + * Finally, each span's "service name" is derived either from the tracer's + * configuration or a hard-coded default, which is "envoy". + * + * The tests in this file verify all of this behavior for a variety of + * scenarios where spans are created or modified. + */ + +#include "source/extensions/tracers/datadog/config.h" +#include "source/extensions/tracers/datadog/span.h" +#include "source/extensions/tracers/datadog/tracer.h" + +#include "test/mocks/stream_info/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace Datadog { +namespace { + +template Config makeConfig(const std::string& yaml) { + Config result; + TestUtility::loadFromYaml(yaml, result); + return result; +} + +class DatadogTracerNamingTest : public testing::Test { +public: + DatadogTracerNamingTest(); + +protected: + /** + * Verify that a tracer configured using the specified \p config_yaml + * produces spans and child spans having the specified + * \p expected_service_name. + * @param config_yaml YAML representation of a + * \c type.googleapis.com/envoy.config.trace.v3.DatadogConfig + * @param expected_service_name service name to expect each span to have + */ + void serviceNameTest(const std::string& config_yaml, const std::string& expected_service_name); + + /** + * Assign through the specified \p result a pointer to the underlying + * \c datadog::tracing::Span to which the specified \p span refers. + * If \p span does not refer to a Datadog span, then this function triggers a + * fatal test assertion. + * An output parameter is used because the \c ASSERT_* macros require that + * the enclosing function have \c void return type. + * @param result pointer to the output value to overwrite + * @param span pointer to an Envoy span that refers to a Datadog span + */ + static void asDatadogSpan(const datadog::tracing::Span** result, const Tracing::SpanPtr& span); + + NiceMock cluster_manager_; + Stats::TestUtil::TestStore store_; + NiceMock thread_local_slot_allocator_; + Event::SimulatedTimeSystem time_; + NiceMock stream_info_; +}; + +DatadogTracerNamingTest::DatadogTracerNamingTest() { + cluster_manager_.initializeClusters({"fake_cluster"}, {}); + cluster_manager_.thread_local_cluster_.cluster_.info_->name_ = "fake_cluster"; + cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); +} + +void DatadogTracerNamingTest::asDatadogSpan(const datadog::tracing::Span** result, + const Tracing::SpanPtr& span) { + ASSERT_TRUE(span); + const auto as_dd_span_wrapper = dynamic_cast(span.get()); + ASSERT_NE(nullptr, as_dd_span_wrapper); + + const datadog::tracing::Optional& maybe_dd_span = + as_dd_span_wrapper->impl(); + ASSERT_TRUE(maybe_dd_span); + *result = &*maybe_dd_span; +} + +void DatadogTracerNamingTest::serviceNameTest(const std::string& config_yaml, + const std::string& expected_service_name) { + auto config_proto = makeConfig(config_yaml); + + Tracer tracer{config_proto.collector_cluster(), + config_proto.collector_hostname(), + DatadogTracerFactory::makeConfig(config_proto), + cluster_manager_, + *store_.rootScope(), + thread_local_slot_allocator_}; + + // Any values will do for the sake of this test. What we care about is the + // `expected_service_name`. + Tracing::Decision decision; + decision.reason = Tracing::Reason::Sampling; + decision.traced = true; + const std::string operation_name = "some.operation.name"; + Tracing::TestTraceContextImpl context{}; + + const Tracing::SpanPtr span = + tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, operation_name, decision); + const datadog::tracing::Span* dd_span; + asDatadogSpan(&dd_span, span); + + EXPECT_EQ(expected_service_name, dd_span->service_name()); + + const auto child_start = time_.timeSystem().systemTime(); + const std::string child_operation_name = "some.other.operation.name"; + const Tracing::SpanPtr child = + span->spawnChild(Tracing::MockConfig{}, child_operation_name, child_start); + const datadog::tracing::Span* dd_child; + asDatadogSpan(&dd_child, child); + + EXPECT_EQ(expected_service_name, dd_child->service_name()); +} + +TEST_F(DatadogTracerNamingTest, ServiceNameConfigured) { + // If you specify a `service_name` in the tracer configuration, then spans + // created will have that service name. + serviceNameTest(R"EOF( + collector_cluster: fake_cluster + service_name: mr_bigglesworth + )EOF", + "mr_bigglesworth"); +} + +TEST_F(DatadogTracerNamingTest, ServiceNameDefault) { + // If you don't specify a `service_name` in the tracer configuration, then + // spans created will have the default service name, which is "envoy". + serviceNameTest(R"EOF( + collector_cluster: fake_cluster + )EOF", + "envoy"); +} + +TEST_F(DatadogTracerNamingTest, OperationNameAndResourceName) { + // Concerns: + // + // - The span returned by `Tracer::startSpan` has as its resource name the + // operation name passed to `Tracer::startSpan`, and has as its operation + // name "envoy.proxy". + // - The span returned by `Span::spawnChild` has as its resource name the + // operation name passed to `Tracer::spawnChild`, and has as its operation + // name "envoy.proxy". + // - `Span::setOperation` sets the resource name of the span, but does not + // change the operation name. + + auto config_proto = makeConfig(R"EOF( + collector_cluster: fake_cluster + )EOF"); + + Tracer tracer{config_proto.collector_cluster(), + config_proto.collector_hostname(), + DatadogTracerFactory::makeConfig(config_proto), + cluster_manager_, + *store_.rootScope(), + thread_local_slot_allocator_}; + + // Any values will do for the sake of this test. What we care about are the + // operation names and the resource names. + Tracing::Decision decision; + decision.reason = Tracing::Reason::Sampling; + decision.traced = true; + Tracing::TestTraceContextImpl context{}; + + const std::string operation_name = "some.operation.name"; + const Tracing::SpanPtr span = + tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, operation_name, decision); + const datadog::tracing::Span* dd_span; + asDatadogSpan(&dd_span, span); + + EXPECT_EQ("envoy.proxy", dd_span->name()); + EXPECT_EQ(operation_name, dd_span->resource_name()); + + const std::string new_operation_name = "some.new.operation.name"; + span->setOperation(new_operation_name); + + EXPECT_EQ("envoy.proxy", dd_span->name()); + EXPECT_EQ(new_operation_name, dd_span->resource_name()); + + const auto child_start = time_.timeSystem().systemTime(); + const std::string child_operation_name = "some.child.operation.name"; + const Tracing::SpanPtr child = + span->spawnChild(Tracing::MockConfig{}, child_operation_name, child_start); + const datadog::tracing::Span* dd_child; + asDatadogSpan(&dd_child, child); + + EXPECT_EQ("envoy.proxy", dd_child->name()); + EXPECT_EQ(child_operation_name, dd_child->resource_name()); + + const std::string child_new_operation_name = "some.child.new.operation.name"; + child->setOperation(child_new_operation_name); + + EXPECT_EQ("envoy.proxy", dd_child->name()); + EXPECT_EQ(child_new_operation_name, dd_child->resource_name()); +} + +} // namespace +} // namespace Datadog +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/datadog/span_test.cc b/test/extensions/tracers/datadog/span_test.cc index 70b4fafd9352..436685b36f3a 100644 --- a/test/extensions/tracers/datadog/span_test.cc +++ b/test/extensions/tracers/datadog/span_test.cc @@ -84,8 +84,7 @@ class MockIDGenerator : public datadog::tracing::IDGenerator { class DatadogTracerSpanTest : public testing::Test { public: DatadogTracerSpanTest() - : id_(0xcafebabe), collector_(std::make_shared()), - config_(makeConfig(collector_)), + : collector_(std::make_shared()), config_(makeConfig(collector_)), tracer_( // Override the tracer's ID generator so that all trace IDs and span // IDs are 0xcafebabe. @@ -108,7 +107,7 @@ class DatadogTracerSpanTest : public testing::Test { } protected: - const std::uint64_t id_; + const std::uint64_t id_{0xcafebabe}; const std::shared_ptr collector_; const datadog::tracing::TracerConfig config_; datadog::tracing::Tracer tracer_; @@ -128,7 +127,10 @@ TEST_F(DatadogTracerSpanTest, SetOperation) { ASSERT_NE(nullptr, data_ptr); const datadog::tracing::SpanData& data = *data_ptr; - EXPECT_EQ("gastric bypass", data.name); + // Setting the operation name actually sets the resource name, because Envoy's + // notion of operation name more closely matches Datadog's notion of resource + // name. + EXPECT_EQ("gastric bypass", data.resource); } TEST_F(DatadogTracerSpanTest, SetTag) { @@ -136,6 +138,7 @@ TEST_F(DatadogTracerSpanTest, SetTag) { span.setTag("foo", "bar"); span.setTag("boom", "bam"); span.setTag("foo", "new"); + span.setTag("resource.name", "vespene gas"); span.finishSpan(); ASSERT_EQ(1, collector_->chunks.size()); @@ -152,6 +155,12 @@ TEST_F(DatadogTracerSpanTest, SetTag) { found = data.tags.find("boom"); ASSERT_NE(data.tags.end(), found); EXPECT_EQ("bam", found->second); + + // The "resource.name" tag is special. It doesn't set a tag, but instead the + // span's resource name. + found = data.tags.find("resource.name"); + ASSERT_EQ(data.tags.end(), found); + EXPECT_EQ("vespene gas", data.resource); } TEST_F(DatadogTracerSpanTest, InjectContext) { @@ -197,7 +206,11 @@ TEST_F(DatadogTracerSpanTest, SpawnChild) { EXPECT_NE(nullptr, child_ptr); const datadog::tracing::SpanData& child = *child_ptr; EXPECT_EQ(estimateTime(child_start).wall, child.start.wall); - EXPECT_EQ("child", child.name); + // Setting the operation name actually sets the resource name, because + // Envoy's notion of operation name more closely matches Datadog's notion of + // resource name. The actual operation name is hard-coded as "envoy.proxy". + EXPECT_EQ("child", child.resource); + EXPECT_EQ("envoy.proxy", child.name); EXPECT_EQ(id_, child.trace_id); EXPECT_EQ(id_, child.span_id); EXPECT_EQ(id_, child.parent_id); diff --git a/test/extensions/tracers/datadog/tracer_test.cc b/test/extensions/tracers/datadog/tracer_test.cc index 1247b8a44f28..7cd37a8a3329 100644 --- a/test/extensions/tracers/datadog/tracer_test.cc +++ b/test/extensions/tracers/datadog/tracer_test.cc @@ -1,3 +1,5 @@ +#include + #include "envoy/tracing/trace_reason.h" #include "source/common/tracing/null_span_impl.h" @@ -7,11 +9,14 @@ #include "test/mocks/stream_info/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/environment.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "datadog/error.h" #include "datadog/expected.h" +#include "datadog/optional.h" +#include "datadog/propagation_style.h" #include "datadog/sampling_priority.h" #include "datadog/trace_segment.h" #include "datadog/tracer_config.h" @@ -23,6 +28,30 @@ namespace Tracers { namespace Datadog { namespace { +class EnvVarGuard { +public: + EnvVarGuard(const std::string& name, const std::string& value) : name_(name) { + if (const char* const previous = std::getenv(name.c_str())) { + previous_value_ = previous; + } + const int overwrite = 1; // Yes, overwrite it. + TestEnvironment::setEnvVar(name, value, overwrite); + } + + ~EnvVarGuard() { + if (previous_value_) { + const int overwrite = 1; // Yes, overwrite it. + TestEnvironment::setEnvVar(name_, *previous_value_, overwrite); + } else { + TestEnvironment::unsetEnvVar(name_); + } + } + +private: + std::string name_; + datadog::tracing::Optional previous_value_; +}; + class DatadogTracerTest : public testing::Test { public: DatadogTracerTest() { @@ -116,9 +145,13 @@ TEST_F(DatadogTracerTest, SpanProperties) { ASSERT_TRUE(maybe_dd_span); const datadog::tracing::Span& dd_span = *maybe_dd_span; - // Verify that the span has the expected service name, operation name, start - // time, and sampling decision. - EXPECT_EQ("do.thing", dd_span.name()); + // Verify that the span has the expected service name, operation name, + // resource name, start time, and sampling decision. + // Note that the `operation_name` we specified above becomes the + // `resource_name()` of the resulting Datadog span, while the Datadog span's + // `name()` (operation name) is hard-coded to "envoy.proxy." + EXPECT_EQ("envoy.proxy", dd_span.name()); + EXPECT_EQ("do.thing", dd_span.resource_name()); EXPECT_EQ("envoy", dd_span.service_name()); ASSERT_TRUE(dd_span.trace_segment().sampling_decision()); EXPECT_EQ(int(datadog::tracing::SamplingPriority::USER_DROP), @@ -199,6 +232,145 @@ TEST_F(DatadogTracerTest, ExtractionFailure) { ASSERT_TRUE(maybe_dd_span); } +TEST_F(DatadogTracerTest, EnvoySamplingVersusExtractedSampling) { + // Verify that sampling decisions extracted from incoming requests are honored + // regardless of the sampling decision made by Envoy (i.e. `bool + // Tracing::Decision::traced`). + // + // We test two styles of extraction: OpenTelemetry's W3C "tracecontext" style + // and Datadog's "datadog" style. When trace context is extracted in either of + // these styles, a sampling decision might be present. If a sampling decision + // is present, then the resulting sampling priority in the extracted trace + // must be the same as that which was extracted. + // + // If a sampling decision is not present in the extracted trace context, then + // an Envoy decision of "drop" is honored. An Envoy decision of "keep" + // delegates the sampling decision to the underlying Datadog tracer, which + // will not make a sampling decision immediately. + + struct Case { + int line; + datadog::tracing::Optional extracted_sampling_priority; + bool envoy_decision_keep; + datadog::tracing::PropagationStyle extraction_style; + // `resulting_sampling_priority` is the sampling priority that results from + // trace context extraction. + // It's not necessarily the sampling priority that would be sent to the + // Datadog Agent. + // If `resulting_sampling_priority` is null, then that means that the tracer + // does not make an initial sampling decision, though it will make one by + // the time is sends spans to the Datadog Agent or injects trace context + // into an outgoing request. + datadog::tracing::Optional resulting_sampling_priority; + } cases[] = { + {__LINE__, datadog::tracing::nullopt, true, datadog::tracing::PropagationStyle::DATADOG, + datadog::tracing::nullopt}, + // Note that the `resulting_sampling_priority` in this case is an artifact + // of "traceparent" always containing a sampling decision in its flags. See + // the main body of the test, below, for more information. + {__LINE__, datadog::tracing::nullopt, true, datadog::tracing::PropagationStyle::W3C, 0}, + // This is the only case, at least in this test, where Envoy's decision + // affects the resulting sampling priority. + {__LINE__, datadog::tracing::nullopt, false, datadog::tracing::PropagationStyle::DATADOG, -1}, + {__LINE__, datadog::tracing::nullopt, false, datadog::tracing::PropagationStyle::W3C, 0}, + + {__LINE__, -1, true, datadog::tracing::PropagationStyle::DATADOG, -1}, + {__LINE__, -1, true, datadog::tracing::PropagationStyle::W3C, -1}, + {__LINE__, -1, false, datadog::tracing::PropagationStyle::DATADOG, -1}, + {__LINE__, -1, false, datadog::tracing::PropagationStyle::W3C, -1}, + + {__LINE__, 0, true, datadog::tracing::PropagationStyle::DATADOG, 0}, + {__LINE__, 0, true, datadog::tracing::PropagationStyle::W3C, 0}, + {__LINE__, 0, false, datadog::tracing::PropagationStyle::DATADOG, 0}, + {__LINE__, 0, false, datadog::tracing::PropagationStyle::W3C, 0}, + + {__LINE__, 1, true, datadog::tracing::PropagationStyle::DATADOG, 1}, + {__LINE__, 1, true, datadog::tracing::PropagationStyle::W3C, 1}, + {__LINE__, 1, false, datadog::tracing::PropagationStyle::DATADOG, 1}, + {__LINE__, 1, false, datadog::tracing::PropagationStyle::W3C, 1}, + + {__LINE__, 2, true, datadog::tracing::PropagationStyle::DATADOG, 2}, + {__LINE__, 2, true, datadog::tracing::PropagationStyle::W3C, 2}, + {__LINE__, 2, false, datadog::tracing::PropagationStyle::DATADOG, 2}, + {__LINE__, 2, false, datadog::tracing::PropagationStyle::W3C, 2}, + }; + + for (const Case& test_case : cases) { + std::ostringstream failure_context; + failure_context << "Failure occurred for test entry on line " << test_case.line; + + std::string style_name; + if (test_case.extraction_style == datadog::tracing::PropagationStyle::DATADOG) { + style_name = "datadog"; + } else { + ASSERT_EQ(test_case.extraction_style, datadog::tracing::PropagationStyle::W3C) + << failure_context.str(); + style_name = "tracecontext"; + } + + EnvVarGuard guard{"DD_TRACE_PROPAGATION_STYLE", style_name}; + datadog::tracing::TracerConfig config; + config.defaults.service = "envoy"; + Tracer tracer("fake_cluster", "test_host", config, cluster_manager_, *store_.rootScope(), + thread_local_slot_allocator_); + + Tracing::Decision envoy_decision; + envoy_decision.reason = Tracing::Reason::Sampling; + envoy_decision.traced = test_case.envoy_decision_keep; + + const std::string operation_name = "do.thing"; + + Tracing::TestTraceContextImpl context{{}}; + if (test_case.extraction_style == datadog::tracing::PropagationStyle::DATADOG) { + context.context_map_["x-datadog-trace-id"] = "123"; + context.context_map_["x-datadog-parent-id"] = "456"; + if (test_case.extracted_sampling_priority) { + context.context_map_["x-datadog-sampling-priority"] = + std::to_string(*test_case.extracted_sampling_priority); + } + } else { + ASSERT_EQ(test_case.extraction_style, datadog::tracing::PropagationStyle::W3C) + << failure_context.str(); + std::string flags; + if (test_case.extracted_sampling_priority) { + const int priority = *test_case.extracted_sampling_priority; + flags = priority <= 0 ? "00" : "01"; + context.context_map_["tracestate"] = "dd=s:" + std::to_string(priority); + } else { + // There's no such thing as the absence of a sampling decision with + // "traceparent," so default to "drop." + flags = "00"; + } + context.context_map_["traceparent"] = + "00-0000000000000000000000000000007b-00000000000001c8-" + flags; + } + + const Tracing::SpanPtr span = tracer.startSpan(Tracing::MockConfig{}, context, stream_info_, + operation_name, envoy_decision); + ASSERT_TRUE(span) << failure_context.str(); + const auto as_dd_span_wrapper = dynamic_cast(span.get()); + EXPECT_NE(nullptr, as_dd_span_wrapper) << failure_context.str(); + + const datadog::tracing::Optional& maybe_dd_span = + as_dd_span_wrapper->impl(); + ASSERT_TRUE(maybe_dd_span) << failure_context.str(); + const datadog::tracing::Span& dd_span = *maybe_dd_span; + + const datadog::tracing::Optional decision = + dd_span.trace_segment().sampling_decision(); + if (test_case.resulting_sampling_priority) { + // We expect that the tracer made a sampling decision immediately, and + // that it has the expected sampling priority. + ASSERT_NE(datadog::tracing::nullopt, decision) << failure_context.str(); + EXPECT_EQ(*test_case.resulting_sampling_priority, decision->priority) + << failure_context.str(); + } else { + // We expect that the tracer did not immediately make a sampling decision. + EXPECT_EQ(datadog::tracing::nullopt, decision) << failure_context.str(); + } + } +} + } // namespace } // namespace Datadog } // namespace Tracers diff --git a/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc b/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc index dc16e9434c45..c6723ed0c700 100644 --- a/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/dynamic_ot/dynamic_opentracing_driver_impl_test.cc @@ -41,7 +41,7 @@ class DynamicOpenTracingDriverTest : public testing::Test { Stats::IsolatedStoreImpl stats_; const std::string operation_name_{"test"}; - Http::TestRequestHeaderMapImpl request_headers_{ + Tracing::TestTraceContextImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; NiceMock config_; diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 13eeffd2e7c7..0548b99bc569 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -107,7 +107,7 @@ TEST(OpenCensusTracerTest, Span) { new OpenCensus::Driver(oc_config, local_info, *Api::createApiForTest())); NiceMock config; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; const std::string operation_name{"my_operation_1"}; SystemTime fake_system_time; @@ -177,10 +177,10 @@ MATCHER_P2(ContainHeader, header, expected_value, "contains the header " + PrintToString(header) + " with value " + PrintToString(expected_value)) { const auto found_value = arg.get(Http::LowerCaseString(header)); - if (found_value.empty()) { + if (!found_value.has_value()) { return false; } - return found_value[0]->value().getStringView() == expected_value; + return found_value.value() == expected_value; } // Given incoming headers, test that trace context propagation works and generates all the expected @@ -203,18 +203,18 @@ void testIncomingHeaders( std::unique_ptr driver( new OpenCensus::Driver(oc_config, local_info, *Api::createApiForTest())); NiceMock config; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}, }; for (const auto& kv : headers) { - request_headers.addCopy(Http::LowerCaseString(kv.first), kv.second); + request_headers.set(Http::LowerCaseString(kv.first), kv.second); } const std::string operation_name{"my_operation_2"}; NiceMock stream_info; - Http::TestRequestHeaderMapImpl injected_headers; + Tracing::TestTraceContextImpl injected_headers{}; { Tracing::SpanPtr span = driver->startSpan(config, request_headers, stream_info, operation_name, {Tracing::Reason::Sampling, false}); @@ -289,7 +289,7 @@ namespace { // Create a Span using the given config and return how many spans made it to // the exporter (either zero or one). -int SamplerTestHelper(const OpenCensusConfig& oc_config) { +int samplerTestHelper(const OpenCensusConfig& oc_config) { registerSpanCatcher(); NiceMock local_info; std::unique_ptr driver( @@ -313,7 +313,14 @@ TEST(OpenCensusTracerTest, ConstantSamplerAlwaysOn) { OpenCensusConfig oc_config; oc_config.mutable_trace_config()->mutable_constant_sampler()->set_decision( ::opencensus::proto::trace::v1::ConstantSampler::ALWAYS_ON); - EXPECT_EQ(1, SamplerTestHelper(oc_config)); + EXPECT_EQ(1, samplerTestHelper(oc_config)); +} + +// Test no sampler set. +TEST(OpenCensusTracerTest, NoSamplerSet) { + OpenCensusConfig oc_config; + oc_config.mutable_trace_config(); + samplerTestHelper(oc_config); } // Test constant_sampler that's always off. @@ -321,21 +328,21 @@ TEST(OpenCensusTracerTest, ConstantSamplerAlwaysOff) { OpenCensusConfig oc_config; oc_config.mutable_trace_config()->mutable_constant_sampler()->set_decision( ::opencensus::proto::trace::v1::ConstantSampler::ALWAYS_OFF); - EXPECT_EQ(0, SamplerTestHelper(oc_config)); + EXPECT_EQ(0, samplerTestHelper(oc_config)); } // Test probability_sampler that's always on. TEST(OpenCensusTracerTest, ProbabilitySamplerAlwaysOn) { OpenCensusConfig oc_config; oc_config.mutable_trace_config()->mutable_probability_sampler()->set_samplingprobability(1.0); - EXPECT_EQ(1, SamplerTestHelper(oc_config)); + EXPECT_EQ(1, samplerTestHelper(oc_config)); } // Test probability_sampler that's always off. TEST(OpenCensusTracerTest, ProbabilitySamplerAlwaysOff) { OpenCensusConfig oc_config; oc_config.mutable_trace_config()->mutable_probability_sampler()->set_samplingprobability(0.0); - EXPECT_EQ(0, SamplerTestHelper(oc_config)); + EXPECT_EQ(0, samplerTestHelper(oc_config)); } } // namespace OpenCensus diff --git a/test/extensions/tracers/opentelemetry/BUILD b/test/extensions/tracers/opentelemetry/BUILD index c1ac977443d6..96e7e5d91537 100644 --- a/test/extensions/tracers/opentelemetry/BUILD +++ b/test/extensions/tracers/opentelemetry/BUILD @@ -73,9 +73,23 @@ envoy_extension_cc_test( srcs = ["grpc_trace_exporter_test.cc"], extension_names = ["envoy.tracers.opentelemetry"], deps = [ - "//source/extensions/tracers/opentelemetry:grpc_trace_exporter", + "//source/extensions/tracers/opentelemetry:trace_exporter", "//test/mocks/grpc:grpc_mocks", "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", ], ) + +envoy_extension_cc_test( + name = "http_trace_exporter_test", + srcs = ["http_trace_exporter_test.cc"], + extension_names = ["envoy.tracers.opentelemetry"], + deps = [ + "//source/extensions/tracers/opentelemetry:trace_exporter", + "//test/mocks/http:http_mocks", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/mocks/stats:stats_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/tracers/opentelemetry/config_test.cc b/test/extensions/tracers/opentelemetry/config_test.cc index 72206f07910d..839107f24e0e 100644 --- a/test/extensions/tracers/opentelemetry/config_test.cc +++ b/test/extensions/tracers/opentelemetry/config_test.cc @@ -16,7 +16,7 @@ namespace Extensions { namespace Tracers { namespace OpenTelemetry { -TEST(OpenTelemetryTracerConfigTest, OpenTelemetryHttpTracer) { +TEST(OpenTelemetryTracerConfigTest, OpenTelemetryTracerWithGrpcExporter) { NiceMock context; context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); OpenTelemetryTracerFactory factory; @@ -41,7 +41,7 @@ TEST(OpenTelemetryTracerConfigTest, OpenTelemetryHttpTracer) { EXPECT_NE(nullptr, opentelemetry_tracer); } -TEST(OpenTelemetryTracerConfigTest, OpenTelemetryHttpTracerNoExporter) { +TEST(OpenTelemetryTracerConfigTest, OpenTelemetryTracerWithHttpExporter) { NiceMock context; context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); OpenTelemetryTracerFactory factory; @@ -51,6 +51,15 @@ TEST(OpenTelemetryTracerConfigTest, OpenTelemetryHttpTracerNoExporter) { name: envoy.tracers.opentelemetry typed_config: "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig + http_service: + http_uri: + uri: "https://some-o11y.com//otlp/v1/traces" + cluster: "my_o11y_backend" + timeout: 0.250s + request_headers_to_add: + - header: + key: "Authorization" + value: "auth-token" )EOF"; envoy::config::trace::v3::Tracing configuration; TestUtility::loadFromYaml(yaml_string, configuration); @@ -61,6 +70,27 @@ TEST(OpenTelemetryTracerConfigTest, OpenTelemetryHttpTracerNoExporter) { EXPECT_NE(nullptr, opentelemetry_tracer); } +TEST(OpenTelemetryTracerConfigTest, OpenTelemetryTracerNoExporter) { + NiceMock context; + context.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + OpenTelemetryTracerFactory factory; + + const std::string yaml_string = R"EOF( + http: + name: envoy.tracers.opentelemetry + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig + )EOF"; + envoy::config::trace::v3::Tracing configuration; + TestUtility::loadFromYaml(yaml_string, configuration); + + auto message = Config::Utility::translateToFactoryConfig( + configuration.http(), ProtobufMessage::getStrictValidationVisitor(), factory); + + auto opentelemetry_tracer = factory.createTracerDriver(*message, context); + EXPECT_NE(nullptr, opentelemetry_tracer); +} + } // namespace OpenTelemetry } // namespace Tracers } // namespace Extensions diff --git a/test/extensions/tracers/opentelemetry/http_trace_exporter_test.cc b/test/extensions/tracers/opentelemetry/http_trace_exporter_test.cc new file mode 100644 index 000000000000..b6d5702c27e5 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/http_trace_exporter_test.cc @@ -0,0 +1,146 @@ +#include + +#include "source/common/buffer/zero_copy_input_stream_impl.h" +#include "source/extensions/tracers/opentelemetry/http_trace_exporter.h" + +#include "test/mocks/common.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +using testing::_; +using testing::Invoke; +using testing::Return; +using testing::ReturnRef; + +class OpenTelemetryHttpTraceExporterTest : public testing::Test { +public: + OpenTelemetryHttpTraceExporterTest() = default; + + void setup(envoy::config::core::v3::HttpService http_service) { + cluster_manager_.thread_local_cluster_.cluster_.info_->name_ = "my_o11y_backend"; + cluster_manager_.initializeThreadLocalClusters({"my_o11y_backend"}); + ON_CALL(cluster_manager_.thread_local_cluster_, httpAsyncClient()) + .WillByDefault(ReturnRef(cluster_manager_.thread_local_cluster_.async_client_)); + + cluster_manager_.initializeClusters({"my_o11y_backend"}, {}); + + trace_exporter_ = + std::make_unique(cluster_manager_, http_service); + } + +protected: + NiceMock cluster_manager_; + std::unique_ptr trace_exporter_; + NiceMock context_; + NiceMock& mock_scope_ = context_.server_factory_context_.store_; +}; + +// Test exporting an OTLP message via HTTP containing one span +TEST_F(OpenTelemetryHttpTraceExporterTest, CreateExporterAndExportSpan) { + std::string yaml_string = fmt::format(R"EOF( + http_uri: + uri: "https://some-o11y.com/otlp/v1/traces" + cluster: "my_o11y_backend" + timeout: 0.250s + request_headers_to_add: + - header: + key: "Authorization" + value: "auth-token" + - header: + key: "x-custom-header" + value: "custom-value" + )EOF"); + + envoy::config::core::v3::HttpService http_service; + TestUtility::loadFromYaml(yaml_string, http_service); + setup(http_service); + + Http::MockAsyncClientRequest request(&cluster_manager_.thread_local_cluster_.async_client_); + Http::AsyncClient::Callbacks* callback; + + EXPECT_CALL( + cluster_manager_.thread_local_cluster_.async_client_, + send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(250)))) + .WillOnce( + Invoke([&](Http::RequestMessagePtr& message, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callback = &callbacks; + + EXPECT_EQ(Http::Headers::get().MethodValues.Post, message->headers().getMethodValue()); + EXPECT_EQ(Http::Headers::get().ContentTypeValues.Protobuf, + message->headers().getContentTypeValue()); + + EXPECT_EQ("/otlp/v1/traces", message->headers().getPathValue()); + EXPECT_EQ("some-o11y.com", message->headers().getHostValue()); + + // Custom headers provided in the configuration + EXPECT_EQ("auth-token", message->headers() + .get(Http::LowerCaseString("authorization"))[0] + ->value() + .getStringView()); + EXPECT_EQ("custom-value", message->headers() + .get(Http::LowerCaseString("x-custom-header"))[0] + ->value() + .getStringView()); + + return &request; + })); + + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest + export_trace_service_request; + opentelemetry::proto::trace::v1::Span span; + span.set_name("test"); + *export_trace_service_request.add_resource_spans()->add_scope_spans()->add_spans() = span; + EXPECT_TRUE(trace_exporter_->log(export_trace_service_request)); + + Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( + Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "202"}}})); + // onBeforeFinalizeUpstreamSpan is a noop — included for coverage + Tracing::NullSpan null_span; + callback->onBeforeFinalizeUpstreamSpan(null_span, nullptr); + + callback->onSuccess(request, std::move(msg)); + callback->onFailure(request, Http::AsyncClient::FailureReason::Reset); +} + +// Test export is aborted when cluster is not found +TEST_F(OpenTelemetryHttpTraceExporterTest, UnsuccessfulLogWithoutThreadLocalCluster) { + std::string yaml_string = fmt::format(R"EOF( + http_uri: + uri: "https://some-o11y.com/otlp/v1/traces" + cluster: "my_o11y_backend" + timeout: 10s + )EOF"); + + envoy::config::core::v3::HttpService http_service; + TestUtility::loadFromYaml(yaml_string, http_service); + setup(http_service); + + Http::MockAsyncClientRequest request(&cluster_manager_.thread_local_cluster_.async_client_); + + ON_CALL(cluster_manager_, getThreadLocalCluster(absl::string_view("my_o11y_backend"))) + .WillByDefault(Return(nullptr)); + + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest + export_trace_service_request; + opentelemetry::proto::trace::v1::Span span; + span.set_name("test"); + *export_trace_service_request.add_resource_spans()->add_scope_spans()->add_spans() = span; + EXPECT_FALSE(trace_exporter_->log(export_trace_service_request)); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc b/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc index 06f3a6283f1b..223c74352756 100644 --- a/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc +++ b/test/extensions/tracers/opentelemetry/opentelemetry_tracer_impl_test.cc @@ -4,6 +4,7 @@ #include "source/common/tracing/http_tracer_impl.h" #include "source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h" +#include "source/extensions/tracers/opentelemetry/span_context_extractor.h" #include "test/mocks/common.h" #include "test/mocks/server/tracer_factory_context.h" @@ -27,6 +28,14 @@ using testing::NiceMock; using testing::Return; using testing::ReturnRef; +class MockResourceProvider : public ResourceProvider { +public: + MOCK_METHOD(Resource, getResource, + (const envoy::config::trace::v3::OpenTelemetryConfig& opentelemetry_config, + Server::Configuration::TracerFactoryContext& context), + (const)); +}; + class OpenTelemetryDriverTest : public testing::Test { public: OpenTelemetryDriverTest() = default; @@ -44,7 +53,13 @@ class OpenTelemetryDriverTest : public testing::Test { .WillByDefault(Return(ByMove(std::move(mock_client_factory)))); ON_CALL(factory_context, scope()).WillByDefault(ReturnRef(scope_)); - driver_ = std::make_unique(opentelemetry_config, context_); + Resource resource; + resource.attributes_.insert(std::pair("key1", "val1")); + + auto mock_resource_provider = NiceMock(); + EXPECT_CALL(mock_resource_provider, getResource(_, _)).WillRepeatedly(Return(resource)); + + driver_ = std::make_unique(opentelemetry_config, context_, mock_resource_provider); } void setupValidDriver() { @@ -60,6 +75,24 @@ class OpenTelemetryDriverTest : public testing::Test { setup(opentelemetry_config); } + void setupValidDriverWithHttpExporter() { + const std::string yaml_string = R"EOF( + http_service: + http_uri: + cluster: "my_o11y_backend" + uri: "https://some-o11y.com/otlp/v1/traces" + timeout: 0.250s + request_headers_to_add: + - header: + key: "Authorization" + value: "auth-token" + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + setup(opentelemetry_config); + } + protected: const std::string operation_name_{"test"}; NiceMock context_; @@ -75,17 +108,47 @@ class OpenTelemetryDriverTest : public testing::Test { Stats::Scope& scope_{*stats_.rootScope()}; }; +// Tests the tracer initialization with the gRPC exporter TEST_F(OpenTelemetryDriverTest, InitializeDriverValidConfig) { setupValidDriver(); EXPECT_NE(driver_, nullptr); } +// Tests the tracer initialization with the HTTP exporter +TEST_F(OpenTelemetryDriverTest, InitializeDriverValidConfigHttpExporter) { + setupValidDriverWithHttpExporter(); + EXPECT_NE(driver_, nullptr); +} + +// Verifies that the tracer cannot be configured with two exporters at the same time +TEST_F(OpenTelemetryDriverTest, BothGrpcAndHttpExportersConfigured) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + http_service: + http_uri: + cluster: "my_o11y_backend" + uri: "https://some-o11y.com/otlp/v1/traces" + timeout: 0.250s + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + EXPECT_THROW_WITH_MESSAGE(setup(opentelemetry_config), EnvoyException, + "OpenTelemetry Tracer cannot have both gRPC and HTTP exporters " + "configured. OpenTelemetry tracer will be disabled."); + EXPECT_EQ(driver_, nullptr); +} + +// Verifies traceparent/tracestate headers are properly parsed and propagated TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { // Set up driver setupValidDriver(); // Add the OTLP headers to the request headers - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; // traceparent header is "version-trace_id-parent_id-trace_flags" // See https://w3c.github.io/trace-context/#traceparent-header @@ -99,9 +162,9 @@ TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { const std::vector v = {version, trace_id_hex, Hex::uint64ToHex(parent_span_id), trace_flags}; const std::string parent_trace_header = absl::StrJoin(v, "-"); - request_headers.addReferenceKey(OpenTelemetryConstants::get().TRACE_PARENT, parent_trace_header); + request_headers.set(OpenTelemetryConstants::get().TRACE_PARENT.key(), parent_trace_header); // Also add tracestate. - request_headers.addReferenceKey(OpenTelemetryConstants::get().TRACE_STATE, "test=foo"); + request_headers.set(OpenTelemetryConstants::get().TRACE_STATE.key(), "test=foo"); // Mock the random call for generating span ID so we can check it later. const uint64_t new_span_id = 3; @@ -115,19 +178,20 @@ TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { EXPECT_EQ(span->getTraceIdAsHex(), trace_id_hex); // Remove headers, then inject context into header from the span. - request_headers.remove(OpenTelemetryConstants::get().TRACE_PARENT); - request_headers.remove(OpenTelemetryConstants::get().TRACE_STATE); + request_headers.remove(OpenTelemetryConstants::get().TRACE_PARENT.key()); + request_headers.remove(OpenTelemetryConstants::get().TRACE_STATE.key()); span->injectContext(request_headers, nullptr); - auto sampled_entry = request_headers.get(OpenTelemetryConstants::get().TRACE_PARENT); - EXPECT_EQ(sampled_entry.size(), 1); + auto sampled_entry = request_headers.get(OpenTelemetryConstants::get().TRACE_PARENT.key()); + EXPECT_EQ(sampled_entry.has_value(), true); EXPECT_EQ( - sampled_entry[0]->value().getStringView(), + sampled_entry.value(), absl::StrJoin({version, trace_id_hex, Hex::uint64ToHex(new_span_id), trace_flags}, "-")); - auto sampled_tracestate_entry = request_headers.get(OpenTelemetryConstants::get().TRACE_STATE); - EXPECT_EQ(sampled_tracestate_entry.size(), 1); - EXPECT_EQ(sampled_tracestate_entry[0]->value().getStringView(), "test=foo"); + auto sampled_tracestate_entry = + request_headers.get(OpenTelemetryConstants::get().TRACE_STATE.key()); + EXPECT_EQ(sampled_tracestate_entry.has_value(), true); + EXPECT_EQ(sampled_tracestate_entry.value(), "test=foo"); constexpr absl::string_view request_yaml = R"( resource_spans: resource: @@ -135,6 +199,9 @@ TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { key: "service.name" value: string_value: "unknown_service:envoy" + key: "key1" + value: + string_value: "val1" scope_spans: spans: trace_id: "AAA" @@ -167,12 +234,13 @@ TEST_F(OpenTelemetryDriverTest, ParseSpanContextFromHeadersTest) { EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies span is properly created when the incoming request has no traceparent/tracestate headers TEST_F(OpenTelemetryDriverTest, GenerateSpanContextWithoutHeadersTest) { // Set up driver setupValidDriver(); // Add the OTLP headers to the request headers - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; // Mock the random call for generating trace and span IDs so we can check it later. @@ -194,25 +262,25 @@ TEST_F(OpenTelemetryDriverTest, GenerateSpanContextWithoutHeadersTest) { operation_name_, {Tracing::Reason::Sampling, true}); // Remove headers, then inject context into header from the span. - request_headers.remove(OpenTelemetryConstants::get().TRACE_PARENT); + request_headers.remove(OpenTelemetryConstants::get().TRACE_PARENT.key()); span->injectContext(request_headers, nullptr); - auto sampled_entry = request_headers.get(OpenTelemetryConstants::get().TRACE_PARENT); + auto sampled_entry = request_headers.get(OpenTelemetryConstants::get().TRACE_PARENT.key()); // Ends in 01 because span should be sampled. See // https://w3c.github.io/trace-context/#trace-flags. - EXPECT_EQ(sampled_entry.size(), 1); - EXPECT_EQ(sampled_entry[0]->value().getStringView(), - "00-00000000000000010000000000000002-0000000000000003-01"); + EXPECT_EQ(sampled_entry.has_value(), true); + EXPECT_EQ(sampled_entry.value(), "00-00000000000000010000000000000002-0000000000000003-01"); } +// Verifies a span it not created when an invalid traceparent header is received TEST_F(OpenTelemetryDriverTest, NullSpanWithPropagationHeaderError) { setupValidDriver(); // Add an invalid OTLP header to the request headers. - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; - request_headers.addReferenceKey(OpenTelemetryConstants::get().TRACE_PARENT, - "invalid00-0000000000000003-01"); + request_headers.set(OpenTelemetryConstants::get().TRACE_PARENT.key(), + "invalid00-0000000000000003-01"); Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -221,10 +289,11 @@ TEST_F(OpenTelemetryDriverTest, NullSpanWithPropagationHeaderError) { EXPECT_EQ(typeid(null_span).name(), typeid(Tracing::NullSpan).name()); } +// Verifies the export happens after one span is created TEST_F(OpenTelemetryDriverTest, ExportOTLPSpan) { // Set up driver setupValidDriver(); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, @@ -248,10 +317,11 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpan) { EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies the export happens only when a second span is created TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithBuffer) { // Set up driver setupValidDriver(); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, @@ -275,6 +345,7 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithBuffer) { EXPECT_EQ(2U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies the export happens after a timeout TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithFlushTimeout) { timer_ = new NiceMock(&context_.server_factory_context_.thread_local_.dispatcher_); @@ -283,7 +354,7 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithFlushTimeout) { EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000), _)); // Set up driver setupValidDriver(); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, @@ -307,10 +378,11 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithFlushTimeout) { EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.timer_flushed").value()); } +// Verifies child span is related to parent span TEST_F(OpenTelemetryDriverTest, SpawnChildSpan) { // Set up driver setupValidDriver(); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; // Mock the random call for generating the parent span's IDs so we can check it later. @@ -347,9 +419,128 @@ TEST_F(OpenTelemetryDriverTest, SpawnChildSpan) { EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies the span types +TEST_F(OpenTelemetryDriverTest, SpanType) { + // Set up driver + setupValidDriver(); + Tracing::TestTraceContextImpl request_headers{ + {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; + + // Mock the random call for generating the parent span's IDs so we can check it later. + const uint64_t parent_trace_id_high = 0; + const uint64_t parent_trace_id_low = 2; + const uint64_t parent_span_id = 3; + + { + mock_tracing_config_.spawn_upstream_span_ = false; + mock_tracing_config_.operation_name_ = Tracing::OperationName::Ingress; + + NiceMock& mock_random_generator_ = + context_.server_factory_context_.api_.random_; + { + InSequence s; + + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_trace_id_high)); + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_trace_id_low)); + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_span_id)); + } + + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); + EXPECT_NE(span.get(), nullptr); + + // The span type should be SERVER because spawn_upstream_span_ is false and operation_name_ is + // Ingress. + EXPECT_EQ(dynamic_cast(span.get())->spanForTest().kind(), + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER); + + // The child should only generate a span ID for itself; the trace id should come from the + // parent. + const uint64_t child_span_id = 3; + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(child_span_id)); + Tracing::SpanPtr child_span = + span->spawnChild(mock_tracing_config_, operation_name_, time_system_.systemTime()); + + // The child span should also be a CLIENT span. + EXPECT_EQ(dynamic_cast(child_span.get())->spanForTest().kind(), + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT); + } + + { + mock_tracing_config_.spawn_upstream_span_ = false; + mock_tracing_config_.operation_name_ = Tracing::OperationName::Egress; + + NiceMock& mock_random_generator_ = + context_.server_factory_context_.api_.random_; + { + InSequence s; + + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_trace_id_high)); + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_trace_id_low)); + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_span_id)); + } + + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); + EXPECT_NE(span.get(), nullptr); + + // The span type should be CLIENT because spawn_upstream_span_ is false and operation_name_ is + // Egress. + EXPECT_EQ(dynamic_cast(span.get())->spanForTest().kind(), + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT); + + // The child should only generate a span ID for itself; the trace id should come from the + // parent. + const uint64_t child_span_id = 3; + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(child_span_id)); + Tracing::SpanPtr child_span = + span->spawnChild(mock_tracing_config_, operation_name_, time_system_.systemTime()); + + // The child span should also be a CLIENT span. + EXPECT_EQ(dynamic_cast(child_span.get())->spanForTest().kind(), + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT); + } + + { + mock_tracing_config_.spawn_upstream_span_ = true; + mock_tracing_config_.operation_name_ = Tracing::OperationName::Egress; + + NiceMock& mock_random_generator_ = + context_.server_factory_context_.api_.random_; + { + InSequence s; + + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_trace_id_high)); + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_trace_id_low)); + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(parent_span_id)); + } + + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); + EXPECT_NE(span.get(), nullptr); + + // The span type should be SERVER because spawn_upstream_span_ is true and this is + // downstream span. + EXPECT_EQ(dynamic_cast(span.get())->spanForTest().kind(), + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER); + + // The child should only generate a span ID for itself; the trace id should come from the + // parent. + const uint64_t child_span_id = 3; + EXPECT_CALL(mock_random_generator_, random()).WillOnce(Return(child_span_id)); + Tracing::SpanPtr child_span = + span->spawnChild(mock_tracing_config_, operation_name_, time_system_.systemTime()); + + // The child span should also be a CLIENT span. + EXPECT_EQ(dynamic_cast(child_span.get())->spanForTest().kind(), + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_CLIENT); + } +} + +// Verifies spans are exported with their attributes TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithAttributes) { setupValidDriver(); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; NiceMock& mock_random_generator_ = context_.server_factory_context_.api_.random_; @@ -377,6 +568,9 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithAttributes) { key: "service.name" value: string_value: "unknown_service:envoy" + key: "key1" + value: + string_value: "val1" scope_spans: spans: trace_id: "AAA" @@ -412,9 +606,10 @@ TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanWithAttributes) { EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Not sampled spans are ignored TEST_F(OpenTelemetryDriverTest, IgnoreNotSampledSpan) { setupValidDriver(); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -428,13 +623,14 @@ TEST_F(OpenTelemetryDriverTest, IgnoreNotSampledSpan) { EXPECT_EQ(0U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies tracer is "disabled" when no exporter is configured TEST_F(OpenTelemetryDriverTest, NoExportWithoutGrpcService) { const std::string yaml_string = "{}"; envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; TestUtility::loadFromYaml(yaml_string, opentelemetry_config); setup(opentelemetry_config); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, @@ -451,6 +647,7 @@ TEST_F(OpenTelemetryDriverTest, NoExportWithoutGrpcService) { EXPECT_EQ(0U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies a custom service name is properly set on exported spans TEST_F(OpenTelemetryDriverTest, ExportSpanWithCustomServiceName) { const std::string yaml_string = R"EOF( grpc_service: @@ -463,7 +660,7 @@ TEST_F(OpenTelemetryDriverTest, ExportSpanWithCustomServiceName) { TestUtility::loadFromYaml(yaml_string, opentelemetry_config); setup(opentelemetry_config); - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; NiceMock& mock_random_generator_ = context_.server_factory_context_.api_.random_; @@ -483,6 +680,9 @@ TEST_F(OpenTelemetryDriverTest, ExportSpanWithCustomServiceName) { key: "service.name" value: string_value: "test-service-name" + key: "key1" + value: + string_value: "val1" scope_spans: spans: trace_id: "AAA" @@ -511,6 +711,38 @@ TEST_F(OpenTelemetryDriverTest, ExportSpanWithCustomServiceName) { EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.spans_sent").value()); } +// Verifies the export using the HTTP exporter +TEST_F(OpenTelemetryDriverTest, ExportOTLPSpanHTTP) { + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_->name_ = + "my_o11y_backend"; + context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters( + {"my_o11y_backend"}); + ON_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_, + httpAsyncClient()) + .WillByDefault(ReturnRef( + context_.server_factory_context_.cluster_manager_.thread_local_cluster_.async_client_)); + context_.server_factory_context_.cluster_manager_.initializeClusters({"my_o11y_backend"}, {}); + setupValidDriverWithHttpExporter(); + + Tracing::TestTraceContextImpl request_headers{ + {":authority", "test.com"}, {":path", "/"}, {":method", "GET"}}; + Tracing::SpanPtr span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, + operation_name_, {Tracing::Reason::Sampling, true}); + EXPECT_NE(span.get(), nullptr); + + // Flush after a single span. + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.opentelemetry.min_flush_spans", 5U)) + .Times(1) + .WillRepeatedly(Return(1)); + // We should see a call to the async client to export that single span. + EXPECT_CALL(context_.server_factory_context_.cluster_manager_.thread_local_cluster_.async_client_, + send_(_, _, _)); + + span->finishSpan(); + + EXPECT_EQ(1U, stats_.counter("tracing.opentelemetry.spans_sent").value()); +} + } // namespace OpenTelemetry } // namespace Tracers } // namespace Extensions diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/BUILD b/test/extensions/tracers/opentelemetry/resource_detectors/BUILD new file mode 100644 index 000000000000..b91bdda9b2e2 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "resource_provider_test", + srcs = ["resource_provider_test.cc"], + deps = [ + "//envoy/registry", + "//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:registry_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/BUILD b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/BUILD new file mode 100644 index 000000000000..b02e8581f291 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/BUILD @@ -0,0 +1,57 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.tracers.opentelemetry.resource_detectors.dynatrace"], + deps = [ + "//envoy/registry", + "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:config", + "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:dynatrace_resource_detector_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "dynatrace_resource_detector_test", + srcs = [ + "dynatrace_metadata_file_reader_test.cc", + "dynatrace_resource_detector_test.cc", + ], + extension_names = ["envoy.tracers.opentelemetry.resource_detectors.dynatrace"], + deps = [ + "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:config", + "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:dynatrace_resource_detector_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "dynatrace_resource_detector_integration_test", + srcs = [ + "dynatrace_resource_detector_integration_test.cc", + ], + extension_names = ["envoy.tracers.opentelemetry.resource_detectors.dynatrace"], + deps = [ + "//source/exe:main_common_lib", + "//source/extensions/tracers/opentelemetry:config", + "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config_test.cc new file mode 100644 index 000000000000..9d769cc62dc6 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config_test.cc @@ -0,0 +1,36 @@ +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/config.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +TEST(DynatraceResourceDetectorFactoryTest, Basic) { + auto* factory = Registry::FactoryRegistry::getFactory( + "envoy.tracers.opentelemetry.resource_detectors.dynatrace"); + ASSERT_NE(factory, nullptr); + + envoy::config::core::v3::TypedExtensionConfig typed_config; + const std::string yaml = R"EOF( + name: envoy.tracers.opentelemetry.resource_detectors.dynatrace + typed_config: + "@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.DynatraceResourceDetectorConfig + )EOF"; + TestUtility::loadFromYaml(yaml, typed_config); + + NiceMock context; + EXPECT_NE(factory->createResourceDetector(typed_config.typed_config(), context), nullptr); + EXPECT_STREQ(factory->name().c_str(), "envoy.tracers.opentelemetry.resource_detectors.dynatrace"); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader_test.cc new file mode 100644 index 000000000000..58132e907064 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader_test.cc @@ -0,0 +1,43 @@ +#include +#include + +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h" + +#include "test/test_common/environment.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +TEST(DynatraceMetadataFileReaderTest, DynatraceNotDeployed) { + const std::string indirection_file = "dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties"; + + DynatraceMetadataFileReaderPtr reader = std::make_unique(); + + EXPECT_THROW(reader->readEnrichmentFile(indirection_file), EnvoyException); +} + +TEST(DynatraceMetadataFileReaderTest, DtMetadataFile) { + const std::string expected_data = "attribute=value"; + const std::string metadata_file = "dt_metadata.properties"; + + TestEnvironment::writeStringToFileForTest(metadata_file, expected_data); + + DynatraceMetadataFileReaderPtr reader = std::make_unique(); + + std::string actual_data = + reader->readEnrichmentFile(TestEnvironment::temporaryPath(metadata_file)); + + EXPECT_EQ(actual_data, expected_data); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector_integration_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector_integration_test.cc new file mode 100644 index 000000000000..6b43580101f5 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector_integration_test.cc @@ -0,0 +1,76 @@ +#include +#include + +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" + +#include "test/integration/http_integration.h" +#include "test/test_common/utility.h" + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { +namespace { + +class DynatraceResourceDetectorIntegrationTest + : public Envoy::HttpIntegrationTest, + public testing::TestWithParam { +public: + DynatraceResourceDetectorIntegrationTest() + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { + + const std::string yaml_string = R"EOF( + provider: + name: envoy.tracers.opentelemetry + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig + grpc_service: + envoy_grpc: + cluster_name: opentelemetry_collector + timeout: 0.250s + service_name: "a_service_name" + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.dynatrace + typed_config: + "@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.DynatraceResourceDetectorConfig + )EOF"; + + auto tracing_config = + std::make_unique<::envoy::extensions::filters::network::http_connection_manager::v3:: + HttpConnectionManager_Tracing>(); + TestUtility::loadFromYaml(yaml_string, *tracing_config.get()); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.set_allocated_tracing(tracing_config.release()); }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, DynatraceResourceDetectorIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Verify Envoy starts even when the Dynatrace resource detector wasn't able to detect any +// attributes +TEST_P(DynatraceResourceDetectorIntegrationTest, TestWithNoDynatraceDeployed) { + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/test/long/url"}, {":scheme", "http"}, {":authority", "host"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); +} + +} // namespace +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector_test.cc new file mode 100644 index 000000000000..2db7439d0021 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector_test.cc @@ -0,0 +1,131 @@ +#include +#include + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/dynatrace_resource_detector.pb.h" +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_metadata_file_reader.h" +#include "source/extensions/tracers/opentelemetry/resource_detectors/dynatrace/dynatrace_resource_detector.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/environment.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +class MockDynatraceFileReader : public DynatraceMetadataFileReader { +public: + MOCK_METHOD(std::string, readEnrichmentFile, (const std::string& file_path)); +}; + +TEST(DynatraceResourceDetectorTest, DynatraceNotDeployed) { + NiceMock context; + + auto dt_file_reader = std::make_unique>(); + EXPECT_CALL(*dt_file_reader, readEnrichmentFile(_)).WillRepeatedly(Return("")); + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3::DynatraceResourceDetectorConfig + config; + + auto detector = std::make_shared(config, std::move(dt_file_reader)); + Resource resource = detector->detect(); + + EXPECT_EQ(resource.schema_url_, ""); + EXPECT_EQ(0, resource.attributes_.size()); +} + +TEST(DynatraceResourceDetectorTest, OnlyOneAgentInstalled) { + NiceMock context; + ResourceAttributes expected_attributes = { + {"dt.entity.host", "HOST-abc"}, + {"dt.entity.process_group_instance", "PROCESS_GROUP_INSTANCE-abc"}}; + + auto dt_file_reader = std::make_unique>(); + + std::string host_attrs = fmt::format(R"EOF( +dt.entity.host=HOST-abc +dt.host_group.id= +)EOF"); + + std::string process_attrs = fmt::format(R"EOF( +dt.entity.process_group_instance=PROCESS_GROUP_INSTANCE-abc +)EOF"); + + EXPECT_CALL(*dt_file_reader, + readEnrichmentFile("dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties")) + .WillRepeatedly(Return(process_attrs)); + EXPECT_CALL(*dt_file_reader, + readEnrichmentFile("/var/lib/dynatrace/enrichment/dt_host_metadata.properties")) + .WillRepeatedly(Return(host_attrs)); + + EXPECT_CALL(*dt_file_reader, + readEnrichmentFile("/var/lib/dynatrace/enrichment/dt_metadata.properties")) + .WillRepeatedly(Return("")); + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3::DynatraceResourceDetectorConfig + config; + + auto detector = std::make_shared(config, std::move(dt_file_reader)); + Resource resource = detector->detect(); + + EXPECT_EQ(resource.schema_url_, ""); + EXPECT_EQ(2, resource.attributes_.size()); + + for (auto& actual : resource.attributes_) { + auto expected = expected_attributes.find(actual.first); + + EXPECT_TRUE(expected != expected_attributes.end()); + EXPECT_EQ(expected->second, actual.second); + } +} + +TEST(DynatraceResourceDetectorTest, Dynatracek8sOperator) { + NiceMock context; + ResourceAttributes expected_attributes = {{"k8s.pod.uid", "123"}, {"k8s.pod.name", "envoy"}}; + + auto dt_file_reader = std::make_unique>(); + + std::string k8s_attrs = fmt::format(R"EOF( +k8s.pod.uid=123 +k8s.pod.name=envoy +)EOF"); + + EXPECT_CALL(*dt_file_reader, + readEnrichmentFile("dt_metadata_e617c525669e072eebe3d0f08212e8f2.properties")) + .WillRepeatedly(Return("")); + EXPECT_CALL(*dt_file_reader, + readEnrichmentFile("/var/lib/dynatrace/enrichment/dt_host_metadata.properties")) + .WillRepeatedly(Return("")); + EXPECT_CALL(*dt_file_reader, + readEnrichmentFile("/var/lib/dynatrace/enrichment/dt_metadata.properties")) + .WillRepeatedly(Return(k8s_attrs)); + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3::DynatraceResourceDetectorConfig + config; + + auto detector = std::make_shared(config, std::move(dt_file_reader)); + Resource resource = detector->detect(); + + EXPECT_EQ(resource.schema_url_, ""); + EXPECT_EQ(2, resource.attributes_.size()); + + for (auto& actual : resource.attributes_) { + auto expected = expected_attributes.find(actual.first); + + EXPECT_TRUE(expected != expected_attributes.end()); + EXPECT_EQ(expected->second, actual.second); + } +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/environment/BUILD b/test/extensions/tracers/opentelemetry/resource_detectors/environment/BUILD new file mode 100644 index 000000000000..2e6598200c38 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/environment/BUILD @@ -0,0 +1,37 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.tracers.opentelemetry.resource_detectors.environment"], + deps = [ + "//envoy/registry", + "//source/extensions/tracers/opentelemetry/resource_detectors/environment:config", + "//source/extensions/tracers/opentelemetry/resource_detectors/environment:environment_resource_detector_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "environment_resource_detector_test", + srcs = ["environment_resource_detector_test.cc"], + extension_names = ["envoy.tracers.opentelemetry.resource_detectors.environment"], + deps = [ + "//source/extensions/tracers/opentelemetry/resource_detectors/environment:environment_resource_detector_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/resource_detectors/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/environment/config_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/environment/config_test.cc new file mode 100644 index 000000000000..7e9ada0850eb --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/environment/config_test.cc @@ -0,0 +1,36 @@ +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/environment/config.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +// Test create resource detector via factory +TEST(EnvironmentResourceDetectorFactoryTest, Basic) { + auto* factory = Registry::FactoryRegistry::getFactory( + "envoy.tracers.opentelemetry.resource_detectors.environment"); + ASSERT_NE(factory, nullptr); + + envoy::config::core::v3::TypedExtensionConfig typed_config; + const std::string yaml = R"EOF( + name: envoy.tracers.opentelemetry.resource_detectors.environment + typed_config: + "@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.EnvironmentResourceDetectorConfig + )EOF"; + TestUtility::loadFromYaml(yaml, typed_config); + + NiceMock context; + EXPECT_NE(factory->createResourceDetector(typed_config.typed_config(), context), nullptr); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector_test.cc new file mode 100644 index 000000000000..3a596f3d0e22 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector_test.cc @@ -0,0 +1,109 @@ +#include + +#include "envoy/extensions/tracers/opentelemetry/resource_detectors/v3/environment_resource_detector.pb.h" +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/environment/environment_resource_detector.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/environment.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +const std::string kOtelResourceAttributesEnv = "OTEL_RESOURCE_ATTRIBUTES"; + +// Test detector when env variable is not present +TEST(EnvironmentResourceDetectorTest, EnvVariableNotPresent) { + NiceMock context; + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + EnvironmentResourceDetectorConfig config; + + auto detector = std::make_unique(config, context); + EXPECT_THROW_WITH_MESSAGE(detector->detect(), EnvoyException, + "Environment variable doesn't exist: OTEL_RESOURCE_ATTRIBUTES"); +} + +// Test detector when env variable is present but contains an empty value +TEST(EnvironmentResourceDetectorTest, EnvVariablePresentButEmpty) { + NiceMock context; + TestEnvironment::setEnvVar(kOtelResourceAttributesEnv, "", 1); + Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar(kOtelResourceAttributesEnv); }); + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + EnvironmentResourceDetectorConfig config; + + auto detector = std::make_unique(config, context); + +#ifdef WIN32 + EXPECT_THROW_WITH_MESSAGE(detector->detect(), EnvoyException, + "Environment variable doesn't exist: OTEL_RESOURCE_ATTRIBUTES"); +#else + EXPECT_THROW_WITH_MESSAGE(detector->detect(), EnvoyException, + "The OpenTelemetry environment resource detector is configured but the " + "'OTEL_RESOURCE_ATTRIBUTES'" + " environment variable is empty."); +#endif +} + +// Test detector with valid values in the env variable +TEST(EnvironmentResourceDetectorTest, EnvVariablePresentAndWithAttributes) { + NiceMock context; + TestEnvironment::setEnvVar(kOtelResourceAttributesEnv, "key1=val1,key2=val2", 1); + Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar(kOtelResourceAttributesEnv); }); + ResourceAttributes expected_attributes = {{"key1", "val1"}, {"key2", "val2"}}; + + Api::ApiPtr api = Api::createApiForTest(); + EXPECT_CALL(context.server_factory_context_, api()).WillRepeatedly(ReturnRef(*api)); + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + EnvironmentResourceDetectorConfig config; + + auto detector = std::make_unique(config, context); + Resource resource = detector->detect(); + + EXPECT_EQ(resource.schema_url_, ""); + EXPECT_EQ(2, resource.attributes_.size()); + + for (auto& actual : resource.attributes_) { + auto expected = expected_attributes.find(actual.first); + + EXPECT_TRUE(expected != expected_attributes.end()); + EXPECT_EQ(expected->second, actual.second); + } +} + +// Test detector with invalid values mixed with valid ones in the env variable +TEST(EnvironmentResourceDetectorTest, EnvVariablePresentAndWithAttributesWrongFormat) { + NiceMock context; + TestEnvironment::setEnvVar(kOtelResourceAttributesEnv, "key1=val1,key2val2,key3/val3, , key", 1); + Envoy::Cleanup cleanup([]() { TestEnvironment::unsetEnvVar(kOtelResourceAttributesEnv); }); + ResourceAttributes expected_attributes = {{"key1", "val"}}; + + Api::ApiPtr api = Api::createApiForTest(); + EXPECT_CALL(context.server_factory_context_, api()).WillRepeatedly(ReturnRef(*api)); + + envoy::extensions::tracers::opentelemetry::resource_detectors::v3:: + EnvironmentResourceDetectorConfig config; + + auto detector = std::make_unique(config, context); + + EXPECT_THROW_WITH_MESSAGE(detector->detect(), EnvoyException, + "The OpenTelemetry environment resource detector is configured but the " + "'OTEL_RESOURCE_ATTRIBUTES'" + " environment variable has an invalid format."); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/resource_detectors/resource_provider_test.cc b/test/extensions/tracers/opentelemetry/resource_detectors/resource_provider_test.cc new file mode 100644 index 000000000000..657365baa889 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/resource_detectors/resource_provider_test.cc @@ -0,0 +1,424 @@ +#include + +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/resource_detectors/resource_provider.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/environment.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::Return; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { +namespace { + +class SampleDetector : public ResourceDetector { +public: + MOCK_METHOD(Resource, detect, ()); +}; + +class DetectorFactoryA : public ResourceDetectorFactory { +public: + MOCK_METHOD(ResourceDetectorPtr, createResourceDetector, + (const Protobuf::Message& message, + Server::Configuration::TracerFactoryContext& context)); + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "envoy.tracers.opentelemetry.resource_detectors.a"; } +}; + +class DetectorFactoryB : public ResourceDetectorFactory { +public: + MOCK_METHOD(ResourceDetectorPtr, createResourceDetector, + (const Protobuf::Message& message, + Server::Configuration::TracerFactoryContext& context)); + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "envoy.tracers.opentelemetry.resource_detectors.b"; } +}; + +const std::string kOtelResourceAttributesEnv = "OTEL_RESOURCE_ATTRIBUTES"; + +class ResourceProviderTest : public testing::Test { +public: + ResourceProviderTest() { + resource_a_.attributes_.insert(std::pair("key1", "val1")); + resource_b_.attributes_.insert(std::pair("key2", "val2")); + } + NiceMock context_; + Resource resource_a_; + Resource resource_b_; +}; + +// Verifies a resource with the static service name is returned when no detectors are configured +TEST_F(ResourceProviderTest, NoResourceDetectorsConfigured) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + EXPECT_EQ(resource.schema_url_, ""); + + // Only the service name was added to the resource + EXPECT_EQ(1, resource.attributes_.size()); +} + +// Verifies a resource with the default service name is returned when no detectors + static service +// name are configured +TEST_F(ResourceProviderTest, ServiceNameNotProvided) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + EXPECT_EQ(resource.schema_url_, ""); + + // service.name receives the unknown value when not configured + EXPECT_EQ(1, resource.attributes_.size()); + auto service_name = resource.attributes_.find("service.name"); + EXPECT_EQ("unknown_service:envoy", service_name->second); +} + +// Verifies it is possible to configure multiple resource detectors +TEST_F(ResourceProviderTest, MultipleResourceDetectorsConfigured) { + auto detector_a = std::make_unique>(); + EXPECT_CALL(*detector_a, detect()).WillOnce(Return(resource_a_)); + + auto detector_b = std::make_unique>(); + EXPECT_CALL(*detector_b, detect()).WillOnce(Return(resource_b_)); + + DetectorFactoryA factory_a; + Registry::InjectFactory factory_a_registration(factory_a); + + DetectorFactoryB factory_b; + Registry::InjectFactory factory_b_registration(factory_b); + + EXPECT_CALL(factory_a, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_a)))); + EXPECT_CALL(factory_b, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_b)))); + + // Expected merged attributes from all detectors + ResourceAttributes expected_attributes = { + {"service.name", "my-service"}, {"key1", "val1"}, {"key2", "val2"}}; + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.a + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + - name: envoy.tracers.opentelemetry.resource_detectors.b + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + EXPECT_EQ(resource.schema_url_, ""); + + // The resource should contain all 3 merged attributes + // service.name + 1 for each detector + EXPECT_EQ(3, resource.attributes_.size()); + + for (auto& actual : resource.attributes_) { + auto expected = expected_attributes.find(actual.first); + + EXPECT_TRUE(expected != expected_attributes.end()); + EXPECT_EQ(expected->second, actual.second); + } +} + +// Verifies Envoy fails when an unknown resource detector is configured +TEST_F(ResourceProviderTest, UnknownResourceDetectors) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.UnkownResourceDetector + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + EXPECT_THROW_WITH_MESSAGE( + resource_provider.getResource(opentelemetry_config, context_), EnvoyException, + "Resource detector factory not found: " + "'envoy.tracers.opentelemetry.resource_detectors.UnkownResourceDetector'"); +} + +// Verifies Envoy fails when an error occurs while instantiating a resource detector +TEST_F(ResourceProviderTest, ProblemCreatingResourceDetector) { + DetectorFactoryA factory; + Registry::InjectFactory factory_registration(factory); + + // Simulating having a problem when creating the resource detector + EXPECT_CALL(factory, createResourceDetector(_, _)).WillOnce(Return(testing::ByMove(nullptr))); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-clusterdetector_a + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.a + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; + + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + EXPECT_THROW_WITH_MESSAGE(resource_provider.getResource(opentelemetry_config, context_), + EnvoyException, + "Resource detector could not be created: " + "'envoy.tracers.opentelemetry.resource_detectors.a'"); +} + +// Test merge when old schema url is empty but updating is not +TEST_F(ResourceProviderTest, OldSchemaEmptyUpdatingSet) { + std::string expected_schema_url = "my.schema/v1"; + Resource old_resource = resource_a_; + + // Updating resource is empty (no attributes) + Resource updating_resource; + updating_resource.schema_url_ = expected_schema_url; + + auto detector_a = std::make_unique>(); + EXPECT_CALL(*detector_a, detect()).WillOnce(Return(old_resource)); + + auto detector_b = std::make_unique>(); + EXPECT_CALL(*detector_b, detect()).WillOnce(Return(updating_resource)); + + DetectorFactoryA factory_a; + Registry::InjectFactory factory_a_registration(factory_a); + + DetectorFactoryB factory_b; + Registry::InjectFactory factory_b_registration(factory_b); + + EXPECT_CALL(factory_a, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_a)))); + EXPECT_CALL(factory_b, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_b)))); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.a + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + - name: envoy.tracers.opentelemetry.resource_detectors.b + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + // OTel spec says the updating schema should be used + EXPECT_EQ(expected_schema_url, resource.schema_url_); +} + +// Test merge when old schema url is not empty but updating is +TEST_F(ResourceProviderTest, OldSchemaSetUpdatingEmpty) { + std::string expected_schema_url = "my.schema/v1"; + Resource old_resource = resource_a_; + old_resource.schema_url_ = expected_schema_url; + + Resource updating_resource = resource_b_; + updating_resource.schema_url_ = ""; + + auto detector_a = std::make_unique>(); + EXPECT_CALL(*detector_a, detect()).WillOnce(Return(old_resource)); + + auto detector_b = std::make_unique>(); + EXPECT_CALL(*detector_b, detect()).WillOnce(Return(updating_resource)); + + DetectorFactoryA factory_a; + Registry::InjectFactory factory_a_registration(factory_a); + + DetectorFactoryB factory_b; + Registry::InjectFactory factory_b_registration(factory_b); + + EXPECT_CALL(factory_a, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_a)))); + EXPECT_CALL(factory_b, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_b)))); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.a + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + - name: envoy.tracers.opentelemetry.resource_detectors.b + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + // OTel spec says the updating schema should be used + EXPECT_EQ(expected_schema_url, resource.schema_url_); +} + +// Test merge when both old and updating schema url are set and equal +TEST_F(ResourceProviderTest, OldAndUpdatingSchemaAreEqual) { + std::string expected_schema_url = "my.schema/v1"; + Resource old_resource = resource_a_; + old_resource.schema_url_ = expected_schema_url; + + Resource updating_resource = resource_b_; + updating_resource.schema_url_ = expected_schema_url; + + auto detector_a = std::make_unique>(); + EXPECT_CALL(*detector_a, detect()).WillOnce(Return(old_resource)); + + auto detector_b = std::make_unique>(); + EXPECT_CALL(*detector_b, detect()).WillOnce(Return(updating_resource)); + + DetectorFactoryA factory_a; + Registry::InjectFactory factory_a_registration(factory_a); + + DetectorFactoryB factory_b; + Registry::InjectFactory factory_b_registration(factory_b); + + EXPECT_CALL(factory_a, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_a)))); + EXPECT_CALL(factory_b, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_b)))); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.a + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + - name: envoy.tracers.opentelemetry.resource_detectors.b + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + EXPECT_EQ(expected_schema_url, resource.schema_url_); +} + +// Test merge when both old and updating schema url are set but different +TEST_F(ResourceProviderTest, OldAndUpdatingSchemaAreDifferent) { + std::string expected_schema_url = "my.schema/v1"; + Resource old_resource = resource_a_; + old_resource.schema_url_ = expected_schema_url; + + Resource updating_resource = resource_b_; + updating_resource.schema_url_ = "my.schema/v2"; + + auto detector_a = std::make_unique>(); + EXPECT_CALL(*detector_a, detect()).WillOnce(Return(old_resource)); + + auto detector_b = std::make_unique>(); + EXPECT_CALL(*detector_b, detect()).WillOnce(Return(updating_resource)); + + DetectorFactoryA factory_a; + Registry::InjectFactory factory_a_registration(factory_a); + + DetectorFactoryB factory_b; + Registry::InjectFactory factory_b_registration(factory_b); + + EXPECT_CALL(factory_a, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_a)))); + EXPECT_CALL(factory_b, createResourceDetector(_, _)) + .WillOnce(Return(testing::ByMove(std::move(detector_b)))); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + resource_detectors: + - name: envoy.tracers.opentelemetry.resource_detectors.a + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + - name: envoy.tracers.opentelemetry.resource_detectors.b + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue + )EOF"; + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + ResourceProviderImpl resource_provider; + Resource resource = resource_provider.getResource(opentelemetry_config, context_); + + // OTel spec says Old schema should be used + EXPECT_EQ(expected_schema_url, resource.schema_url_); +} + +} // namespace +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/samplers/BUILD b/test/extensions/tracers/opentelemetry/samplers/BUILD new file mode 100644 index 000000000000..55414e7854c9 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/samplers/BUILD @@ -0,0 +1,23 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "sampler_test", + srcs = ["sampler_test.cc"], + deps = [ + "//envoy/registry", + "//source/extensions/tracers/opentelemetry:opentelemetry_tracer_lib", + "//source/extensions/tracers/opentelemetry/samplers:sampler_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:registry_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/tracers/opentelemetry/samplers/always_on/BUILD b/test/extensions/tracers/opentelemetry/samplers/always_on/BUILD new file mode 100644 index 000000000000..3379c6fb5bb6 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/samplers/always_on/BUILD @@ -0,0 +1,55 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_names = ["envoy.tracers.opentelemetry.samplers.always_on"], + deps = [ + "//envoy/registry", + "//source/extensions/tracers/opentelemetry/samplers/always_on:always_on_sampler_lib", + "//source/extensions/tracers/opentelemetry/samplers/always_on:config", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "always_on_sampler_test", + srcs = ["always_on_sampler_test.cc"], + extension_names = ["envoy.tracers.opentelemetry.samplers.always_on"], + deps = [ + "//source/extensions/tracers/opentelemetry/samplers/always_on:always_on_sampler_lib", + "//source/extensions/tracers/opentelemetry/samplers/always_on:config", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/tracers/opentelemetry/samplers/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "always_on_sampler_integration_test", + srcs = [ + "always_on_sampler_integration_test.cc", + ], + extension_names = ["envoy.tracers.opentelemetry.samplers.always_on"], + deps = [ + "//source/exe:main_common_lib", + "//source/extensions/tracers/opentelemetry:config", + "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:config", + "//source/extensions/tracers/opentelemetry/samplers/always_on:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler_integration_test.cc b/test/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler_integration_test.cc new file mode 100644 index 000000000000..051a21b6846f --- /dev/null +++ b/test/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler_integration_test.cc @@ -0,0 +1,142 @@ +#include +#include + +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" + +#include "test/integration/http_integration.h" +#include "test/test_common/utility.h" + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { +namespace { + +const char* TRACEPARENT_VALUE = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; +const char* TRACEPARENT_VALUE_START = "00-0af7651916cd43dd8448eb211c80319c"; + +class AlwaysOnSamplerIntegrationTest : public Envoy::HttpIntegrationTest, + public testing::TestWithParam { +public: + AlwaysOnSamplerIntegrationTest() : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam()) { + + const std::string yaml_string = R"EOF( + provider: + name: envoy.tracers.opentelemetry + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig + grpc_service: + envoy_grpc: + cluster_name: opentelemetry_collector + timeout: 0.250s + service_name: "a_service_name" + sampler: + name: envoy.tracers.opentelemetry.samplers.dynatrace + typed_config: + "@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.samplers.v3.AlwaysOnSamplerConfig + )EOF"; + + auto tracing_config = + std::make_unique<::envoy::extensions::filters::network::http_connection_manager::v3:: + HttpConnectionManager_Tracing>(); + TestUtility::loadFromYaml(yaml_string, *tracing_config.get()); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.set_allocated_tracing(tracing_config.release()); }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, AlwaysOnSamplerIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Sends a request with traceparent and tracestate header. +TEST_P(AlwaysOnSamplerIntegrationTest, TestWithTraceparentAndTracestate) { + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/test/long/url"}, {":scheme", "http"}, + {":authority", "host"}, {"tracestate", "key=value"}, {"traceparent", TRACEPARENT_VALUE}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); + + // traceparent should be set: traceid should be re-used, span id should be different + absl::string_view traceparent_value = upstream_request_->headers() + .get(Http::LowerCaseString("traceparent"))[0] + ->value() + .getStringView(); + EXPECT_TRUE(absl::StartsWith(traceparent_value, TRACEPARENT_VALUE_START)); + EXPECT_NE(TRACEPARENT_VALUE, traceparent_value); + // tracestate should be forwarded + absl::string_view tracestate_value = upstream_request_->headers() + .get(Http::LowerCaseString("tracestate"))[0] + ->value() + .getStringView(); + EXPECT_EQ("key=value", tracestate_value); +} + +// Sends a request with traceparent but no tracestate header. +TEST_P(AlwaysOnSamplerIntegrationTest, TestWithTraceparentOnly) { + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"traceparent", TRACEPARENT_VALUE}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); + + // traceparent should be set: traceid should be re-used, span id should be different + absl::string_view traceparent_value = upstream_request_->headers() + .get(Http::LowerCaseString("traceparent"))[0] + ->value() + .getStringView(); + EXPECT_TRUE(absl::StartsWith(traceparent_value, TRACEPARENT_VALUE_START)); + EXPECT_NE(TRACEPARENT_VALUE, traceparent_value); + // OTLP tracer adds an empty tracestate + absl::string_view tracestate_value = upstream_request_->headers() + .get(Http::LowerCaseString("tracestate"))[0] + ->value() + .getStringView(); + EXPECT_EQ("", tracestate_value); +} + +// Sends a request without traceparent and tracestate header. +TEST_P(AlwaysOnSamplerIntegrationTest, TestWithoutTraceparentAndTracestate) { + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/test/long/url"}, {":scheme", "http"}, {":authority", "host"}}; + + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); + + // traceparent will be added, trace_id and span_id will be generated, so there is nothing we can + // assert + EXPECT_EQ(upstream_request_->headers().get(::Envoy::Http::LowerCaseString("traceparent")).size(), + 1); + // OTLP tracer adds an empty tracestate + absl::string_view tracestate_value = upstream_request_->headers() + .get(Http::LowerCaseString("tracestate"))[0] + ->value() + .getStringView(); + EXPECT_EQ("", tracestate_value); +} + +} // namespace +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler_test.cc b/test/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler_test.cc new file mode 100644 index 000000000000..79d37ca9bfe8 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler_test.cc @@ -0,0 +1,56 @@ +#include + +#include "envoy/extensions/tracers/opentelemetry/samplers/v3/always_on_sampler.pb.h" + +#include "source/extensions/tracers/opentelemetry/samplers/always_on/always_on_sampler.h" +#include "source/extensions/tracers/opentelemetry/span_context.h" + +#include "test/mocks/server/tracer_factory_context.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +// Verify sampler being invoked with an invalid span context +TEST(AlwaysOnSamplerTest, TestWithInvalidParentContext) { + envoy::extensions::tracers::opentelemetry::samplers::v3::AlwaysOnSamplerConfig config; + NiceMock context; + auto sampler = std::make_shared(config, context); + EXPECT_STREQ(sampler->getDescription().c_str(), "AlwaysOnSampler"); + + auto sampling_result = + sampler->shouldSample(absl::nullopt, "operation_name", "12345", + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER, {}, {}); + EXPECT_EQ(sampling_result.decision, Decision::RECORD_AND_SAMPLE); + EXPECT_EQ(sampling_result.attributes, nullptr); + EXPECT_STREQ(sampling_result.tracestate.c_str(), ""); + EXPECT_TRUE(sampling_result.isRecording()); + EXPECT_TRUE(sampling_result.isSampled()); +} + +// Verify sampler being invoked with a valid span context +TEST(AlwaysOnSamplerTest, TestWithValidParentContext) { + envoy::extensions::tracers::opentelemetry::samplers::v3::AlwaysOnSamplerConfig config; + NiceMock context; + auto sampler = std::make_shared(config, context); + EXPECT_STREQ(sampler->getDescription().c_str(), "AlwaysOnSampler"); + + SpanContext span_context("0", "12345", "45678", false, "some_tracestate"); + auto sampling_result = + sampler->shouldSample(span_context, "operation_name", "12345", + ::opentelemetry::proto::trace::v1::Span::SPAN_KIND_SERVER, {}, {}); + EXPECT_EQ(sampling_result.decision, Decision::RECORD_AND_SAMPLE); + EXPECT_EQ(sampling_result.attributes, nullptr); + EXPECT_STREQ(sampling_result.tracestate.c_str(), "some_tracestate"); + EXPECT_TRUE(sampling_result.isRecording()); + EXPECT_TRUE(sampling_result.isSampled()); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/samplers/always_on/config_test.cc b/test/extensions/tracers/opentelemetry/samplers/always_on/config_test.cc new file mode 100644 index 000000000000..226cd58e34f3 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/samplers/always_on/config_test.cc @@ -0,0 +1,38 @@ +#include "envoy/registry/registry.h" + +#include "source/extensions/tracers/opentelemetry/samplers/always_on/config.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +// Test create sampler via factory +TEST(AlwaysOnSamplerFactoryTest, Test) { + auto* factory = Registry::FactoryRegistry::getFactory( + "envoy.tracers.opentelemetry.samplers.always_on"); + ASSERT_NE(factory, nullptr); + EXPECT_STREQ(factory->name().c_str(), "envoy.tracers.opentelemetry.samplers.always_on"); + EXPECT_NE(factory->createEmptyConfigProto(), nullptr); + + envoy::config::core::v3::TypedExtensionConfig typed_config; + const std::string yaml = R"EOF( + name: envoy.tracers.opentelemetry.samplers.always_on + typed_config: + "@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.samplers.v3.AlwaysOnSamplerConfig + )EOF"; + TestUtility::loadFromYaml(yaml, typed_config); + NiceMock context; + EXPECT_NE(factory->createSampler(typed_config.typed_config(), context), nullptr); + EXPECT_STREQ(factory->name().c_str(), "envoy.tracers.opentelemetry.samplers.always_on"); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/samplers/sampler_test.cc b/test/extensions/tracers/opentelemetry/samplers/sampler_test.cc new file mode 100644 index 000000000000..a8ff62c71d85 --- /dev/null +++ b/test/extensions/tracers/opentelemetry/samplers/sampler_test.cc @@ -0,0 +1,225 @@ +#include + +#include "envoy/common/optref.h" +#include "envoy/config/trace/v3/opentelemetry.pb.h" +#include "envoy/registry/registry.h" + +#include "source/common/tracing/http_tracer_impl.h" +#include "source/extensions/tracers/opentelemetry/opentelemetry_tracer_impl.h" +#include "source/extensions/tracers/opentelemetry/samplers/sampler.h" +#include "source/extensions/tracers/opentelemetry/span_context.h" + +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace OpenTelemetry { + +using ::testing::NiceMock; +using ::testing::StrictMock; + +class TestSampler : public Sampler { +public: + MOCK_METHOD(SamplingResult, shouldSample, + ((const absl::optional), (const std::string&), (const std::string&), + (OTelSpanKind), (OptRef), + (const std::vector&)), + (override)); + MOCK_METHOD(std::string, getDescription, (), (const, override)); +}; + +class TestSamplerFactory : public SamplerFactory { +public: + MOCK_METHOD(SamplerSharedPtr, createSampler, + (const Protobuf::Message& message, + Server::Configuration::TracerFactoryContext& context)); + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "envoy.tracers.opentelemetry.samplers.testsampler"; } +}; + +class SamplerFactoryTest : public testing::Test { + +protected: + NiceMock config; + NiceMock stream_info; + Tracing::TestTraceContextImpl trace_context{}; + NiceMock context; +}; + +// Test OTLP tracer without a sampler +TEST_F(SamplerFactoryTest, TestWithoutSampler) { + // using StrictMock, calls to SamplerFactory would cause a test failure + auto test_sampler = std::make_shared>(); + StrictMock sampler_factory; + Registry::InjectFactory sampler_factory_registration(sampler_factory); + + // no sampler configured + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + )EOF"; + + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + auto driver = std::make_unique(opentelemetry_config, context); + + driver->startSpan(config, trace_context, stream_info, "operation_name", + {Tracing::Reason::Sampling, true}); +} + +// Test config containing an unknown sampler +TEST_F(SamplerFactoryTest, TestWithInvalidSampler) { + // using StrictMock, calls to SamplerFactory would cause a test failure + auto test_sampler = std::make_shared>(); + StrictMock sampler_factory; + Registry::InjectFactory sampler_factory_registration(sampler_factory); + + // invalid sampler configured + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + sampler: + name: envoy.tracers.opentelemetry.samplers.testsampler + typed_config: + "@type": type.googleapis.com/google.protobuf.Value + )EOF"; + + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + EXPECT_THROW(std::make_unique(opentelemetry_config, context), EnvoyException); +} + +// Test OTLP tracer with a sampler +TEST_F(SamplerFactoryTest, TestWithSampler) { + auto test_sampler = std::make_shared>(); + TestSamplerFactory sampler_factory; + Registry::InjectFactory sampler_factory_registration(sampler_factory); + + EXPECT_CALL(sampler_factory, createSampler(_, _)).WillOnce(Return(test_sampler)); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + sampler: + name: envoy.tracers.opentelemetry.samplers.testsampler + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; + + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + auto driver = std::make_unique(opentelemetry_config, context); + + // shouldSample returns a result without additional attributes and Decision::RECORD_AND_SAMPLE + EXPECT_CALL(*test_sampler, shouldSample(_, _, _, _, _, _)) + .WillOnce([](const absl::optional, const std::string&, const std::string&, + OTelSpanKind, OptRef, + const std::vector&) { + SamplingResult res; + res.decision = Decision::RECORD_AND_SAMPLE; + res.tracestate = "this_is=tracesate"; + return res; + }); + + Tracing::SpanPtr tracing_span = driver->startSpan( + config, trace_context, stream_info, "operation_name", {Tracing::Reason::Sampling, true}); + // startSpan returns a Tracing::SpanPtr. Tracing::Span has no sampled() method. + // We know that the underlying span is Extensions::Tracers::OpenTelemetry::Span + // So the dynamic_cast should be safe. + std::unique_ptr span(dynamic_cast(tracing_span.release())); + EXPECT_TRUE(span->sampled()); + EXPECT_STREQ(span->tracestate().c_str(), "this_is=tracesate"); + + // shouldSamples return a result containing additional attributes and Decision::DROP + EXPECT_CALL(*test_sampler, shouldSample(_, _, _, _, _, _)) + .WillOnce([](const absl::optional, const std::string&, const std::string&, + OTelSpanKind, OptRef, + const std::vector&) { + SamplingResult res; + res.decision = Decision::DROP; + std::map attributes; + attributes["key"] = "value"; + attributes["another_key"] = "another_value"; + res.attributes = + std::make_unique>(std::move(attributes)); + res.tracestate = "this_is=another_tracesate"; + return res; + }); + tracing_span = driver->startSpan(config, trace_context, stream_info, "operation_name", + {Tracing::Reason::Sampling, true}); + std::unique_ptr unsampled_span(dynamic_cast(tracing_span.release())); + EXPECT_FALSE(unsampled_span->sampled()); + EXPECT_STREQ(unsampled_span->tracestate().c_str(), "this_is=another_tracesate"); +} + +// Test that sampler receives trace_context +TEST_F(SamplerFactoryTest, TestInitialAttributes) { + auto test_sampler = std::make_shared>(); + TestSamplerFactory sampler_factory; + Registry::InjectFactory sampler_factory_registration(sampler_factory); + + EXPECT_CALL(sampler_factory, createSampler(_, _)).WillOnce(Return(test_sampler)); + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake-cluster + timeout: 0.250s + service_name: my-service + sampler: + name: envoy.tracers.opentelemetry.samplers.testsampler + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + )EOF"; + + envoy::config::trace::v3::OpenTelemetryConfig opentelemetry_config; + TestUtility::loadFromYaml(yaml_string, opentelemetry_config); + + auto driver = std::make_unique(opentelemetry_config, context); + + auto expected = makeOptRef(trace_context); + EXPECT_CALL(*test_sampler, shouldSample(_, _, _, _, expected, _)); + driver->startSpan(config, trace_context, stream_info, "operation_name", + {Tracing::Reason::Sampling, true}); +} + +// Test sampling result decision +TEST(SamplingResultTest, TestSamplingResult) { + SamplingResult result; + result.decision = Decision::RECORD_AND_SAMPLE; + EXPECT_TRUE(result.isRecording()); + EXPECT_TRUE(result.isSampled()); + result.decision = Decision::RECORD_ONLY; + EXPECT_TRUE(result.isRecording()); + EXPECT_FALSE(result.isSampled()); + result.decision = Decision::DROP; + EXPECT_FALSE(result.isRecording()); + EXPECT_FALSE(result.isSampled()); +} + +} // namespace OpenTelemetry +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/opentelemetry/span_context_extractor_test.cc b/test/extensions/tracers/opentelemetry/span_context_extractor_test.cc index a09fba2e3564..b87f984768eb 100644 --- a/test/extensions/tracers/opentelemetry/span_context_extractor_test.cc +++ b/test/extensions/tracers/opentelemetry/span_context_extractor_test.cc @@ -1,3 +1,4 @@ +#include "source/common/tracing/http_tracer_impl.h" #include "source/extensions/tracers/opentelemetry/span_context_extractor.h" #include "test/test_common/status_utility.h" @@ -21,8 +22,9 @@ constexpr absl::string_view parent_id = "0000000000000003"; constexpr absl::string_view trace_flags = "01"; TEST(SpanContextExtractorTest, ExtractSpanContext) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", version, trace_id, parent_id, trace_flags)}}; + SpanContextExtractor span_context_extractor(request_headers); absl::StatusOr span_context = span_context_extractor.extractSpanContext(); @@ -35,7 +37,7 @@ TEST(SpanContextExtractorTest, ExtractSpanContext) { TEST(SpanContextExtractorTest, ExtractSpanContextNotSampled) { const std::string trace_flags_unsampled{"00"}; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", version, trace_id, parent_id, trace_flags_unsampled)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -49,7 +51,7 @@ TEST(SpanContextExtractorTest, ExtractSpanContextNotSampled) { } TEST(SpanContextExtractorTest, ThrowsExceptionWithoutHeader) { - Http::TestRequestHeaderMapImpl request_headers{{}}; + Tracing::TestTraceContextImpl request_headers{{}}; SpanContextExtractor span_context_extractor(request_headers); absl::StatusOr span_context = span_context_extractor.extractSpanContext(); @@ -59,7 +61,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithoutHeader) { } TEST(SpanContextExtractorTest, ThrowsExceptionWithTooLongHeader) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("000{}-{}-{}-{}", version, trace_id, parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -70,7 +72,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithTooLongHeader) { } TEST(SpanContextExtractorTest, ThrowsExceptionWithTooShortHeader) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}", trace_id, parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -81,7 +83,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithTooShortHeader) { } TEST(SpanContextExtractorTest, ThrowsExceptionWithInvalidHyphenation) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}{}-{}-{}", version, trace_id, parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -94,7 +96,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithInvalidHyphenation) { TEST(SpanContextExtractorTest, ThrowsExceptionWithInvalidSizes) { const std::string invalid_version{"0"}; const std::string invalid_trace_flags{"001"}; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", invalid_version, trace_id, parent_id, invalid_trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -107,7 +109,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithInvalidSizes) { TEST(SpanContextExtractorTest, ThrowsExceptionWithInvalidHex) { const std::string invalid_version{"ZZ"}; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", invalid_version, trace_id, parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -120,7 +122,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithInvalidHex) { TEST(SpanContextExtractorTest, ThrowsExceptionWithAllZeroTraceId) { const std::string invalid_trace_id{"00000000000000000000000000000000"}; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", version, invalid_trace_id, parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -133,7 +135,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithAllZeroTraceId) { TEST(SpanContextExtractorTest, ThrowsExceptionWithAllZeroParentId) { const std::string invalid_parent_id{"0000000000000000"}; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", version, trace_id, invalid_parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); @@ -145,7 +147,7 @@ TEST(SpanContextExtractorTest, ThrowsExceptionWithAllZeroParentId) { } TEST(SpanContextExtractorTest, ExtractSpanContextWithEmptyTracestate) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", version, trace_id, parent_id, trace_flags)}}; SpanContextExtractor span_context_extractor(request_headers); absl::StatusOr span_context = span_context_extractor.extractSpanContext(); @@ -155,7 +157,7 @@ TEST(SpanContextExtractorTest, ExtractSpanContextWithEmptyTracestate) { } TEST(SpanContextExtractorTest, ExtractSpanContextWithTracestate) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"traceparent", fmt::format("{}-{}-{}-{}", version, trace_id, parent_id, trace_flags)}, {"tracestate", "sample-tracestate"}}; SpanContextExtractor span_context_extractor(request_headers); @@ -166,7 +168,7 @@ TEST(SpanContextExtractorTest, ExtractSpanContextWithTracestate) { } TEST(SpanContextExtractorTest, IgnoreTracestateWithoutTraceparent) { - Http::TestRequestHeaderMapImpl request_headers{{"tracestate", "sample-tracestate"}}; + Tracing::TestTraceContextImpl request_headers{{"tracestate", "sample-tracestate"}}; SpanContextExtractor span_context_extractor(request_headers); absl::StatusOr span_context = span_context_extractor.extractSpanContext(); @@ -179,7 +181,8 @@ TEST(SpanContextExtractorTest, ExtractSpanContextWithMultipleTracestateEntries) {"traceparent", fmt::format("{}-{}-{}-{}", version, trace_id, parent_id, trace_flags)}, {"tracestate", "sample-tracestate"}, {"tracestate", "sample-tracestate-2"}}; - SpanContextExtractor span_context_extractor(request_headers); + Tracing::HttpTraceContext trace_context(request_headers); + SpanContextExtractor span_context_extractor(trace_context); absl::StatusOr span_context = span_context_extractor.extractSpanContext(); EXPECT_OK(span_context); diff --git a/test/extensions/tracers/skywalking/BUILD b/test/extensions/tracers/skywalking/BUILD index fcebfe3bee30..36f31a7bcea0 100644 --- a/test/extensions/tracers/skywalking/BUILD +++ b/test/extensions/tracers/skywalking/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_library", "envoy_package", ) load( @@ -41,14 +42,19 @@ envoy_extension_cc_test( ], ) +envoy_cc_library( + name = "skywalking_test_helper_lib", + hdrs = ["skywalking_test_helper.h"], +) + envoy_extension_cc_test( name = "skywalking_test_helper", - srcs = ["skywalking_test_helper.h"], extension_names = ["envoy.tracers.skywalking"], external_deps = [ "cpp2sky", ], deps = [ + ":skywalking_test_helper_lib", "//source/common/common:base64_lib", "//source/common/common:hex_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc b/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc index c6687af8e9cd..7d4726edcdb4 100644 --- a/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc +++ b/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc @@ -77,10 +77,10 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { auto previous_header_value = SkyWalkingTestHelper::createPropagatedSW8HeaderValue(false, ""); - Http::TestRequestHeaderMapImpl request_headers{{"sw8", previous_header_value}, - {":path", "/path"}, - {":method", "GET"}, - {":authority", "test.com"}}; + Tracing::TestTraceContextImpl request_headers{{"sw8", previous_header_value}, + {":path", "/path"}, + {":method", "GET"}, + {":authority", "test.com"}}; ON_CALL(mock_tracing_config_, operationName()) .WillByDefault(Return(Tracing::OperationName::Ingress)); @@ -110,7 +110,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { // Create new span segment with no previous span context. - Http::TestRequestHeaderMapImpl new_request_headers{ + Tracing::TestTraceContextImpl new_request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; Tracing::SpanPtr org_span = @@ -132,11 +132,10 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { // Create new span segment with error propagation header. - Http::TestRequestHeaderMapImpl error_request_headers{ - {":path", "/path"}, - {":method", "GET"}, - {":authority", "test.com"}, - {"sw8", "xxxxxx-error-propagation-header"}}; + Tracing::TestTraceContextImpl error_request_headers{{":path", "/path"}, + {":method", "GET"}, + {":authority", "test.com"}, + {"sw8", "xxxxxx-error-propagation-header"}}; Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, error_request_headers, stream_info_, "TEST_OP", decision); Span* span = dynamic_cast(org_span.get()); @@ -155,7 +154,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { // Create new span segment with error propagation header. - Http::TestRequestHeaderMapImpl error_request_headers{ + Tracing::TestTraceContextImpl error_request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}, @@ -180,7 +179,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { // Create null span with disabled tracing. decision.traced = false; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; Tracing::SpanPtr org_null_span = driver_->startSpan(mock_tracing_config_, request_headers, stream_info_, "TEST_OP", decision); @@ -194,11 +193,10 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { // Create null span with disabled tracing. decision.traced = false; - Http::TestRequestHeaderMapImpl error_request_headers{ - {":path", "/path"}, - {":method", "GET"}, - {":authority", "test.com"}, - {"sw8", "xxxxxx-error-propagation-header"}}; + Tracing::TestTraceContextImpl error_request_headers{{":path", "/path"}, + {":method", "GET"}, + {":authority", "test.com"}, + {"sw8", "xxxxxx-error-propagation-header"}}; Tracing::SpanPtr org_null_span = driver_->startSpan(mock_tracing_config_, error_request_headers, stream_info_, "TEST_OP", decision); @@ -211,7 +209,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { { // Create null span with disabled tracing. decision.traced = false; - Http::TestRequestHeaderMapImpl error_request_headers{ + Tracing::TestTraceContextImpl error_request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}, @@ -239,7 +237,7 @@ TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestNoClientConfig) { decision.reason = Tracing::Reason::Sampling; decision.traced = true; - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; Tracing::SpanPtr org_span = diff --git a/test/extensions/tracers/skywalking/tracer_test.cc b/test/extensions/tracers/skywalking/tracer_test.cc index bee015a11442..555f28c59d84 100644 --- a/test/extensions/tracers/skywalking/tracer_test.cc +++ b/test/extensions/tracers/skywalking/tracer_test.cc @@ -152,8 +152,8 @@ TEST_F(TracerTest, TracerTestCreateNewSpanWithNoPropagationHeaders) { // child span (EXIT span). EXPECT_EQ(span->spanEntity()->operationName(), first_child_span->spanEntity()->operationName()); - Http::TestRequestHeaderMapImpl first_child_headers{{":authority", "test.com"}, - {":path", "/upstream/path"}}; + Tracing::TestTraceContextImpl first_child_headers{{":authority", "test.com"}, + {":path", "/upstream/path"}}; Upstream::HostDescriptionConstSharedPtr host{ new testing::NiceMock()}; @@ -162,7 +162,7 @@ TEST_F(TracerTest, TracerTestCreateNewSpanWithNoPropagationHeaders) { // request. EXPECT_EQ("/upstream/path", first_child_span->spanEntity()->operationName()); - auto sp = createSpanContext(first_child_headers.get_("sw8")); + auto sp = createSpanContext(std::string(first_child_headers.get("sw8").value())); EXPECT_EQ("CURR#SERVICE", sp->service()); EXPECT_EQ("CURR#INSTANCE", sp->serviceInstance()); EXPECT_EQ("/downstream/path", sp->endpoint()); @@ -189,10 +189,10 @@ TEST_F(TracerTest, TracerTestCreateNewSpanWithNoPropagationHeaders) { EXPECT_EQ(span->spanEntity()->operationName(), second_child_span->spanEntity()->operationName()); - Http::TestRequestHeaderMapImpl second_child_headers{{":authority", "test.com"}}; + Tracing::TestTraceContextImpl second_child_headers{{":authority", "test.com"}}; second_child_span->injectContext(second_child_headers, nullptr); - auto sp = createSpanContext(second_child_headers.get_("sw8")); + auto sp = createSpanContext(std::string(second_child_headers.get("sw8").value())); EXPECT_EQ("CURR#SERVICE", sp->service()); EXPECT_EQ("CURR#INSTANCE", sp->serviceInstance()); EXPECT_EQ("/downstream/path", sp->endpoint()); diff --git a/test/extensions/tracers/xray/tracer_test.cc b/test/extensions/tracers/xray/tracer_test.cc index f75b3d649d44..a9bed48890cb 100644 --- a/test/extensions/tracers/xray/tracer_test.cc +++ b/test/extensions/tracers/xray/tracer_test.cc @@ -507,13 +507,13 @@ TEST_F(XRayTracerTest, SpanInjectContextHasXRayHeader) { auto span = tracer.startSpan(config_, "ingress", server_.timeSource().systemTime(), absl::nullopt /*headers*/, absl::nullopt /*client_ip from x-forwarded-for header*/); - Http::TestRequestHeaderMapImpl request_headers; + Tracing::TestTraceContextImpl request_headers{}; span->injectContext(request_headers, nullptr); - auto header = request_headers.get(Http::LowerCaseString{XRayTraceHeader}); - ASSERT_FALSE(header.empty()); - EXPECT_NE(header[0]->value().getStringView().find("Root="), absl::string_view::npos); - EXPECT_NE(header[0]->value().getStringView().find("Parent="), absl::string_view::npos); - EXPECT_NE(header[0]->value().getStringView().find("Sampled=1"), absl::string_view::npos); + auto header = request_headers.get(xRayTraceHeader().key()); + ASSERT_FALSE(!header.has_value()); + EXPECT_NE(header.value().find("Root="), absl::string_view::npos); + EXPECT_NE(header.value().find("Parent="), absl::string_view::npos); + EXPECT_NE(header.value().find("Sampled=1"), absl::string_view::npos); } TEST_F(XRayTracerTest, SpanInjectContextHasXRayHeaderNonSampled) { @@ -525,13 +525,13 @@ TEST_F(XRayTracerTest, SpanInjectContextHasXRayHeaderNonSampled) { server_.timeSource(), server_.api().randomGenerator()}; auto span = tracer.createNonSampledSpan(absl::nullopt /*headers*/); - Http::TestRequestHeaderMapImpl request_headers; + Tracing::TestTraceContextImpl request_headers{}; span->injectContext(request_headers, nullptr); - auto header = request_headers.get(Http::LowerCaseString{XRayTraceHeader}); - ASSERT_FALSE(header.empty()); - EXPECT_NE(header[0]->value().getStringView().find("Root="), absl::string_view::npos); - EXPECT_NE(header[0]->value().getStringView().find("Parent="), absl::string_view::npos); - EXPECT_NE(header[0]->value().getStringView().find("Sampled=0"), absl::string_view::npos); + auto header = request_headers.get(xRayTraceHeader().key()); + ASSERT_FALSE(!header.has_value()); + EXPECT_NE(header.value().find("Root="), absl::string_view::npos); + EXPECT_NE(header.value().find("Parent="), absl::string_view::npos); + EXPECT_NE(header.value().find("Sampled=0"), absl::string_view::npos); } TEST_F(XRayTracerTest, TraceIDFormatTest) { diff --git a/test/extensions/tracers/xray/xray_tracer_impl_test.cc b/test/extensions/tracers/xray/xray_tracer_impl_test.cc index a4300d367107..4a7a2c995c32 100644 --- a/test/extensions/tracers/xray/xray_tracer_impl_test.cc +++ b/test/extensions/tracers/xray/xray_tracer_impl_test.cc @@ -38,12 +38,12 @@ class XRayDriverTest : public ::testing::Test { NiceMock context_; NiceMock tls_; NiceMock tracing_config_; - Http::TestRequestHeaderMapImpl request_headers_{ + Tracing::TestTraceContextImpl request_headers_{ {":authority", "api.amazon.com"}, {":path", "/"}, {":method", "GET"}}; }; TEST_F(XRayDriverTest, XRayTraceHeaderNotSampled) { - request_headers_.addCopy(std::string(XRayTraceHeader), "Root=1-272793;Parent=5398ad8;Sampled=0"); + request_headers_.set(xRayTraceHeader().key(), "Root=1-272793;Parent=5398ad8;Sampled=0"); XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/, "" /*origin*/, aws_metadata_}; @@ -59,7 +59,7 @@ TEST_F(XRayDriverTest, XRayTraceHeaderNotSampled) { } TEST_F(XRayDriverTest, XRayTraceHeaderSampled) { - request_headers_.addCopy(std::string(XRayTraceHeader), "Root=1-272793;Parent=5398ad8;Sampled=1"); + request_headers_.set(xRayTraceHeader().key(), "Root=1-272793;Parent=5398ad8;Sampled=1"); XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/, "" /*origin*/, aws_metadata_}; @@ -73,7 +73,7 @@ TEST_F(XRayDriverTest, XRayTraceHeaderSampled) { } TEST_F(XRayDriverTest, XRayTraceHeaderSamplingUnknown) { - request_headers_.addCopy(std::string(XRayTraceHeader), "Root=1-272793;Parent=5398ad8;Sampled="); + request_headers_.set(xRayTraceHeader().key(), "Root=1-272793;Parent=5398ad8;Sampled="); XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/, "" /*origin*/, aws_metadata_}; @@ -91,7 +91,7 @@ TEST_F(XRayDriverTest, XRayTraceHeaderSamplingUnknown) { } TEST_F(XRayDriverTest, XRayTraceHeaderWithoutSamplingDecision) { - request_headers_.addCopy(std::string(XRayTraceHeader), "Root=1-272793;Parent=5398ad8;"); + request_headers_.set(xRayTraceHeader().key(), "Root=1-272793;Parent=5398ad8;"); // sampling rules with default fixed_target = 0 & rate = 0 XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", R"EOF( { @@ -134,7 +134,7 @@ TEST_F(XRayDriverTest, NoXRayTracerHeader) { } TEST_F(XRayDriverTest, XForwardedForHeaderSet) { - request_headers_.addCopy(std::string(XForwardedForHeader), "191.251.191.251"); + request_headers_.set(xForwardedForHeader().key(), "191.251.191.251"); XRayConfiguration config{"" /*daemon_endpoint*/, "test_segment_name", "" /*sampling_rules*/, "" /*origin*/, aws_metadata_}; Driver driver(config, context_); diff --git a/test/extensions/tracers/zipkin/config_test.cc b/test/extensions/tracers/zipkin/config_test.cc index 7a44de3aca28..073d9af2246c 100644 --- a/test/extensions/tracers/zipkin/config_test.cc +++ b/test/extensions/tracers/zipkin/config_test.cc @@ -11,8 +11,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using ::testing::Eq; - namespace Envoy { namespace Extensions { namespace Tracers { diff --git a/test/extensions/tracers/zipkin/span_context_extractor_test.cc b/test/extensions/tracers/zipkin/span_context_extractor_test.cc index 1b6f41bf6ea0..393b995f7167 100644 --- a/test/extensions/tracers/zipkin/span_context_extractor_test.cc +++ b/test/extensions/tracers/zipkin/span_context_extractor_test.cc @@ -20,7 +20,7 @@ const std::string parent_id{"0000000000000002"}; } // namespace TEST(ZipkinSpanContextExtractorTest, Largest) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}-{}-1-{}", trace_id_high, trace_id, span_id, parent_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); @@ -35,7 +35,7 @@ TEST(ZipkinSpanContextExtractorTest, Largest) { } TEST(ZipkinSpanContextExtractorTest, WithoutParentDebug) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}-{}-d", trace_id_high, trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); @@ -50,7 +50,7 @@ TEST(ZipkinSpanContextExtractorTest, WithoutParentDebug) { } TEST(ZipkinSpanContextExtractorTest, MalformedUuid) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "b970dafd-0d95-40aa-95d8-1d8725aebe40"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "b970dafd-0d95-40aa-95d8-1d8725aebe40"}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: invalid trace id b970dafd-0d95-40"); @@ -58,7 +58,7 @@ TEST(ZipkinSpanContextExtractorTest, MalformedUuid) { } TEST(ZipkinSpanContextExtractorTest, MiddleOfString) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}-{},", trace_id, trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -67,7 +67,7 @@ TEST(ZipkinSpanContextExtractorTest, MiddleOfString) { } TEST(ZipkinSpanContextExtractorTest, DebugOnly) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "d"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "d"}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); EXPECT_FALSE(context.second); @@ -81,7 +81,7 @@ TEST(ZipkinSpanContextExtractorTest, DebugOnly) { } TEST(ZipkinSpanContextExtractorTest, Sampled) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "1"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "1"}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); EXPECT_FALSE(context.second); @@ -95,7 +95,7 @@ TEST(ZipkinSpanContextExtractorTest, Sampled) { } TEST(ZipkinSpanContextExtractorTest, SampledFalse) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "0"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "0"}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); EXPECT_FALSE(context.second); @@ -109,7 +109,7 @@ TEST(ZipkinSpanContextExtractorTest, SampledFalse) { } TEST(ZipkinSpanContextExtractorTest, IdNotYetSampled128) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}-{}", trace_id_high, trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); @@ -124,7 +124,7 @@ TEST(ZipkinSpanContextExtractorTest, IdNotYetSampled128) { } TEST(ZipkinSpanContextExtractorTest, IdsUnsampled) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", fmt::format("{}-{}-0", trace_id, span_id)}}; + Tracing::TestTraceContextImpl request_headers{{"b3", fmt::format("{}-{}-0", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); EXPECT_TRUE(context.second); @@ -138,7 +138,7 @@ TEST(ZipkinSpanContextExtractorTest, IdsUnsampled) { } TEST(ZipkinSpanContextExtractorTest, ParentUnsampled) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-0-{}", trace_id, span_id, parent_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); @@ -153,7 +153,7 @@ TEST(ZipkinSpanContextExtractorTest, ParentUnsampled) { } TEST(ZipkinSpanContextExtractorTest, ParentDebug) { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-d-{}", trace_id, span_id, parent_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); @@ -168,7 +168,7 @@ TEST(ZipkinSpanContextExtractorTest, ParentDebug) { } TEST(ZipkinSpanContextExtractorTest, IdsWithDebug) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", fmt::format("{}-{}-d", trace_id, span_id)}}; + Tracing::TestTraceContextImpl request_headers{{"b3", fmt::format("{}-{}-d", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(true); EXPECT_TRUE(context.second); @@ -182,7 +182,7 @@ TEST(ZipkinSpanContextExtractorTest, IdsWithDebug) { } TEST(ZipkinSpanContextExtractorTest, WithoutSampled) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", fmt::format("{}-{}", trace_id, span_id)}}; + Tracing::TestTraceContextImpl request_headers{{"b3", fmt::format("{}-{}", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); auto context = extractor.extractSpanContext(false); EXPECT_TRUE(context.second); @@ -197,7 +197,7 @@ TEST(ZipkinSpanContextExtractorTest, WithoutSampled) { TEST(ZipkinSpanContextExtractorTest, TooBig) { { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}{}-{}-{}", trace_id, trace_id, trace_id, span_id, trace_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -206,7 +206,7 @@ TEST(ZipkinSpanContextExtractorTest, TooBig) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}-{}-1-{}a", trace_id_high, trace_id, span_id, parent_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -215,7 +215,7 @@ TEST(ZipkinSpanContextExtractorTest, TooBig) { } TEST(ZipkinSpanContextExtractorTest, Empty) { - Http::TestRequestHeaderMapImpl request_headers{{"b3", ""}}; + Tracing::TestTraceContextImpl request_headers{{"b3", ""}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: empty"); @@ -223,8 +223,8 @@ TEST(ZipkinSpanContextExtractorTest, Empty) { TEST(ZipkinSpanContextExtractorTest, InvalidInput) { { - Http::TestRequestHeaderMapImpl request_headers{ - {"X-B3-TraceId", trace_id_high + trace_id.substr(0, 15) + "!"}, {"X-B3-SpanId", span_id}}; + Tracing::TestTraceContextImpl request_headers{ + {"x-b3-traceid", trace_id_high + trace_id.substr(0, 15) + "!"}, {"x-b3-spanid", span_id}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, fmt::format("Invalid traceid_high {} or tracid {}", trace_id_high, @@ -232,7 +232,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}!{}-{}", trace_id.substr(0, 15), trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE( @@ -241,7 +241,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}{}!-{}", trace_id, trace_id.substr(0, 15), span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE( @@ -250,7 +250,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}!-{}", trace_id.substr(0, 15), span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE( @@ -259,14 +259,14 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", fmt::format("{}!{}", trace_id, span_id)}}; + Tracing::TestTraceContextImpl request_headers{{"b3", fmt::format("{}!{}", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: not exists span id"); } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}!", trace_id, span_id.substr(0, 15))}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE( @@ -275,7 +275,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}!0", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -283,7 +283,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-c", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -291,7 +291,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-d!{}", trace_id, span_id, parent_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -299,7 +299,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-d-{}!", trace_id, span_id, parent_id.substr(0, 15))}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE( @@ -308,7 +308,7 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "-"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "-"}}; SpanContextExtractor extractor(request_headers); EXPECT_TRUE(extractor.extractSampled({Tracing::Reason::Sampling, true})); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -318,49 +318,49 @@ TEST(ZipkinSpanContextExtractorTest, InvalidInput) { TEST(ZipkinSpanContextExtractorTest, Truncated) { { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "-1"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "-1"}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "1-"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "1-"}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", "1-"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", "1-"}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", trace_id.substr(0, 15)}}; + Tracing::TestTraceContextImpl request_headers{{"b3", trace_id.substr(0, 15)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", trace_id}}; + Tracing::TestTraceContextImpl request_headers{{"b3", trace_id}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{{"b3", trace_id + "-"}}; + Tracing::TestTraceContextImpl request_headers{{"b3", trace_id + "-"}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}", trace_id.substr(0, 15), span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -368,7 +368,7 @@ TEST(ZipkinSpanContextExtractorTest, Truncated) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}", trace_id, span_id.substr(0, 15))}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -376,15 +376,14 @@ TEST(ZipkinSpanContextExtractorTest, Truncated) { } { - Http::TestRequestHeaderMapImpl request_headers{ - {"b3", fmt::format("{}-{}-", trace_id, span_id)}}; + Tracing::TestTraceContextImpl request_headers{{"b3", fmt::format("{}-{}-", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, "Invalid input: truncated"); } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-1-", trace_id, span_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -392,7 +391,7 @@ TEST(ZipkinSpanContextExtractorTest, Truncated) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-1-{}", trace_id, span_id, parent_id.substr(0, 15))}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, @@ -400,7 +399,7 @@ TEST(ZipkinSpanContextExtractorTest, Truncated) { } { - Http::TestRequestHeaderMapImpl request_headers{ + Tracing::TestTraceContextImpl request_headers{ {"b3", fmt::format("{}-{}-{}{}", trace_id, span_id, trace_id, trace_id)}}; SpanContextExtractor extractor(request_headers); EXPECT_THROW_WITH_MESSAGE(extractor.extractSpanContext(true), ExtractorException, diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index b77dd99abb5d..37dd69a83ea1 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -152,7 +152,7 @@ class ZipkinDriverTest : public testing::Test { uint64_t generateRandom64() { return Util::generateRandom64(time_source_); } const std::string operation_name_{"test"}; - Http::TestRequestHeaderMapImpl request_headers_{ + Tracing::TestTraceContextImpl request_headers_{ {":authority", "api.lyft.com"}, {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}}; NiceMock stream_info_; @@ -496,9 +496,9 @@ TEST_F(ZipkinDriverTest, FlushSpansTimer) { TEST_F(ZipkinDriverTest, NoB3ContextSampledTrue) { setupValidDriver("HTTP_JSON"); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()).has_value()); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -510,9 +510,9 @@ TEST_F(ZipkinDriverTest, NoB3ContextSampledTrue) { TEST_F(ZipkinDriverTest, NoB3ContextSampledFalse) { setupValidDriver("HTTP_JSON"); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()).has_value()); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, false}); @@ -524,11 +524,11 @@ TEST_F(ZipkinDriverTest, NoB3ContextSampledFalse) { TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleTrue) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()).has_value()); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -540,11 +540,11 @@ TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleTrue) { TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleFalse) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED).empty()); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()).has_value()); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, false}); @@ -556,76 +556,76 @@ TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleFalse) { TEST_F(ZipkinDriverTest, PropagateB3NotSampled) { setupValidDriver("HTTP_JSON"); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID.key()).has_value()); // Only context header set is B3 sampled to indicate trace should not be sampled - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, NOT_SAMPLED); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SAMPLED.key(), NOT_SAMPLED); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); - request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); + request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); span->injectContext(request_headers_, nullptr); - auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED); + auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); // Check B3 sampled flag is set to not sample - EXPECT_EQ(NOT_SAMPLED, sampled_entry[0]->value().getStringView()); + EXPECT_EQ(NOT_SAMPLED, sampled_entry.value()); } TEST_F(ZipkinDriverTest, PropagateB3NotSampledWithFalse) { setupValidDriver("HTTP_JSON"); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID.key()).has_value()); // Only context header set is B3 sampled to indicate trace should not be sampled (using legacy // 'false' value) const std::string sampled = "false"; - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, sampled); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SAMPLED.key(), sampled); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); - request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); + request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); span->injectContext(request_headers_, nullptr); - auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED); + auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); // Check B3 sampled flag is set to not sample - EXPECT_EQ(NOT_SAMPLED, sampled_entry[0]->value().getStringView()); + EXPECT_EQ(NOT_SAMPLED, sampled_entry.value()); } TEST_F(ZipkinDriverTest, PropagateB3SampledWithTrue) { setupValidDriver("HTTP_JSON"); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID).empty()); - EXPECT_TRUE(request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID).empty()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID.key()).has_value()); + EXPECT_TRUE(!request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID.key()).has_value()); // Only context header set is B3 sampled to indicate trace should be sampled (using legacy // 'true' value) const std::string sampled = "true"; - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, sampled); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SAMPLED.key(), sampled); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, false}); - request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); + request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); span->injectContext(request_headers_, nullptr); - auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED); + auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); // Check B3 sampled flag is set to sample - EXPECT_EQ(SAMPLED, sampled_entry[0]->value().getStringView()); + EXPECT_EQ(SAMPLED, sampled_entry.value()); } TEST_F(ZipkinDriverTest, PropagateB3SampleFalse) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, NOT_SAMPLED); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SAMPLED.key(), NOT_SAMPLED); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -664,7 +664,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { const std::string parent_id = Hex::uint64ToHex(generateRandom64()); const std::string context = trace_id + ";" + span_id + ";" + parent_id + ";" + CLIENT_SEND; - request_headers_.setCopy(Http::CustomHeaders::get().OtSpanContext, context); + request_headers_.set(Http::CustomHeaders::get().OtSpanContext, context); // New span will have an SR annotation Tracing::SpanPtr span2 = driver_->startSpan(config_, request_headers_, stream_info_, @@ -735,9 +735,9 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersTest) { const std::string span_id = Hex::uint64ToHex(generateRandom64()); const std::string parent_id = Hex::uint64ToHex(generateRandom64()); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, trace_id); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, span_id); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, parent_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), trace_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), span_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), parent_id); // New span will have an SR annotation - so its span and parent ids will be // the same as the supplied span context (i.e. shared context) @@ -757,13 +757,13 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersEmptyParentSpanTest) { // Root span so have same trace and span id const std::string id = Hex::uint64ToHex(generateRandom64()); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, id); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, id); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SAMPLED, SAMPLED); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SAMPLED.key(), SAMPLED); // Set parent span id to empty string, to ensure it is ignored const std::string parent_span_id = ""; - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, parent_span_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), parent_span_id); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -781,9 +781,9 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3Headers128TraceIdTest) { const std::string span_id = Hex::uint64ToHex(generateRandom64()); const std::string parent_id = Hex::uint64ToHex(generateRandom64()); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, trace_id); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, span_id); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, parent_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), trace_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), span_id); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), parent_id); // New span will have an SR annotation - so its span and parent ids will be // the same as the supplied span context (i.e. shared context) @@ -803,11 +803,11 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3Headers128TraceIdTest) { TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidTraceIdB3HeadersTest) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, std::string("xyz")); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), std::string("xyz")); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -817,11 +817,11 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidTraceIdB3HeadersTest) { TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidSpanIdB3HeadersTest) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, std::string("xyz")); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), std::string("xyz")); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -831,12 +831,11 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidSpanIdB3HeadersTest) { TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidParentIdB3HeadersTest) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, - std::string("xyz")); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), std::string("xyz")); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); @@ -851,13 +850,13 @@ TEST_F(ZipkinDriverTest, ExplicitlySetSampledFalse) { span->setSampled(false); - request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); + request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); span->injectContext(request_headers_, nullptr); - auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED); + auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); // Check B3 sampled flag is set to not sample - EXPECT_EQ(NOT_SAMPLED, sampled_entry[0]->value().getStringView()); + EXPECT_EQ(NOT_SAMPLED, sampled_entry.value()); } TEST_F(ZipkinDriverTest, ExplicitlySetSampledTrue) { @@ -868,23 +867,23 @@ TEST_F(ZipkinDriverTest, ExplicitlySetSampledTrue) { span->setSampled(true); - request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED); + request_headers_.remove(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); span->injectContext(request_headers_, nullptr); - auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED); + auto sampled_entry = request_headers_.get(ZipkinCoreConstants::get().X_B3_SAMPLED.key()); // Check B3 sampled flag is set to sample - EXPECT_EQ(SAMPLED, sampled_entry[0]->value().getStringView()); + EXPECT_EQ(SAMPLED, sampled_entry.value()); } TEST_F(ZipkinDriverTest, DuplicatedHeader) { setupValidDriver("HTTP_JSON"); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); - request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID, - Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_TRACE_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); + request_headers_.set(ZipkinCoreConstants::get().X_B3_PARENT_SPAN_ID.key(), + Hex::uint64ToHex(generateRandom64())); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, false}); @@ -900,11 +899,10 @@ TEST_F(ZipkinDriverTest, DuplicatedHeader) { span->setSampled(true); span->injectContext(request_headers_, nullptr); - request_headers_.iterate( - [&dup_callback](const Http::HeaderEntry& header) -> Http::HeaderMap::Iterate { - dup_callback(header.key().getStringView()); - return Http::HeaderMap::Iterate::Continue; - }); + request_headers_.forEach([&dup_callback](absl::string_view key, absl::string_view) -> bool { + dup_callback(key); + return true; + }); } } // namespace diff --git a/test/extensions/transport_sockets/alts/BUILD b/test/extensions/transport_sockets/alts/BUILD index f46c6907f15a..d36c65499448 100644 --- a/test/extensions/transport_sockets/alts/BUILD +++ b/test/extensions/transport_sockets/alts/BUILD @@ -19,12 +19,67 @@ envoy_extension_cc_test( deps = [ "//source/common/singleton:manager_impl_lib", "//source/extensions/transport_sockets/alts:config", + "//source/extensions/transport_sockets/alts:tsi_socket", "//test/mocks/event:event_mocks", "//test/mocks/network:network_mocks", "//test/mocks/server:transport_socket_factory_context_mocks", ], ) +envoy_extension_cc_test( + name = "alts_channel_pool_test", + srcs = ["alts_channel_pool_test.cc"], + extension_names = ["envoy.transport_sockets.alts"], + deps = [ + "//envoy/network:address_interface", + "//source/extensions/transport_sockets/alts:alts_channel_pool", + "//source/extensions/transport_sockets/alts:handshaker_cc_grpc", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:utility_lib", + "@com_github_grpc_grpc//:grpc++", + ], +) + +envoy_extension_cc_test( + name = "alts_proxy_test", + srcs = ["alts_proxy_test.cc"], + extension_names = ["envoy.transport_sockets.alts"], + deps = [ + "//envoy/network:address_interface", + "//source/extensions/transport_sockets/alts:alts_proxy", + "//source/extensions/transport_sockets/alts:handshaker_cc_grpc", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:status_utility_lib", + "//test/test_common:utility_lib", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + ], +) + +envoy_extension_cc_test( + name = "alts_tsi_handshaker_test", + srcs = ["alts_tsi_handshaker_test.cc"], + extension_names = ["envoy.transport_sockets.alts"], + deps = [ + "//envoy/network:address_interface", + "//source/extensions/transport_sockets/alts:alts_proxy", + "//source/extensions/transport_sockets/alts:alts_tsi_handshaker", + "//source/extensions/transport_sockets/alts:handshaker_cc_grpc", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:status_utility_lib", + "//test/test_common:utility_lib", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + ], +) + envoy_extension_cc_test( name = "tsi_frame_protector_test", srcs = ["tsi_frame_protector_test.cc"], @@ -41,9 +96,20 @@ envoy_extension_cc_test( extension_names = ["envoy.transport_sockets.alts"], deps = [ "//envoy/event:dispatcher_interface", + "//envoy/network:address_interface", + "//source/extensions/transport_sockets/alts:alts_proxy", + "//source/extensions/transport_sockets/alts:alts_tsi_handshaker", + "//source/extensions/transport_sockets/alts:handshaker_cc_grpc", "//source/extensions/transport_sockets/alts:tsi_handshaker", "//test/mocks/buffer:buffer_mocks", "//test/mocks/event:event_mocks", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:status_utility_lib", + "//test/test_common:utility_lib", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", ], ) @@ -51,12 +117,20 @@ envoy_extension_cc_test( name = "tsi_socket_test", srcs = ["tsi_socket_test.cc"], extension_names = ["envoy.transport_sockets.alts"], + external_deps = [ + "grpc", + ], deps = [ "//envoy/event:dispatcher_interface", + "//envoy/network:address_interface", + "//source/extensions/transport_sockets/alts:handshaker_cc_grpc", "//source/extensions/transport_sockets/alts:tsi_socket", "//test/mocks/buffer:buffer_mocks", "//test/mocks/event:event_mocks", "//test/mocks/network:network_mocks", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:utility_lib", ], ) @@ -76,9 +150,7 @@ envoy_extension_cc_test( srcs = envoy_select_google_grpc(["alts_integration_test.cc"]), extension_names = ["envoy.transport_sockets.alts"], external_deps = [ - "grpc_alts_fake_handshaker_server", - "grpc_alts_handshaker_proto", - "grpc_alts_transport_security_common_proto", + "grpc", ], deps = [ "//source/common/common:utility_lib", @@ -87,12 +159,14 @@ envoy_extension_cc_test( "//source/common/network:connection_lib", "//source/common/network:utility_lib", "//source/extensions/transport_sockets/alts:config", + "//source/extensions/transport_sockets/alts:handshaker_cc_grpc", "//source/extensions/transport_sockets/alts:tsi_socket", "//test/integration:http_integration_lib", "//test/integration/filters:decode_dynamic_metadata_filter_lib", "//test/mocks/runtime:runtime_mocks", "//test/mocks/secret:secret_mocks", "//test/mocks/server:server_mocks", + "//test/test_common:status_utility_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/transport_socket/alts/v2alpha:pkg_cc_proto", ], diff --git a/test/extensions/transport_sockets/alts/alts_channel_pool_test.cc b/test/extensions/transport_sockets/alts/alts_channel_pool_test.cc new file mode 100644 index 000000000000..86c76e0b4e2d --- /dev/null +++ b/test/extensions/transport_sockets/alts/alts_channel_pool_test.cc @@ -0,0 +1,123 @@ +#include +#include +#include + +#include "envoy/network/address.h" + +#include "source/extensions/transport_sockets/alts/alts_channel_pool.h" + +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "grpcpp/client_context.h" +#include "grpcpp/server.h" +#include "grpcpp/server_builder.h" +#include "grpcpp/server_context.h" +#include "grpcpp/support/status.h" +#include "grpcpp/support/sync_stream.h" +#include "gtest/gtest.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { +namespace { + +using ::grpc::gcp::HandshakerReq; +using ::grpc::gcp::HandshakerResp; +using ::grpc::gcp::HandshakerService; +using ::testing::NotNull; +using ::testing::Test; + +class FakeHandshakerService final : public HandshakerService::Service { +public: + FakeHandshakerService() = default; + + grpc::Status + DoHandshake(grpc::ServerContext*, + grpc::ServerReaderWriter* stream) override { + HandshakerReq request; + while (stream->Read(&request)) { + HandshakerResp response; + EXPECT_TRUE(stream->Write(response)); + } + return grpc::Status::OK; + } +}; + +class AltsChannelPoolTest : public testing::TestWithParam { +protected: + AltsChannelPoolTest() : version_(GetParam()){}; + void startFakeHandshakerService() { + server_address_ = absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + testing::internal::Notification notification; + server_thread_ = std::make_unique([this, ¬ification]() { + FakeHandshakerService fake_handshaker_service; + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(server_address_, grpc::InsecureServerCredentials(), + &listening_port); + server_builder.RegisterService(&fake_handshaker_service); + server_ = server_builder.BuildAndStart(); + EXPECT_THAT(server_, NotNull()); + EXPECT_NE(listening_port, -1); + server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + (¬ification)->Notify(); + server_->Wait(); + }); + notification.WaitForNotification(); + } + + void TearDown() override { + if (server_thread_) { + server_->Shutdown(); + server_thread_->join(); + } + } + + std::string serverAddress() { return server_address_; } + +private: + std::string server_address_; + std::unique_ptr server_; + std::unique_ptr server_thread_; + Network::Address::IpVersion version_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, AltsChannelPoolTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(AltsChannelPoolTest, SuccessWithDefaultChannels) { + startFakeHandshakerService(); + + // Create a channel pool and check that it has the correct dimensions. + auto channel_pool = AltsChannelPool::create(serverAddress()); + EXPECT_THAT(channel_pool, NotNull()); + EXPECT_THAT(channel_pool->getChannel(), NotNull()); + EXPECT_EQ(channel_pool->getChannelPoolSize(), 10); + + // Check that we can write to and read from the channel multiple times. + for (int i = 0; i < 10; ++i) { + auto channel = channel_pool->getChannel(); + EXPECT_THAT(channel, NotNull()); + grpc::ClientContext client_context; + auto stub = HandshakerService::NewStub(channel); + auto stream = stub->DoHandshake(&client_context); + HandshakerReq request; + EXPECT_TRUE(stream->Write(request)); + HandshakerResp response; + EXPECT_TRUE(stream->Read(&response)); + } +} + +} // namespace +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/alts/alts_integration_test.cc b/test/extensions/transport_sockets/alts/alts_integration_test.cc index 59c65c08c5b3..0b86fe77e5b0 100644 --- a/test/extensions/transport_sockets/alts/alts_integration_test.cc +++ b/test/extensions/transport_sockets/alts/alts_integration_test.cc @@ -1,3 +1,10 @@ +#include +#include +#include +#include +#include +#include + #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/extensions/transport_sockets/alts/v3/alts.pb.h" @@ -5,6 +12,10 @@ #include "source/extensions/transport_sockets/alts/config.h" #include "source/extensions/transport_sockets/alts/tsi_socket.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" + #ifdef major #undef major #endif @@ -12,11 +23,6 @@ #undef minor #endif -#include "test/core/tsi/alts/fake_handshaker/fake_handshaker_server.h" -#include "test/core/tsi/alts/fake_handshaker/handshaker.grpc.pb.h" -#include "test/core/tsi/alts/fake_handshaker/handshaker.pb.h" -#include "test/core/tsi/alts/fake_handshaker/transport_security_common.pb.h" - #include "test/integration/http_integration.h" #include "test/integration/integration.h" #include "test/integration/server.h" @@ -24,6 +30,7 @@ #include "test/mocks/server/transport_socket_factory_context.h" #include "test/test_common/network_utility.h" +#include "test/test_common/status_utility.h" #include "test/test_common/utility.h" #include "absl/strings/match.h" @@ -31,8 +38,14 @@ #include "gmock/gmock.h" #include "grpcpp/grpcpp.h" #include "grpcpp/impl/codegen/service_type.h" +#include "grpcpp/security/server_credentials.h" +#include "grpcpp/server_builder.h" +#include "grpcpp/server_context.h" +#include "grpcpp/support/sync_stream.h" #include "gtest/gtest.h" +using ::grpc::Service; +using ::grpc::gcp::HandshakerService; using ::testing::ReturnRef; namespace Envoy { @@ -41,8 +54,13 @@ namespace TransportSockets { namespace Alts { namespace { -// Fake handshaker message, copied from grpc::gcp::FakeHandshakerService implementation. +// Fake handshake messages. constexpr char kClientInitFrame[] = "ClientInit"; +constexpr char kServerFrame[] = "ServerInitAndFinished"; +constexpr char kClientFinishFrame[] = "ClientFinished"; +// Error messages. +constexpr char kInvalidFrameError[] = "Invalid input frame."; +constexpr char kWrongStateError[] = "Wrong handshake state."; // Hollowed out implementation of HandshakerService that is dysfunctional, but // responds correctly to the first client request, capturing client and server @@ -87,6 +105,204 @@ class CapturingHandshakerService : public grpc::gcp::HandshakerService::Service size_t server_max_frame_size{0}; }; +// FakeHandshakeService implements a fake handshaker service using a fake key +// exchange protocol. The fake key exchange protocol is a 3-message protocol: +// - Client first sends ClientInit message to Server. +// - Server then sends ServerInitAndFinished message back to Client. +// - Client finally sends ClientFinished message to Server. +// This fake handshaker service is intended for ALTS integration testing without +// relying on real ALTS handshaker service inside GCE. +// It is thread-safe. +class FakeHandshakerService : public HandshakerService::Service { +public: + explicit FakeHandshakerService(const std::string& peer_identity) + : peer_identity_(peer_identity) {} + + grpc::Status + DoHandshake(grpc::ServerContext* /*server_context*/, + grpc::ServerReaderWriter* stream) + override { + grpc::Status status; + HandshakerContext context; + grpc::gcp::HandshakerReq request; + grpc::gcp::HandshakerResp response; + while (stream->Read(&request)) { + status = ProcessRequest(&context, request, &response); + if (!status.ok()) + return WriteErrorResponse(stream, status); + stream->Write(response); + if (context.state == HandshakeState::COMPLETED) + return grpc::Status::OK; + request.Clear(); + } + return grpc::Status::OK; + } + +private: + // HandshakeState is used by fake handshaker server to keep track of client's + // handshake status. In the beginning of a handshake, the state is INITIAL. + // If start_client or start_server request is called, the state becomes at + // least STARTED. When the handshaker server produces the first fame, the + // state becomes SENT. After the handshaker server processes the final frame + // from the peer, the state becomes COMPLETED. + enum class HandshakeState { INITIAL, STARTED, SENT, COMPLETED }; + + struct HandshakerContext { + bool is_client = true; + HandshakeState state = HandshakeState::INITIAL; + }; + + grpc::Status ProcessRequest(HandshakerContext* context, const grpc::gcp::HandshakerReq& request, + grpc::gcp::HandshakerResp* response) { + response->Clear(); + if (request.has_client_start()) { + return ProcessClientStart(context, request.client_start(), response); + } else if (request.has_server_start()) { + return ProcessServerStart(context, request.server_start(), response); + } else if (request.has_next()) { + return ProcessNext(context, request.next(), response); + } + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Request is empty."); + } + + grpc::Status ProcessClientStart(HandshakerContext* context, + const grpc::gcp::StartClientHandshakeReq& request, + grpc::gcp::HandshakerResp* response) { + // Checks request. + if (context->state != HandshakeState::INITIAL) { + return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, kWrongStateError); + } + if (request.application_protocols_size() == 0) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "At least one application protocol needed."); + } + if (request.record_protocols_size() == 0) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "At least one record protocol needed."); + } + // Sets response. + response->set_out_frames(kClientInitFrame); + response->set_bytes_consumed(0); + response->mutable_status()->set_code(grpc::StatusCode::OK); + // Updates handshaker context. + context->is_client = true; + context->state = HandshakeState::SENT; + return grpc::Status::OK; + } + + grpc::Status ProcessServerStart(HandshakerContext* context, + const grpc::gcp::StartServerHandshakeReq& request, + grpc::gcp::HandshakerResp* response) { + // Checks request. + if (context->state != HandshakeState::INITIAL) { + return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, kWrongStateError); + } + if (request.application_protocols_size() == 0) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "At least one application protocol needed."); + } + if (request.handshake_parameters().empty()) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "At least one set of handshake parameters needed."); + } + // Sets response. + if (request.in_bytes().empty()) { + // start_server request does not have in_bytes. + response->set_bytes_consumed(0); + context->state = HandshakeState::STARTED; + } else { + // start_server request has in_bytes. + if (request.in_bytes() == kClientInitFrame) { + response->set_out_frames(kServerFrame); + response->set_bytes_consumed(strlen(kClientInitFrame)); + context->state = HandshakeState::SENT; + } else { + return grpc::Status(grpc::StatusCode::UNKNOWN, kInvalidFrameError); + } + } + response->mutable_status()->set_code(grpc::StatusCode::OK); + context->is_client = false; + return grpc::Status::OK; + } + + grpc::Status ProcessNext(HandshakerContext* context, + const grpc::gcp::NextHandshakeMessageReq& request, + grpc::gcp::HandshakerResp* response) { + if (context->is_client) { + // Processes next request on client side. + if (context->state != HandshakeState::SENT) { + return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, kWrongStateError); + } + if (request.in_bytes() != kServerFrame) { + return grpc::Status(grpc::StatusCode::UNKNOWN, kInvalidFrameError); + } + response->set_out_frames(kClientFinishFrame); + response->set_bytes_consumed(strlen(kServerFrame)); + context->state = HandshakeState::COMPLETED; + } else { + // Processes next request on server side. + HandshakeState current_state = context->state; + if (current_state == HandshakeState::STARTED) { + if (request.in_bytes() != kClientInitFrame) { + return grpc::Status(grpc::StatusCode::UNKNOWN, kInvalidFrameError); + } + response->set_out_frames(kServerFrame); + response->set_bytes_consumed(strlen(kClientInitFrame)); + context->state = HandshakeState::SENT; + } else if (current_state == HandshakeState::SENT) { + // Client finish frame may be sent along with the first payload from the + // client, handshaker only consumes the client finish frame. + if (request.in_bytes().substr(0, strlen(kClientFinishFrame)) != kClientFinishFrame) { + return grpc::Status(grpc::StatusCode::UNKNOWN, kInvalidFrameError); + } + response->set_bytes_consumed(strlen(kClientFinishFrame)); + context->state = HandshakeState::COMPLETED; + } else { + return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, kWrongStateError); + } + } + // At this point, processing next request succeeded. + response->mutable_status()->set_code(grpc::StatusCode::OK); + if (context->state == HandshakeState::COMPLETED) { + *response->mutable_result() = GetHandshakerResult(); + } + return grpc::Status::OK; + } + + grpc::Status WriteErrorResponse( + grpc::ServerReaderWriter* stream, + const grpc::Status& status) { + EXPECT_TRUE(status.ok()); + grpc::gcp::HandshakerResp response; + response.mutable_status()->set_code(status.error_code()); + response.mutable_status()->set_details(status.error_message()); + stream->Write(response); + return status; + } + + grpc::gcp::HandshakerResult GetHandshakerResult() { + grpc::gcp::HandshakerResult result; + result.set_application_protocol("grpc"); + result.set_record_protocol("ALTSRP_GCM_AES128_REKEY"); + result.mutable_peer_identity()->set_service_account(peer_identity_); + result.mutable_local_identity()->set_service_account("local_identity"); + std::string key(1024, '\0'); + result.set_key_data(key); + result.set_max_frame_size(16384); + result.mutable_peer_rpc_versions()->mutable_max_rpc_version()->set_major(2); + result.mutable_peer_rpc_versions()->mutable_max_rpc_version()->set_minor(1); + result.mutable_peer_rpc_versions()->mutable_min_rpc_version()->set_major(2); + result.mutable_peer_rpc_versions()->mutable_min_rpc_version()->set_minor(1); + return result; + } + + const std::string peer_identity_; +}; + +std::unique_ptr CreateFakeHandshakerService(const std::string& peer_identity) { + return std::unique_ptr{new FakeHandshakerService(peer_identity)}; +} + class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, public testing::TestWithParam, public HttpIntegrationTest { @@ -131,9 +347,7 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, service = std::unique_ptr{capturing_handshaker_service_}; } else { capturing_handshaker_service_ = nullptr; - // If max_expected_concurrent_rpcs is zero, the fake handshaker service will not track - // concurrent RPCs and abort if it exceeds the value. - service = grpc::gcp::CreateFakeHandshakerService("peer_identity"); + service = CreateFakeHandshakerService("peer_identity"); } std::string server_address = Network::Test::getLoopbackAddressUrlString(version_) + ":0"; @@ -152,12 +366,17 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, NiceMock mock_factory_ctx; // We fake the singleton manager for the client, since it doesn't need to manage ALTS global // state, this is done by the test server instead. - // TODO(htuch): Make this a proper mock. class FakeSingletonManager : public Singleton::Manager { public: - Singleton::InstanceSharedPtr get(const std::string&, Singleton::SingletonFactoryCb) override { - return nullptr; + Singleton::InstanceSharedPtr get(const std::string&, Singleton::SingletonFactoryCb cb, + bool pin) override { + auto singleton = cb(); + if (pin) { + pinned_singletons_.push_back(singleton); + } + return singleton; } + std::vector pinned_singletons_; }; FakeSingletonManager fsm; ON_CALL(mock_factory_ctx.server_context_, singletonManager()).WillByDefault(ReturnRef(fsm)); @@ -185,11 +404,8 @@ class AltsIntegrationTestBase : public Event::TestUsingSimulatedTime, } Network::TransportSocketPtr makeAltsTransportSocket() { - auto client_transport_socket = client_alts_->createTransportSocket(nullptr, nullptr); - client_tsi_socket_ = dynamic_cast(client_transport_socket.get()); - client_tsi_socket_->setActualFrameSizeToUse(16384); - client_tsi_socket_->setFrameOverheadSize(4); - return client_transport_socket; + return client_alts_->createTransportSocket(/*options=*/nullptr, + /*host=*/nullptr); } Network::ClientConnectionPtr makeAltsConnection() { @@ -413,8 +629,8 @@ TEST_P(AltsIntegrationTestCapturingHandshaker, CheckMaxFrameSize) { initialize(); codec_client_ = makeRawHttpConnection(makeAltsConnection(), absl::nullopt); EXPECT_FALSE(codec_client_->connected()); - EXPECT_EQ(capturing_handshaker_service_->client_max_frame_size, 16384); - EXPECT_EQ(capturing_handshaker_service_->server_max_frame_size, 16384); + EXPECT_EQ(capturing_handshaker_service_->client_max_frame_size, 1024 * 1024); + EXPECT_EQ(capturing_handshaker_service_->server_max_frame_size, 1024 * 1024); } } // namespace diff --git a/test/extensions/transport_sockets/alts/alts_proxy_test.cc b/test/extensions/transport_sockets/alts/alts_proxy_test.cc new file mode 100644 index 000000000000..1017d15f7473 --- /dev/null +++ b/test/extensions/transport_sockets/alts/alts_proxy_test.cc @@ -0,0 +1,438 @@ +#include +#include +#include +#include + +#include "envoy/network/address.h" + +#include "source/extensions/transport_sockets/alts/alts_proxy.h" + +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/status_utility.h" +#include "test/test_common/utility.h" + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/notification.h" +#include "absl/types/span.h" +#include "gmock/gmock.h" +#include "grpcpp/channel.h" +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" +#include "grpcpp/server.h" +#include "grpcpp/server_builder.h" +#include "grpcpp/server_context.h" +#include "grpcpp/support/status.h" +#include "grpcpp/support/sync_stream.h" +#include "gtest/gtest.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { +namespace { + +using ::Envoy::StatusHelpers::StatusIs; +using ::grpc::gcp::HandshakeProtocol; +using ::grpc::gcp::HandshakerReq; +using ::grpc::gcp::HandshakerResp; +using ::grpc::gcp::HandshakerService; +using ::grpc::gcp::ServerHandshakeParameters; +using ::grpc::gcp::StartClientHandshakeReq; +using ::grpc::gcp::StartServerHandshakeReq; + +constexpr absl::string_view ClientStartResponse = "client_start_response"; +constexpr absl::string_view ServerStartResponse = "server_start_response"; +constexpr absl::string_view NextResponse = "next_response"; + +HandshakerResp expectedClientStartResponse() { + HandshakerResp response; + response.set_out_frames(ClientStartResponse); + return response; +} + +HandshakerResp expectedServerStartResponse() { + HandshakerResp response; + response.set_out_frames(ServerStartResponse); + return response; +} + +HandshakerResp expectedNextResponse() { + HandshakerResp response; + response.set_out_frames(NextResponse); + return response; +} + +class FakeHandshakerService final : public HandshakerService::Service { +public: + FakeHandshakerService(const std::vector& expected_requests, + grpc::Status status_to_return, bool return_error_response) + : expected_requests_(expected_requests), status_to_return_(status_to_return), + return_error_response_(return_error_response) {} + + grpc::Status + DoHandshake(grpc::ServerContext*, + grpc::ServerReaderWriter* stream) override { + HandshakerReq request; + int request_number = 0; + while (stream->Read(&request)) { + if (!status_to_return_.ok()) { + return status_to_return_; + } + if (request_number < static_cast(expected_requests_.size())) { + EXPECT_TRUE(TestUtility::protoEqual(request, expected_requests_[request_number])); + request_number++; + } + HandshakerResp response; + if (return_error_response_) { + response.mutable_status()->set_code(static_cast(grpc::StatusCode::INTERNAL)); + response.mutable_status()->set_details("An internal error occurred."); + } else if (request.has_client_start()) { + response.set_out_frames(ClientStartResponse); + } else if (request.has_server_start()) { + response.set_out_frames(ServerStartResponse); + } else if (request.has_next()) { + response.set_out_frames(NextResponse); + } + EXPECT_TRUE(stream->Write(response)); + } + return grpc::Status::OK; + } + +private: + std::vector expected_requests_; + grpc::Status status_to_return_; + bool return_error_response_; +}; + +class AltsProxyTest : public testing::TestWithParam { +protected: + AltsProxyTest() : version_(GetParam()){}; + void startFakeHandshakerService(const std::vector& expected_requests, + grpc::Status status_to_return, + bool return_error_response = false) { + server_address_ = absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + absl::Notification notification; + server_thread_ = std::make_unique([this, ¬ification, expected_requests, + status_to_return, return_error_response]() { + FakeHandshakerService fake_handshaker_service(expected_requests, status_to_return, + return_error_response); + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(server_address_, grpc::InsecureServerCredentials(), + &listening_port); + server_builder.RegisterService(&fake_handshaker_service); + server_ = server_builder.BuildAndStart(); + EXPECT_THAT(server_, ::testing::NotNull()); + EXPECT_NE(listening_port, -1); + server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + (¬ification)->Notify(); + server_->Wait(); + }); + notification.WaitForNotification(); + } + + void TearDown() override { + if (server_thread_) { + server_->Shutdown(); + server_thread_->join(); + } + } + + std::shared_ptr getChannel() { + return grpc::CreateChannel(server_address_, + grpc::InsecureChannelCredentials()); // NOLINT + } + +private: + std::string server_address_; + std::unique_ptr server_; + std::unique_ptr server_thread_; + Network::Address::IpVersion version_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, AltsProxyTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Verify that a StartClientHandshakeReq can successfully be created, sent to +// the handshaker service, and the correct response is received. +TEST_P(AltsProxyTest, ClientStartSuccess) { + HandshakerReq expected_request; + StartClientHandshakeReq* expected_client_start = expected_request.mutable_client_start(); + expected_client_start->set_handshake_security_protocol(grpc::gcp::ALTS); + expected_client_start->add_application_protocols(ApplicationProtocol); + expected_client_start->add_record_protocols(RecordProtocol); + expected_client_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_major( + MaxMajorRpcVersion); + expected_client_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_minor( + MaxMinorRpcVersion); + expected_client_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_major( + MinMajorRpcVersion); + expected_client_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_minor( + MinMinorRpcVersion); + expected_client_start->set_max_frame_size(MaxFrameSize); + startFakeHandshakerService({expected_request}, grpc::Status::OK); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_TRUE(TestUtility::protoEqual((*alts_proxy)->sendStartClientHandshakeReq().value(), + expectedClientStartResponse())); +} + +// Verify that a full client-side ALTS handshake can be performed when talking +// to a fake handshaker service. +TEST_P(AltsProxyTest, ClientFullHandshakeSuccess) { + HandshakerReq expected_request_1; + StartClientHandshakeReq* expected_client_start = expected_request_1.mutable_client_start(); + expected_client_start->set_handshake_security_protocol(grpc::gcp::ALTS); + expected_client_start->add_application_protocols(ApplicationProtocol); + expected_client_start->add_record_protocols(RecordProtocol); + expected_client_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_major( + MaxMajorRpcVersion); + expected_client_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_minor( + MaxMinorRpcVersion); + expected_client_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_major( + MinMajorRpcVersion); + expected_client_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_minor( + MinMinorRpcVersion); + expected_client_start->set_max_frame_size(MaxFrameSize); + HandshakerReq expected_request_2; + expected_request_2.mutable_next()->set_in_bytes(ServerStartResponse); + startFakeHandshakerService({expected_request_1, expected_request_2}, grpc::Status::OK); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + auto resp = (*alts_proxy)->sendStartClientHandshakeReq(); + EXPECT_OK(resp); + grpc::gcp::HandshakerResp& handshaker_resp = resp.value(); + EXPECT_TRUE(TestUtility::protoEqual(handshaker_resp, expectedClientStartResponse())); + + resp = (*alts_proxy) + ->sendNextHandshakeReq( + absl::MakeSpan(reinterpret_cast(ServerStartResponse.data()), + ServerStartResponse.size())); + EXPECT_OK(resp); + handshaker_resp = resp.value(); + EXPECT_TRUE(TestUtility::protoEqual(handshaker_resp, expectedNextResponse())); +} + +// Verify that a StartServerHandshakeReq can successfully be created, sent to +// the handshaker service, and the correct response is received. +TEST_P(AltsProxyTest, ServerStartSuccess) { + HandshakerReq expected_request; + ServerHandshakeParameters server_parameters; + server_parameters.add_record_protocols(RecordProtocol); + StartServerHandshakeReq* expected_server_start = expected_request.mutable_server_start(); + expected_server_start->add_application_protocols(ApplicationProtocol); + (*expected_server_start->mutable_handshake_parameters())[HandshakeProtocol::ALTS] = + server_parameters; + expected_server_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_major( + MaxMajorRpcVersion); + expected_server_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_minor( + MaxMinorRpcVersion); + expected_server_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_major( + MinMajorRpcVersion); + expected_server_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_minor( + MinMinorRpcVersion); + expected_server_start->set_in_bytes(ClientStartResponse); + expected_server_start->set_max_frame_size(MaxFrameSize); + startFakeHandshakerService({expected_request}, grpc::Status::OK); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + + auto resp = (*alts_proxy) + ->sendStartServerHandshakeReq( + absl::MakeSpan(reinterpret_cast(ClientStartResponse.data()), + ClientStartResponse.size())); + EXPECT_OK(resp); + grpc::gcp::HandshakerResp& handshaker_resp = resp.value(); + EXPECT_TRUE(TestUtility::protoEqual(handshaker_resp, expectedServerStartResponse())); +} + +// Verify that a full server-side ALTS handshake can be performed when talking +// to a fake handshaker service. +TEST_P(AltsProxyTest, ServerFullHandshakeSuccess) { + HandshakerReq expected_request_1; + ServerHandshakeParameters server_parameters; + server_parameters.add_record_protocols(RecordProtocol); + StartServerHandshakeReq* expected_server_start = expected_request_1.mutable_server_start(); + expected_server_start->add_application_protocols(ApplicationProtocol); + (*expected_server_start->mutable_handshake_parameters())[HandshakeProtocol::ALTS] = + server_parameters; + expected_server_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_major( + MaxMajorRpcVersion); + expected_server_start->mutable_rpc_versions()->mutable_max_rpc_version()->set_minor( + MaxMinorRpcVersion); + expected_server_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_major( + MinMajorRpcVersion); + expected_server_start->mutable_rpc_versions()->mutable_min_rpc_version()->set_minor( + MinMinorRpcVersion); + expected_server_start->set_in_bytes(ClientStartResponse); + expected_server_start->set_max_frame_size(MaxFrameSize); + HandshakerReq expected_request_2; + expected_request_2.mutable_next()->set_in_bytes(ServerStartResponse); + startFakeHandshakerService({expected_request_1, expected_request_2}, grpc::Status::OK); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_TRUE( + TestUtility::protoEqual((*alts_proxy) + ->sendStartServerHandshakeReq(absl::MakeSpan( + reinterpret_cast(ClientStartResponse.data()), + ClientStartResponse.size())) + .value(), + expectedServerStartResponse())); + EXPECT_TRUE( + TestUtility::protoEqual((*alts_proxy) + ->sendNextHandshakeReq(absl::MakeSpan( + reinterpret_cast(ServerStartResponse.data()), + ServerStartResponse.size())) + .value(), + expectedNextResponse())); +} + +// Check that the AltsProxy cannot be created when the channel to the handshaker +// service is nullptr. +TEST_P(AltsProxyTest, CreateFailsDueToNullChannel) { + EXPECT_THAT(AltsProxy::create(/*handshaker_service_channel=*/nullptr), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +// Check that the AltsProxy correctly handles gRPC-level errors returned by the +// handshaker service when sending a StartClientHandshakeReq. +TEST_P(AltsProxyTest, ClientStartRpcLevelFailure) { + startFakeHandshakerService( + /*expected_requests=*/{}, + grpc::Status(grpc::StatusCode::INTERNAL, "An RPC-level internal error occurred.")); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy)->sendStartClientHandshakeReq(), StatusIs(absl::StatusCode::kInternal)); +} + +// Check that the AltsProxy correctly handles gRPC-level errors returned by the +// handshaker service when sending a StartServerHandshakeReq. +TEST_P(AltsProxyTest, ServerStartRpcLevelFailure) { + startFakeHandshakerService( + /*expected_requests=*/{}, + grpc::Status(grpc::StatusCode::INTERNAL, "An RPC-level internal error occurred.")); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy) + ->sendStartServerHandshakeReq( + absl::MakeSpan(reinterpret_cast(ClientStartResponse.data()), + ClientStartResponse.size())), + StatusIs(absl::StatusCode::kInternal)); +} + +// Check that the AltsProxy correctly handles gRPC-level errors returned by the +// handshaker service when sending a NextHandshakeReq. +TEST_P(AltsProxyTest, NextRpcLevelFailure) { + startFakeHandshakerService( + /*expected_requests=*/{}, + grpc::Status(grpc::StatusCode::INTERNAL, "An RPC-level internal error occurred.")); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy) + ->sendNextHandshakeReq( + absl::MakeSpan(reinterpret_cast(ServerStartResponse.data()), + ServerStartResponse.size())), + StatusIs(absl::StatusCode::kInternal)); +} + +// Check that the AltsProxy correctly handles a handshake-level error returned by the +// handshaker service when sending a StartClientHandshakeReq. +TEST_P(AltsProxyTest, ClientStartRequestLevelFailure) { + startFakeHandshakerService( + /*expected_requests=*/{}, grpc::Status::OK, + /*return_error_response=*/true); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy)->sendStartClientHandshakeReq(), StatusIs(absl::StatusCode::kInternal)); +} + +// Check that the AltsProxy correctly handles a handshake-level error returned by the +// handshaker service when sending a StartServerHandshakeReq. +TEST_P(AltsProxyTest, ServerStartRequestLevelFailure) { + startFakeHandshakerService( + /*expected_requests=*/{}, grpc::Status::OK, + /*return_error_response=*/true); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy) + ->sendStartServerHandshakeReq( + absl::MakeSpan(reinterpret_cast(ClientStartResponse.data()), + ClientStartResponse.size())), + StatusIs(absl::StatusCode::kInternal)); +} + +// Check that the AltsProxy correctly handles a handshake-level error returned by the +// handshaker service when sending a NextHandshakeReq. +TEST_P(AltsProxyTest, NextRequestLevelFailure) { + startFakeHandshakerService( + /*expected_requests=*/{}, grpc::Status::OK, + /*return_error_response=*/true); + + auto alts_proxy = AltsProxy::create(getChannel()); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy) + ->sendNextHandshakeReq( + absl::MakeSpan(reinterpret_cast(ServerStartResponse.data()), + ServerStartResponse.size())), + StatusIs(absl::StatusCode::kInternal)); +} + +// Check that the AltsProxy correctly handles an unreachable handshaker service +// when sending a StartClientHandshakeReq. +TEST_P(AltsProxyTest, HandshakerServiceIsUnreacheableOnClientStart) { + std::string unreacheable_address = absl::StrCat("[::1]:", "1001"); + auto channel = grpc::CreateChannel(unreacheable_address, + grpc::InsecureChannelCredentials()); // NOLINT + auto alts_proxy = AltsProxy::create(channel); + EXPECT_OK(alts_proxy.status()); + EXPECT_THAT((*alts_proxy)->sendStartClientHandshakeReq(), + StatusIs(absl::StatusCode::kUnavailable)); +} + +// Check that the AltsProxy correctly handles an unreachable handshaker service +// when sending a StartServerHandshakeReq. +TEST_P(AltsProxyTest, HandshakerServiceIsUnreacheableOnServerStart) { + std::string unreacheable_address = absl::StrCat("[::1]:", "1001"); + auto channel = grpc::CreateChannel(unreacheable_address, + grpc::InsecureChannelCredentials()); // NOLINT + auto alts_proxy = AltsProxy::create(channel); + EXPECT_OK(alts_proxy.status()); + absl::Span in_bytes; + EXPECT_THAT((*alts_proxy)->sendStartServerHandshakeReq(in_bytes), + StatusIs(absl::StatusCode::kUnavailable)); +} + +// Check that the AltsProxy correctly handles an unreachable handshaker service +// when sending a NextHandshakeReq. +TEST_P(AltsProxyTest, HandshakerServiceIsUnreacheableOnNextRequest) { + std::string unreacheable_address = absl::StrCat("[::1]:", "1001"); + auto channel = grpc::CreateChannel(unreacheable_address, + grpc::InsecureChannelCredentials()); // NOLINT + auto alts_proxy = AltsProxy::create(channel); + EXPECT_OK(alts_proxy.status()); + absl::Span in_bytes; + EXPECT_THAT((*alts_proxy)->sendNextHandshakeReq(in_bytes), + StatusIs(absl::StatusCode::kUnavailable)); +} + +} // namespace +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/alts/alts_tsi_handshaker_test.cc b/test/extensions/transport_sockets/alts/alts_tsi_handshaker_test.cc new file mode 100644 index 000000000000..a60dc4d20a0f --- /dev/null +++ b/test/extensions/transport_sockets/alts/alts_tsi_handshaker_test.cc @@ -0,0 +1,704 @@ +#include +#include +#include +#include + +#include "envoy/network/address.h" + +#include "source/extensions/transport_sockets/alts/alts_proxy.h" +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" + +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/status_utility.h" +#include "test/test_common/utility.h" + +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/notification.h" +#include "gmock/gmock.h" +#include "grpcpp/channel.h" +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" +#include "grpcpp/server.h" +#include "grpcpp/server_builder.h" +#include "grpcpp/server_context.h" +#include "grpcpp/support/status.h" +#include "grpcpp/support/sync_stream.h" +#include "gtest/gtest.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Alts { +namespace { + +using ::Envoy::StatusHelpers::StatusCodeIs; +using ::Envoy::StatusHelpers::StatusIs; +using ::grpc::gcp::HandshakerReq; +using ::grpc::gcp::HandshakerResp; +using ::grpc::gcp::HandshakerResult; +using ::grpc::gcp::HandshakerService; +using ::testing::IsNull; +using ::testing::NotNull; + +constexpr absl::string_view ApplicationData = "APPLICATION_DATA"; +constexpr absl::string_view ClientInit = "CLIENT_INIT"; +constexpr absl::string_view ServerInit = "SERVER_INIT"; +constexpr absl::string_view ClientFinished = "CLIENT_FINISHED"; +constexpr absl::string_view ServerFinished = "SERVER_FINISHED"; + +constexpr absl::string_view KeyData = "fake_key_data_needs_to_be_at_least_44_characters_long"; +constexpr absl::string_view LocalServiceAccount = "local_service_account"; +constexpr absl::string_view PeerServiceAccount = "peer_service_account"; + +void populateHandshakeResult(HandshakerResult* result) { + result->mutable_peer_identity()->set_service_account(PeerServiceAccount); + result->mutable_peer_rpc_versions(); + result->mutable_local_identity()->set_service_account(LocalServiceAccount); + result->set_application_protocol(ApplicationProtocol); + result->set_record_protocol(RecordProtocol); + result->set_key_data(KeyData); +} + +class FakeHandshakerService final : public HandshakerService::Service { +public: + FakeHandshakerService() = default; + + grpc::Status + DoHandshake(grpc::ServerContext*, + grpc::ServerReaderWriter* stream) override { + bool is_assisting_client = false; + bool is_handshake_complete = false; + bool sent_client_finished = false; + bool sent_server_init_and_server_finished = false; + std::string bytes_from_client; + std::string bytes_from_server; + HandshakerReq request; + while (stream->Read(&request)) { + HandshakerResp response; + if (request.has_client_start()) { + // The request contains a StartClientHandshakeReq message. + is_assisting_client = true; + response.set_out_frames(ClientInit); + response.set_bytes_consumed(ClientInit.size()); + } else if (request.has_server_start()) { + // The request contains a StartServerHandshakeReq message. + if (absl::StartsWith(ClientInit, request.server_start().in_bytes()) && + request.server_start().in_bytes() != ClientInit) { + // If the in_bytes contain a subset of the ClientInit message and we + // allow processing of an incomplete ClientInit message, then tell the + // client that we consumed the bytes and are waiting for more. + response.set_bytes_consumed(request.server_start().in_bytes().size()); + } else { + EXPECT_EQ(request.server_start().in_bytes(), ClientInit); + std::string out_frames = absl::StrCat(ServerInit, ServerFinished); + response.set_out_frames(out_frames); + response.set_bytes_consumed(out_frames.size()); + sent_server_init_and_server_finished = true; + } + bytes_from_client.append(request.server_start().in_bytes()); + } else if (request.has_next()) { + std::size_t number_handshake_bytes = request.next().in_bytes().size(); + if (absl::EndsWith(request.next().in_bytes(), ApplicationData)) { + number_handshake_bytes -= ApplicationData.size(); + } + if (is_assisting_client) { + bytes_from_server.append(request.next().in_bytes().substr(0, number_handshake_bytes)); + } else { + bytes_from_client.append(request.next().in_bytes().substr(0, number_handshake_bytes)); + } + response.set_bytes_consumed(number_handshake_bytes); + + // Consider sending the ServerInit and ServerFinished. + if (!is_assisting_client && !sent_server_init_and_server_finished && + bytes_from_client == ClientInit) { + sent_server_init_and_server_finished = true; + response.set_out_frames(absl::StrCat(ServerInit, ServerFinished)); + } + + // Consider sending the ClientFinished. + if (is_assisting_client && !sent_client_finished && + bytes_from_server == absl::StrCat(ServerInit, ServerFinished)) { + sent_client_finished = true; + response.set_out_frames(ClientFinished); + } + + // Check if the handshake is complete and, if so, populate the result. + if (is_assisting_client) { + is_handshake_complete = (bytes_from_server == absl::StrCat(ServerInit, ServerFinished)); + } else { + is_handshake_complete = (bytes_from_client == absl::StrCat(ClientInit, ClientFinished)); + } + if (is_handshake_complete) { + populateHandshakeResult(response.mutable_result()); + } + } else { + response.mutable_status()->set_code( + static_cast(grpc::StatusCode::FAILED_PRECONDITION)); + response.mutable_status()->set_details("Missing body of handshake request."); + } + EXPECT_TRUE(stream->Write(response)); + } + return grpc::Status::OK; + } +}; + +class CapturingHandshaker { +public: + CapturingHandshaker() = default; + + std::string getBytesToSend() { return bytes_to_send_; } + void setBytesToSend(absl::string_view bytes_to_send) { bytes_to_send_ = bytes_to_send; } + + absl::Status getStatus() { return status_; } + void setStatus(const absl::Status& status) { status_ = status; } + + std::unique_ptr getAltsHandshakeResult() { + return std::move(alts_handshake_result_); + } + void setAltsHandshakeResult(std::unique_ptr alts_handshake_result) { + alts_handshake_result_ = std::move(alts_handshake_result); + } + +private: + std::unique_ptr alts_handshake_result_; + std::string bytes_to_send_; + absl::Status status_ = absl::InternalError("Never populated."); +}; + +void onNextDoneImpl(absl::Status status, void* handshaker, const unsigned char* bytes_to_send, + size_t bytes_to_send_size, + std::unique_ptr handshake_result) { + CapturingHandshaker* capturing_handshaker = static_cast(handshaker); + capturing_handshaker->setStatus(status); + absl::string_view bytes(reinterpret_cast(bytes_to_send), bytes_to_send_size); + capturing_handshaker->setBytesToSend(bytes); + capturing_handshaker->setAltsHandshakeResult(std::move(handshake_result)); +} + +class AltsTsiHandshakerTest : public testing::TestWithParam { +protected: + AltsTsiHandshakerTest() : version_(GetParam()){}; + void startFakeHandshakerService() { + server_address_ = absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + absl::Notification notification; + server_thread_ = std::make_unique([this, ¬ification]() { + FakeHandshakerService fake_handshaker_service; + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(server_address_, grpc::InsecureServerCredentials(), + &listening_port); + server_builder.RegisterService(&fake_handshaker_service); + server_ = server_builder.BuildAndStart(); + EXPECT_THAT(server_, NotNull()); + EXPECT_NE(listening_port, -1); + server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + notification.Notify(); + server_->Wait(); + }); + notification.WaitForNotification(); + } + + void TearDown() override { + if (server_thread_) { + server_->Shutdown(); + server_thread_->join(); + } + } + + std::shared_ptr getChannel() { + return grpc::CreateChannel(server_address_, + grpc::InsecureChannelCredentials()); // NOLINT + } + +private: + std::string server_address_; + std::unique_ptr server_; + std::unique_ptr server_thread_; + Network::Address::IpVersion version_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, AltsTsiHandshakerTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Check that a client-side AltsTsiHandshaker can successfully complete a full +// client-side ALTS handshake. +TEST_P(AltsTsiHandshakerTest, ClientSideFullHandshake) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + + // Get the ClientInit. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ClientInit); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the ClientFinished and the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(handshake_message.c_str()), + handshake_message.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ClientFinished); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + EXPECT_EQ(handshake_result->unused_bytes.size(), 0); + } + + // Confirm that the handshake cannot continue. + CapturingHandshaker capturing_handshaker; + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInternal)); +} + +// Check that several client-side handshaker can successfully complete concurrent, full +// client-side ALTS handshakes over the same channel to the handshaker service. +TEST_P(AltsTsiHandshakerTest, ConcurrentClientSideFullHandshakes) { + // Setup. + startFakeHandshakerService(); + + std::vector> handshake_threads; + for (int i = 0; i < 10; ++i) { + auto handshake_thread = std::make_unique([this]() { + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + + // Get the ClientInit. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ClientInit); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the ClientFinished and the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + CapturingHandshaker capturing_handshaker; + EXPECT_OK( + handshaker->next(&capturing_handshaker, + reinterpret_cast(handshake_message.c_str()), + handshake_message.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ClientFinished); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + EXPECT_EQ(handshake_result->unused_bytes.size(), 0); + } + }); + handshake_threads.push_back(std::move(handshake_thread)); + } + for (int i = 0; i < 10; ++i) { + handshake_threads[i]->join(); + } +} + +// Check that a client-side AltsTsiHandshaker can successfully complete a full +// client-side ALTS handshake when there are unused bytes after the handshake. +TEST_P(AltsTsiHandshakerTest, ClientSideFullHandshakeWithUnusedBytes) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + + // Get the ClientInit. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ClientInit); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the ClientFinished and the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished, ApplicationData); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(handshake_message.c_str()), + handshake_message.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ClientFinished); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + absl::string_view unused_bytes( + reinterpret_cast(handshake_result->unused_bytes.data()), + handshake_result->unused_bytes.size()); + EXPECT_EQ(unused_bytes, ApplicationData); + } + + // Confirm that the handshake cannot continue. + CapturingHandshaker capturing_handshaker; + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInternal)); +} + +// Check that a server-side AltsTsiHandshaker can successfully complete a full +// server-side ALTS handshake. +TEST_P(AltsTsiHandshakerTest, ServerSideFullHandshake) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + + // Get the ServerInit and ServerFinished. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientInit.data()), + ClientInit.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), absl::StrCat(ServerInit, ServerFinished)); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientFinished.data()), + ClientFinished.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + EXPECT_EQ(handshake_result->unused_bytes.size(), 0); + } + + // Confirm that the handshake cannot continue. + CapturingHandshaker capturing_handshaker; + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInternal)); +} + +// Check that several server-side handshaker can successfully complete concurrent, full +// server-side ALTS handshakes over the same channel to the handshaker service. +TEST_P(AltsTsiHandshakerTest, ConcurrentServerSideFullHandshakes) { + // Setup. + startFakeHandshakerService(); + + std::vector> handshake_threads; + for (int i = 0; i < 10; ++i) { + auto handshake_thread = std::make_unique([this]() { + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + + // Get the ServerInit and ServerFinished. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientInit.data()), + ClientInit.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), absl::StrCat(ServerInit, ServerFinished)); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientFinished.data()), + ClientFinished.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + EXPECT_EQ(handshake_result->unused_bytes.size(), 0); + } + }); + handshake_threads.push_back(std::move(handshake_thread)); + } + for (int i = 0; i < 10; ++i) { + handshake_threads[i]->join(); + } +} + +// Check that a server-side AltsTsiHandshaker can successfully complete a full +// server-side ALTS handshake when there are unused bytes sent after the +// handshake. +TEST_P(AltsTsiHandshakerTest, ServerSideFullHandshakeWithUnusedBytes) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + + // Get the ServerInit and ServerFinished. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientInit.data()), + ClientInit.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), absl::StrCat(ServerInit, ServerFinished)); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + std::string received_bytes = absl::StrCat(ClientFinished, ApplicationData); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(received_bytes.data()), + received_bytes.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + absl::string_view unused_bytes( + reinterpret_cast(handshake_result->unused_bytes.data()), + handshake_result->unused_bytes.size()); + EXPECT_EQ(unused_bytes, ApplicationData); + } + + // Confirm that the handshake cannot continue. + CapturingHandshaker capturing_handshaker; + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInternal)); +} + +// Check that a server-side AltsTsiHandshaker can successfully complete a full +// server-side ALTS handshake when the initial bytes from the client are missing. +TEST_P(AltsTsiHandshakerTest, ServerSideFullHandshakeWithMissingInitialInBytes) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + + // Fail to get the ServerInit and ServerFinished because the ClientInit has + // not arrived yet. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the ServerInit and ServerFinished now that the ClientInit has arrived. + { + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientInit.data()), + ClientInit.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), absl::StrCat(ServerInit, ServerFinished)); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientFinished.data()), + ClientFinished.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + EXPECT_EQ(handshake_result->unused_bytes.size(), 0); + } + + // Confirm that the handshake cannot continue. + CapturingHandshaker capturing_handshaker; + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInternal)); +} + +// Check that a server-side AltsTsiHandshaker can successfully complete a full +// server-side ALTS handshake when the ClientInit is split across 2 packets on +// the wire. +TEST_P(AltsTsiHandshakerTest, ServerSideFullHandshakeWithClientInitSplit) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + + // Fail to get the ServerInit and ServerFinished because only part of the + // ClientInit has arrived. + { + absl::string_view client_init_first_half = ClientInit.substr(0, ClientInit.size() / 2); + CapturingHandshaker capturing_handshaker; + EXPECT_OK( + handshaker->next(&capturing_handshaker, + reinterpret_cast(client_init_first_half.data()), + client_init_first_half.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the ServerInit and ServerFinished now that the rest of the ClientInit + // has arrived. + { + absl::string_view client_init_second_half = ClientInit.substr(ClientInit.size() / 2); + CapturingHandshaker capturing_handshaker; + EXPECT_OK( + handshaker->next(&capturing_handshaker, + reinterpret_cast(client_init_second_half.data()), + client_init_second_half.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), absl::StrCat(ServerInit, ServerFinished)); + EXPECT_OK(capturing_handshaker.getStatus()); + EXPECT_THAT(capturing_handshaker.getAltsHandshakeResult(), IsNull()); + } + + // Get the handshake result. + { + std::string handshake_message = absl::StrCat(ServerInit, ServerFinished); + CapturingHandshaker capturing_handshaker; + EXPECT_OK(handshaker->next(&capturing_handshaker, + reinterpret_cast(ClientFinished.data()), + ClientFinished.size(), onNextDoneImpl)); + EXPECT_EQ(capturing_handshaker.getBytesToSend(), ""); + EXPECT_OK(capturing_handshaker.getStatus()); + auto handshake_result = capturing_handshaker.getAltsHandshakeResult(); + EXPECT_THAT(handshake_result, NotNull()); + EXPECT_THAT(handshake_result->frame_protector, NotNull()); + EXPECT_EQ(handshake_result->peer_identity, PeerServiceAccount); + EXPECT_EQ(handshake_result->unused_bytes.size(), 0); + } + + // Confirm that the handshake cannot continue. + CapturingHandshaker capturing_handshaker; + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/0, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInternal)); +} + +// Check that AltsTsiHandshaker correctly handles invalid arguments. +TEST_P(AltsTsiHandshakerTest, InvalidArgumentToNext) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + + CapturingHandshaker capturing_handshaker; + std::string received_bytes; + EXPECT_THAT(handshaker->next( + /*handshaker=*/nullptr, + reinterpret_cast(received_bytes.data()), + received_bytes.size(), onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(handshaker->next(&capturing_handshaker, /*received_bytes=*/nullptr, + /*received_bytes_size=*/1, onNextDoneImpl), + StatusCodeIs(absl::StatusCode::kInvalidArgument)); + EXPECT_THAT(handshaker->next(&capturing_handshaker, + reinterpret_cast(received_bytes.data()), + received_bytes.size(), /*on_next_done=*/nullptr), + StatusCodeIs(absl::StatusCode::kInvalidArgument)); +} + +// Check that the handshake result is correctly validated. +TEST_P(AltsTsiHandshakerTest, InvalidHandshakeResult) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + absl::Span received_bytes; + + // Fail due to a missing peer identity. + HandshakerResult handshake_result; + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/0), + StatusIs(absl::StatusCode::kFailedPrecondition)); + + // Fail due to a missing local identity. + handshake_result.mutable_peer_identity(); + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/0), + StatusIs(absl::StatusCode::kFailedPrecondition)); + + // Fail due to a missing peer RPC versions. + handshake_result.mutable_local_identity(); + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/0), + StatusIs(absl::StatusCode::kFailedPrecondition)); + + // Fail due to empty application protocol. + handshake_result.mutable_peer_rpc_versions(); + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/0), + StatusIs(absl::StatusCode::kFailedPrecondition)); + + // Fail due to unsupported record protocol. + handshake_result.set_application_protocol(ApplicationProtocol); + handshake_result.set_record_protocol("ALTSRP_GCM_AES128"); + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/0), + StatusIs(absl::StatusCode::kFailedPrecondition)); + + // Fail due to short key length. + handshake_result.set_record_protocol("ALTSRP_GCM_AES128_REKEY"); + handshake_result.set_key_data(std::string(20, 'a')); + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/0), + StatusIs(absl::StatusCode::kFailedPrecondition)); + + // Fail due to consuming more bytes than we received from the peer. + handshake_result.set_key_data(std::string(44, 'a')); + EXPECT_THAT(handshaker->getHandshakeResult(handshake_result, received_bytes, + /*bytes_consumed=*/1), + StatusIs(absl::StatusCode::kFailedPrecondition)); +} + +// Check that the max frame size is correctly computed. +TEST_P(AltsTsiHandshakerTest, ComputeMaxFrameSize) { + // Max frame size is not set. + HandshakerResult result; + EXPECT_EQ(AltsTsiHandshaker::computeMaxFrameSize(result), AltsMinFrameSize); + + // Max frame size is over the allowed limit. + result.set_max_frame_size(MaxFrameSize + 1); + EXPECT_EQ(AltsTsiHandshaker::computeMaxFrameSize(result), MaxFrameSize); + + // Max frame size is under the allowed limit. + result.set_max_frame_size(AltsMinFrameSize - 1); + EXPECT_EQ(AltsTsiHandshaker::computeMaxFrameSize(result), AltsMinFrameSize); + + // Max frame size is within the allowed limits. + result.set_max_frame_size(AltsMinFrameSize + 1); + EXPECT_EQ(AltsTsiHandshaker::computeMaxFrameSize(result), AltsMinFrameSize + 1); +} + +} // namespace +} // namespace Alts +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc b/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc index 0717f0e3fb04..a02980311ac3 100644 --- a/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc +++ b/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc @@ -58,6 +58,13 @@ TEST_F(TsiFrameProtectorTest, Protect) { "\0\x40\0\0"s + std::string(16380, 'a') + "\x28\x0e\0\0"s + std::string(3620, 'a'); EXPECT_EQ(expected, encrypted.toString()); } + + { + Buffer::OwnedImpl encrypted; + + EXPECT_EQ(frame_protector_.protect(grpc_empty_slice(), encrypted), TSI_OK); + EXPECT_EQ(encrypted.length(), 0); + } } TEST_F(TsiFrameProtectorTest, ProtectError) { @@ -105,7 +112,15 @@ TEST_F(TsiFrameProtectorTest, Unprotect) { EXPECT_EQ(TSI_OK, frame_protector_.unprotect(input, decrypted)); EXPECT_EQ(std::string(20000, 'a'), decrypted.toString()); } + + { + Buffer::OwnedImpl input, decrypted; + + EXPECT_EQ(TSI_OK, frame_protector_.unprotect(input, decrypted)); + EXPECT_EQ(decrypted.length(), 0); + } } + TEST_F(TsiFrameProtectorTest, UnprotectError) { const tsi_zero_copy_grpc_protector_vtable* vtable = raw_frame_protector_->vtable; tsi_zero_copy_grpc_protector_vtable mock_vtable = *raw_frame_protector_->vtable; diff --git a/test/extensions/transport_sockets/alts/tsi_handshaker_test.cc b/test/extensions/transport_sockets/alts/tsi_handshaker_test.cc index 2104d9af0143..32476c23f426 100644 --- a/test/extensions/transport_sockets/alts/tsi_handshaker_test.cc +++ b/test/extensions/transport_sockets/alts/tsi_handshaker_test.cc @@ -1,10 +1,39 @@ +#include +#include +#include +#include + +#include "envoy/network/address.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/transport_sockets/alts/alts_proxy.h" +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" #include "source/extensions/transport_sockets/alts/tsi_handshaker.h" #include "test/mocks/event/mocks.h" - +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/status_utility.h" +#include "test/test_common/utility.h" + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/notification.h" #include "gmock/gmock.h" +#include "grpcpp/channel.h" +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" +#include "grpcpp/security/server_credentials.h" +#include "grpcpp/server.h" +#include "grpcpp/server_builder.h" +#include "grpcpp/server_context.h" +#include "grpcpp/support/status.h" +#include "grpcpp/support/sync_stream.h" #include "gtest/gtest.h" -#include "src/core/tsi/fake_transport_security.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" namespace Envoy { namespace Extensions { @@ -12,163 +41,309 @@ namespace TransportSockets { namespace Alts { namespace { -using testing::_; -using testing::InSequence; -using testing::Invoke; -using testing::NiceMock; +using ::Envoy::StatusHelpers::StatusCodeIs; +using ::grpc::gcp::HandshakerReq; +using ::grpc::gcp::HandshakerResp; +using ::grpc::gcp::HandshakerResult; +using ::grpc::gcp::HandshakerService; +using ::testing::IsNull; +using ::testing::NotNull; +using ::testing::Test; + +constexpr absl::string_view ClientInit = "CLIENT_INIT"; +constexpr absl::string_view ServerInit = "SERVER_INIT"; +constexpr absl::string_view ClientFinished = "CLIENT_FINISHED"; +constexpr absl::string_view ServerFinished = "SERVER_FINISHED"; + +constexpr absl::string_view KeyData = "fake_key_data_needs_to_be_at_least_44_characters_long"; +constexpr absl::string_view LocalServiceAccount = "local_service_account"; +constexpr absl::string_view PeerServiceAccount = "peer_service_account"; + +void populateHandshakeResult(HandshakerResult* result) { + result->mutable_peer_identity()->set_service_account(PeerServiceAccount); + result->mutable_peer_rpc_versions(); + result->mutable_local_identity()->set_service_account(LocalServiceAccount); + result->set_application_protocol(ApplicationProtocol); + result->set_record_protocol(RecordProtocol); + result->set_key_data(KeyData); +} + +class CapturingTsiHandshakerCallbacks final : public TsiHandshakerCallbacks { +public: + CapturingTsiHandshakerCallbacks() = default; + ~CapturingTsiHandshakerCallbacks() override = default; + + void onNextDone(NextResultPtr&& result) override { next_result_ = std::move(result); } + + NextResultPtr getNextResult() { return std::move(next_result_); } -class MockTsiHandshakerCallbacks : public TsiHandshakerCallbacks { +private: + NextResultPtr next_result_; +}; + +class FakeHandshakerService final : public HandshakerService::Service { public: - void onNextDone(NextResultPtr&& result) override { onNextDone_(result.get()); } - MOCK_METHOD(void, onNextDone_, (NextResult*)); - - void expectDone(tsi_result status, Buffer::Instance& to_send, CHandshakerResultPtr& result) { - EXPECT_CALL(*this, onNextDone_(_)) - .WillOnce(Invoke([&, status](TsiHandshakerCallbacks::NextResult* next_result) { - EXPECT_EQ(status, next_result->status_); - to_send.add(*next_result->to_send_); - result.swap(next_result->result_); - })); + FakeHandshakerService() = default; + + grpc::Status + DoHandshake(grpc::ServerContext* context, + grpc::ServerReaderWriter* stream) override { + EXPECT_THAT(context, NotNull()); + HandshakerReq request; + bool is_assisting_client = false; + while (stream->Read(&request)) { + HandshakerResp response; + if (request.has_client_start()) { + // The request contains a StartClientHandshakeReq message. + is_assisting_client = true; + response.set_out_frames(ClientInit); + response.set_bytes_consumed(ClientInit.size()); + } else if (request.has_server_start()) { + // The request contains a StartServerHandshakeReq message. + EXPECT_EQ(request.server_start().in_bytes(), ClientInit); + std::string out_frames = absl::StrCat(ServerInit, ServerFinished); + response.set_out_frames(out_frames); + response.set_bytes_consumed(out_frames.size()); + } else if (request.has_next()) { + // The request contains a NextHandshakeMessageReq message. + std::string expected_in_bytes = is_assisting_client + ? absl::StrCat(ServerInit, ServerFinished) + : std::string(ClientFinished); + EXPECT_EQ(request.next().in_bytes(), expected_in_bytes); + response.set_bytes_consumed(expected_in_bytes.size()); + if (is_assisting_client) { + response.set_out_frames(ClientFinished); + } + populateHandshakeResult(response.mutable_result()); + } else { + response.mutable_status()->set_code( + static_cast(grpc::StatusCode::FAILED_PRECONDITION)); + response.mutable_status()->set_details("Missing body of handshake request."); + } + EXPECT_TRUE(stream->Write(response)); + } + return grpc::Status::OK; } }; -class TsiHandshakerTest : public testing::Test { +class ErrorHandshakerService final : public HandshakerService::Service { public: - TsiHandshakerTest() - : server_handshaker_({tsi_create_fake_handshaker(0)}, dispatcher_), - client_handshaker_({tsi_create_fake_handshaker(1)}, dispatcher_) { - server_handshaker_.setHandshakerCallbacks(server_callbacks_); - client_handshaker_.setHandshakerCallbacks(client_callbacks_); + ErrorHandshakerService() = default; + + grpc::Status + DoHandshake(grpc::ServerContext* context, + grpc::ServerReaderWriter* stream) override { + EXPECT_THAT(context, NotNull()); + HandshakerReq request; + while (stream->Read(&request)) { + HandshakerResp response; + response.mutable_status()->set_code(static_cast(grpc::StatusCode::INTERNAL)); + response.mutable_status()->set_details("Internal error."); + EXPECT_TRUE(stream->Write(response)); + } + return grpc::Status(grpc::StatusCode::INTERNAL, "DoHandshake internal error."); } +}; +class AltsTsiHandshakerTest : public testing::TestWithParam { protected: - NiceMock dispatcher_; - MockTsiHandshakerCallbacks server_callbacks_; - MockTsiHandshakerCallbacks client_callbacks_; - TsiHandshaker server_handshaker_; - TsiHandshaker client_handshaker_; -}; + AltsTsiHandshakerTest() : version_(GetParam()){}; + void startFakeHandshakerService() { + server_address_ = absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + absl::Notification notification; + server_thread_ = std::make_unique([this, ¬ification]() { + FakeHandshakerService fake_handshaker_service; + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(server_address_, grpc::InsecureServerCredentials(), + &listening_port); + server_builder.RegisterService(&fake_handshaker_service); + server_ = server_builder.BuildAndStart(); + EXPECT_THAT(server_, NotNull()); + EXPECT_NE(listening_port, -1); + server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + notification.Notify(); + server_->Wait(); + }); + notification.WaitForNotification(); + } -TEST_F(TsiHandshakerTest, DoHandshake) { - InSequence s; - - Buffer::OwnedImpl server_sent; - Buffer::OwnedImpl client_sent; - - CHandshakerResultPtr client_result; - CHandshakerResultPtr server_result; - - client_callbacks_.expectDone(TSI_OK, client_sent, client_result); - client_handshaker_.next(server_sent); // Initially server_sent is empty. - EXPECT_EQ(nullptr, client_result); - EXPECT_EQ("CLIENT_INIT", client_sent.toString().substr(4)); - - server_callbacks_.expectDone(TSI_OK, server_sent, server_result); - server_handshaker_.next(client_sent); - EXPECT_EQ(nullptr, client_result); - EXPECT_EQ("SERVER_INIT", server_sent.toString().substr(4)); - - client_callbacks_.expectDone(TSI_OK, client_sent, client_result); - client_handshaker_.next(server_sent); - EXPECT_EQ(nullptr, client_result); - EXPECT_EQ("CLIENT_FINISHED", client_sent.toString().substr(4)); - - server_callbacks_.expectDone(TSI_OK, server_sent, server_result); - server_handshaker_.next(client_sent); - EXPECT_NE(nullptr, server_result); - EXPECT_EQ("SERVER_FINISHED", server_sent.toString().substr(4)); - - client_callbacks_.expectDone(TSI_OK, client_sent, client_result); - client_handshaker_.next(server_sent); - EXPECT_NE(nullptr, client_result); - EXPECT_EQ("", client_sent.toString()); - - tsi_peer client_peer; - EXPECT_EQ(TSI_OK, tsi_handshaker_result_extract_peer(client_result.get(), &client_peer)); - EXPECT_EQ(2, client_peer.property_count); - EXPECT_STREQ("certificate_type", client_peer.properties[0].name); - absl::string_view client_certificate_type{client_peer.properties[0].value.data, - client_peer.properties[0].value.length}; - EXPECT_EQ("FAKE", client_certificate_type); - EXPECT_STREQ("security_level", client_peer.properties[1].name); - absl::string_view client_security_level{client_peer.properties[1].value.data, - client_peer.properties[1].value.length}; - EXPECT_EQ("TSI_SECURITY_NONE", client_security_level); - - tsi_peer server_peer; - EXPECT_EQ(TSI_OK, tsi_handshaker_result_extract_peer(server_result.get(), &server_peer)); - EXPECT_EQ(2, server_peer.property_count); - EXPECT_STREQ("certificate_type", server_peer.properties[0].name); - absl::string_view server_certificate_type{server_peer.properties[0].value.data, - server_peer.properties[0].value.length}; - EXPECT_EQ("FAKE", server_certificate_type); - EXPECT_STREQ("security_level", server_peer.properties[1].name); - absl::string_view server_security_level{server_peer.properties[1].value.data, - server_peer.properties[1].value.length}; - EXPECT_EQ("TSI_SECURITY_NONE", server_security_level); - - tsi_peer_destruct(&client_peer); - tsi_peer_destruct(&server_peer); -} + void startErrorHandshakerService() { + server_address_ = absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + absl::Notification notification; + server_thread_ = std::make_unique([this, ¬ification]() { + ErrorHandshakerService error_handshaker_service; + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(server_address_, grpc::InsecureServerCredentials(), + &listening_port); + server_builder.RegisterService(&error_handshaker_service); + server_ = server_builder.BuildAndStart(); + EXPECT_THAT(server_, NotNull()); + EXPECT_NE(listening_port, -1); + server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + notification.Notify(); + server_->Wait(); + }); + notification.WaitForNotification(); + } -TEST_F(TsiHandshakerTest, IncompleteData) { - InSequence s; + void TearDown() override { + if (server_thread_) { + server_->Shutdown(); + server_thread_->join(); + } + } - Buffer::OwnedImpl server_sent; - Buffer::OwnedImpl client_sent; + std::shared_ptr getChannel() { + return grpc::CreateChannel(server_address_, + grpc::InsecureChannelCredentials()); // NOLINT + } - CHandshakerResultPtr client_result; - CHandshakerResultPtr server_result; +private: + std::string server_address_; + std::unique_ptr server_; + std::unique_ptr server_thread_; + Network::Address::IpVersion version_; +}; - client_callbacks_.expectDone(TSI_OK, client_sent, client_result); - client_handshaker_.next(server_sent); // Initially server_sent is empty. - EXPECT_EQ(nullptr, client_result); - EXPECT_EQ("CLIENT_INIT", client_sent.toString().substr(4)); +INSTANTIATE_TEST_SUITE_P(IpVersions, AltsTsiHandshakerTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(AltsTsiHandshakerTest, ClientSideFullHandshake) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + Event::MockDispatcher dispatcher; + auto tsi_handshaker = std::make_unique(std::move(handshaker), dispatcher); + + // Get the ClientInit. + { + CapturingTsiHandshakerCallbacks capturing_callbacks; + tsi_handshaker->setHandshakerCallbacks(capturing_callbacks); + EXPECT_CALL(dispatcher, post(_)).WillOnce([](Envoy::Event::PostCb cb) { cb(); }); + Buffer::OwnedImpl received_bytes; + EXPECT_OK(tsi_handshaker->next(received_bytes)); + + auto result = capturing_callbacks.getNextResult(); + EXPECT_THAT(result, NotNull()); + EXPECT_TRUE(result->status_.ok()); + EXPECT_EQ(result->to_send_->toString(), ClientInit); + EXPECT_THAT(result->result_, IsNull()); + } - client_sent.drain(3); // make data incomplete - server_callbacks_.expectDone(TSI_INCOMPLETE_DATA, server_sent, server_result); - server_handshaker_.next(client_sent); - EXPECT_EQ(nullptr, client_result); - EXPECT_EQ("", server_sent.toString()); + // Get the ClientFinished and the handshake result. + { + CapturingTsiHandshakerCallbacks capturing_callbacks; + tsi_handshaker->setHandshakerCallbacks(capturing_callbacks); + EXPECT_CALL(dispatcher, post(_)).WillOnce([](Envoy::Event::PostCb cb) { cb(); }); + Buffer::OwnedImpl received_bytes; + received_bytes.add(ServerInit.data(), ServerInit.size()); + received_bytes.add(ServerFinished.data(), ServerFinished.size()); + EXPECT_TRUE(tsi_handshaker->next(received_bytes).ok()); + + auto result = capturing_callbacks.getNextResult(); + EXPECT_THAT(result, NotNull()); + EXPECT_TRUE(result->status_.ok()); + EXPECT_EQ(result->to_send_->toString(), ClientFinished); + EXPECT_THAT(result->result_, NotNull()); + EXPECT_THAT(result->result_->frame_protector, NotNull()); + EXPECT_EQ(result->result_->peer_identity, PeerServiceAccount); + EXPECT_EQ(result->result_->unused_bytes.size(), 0); + } } -TEST_F(TsiHandshakerTest, DeferredDelete) { - InSequence s; +TEST_P(AltsTsiHandshakerTest, ServerSideFullHandshake) { + // Setup. + startFakeHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + Event::MockDispatcher dispatcher; + auto tsi_handshaker = std::make_unique(std::move(handshaker), dispatcher); + + // Get the ServerInit and ServerFinished. + { + CapturingTsiHandshakerCallbacks capturing_callbacks; + tsi_handshaker->setHandshakerCallbacks(capturing_callbacks); + EXPECT_CALL(dispatcher, post(_)).WillOnce([](Envoy::Event::PostCb cb) { cb(); }); + Buffer::OwnedImpl received_bytes; + received_bytes.add(ClientInit.data(), ClientInit.size()); + EXPECT_OK(tsi_handshaker->next(received_bytes)); + + auto result = capturing_callbacks.getNextResult(); + EXPECT_THAT(result, NotNull()); + EXPECT_TRUE(result->status_.ok()); + EXPECT_EQ(result->to_send_->toString(), absl::StrCat(ServerInit, ServerFinished)); + EXPECT_THAT(result->result_, IsNull()); + } - TsiHandshakerPtr handshaker{new TsiHandshaker({tsi_create_fake_handshaker(0)}, dispatcher_)}; - handshaker->deferredDelete(); - // The handshaker is now in dispatcher_ to delete queue. - EXPECT_EQ(dispatcher_.to_delete_.back().get(), handshaker.get()); - handshaker.release(); + // Get the handshake result. + { + CapturingTsiHandshakerCallbacks capturing_callbacks; + tsi_handshaker->setHandshakerCallbacks(capturing_callbacks); + EXPECT_CALL(dispatcher, post(_)).WillOnce([](Envoy::Event::PostCb cb) { cb(); }); + Buffer::OwnedImpl received_bytes; + received_bytes.add(ClientFinished.data(), ClientFinished.size()); + EXPECT_TRUE(tsi_handshaker->next(received_bytes).ok()); + + auto result = capturing_callbacks.getNextResult(); + EXPECT_THAT(result, NotNull()); + EXPECT_TRUE(result->status_.ok()); + EXPECT_EQ(result->to_send_->toString(), ""); + EXPECT_THAT(result->result_, NotNull()); + EXPECT_THAT(result->result_->frame_protector, NotNull()); + EXPECT_EQ(result->result_->peer_identity, PeerServiceAccount); + EXPECT_EQ(result->result_->unused_bytes.size(), 0); + } } -TEST_F(TsiHandshakerTest, DeleteOnDone) { - InSequence s; - - TsiHandshakerPtr handshaker(new TsiHandshaker({tsi_create_fake_handshaker(1)}, dispatcher_)); - handshaker->setHandshakerCallbacks(client_callbacks_); - - Buffer::OwnedImpl empty; - Event::PostCb done; - - EXPECT_CALL(dispatcher_, post(_)).WillOnce([&done](Event::PostCb cb) { done = std::move(cb); }); - - handshaker->next(empty); - handshaker->deferredDelete(); - - // Make sure the handshaker is not in dispatcher_ queue, since the next call is not done. - EXPECT_TRUE(dispatcher_.to_delete_.empty()); - - // After deferredDelete, the callback should be never invoked, in real use it might be already - // a dangling pointer. - EXPECT_CALL(client_callbacks_, onNextDone_(_)).Times(0); - - // Simulate the next call is completed. - done(); +TEST_P(AltsTsiHandshakerTest, ClientSideError) { + // Setup. + startErrorHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); + Event::MockDispatcher dispatcher; + auto tsi_handshaker = std::make_unique(std::move(handshaker), dispatcher); + + // Try to get the ClientInit and observe failure. + { + CapturingTsiHandshakerCallbacks capturing_callbacks; + tsi_handshaker->setHandshakerCallbacks(capturing_callbacks); + EXPECT_CALL(dispatcher, post(_)).WillOnce([](Envoy::Event::PostCb cb) { cb(); }); + Buffer::OwnedImpl received_bytes; + ASSERT_THAT(tsi_handshaker->next(received_bytes), StatusCodeIs(absl::StatusCode::kInternal)); + + auto result = capturing_callbacks.getNextResult(); + EXPECT_THAT(result, NotNull()); + EXPECT_THAT(result->status_, StatusCodeIs(absl::StatusCode::kInternal)); + EXPECT_EQ(result->to_send_->toString(), ""); + EXPECT_THAT(result->result_, IsNull()); + } +} - // The handshaker is now in dispatcher_ to delete queue. - EXPECT_EQ(dispatcher_.to_delete_.back().get(), handshaker.get()); - handshaker.release(); +TEST_P(AltsTsiHandshakerTest, ServerSideError) { + // Setup. + startErrorHandshakerService(); + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); + Event::MockDispatcher dispatcher; + auto tsi_handshaker = std::make_unique(std::move(handshaker), dispatcher); + + // Try to get the ServerInit and ServerFinished and observe failure. + { + CapturingTsiHandshakerCallbacks capturing_callbacks; + tsi_handshaker->setHandshakerCallbacks(capturing_callbacks); + EXPECT_CALL(dispatcher, post(_)).WillOnce([](Envoy::Event::PostCb cb) { cb(); }); + Buffer::OwnedImpl received_bytes; + received_bytes.add(ClientInit.data(), ClientInit.size()); + ASSERT_THAT(tsi_handshaker->next(received_bytes), StatusCodeIs(absl::StatusCode::kInternal)); + + auto result = capturing_callbacks.getNextResult(); + EXPECT_THAT(result, NotNull()); + EXPECT_THAT(result->status_, StatusCodeIs(absl::StatusCode::kInternal)); + EXPECT_EQ(result->to_send_->toString(), ""); + EXPECT_THAT(result->result_, IsNull()); + } } } // namespace diff --git a/test/extensions/transport_sockets/alts/tsi_socket_test.cc b/test/extensions/transport_sockets/alts/tsi_socket_test.cc index 18818846f52f..a8b9b675b744 100644 --- a/test/extensions/transport_sockets/alts/tsi_socket_test.cc +++ b/test/extensions/transport_sockets/alts/tsi_socket_test.cc @@ -1,12 +1,49 @@ - - +#include +#include +#include +#include +#include +#include + +#include "envoy/buffer/buffer.h" +#include "envoy/event/dispatcher.h" +#include "envoy/network/address.h" +#include "envoy/network/connection.h" +#include "envoy/network/post_io_action.h" +#include "envoy/network/transport_socket.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/transport_sockets/alts/alts_proxy.h" +#include "source/extensions/transport_sockets/alts/alts_tsi_handshaker.h" +#include "source/extensions/transport_sockets/alts/tsi_handshaker.h" #include "source/extensions/transport_sockets/alts/tsi_socket.h" +#include "test/mocks/event/mocks.h" #include "test/mocks/network/mocks.h" - +#include "test/mocks/network/transport_socket.h" +#include "test/mocks/upstream/cluster_info.h" +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/utility.h" + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/notification.h" #include "gmock/gmock.h" +#include "grpcpp/channel.h" +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" +#include "grpcpp/security/server_credentials.h" +#include "grpcpp/server.h" +#include "grpcpp/server_builder.h" +#include "grpcpp/server_context.h" +#include "grpcpp/support/status.h" +#include "grpcpp/support/sync_stream.h" #include "gtest/gtest.h" -#include "src/core/tsi/fake_transport_security.h" +#include "src/proto/grpc/gcp/handshaker.grpc.pb.h" +#include "src/proto/grpc/gcp/handshaker.pb.h" +#include "src/proto/grpc/gcp/transport_security_common.pb.h" namespace Envoy { namespace Extensions { @@ -14,52 +51,199 @@ namespace TransportSockets { namespace Alts { namespace { -using testing::InSequence; -using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; +using ::grpc::gcp::HandshakerReq; +using ::grpc::gcp::HandshakerResp; +using ::grpc::gcp::HandshakerResult; +using ::grpc::gcp::HandshakerService; +using ::testing::_; +using ::testing::Invoke; +using ::testing::NotNull; +using ::testing::Return; +using ::testing::WithArgs; + +constexpr absl::string_view ApplicationData = "application_data"; +constexpr absl::string_view ClientInit = "CLIENT_INIT"; +constexpr absl::string_view ServerInit = "SERVER_INIT"; +constexpr absl::string_view ClientFinished = "CLIENT_FINISHED"; +constexpr absl::string_view ServerFinished = "SERVER_FINISHED"; + +constexpr absl::string_view KeyData = "fake_key_data_needs_to_be_at_least_44_characters_long"; +constexpr absl::string_view LocalServiceAccount = "local_service_account"; +constexpr absl::string_view PeerServiceAccount = "peer_service_account"; + +constexpr std::size_t AltsFrameOverhead = 24; + +void populateHandshakeResult(HandshakerResult* result) { + result->mutable_peer_identity()->set_service_account(PeerServiceAccount); + result->mutable_peer_rpc_versions(); + result->mutable_local_identity()->set_service_account(LocalServiceAccount); + result->set_application_protocol(ApplicationProtocol); + result->set_record_protocol(RecordProtocol); + result->set_key_data(KeyData); +} -static const std::string ClientToServerData = "hello from client"; -static const std::string ClientToServerDataFirstHalf = "hello fro"; -static const std::string ClientToServerDataSecondHalf = "m client"; -static const std::string ServerToClientData = "hello from server"; -static const uint32_t LargeFrameSize = 100; -static const uint32_t SmallFrameSize = 13; +class FakeHandshakerService final : public HandshakerService::Service { +public: + FakeHandshakerService() = default; + + grpc::Status + DoHandshake(grpc::ServerContext* context, + grpc::ServerReaderWriter* stream) override { + EXPECT_THAT(context, NotNull()); + HandshakerReq request; + bool is_assisting_client = false; + while (stream->Read(&request)) { + HandshakerResp response; + if (request.has_client_start()) { + // The request contains a StartClientHandshakeReq message. + is_assisting_client = true; + response.set_out_frames(ClientInit); + response.set_bytes_consumed(ClientInit.size()); + } else if (request.has_server_start()) { + // The request contains a StartServerHandshakeReq message. + EXPECT_EQ(request.server_start().in_bytes(), ClientInit); + std::string out_frames = absl::StrCat(ServerInit, ServerFinished); + response.set_out_frames(out_frames); + response.set_bytes_consumed(out_frames.size()); + } else if (request.has_next()) { + // The request contains a NextHandshakeMessageReq message. + if (!is_assisting_client) { + EXPECT_TRUE(absl::StartsWith(request.next().in_bytes(), ClientFinished)); + response.set_bytes_consumed(ClientFinished.size()); + } else { + std::string expected_in_bytes = is_assisting_client + ? absl::StrCat(ServerInit, ServerFinished) + : std::string(ClientFinished); + EXPECT_EQ(request.next().in_bytes(), expected_in_bytes); + response.set_bytes_consumed(expected_in_bytes.size()); + } + if (is_assisting_client) { + response.set_out_frames(ClientFinished); + } + populateHandshakeResult(response.mutable_result()); + } else { + response.mutable_status()->set_code( + static_cast(grpc::StatusCode::FAILED_PRECONDITION)); + response.mutable_status()->set_details("Missing body of handshake request."); + } + EXPECT_TRUE(stream->Write(response)); + } + return grpc::Status::OK; + } +}; -class TsiSocketTest : public testing::Test { -protected: - TsiSocketTest() { - server_.handshaker_factory_ = [](Event::Dispatcher& dispatcher, - const Network::Address::InstanceConstSharedPtr&, - const Network::Address::InstanceConstSharedPtr&) { - CHandshakerPtr handshaker{tsi_create_fake_handshaker(/*is_client=*/0)}; +class ErrorHandshakerService final : public HandshakerService::Service { +public: + explicit ErrorHandshakerService(bool keep_stream_alive) : keep_stream_alive_(keep_stream_alive) {} + + grpc::Status + DoHandshake(grpc::ServerContext* context, + grpc::ServerReaderWriter* stream) override { + EXPECT_THAT(context, NotNull()); + HandshakerReq request; + while (stream->Read(&request)) { + if (keep_stream_alive_) { + HandshakerResp response; + response.mutable_status()->set_code(static_cast(grpc::StatusCode::INTERNAL)); + response.mutable_status()->set_details("Internal error."); + EXPECT_TRUE(stream->Write(response)); + } else { + break; + } + } + return grpc::Status(grpc::StatusCode::INTERNAL, "DoHandshake internal error."); + } + + const bool keep_stream_alive_; +}; +class TsiSocketTest : public testing::TestWithParam { +protected: + TsiSocketTest() : version_(GetParam()) { + server_.handshaker_factory_ = [this](Event::Dispatcher& dispatcher, + const Network::Address::InstanceConstSharedPtr&, + const Network::Address::InstanceConstSharedPtr&) { + auto handshaker = AltsTsiHandshaker::createForServer(getChannel()); return std::make_unique(std::move(handshaker), dispatcher); }; - - client_.handshaker_factory_ = [](Event::Dispatcher& dispatcher, - const Network::Address::InstanceConstSharedPtr&, - const Network::Address::InstanceConstSharedPtr&) { - CHandshakerPtr handshaker{tsi_create_fake_handshaker(/*is_client=*/1)}; - + client_.handshaker_factory_ = [this](Event::Dispatcher& dispatcher, + const Network::Address::InstanceConstSharedPtr&, + const Network::Address::InstanceConstSharedPtr&) { + auto handshaker = AltsTsiHandshaker::createForClient(getChannel()); return std::make_unique(std::move(handshaker), dispatcher); }; } void TearDown() override { - client_.tsi_socket_->closeSocket(Network::ConnectionEvent::LocalClose); - server_.tsi_socket_->closeSocket(Network::ConnectionEvent::RemoteClose); + if (client_.tsi_socket_ != nullptr) { + client_.tsi_socket_->closeSocket(Network::ConnectionEvent::LocalClose); + } + if (server_.tsi_socket_ != nullptr) { + server_.tsi_socket_->closeSocket(Network::ConnectionEvent::RemoteClose); + } + if (handshaker_server_thread_) { + handshaker_server_->Shutdown(std::chrono::system_clock::now()); // NO_CHECK_FORMAT(real_time) + handshaker_server_thread_->join(); + } } - void initialize(HandshakeValidator server_validator, HandshakeValidator client_validator) { - server_.raw_socket_ = new Network::MockTransportSocket(); + void startFakeHandshakerService() { + handshaker_server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + absl::Notification notification; + handshaker_server_thread_ = std::make_unique([this, ¬ification]() { + FakeHandshakerService fake_handshaker_service; + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(handshaker_server_address_, grpc::InsecureServerCredentials(), + &listening_port); + server_builder.RegisterService(&fake_handshaker_service); + handshaker_server_ = server_builder.BuildAndStart(); + EXPECT_THAT(handshaker_server_, NotNull()); + EXPECT_NE(listening_port, -1); + handshaker_server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + notification.Notify(); + handshaker_server_->Wait(); + }); + notification.WaitForNotification(); + } + + void startErrorHandshakerService(bool keep_stream_alive) { + handshaker_server_address_ = + absl::StrCat(Network::Test::getLoopbackAddressUrlString(version_), ":0"); + absl::Notification notification; + handshaker_server_thread_ = + std::make_unique([this, keep_stream_alive, ¬ification]() { + ErrorHandshakerService error_handshaker_service(keep_stream_alive); + grpc::ServerBuilder server_builder; + int listening_port = -1; + server_builder.AddListeningPort(handshaker_server_address_, + grpc::InsecureServerCredentials(), &listening_port); + server_builder.RegisterService(&error_handshaker_service); + handshaker_server_ = server_builder.BuildAndStart(); + EXPECT_THAT(handshaker_server_, NotNull()); + EXPECT_NE(listening_port, -1); + handshaker_server_address_ = absl::StrCat( + Network::Test::getLoopbackAddressUrlString(version_), ":", listening_port); + notification.Notify(); + handshaker_server_->Wait(); + }); + notification.WaitForNotification(); + } + std::shared_ptr getChannel() { + return grpc::CreateChannel(handshaker_server_address_, grpc::InsecureChannelCredentials()); + } + + void initializeSockets(HandshakeValidator server_validator, HandshakeValidator client_validator, + bool have_client_raw_socket_write_default = true) { + server_.raw_socket_ = new Network::MockTransportSocket(); server_.tsi_socket_ = std::make_unique(server_.handshaker_factory_, server_validator, Network::TransportSocketPtr{server_.raw_socket_}, true); client_.raw_socket_ = new Network::MockTransportSocket(); - client_.tsi_socket_ = std::make_unique(client_.handshaker_factory_, client_validator, Network::TransportSocketPtr{client_.raw_socket_}, false); @@ -71,12 +255,14 @@ class TsiSocketTest : public testing::Test { ON_CALL(server_.callbacks_, shouldDrainReadBuffer()).WillByDefault(Return(false)); - ON_CALL(*client_.raw_socket_, doWrite(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; - client_to_server_.move(buffer); - return result; - })); + if (have_client_raw_socket_write_default) { + ON_CALL(*client_.raw_socket_, doWrite(_, _)) + .WillByDefault(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + } ON_CALL(*server_.raw_socket_, doWrite(_, _)) .WillByDefault(Invoke([&](Buffer::Instance& buffer, bool) { Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; @@ -103,109 +289,77 @@ class TsiSocketTest : public testing::Test { EXPECT_CALL(*server_.raw_socket_, setTransportSocketCallbacks(_)); server_.tsi_socket_->setTransportSocketCallbacks(server_.callbacks_); - server_.tsi_socket_->setFrameOverheadSize(4); - client_.tsi_socket_->setFrameOverheadSize(4); - } - - void expectIoResult(Network::IoResult expected, Network::IoResult actual) { - EXPECT_EQ(expected.action_, actual.action_); - EXPECT_EQ(expected.bytes_processed_, actual.bytes_processed_); - EXPECT_EQ(expected.end_stream_read_, actual.end_stream_read_); + ON_CALL(dispatcher_, post(_)).WillByDefault(WithArgs<0>(Invoke([](Event::PostCb callback) { + callback(); + }))); } - std::string makeFakeTsiFrame(const std::string& payload) { - uint32_t length = static_cast(payload.length()) + 4; - std::string frame; - frame.reserve(length); - frame.push_back(static_cast(length)); - length >>= 8; - frame.push_back(static_cast(length)); - length >>= 8; - frame.push_back(static_cast(length)); - length >>= 8; - frame.push_back(static_cast(length)); - - frame.append(payload); - return frame; + void expectIoResult(Network::IoResult expected, Network::IoResult actual, + absl::string_view debug_string) { + EXPECT_EQ(expected.action_, actual.action_) << debug_string; + EXPECT_EQ(expected.bytes_processed_, actual.bytes_processed_) << debug_string; + EXPECT_EQ(expected.end_stream_read_, actual.end_stream_read_) << debug_string; } - std::string makeInvalidTsiFrame() { - // For fake frame protector, minimum frame size is 4 bytes. - uint32_t length = 3; - std::string frame; - frame.reserve(4); - frame.push_back(static_cast(length)); - length >>= 8; - frame.push_back(static_cast(length)); - length >>= 8; - frame.push_back(static_cast(length)); - length >>= 8; - frame.push_back(static_cast(length)); - - return frame; - } - - void doFakeInitHandshake() { + void doHandshakeAndExpectSuccess() { + // On the client side, get the ClientInit and write it to the server. EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); client_.tsi_socket_->onConnected(); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame("CLIENT_INIT"), client_to_server_.toString()); + expectIoResult(client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ClientInit); + // On the server side, read the ClientInit and write the ServerInit and the + // ServerFinished to the client. EXPECT_CALL(*server_.raw_socket_, doRead(_)); EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("SERVER_INIT"), server_to_client_.toString()); - EXPECT_EQ(0L, server_.read_buffer_.length()); - } - - void doHandshakeAndExpectSuccess() { - doFakeInitHandshake(); - - EXPECT_CALL(*client_.raw_socket_, doRead(_)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + "While reading ClientInit."); + EXPECT_EQ(server_.read_buffer_.length(), 0L); + EXPECT_EQ(server_to_client_.toString(), absl::StrCat(ServerInit, ServerFinished)); + + // On the client side, read the ServerInit and the ServerFinished, and write + // the ClientFinished to the server. + EXPECT_CALL(*client_.raw_socket_, doRead(_)).Times(2); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("CLIENT_FINISHED"), client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)); - EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); + EXPECT_CALL(client_.callbacks_, raiseEvent(Envoy::Network::ConnectionEvent::Connected)); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doRead(client_.read_buffer_), + "While reading ServerInit and ServerFinished."); + EXPECT_EQ(client_.read_buffer_.length(), 0L); + EXPECT_EQ(client_to_server_.toString(), ClientFinished); + + // On the server side, read the ClientFinished. + EXPECT_CALL(*server_.raw_socket_, doRead(_)).Times(2); EXPECT_CALL(server_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*server_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED"), server_to_client_.toString()); - - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - EXPECT_CALL(client_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + "While reading ClientFinished."); + EXPECT_EQ(server_.read_buffer_.toString(), ""); } - void expectTransferDataFromClientToServer(const std::string& data) { - EXPECT_EQ(0L, server_.read_buffer_.length()); - EXPECT_EQ(0L, client_.read_buffer_.length()); - - EXPECT_EQ("", client_.tsi_socket_->protocol()); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - + void expectTransferDataFromClientToServer(absl::string_view data) { + EXPECT_EQ(server_.read_buffer_.length(), 0); + EXPECT_EQ(client_.read_buffer_.length(), 0); + EXPECT_EQ(client_.tsi_socket_->protocol(), ""); + client_.write_buffer_.add(data); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame(data), client_to_server_.toString()); - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), - true}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, true}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(data, server_.read_buffer_.toString()); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, data.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); + EXPECT_CALL(*server_.raw_socket_, doRead(_)) + .WillOnce(Invoke([&](Envoy::Buffer::Instance& buffer) { + Envoy::Network::IoResult result = {Envoy::Network::PostIoAction::KeepOpen, + client_to_server_.length(), true}; + buffer.move(client_to_server_); + return result; + })); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, data.size(), true}, + server_.tsi_socket_->doRead(server_.read_buffer_), + "While the server is reading application data."); + EXPECT_EQ(server_.read_buffer_.toString(), data); } struct SocketForTest { @@ -217,6 +371,8 @@ class TsiSocketTest : public testing::Test { Buffer::OwnedImpl write_buffer_; }; + Network::Address::IpVersion version_; + SocketForTest client_; SocketForTest server_; @@ -224,197 +380,401 @@ class TsiSocketTest : public testing::Test { Buffer::OwnedImpl server_to_client_; NiceMock dispatcher_; + + std::string handshaker_server_address_; + std::unique_ptr handshaker_server_; + std::unique_ptr handshaker_server_thread_; }; -TEST_F(TsiSocketTest, DoesNotHaveSsl) { - initialize(nullptr, nullptr); - EXPECT_EQ(nullptr, client_.tsi_socket_->ssl()); - EXPECT_FALSE(client_.tsi_socket_->canFlushClose()); +INSTANTIATE_TEST_SUITE_P(IpVersions, TsiSocketTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); - const auto& socket_ = *client_.tsi_socket_; - EXPECT_EQ(nullptr, socket_.ssl()); +TEST_P(TsiSocketTest, ConfigureInitialCongestionWindowIsNoOp) { + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + client_.tsi_socket_->configureInitialCongestionWindow(0, std::chrono::milliseconds(0)); } -TEST_F(TsiSocketTest, HandshakeWithoutValidationAndTransferData) { - // pass a nullptr validator to skip validation. - initialize(nullptr, nullptr); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); +TEST_P(TsiSocketTest, DoesNotStartSecureTransport) { + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + EXPECT_FALSE(client_.tsi_socket_->startSecureTransport()); +} - doHandshakeAndExpectSuccess(); - expectTransferDataFromClientToServer(ClientToServerData); +TEST_P(TsiSocketTest, DoesNotHaveSsl) { + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + EXPECT_EQ(client_.tsi_socket_->ssl(), nullptr); + EXPECT_FALSE(client_.tsi_socket_->canFlushClose()); + EXPECT_EQ(client_.tsi_socket_->ssl(), nullptr); } -TEST_F(TsiSocketTest, HandshakeWithSucessfulValidationAndTransferData) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); +TEST_P(TsiSocketTest, EmptyFailureReason) { + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + EXPECT_EQ(client_.tsi_socket_->failureReason(), ""); +} - InSequence s; +TEST_P(TsiSocketTest, UpstreamHandshakeFactoryFailure) { + auto raw_socket = new Network::MockTransportSocket(); + auto tsi_socket = std::make_unique( + [](Event::Dispatcher&, const Network::Address::InstanceConstSharedPtr&, + const Network::Address::InstanceConstSharedPtr&) { return nullptr; }, + nullptr, Network::TransportSocketPtr{raw_socket}, false); + NiceMock callbacks; + EXPECT_CALL(*raw_socket, setTransportSocketCallbacks(_)); + tsi_socket->setTransportSocketCallbacks(callbacks); + tsi_socket->onConnected(); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + tsi_socket->doWrite(client_.write_buffer_, /*end_stream=*/false), + "While writing ClientInit."); +} - client_.write_buffer_.add(ClientToServerData); +TEST_P(TsiSocketTest, DownstreamHandshakeFactoryFailure) { + auto raw_socket = new Network::MockTransportSocket(); + auto tsi_socket = std::make_unique( + [](Event::Dispatcher&, const Network::Address::InstanceConstSharedPtr&, + const Network::Address::InstanceConstSharedPtr&) { return nullptr; }, + nullptr, Network::TransportSocketPtr{raw_socket}, false); + NiceMock callbacks; + EXPECT_CALL(*raw_socket, setTransportSocketCallbacks(_)); + tsi_socket->setTransportSocketCallbacks(callbacks); + tsi_socket->onConnected(); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + tsi_socket->doWrite(client_.write_buffer_, /*end_stream=*/false), + "While writing ClientInit."); +} +TEST_P(TsiSocketTest, HandshakeSuccessAndTransferData) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); doHandshakeAndExpectSuccess(); - expectTransferDataFromClientToServer(ClientToServerData); + expectTransferDataFromClientToServer(ApplicationData); } -TEST_F(TsiSocketTest, HandshakeWithSucessfulValidationAndTransferInvalidData) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - +TEST_P(TsiSocketTest, HandshakeSuccessAndTransferLargeData) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); doHandshakeAndExpectSuccess(); - client_to_server_.add(makeInvalidTsiFrame()); + std::string large_application_data(1024 * 1024 * 2, 'a'); + expectTransferDataFromClientToServer(large_application_data); +} - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 4UL, true}; +TEST_P(TsiSocketTest, HandshakeSuccessAndTransferDataWithShortWrite) { + startFakeHandshakerService(); + + // Initialize the sockets but do not provide a default action for + // client_.raw_socket's doWrite() API. + server_.raw_socket_ = new Network::MockTransportSocket(); + server_.tsi_socket_ = std::make_unique( + server_.handshaker_factory_, nullptr, Network::TransportSocketPtr{server_.raw_socket_}, true); + client_.raw_socket_ = new Network::MockTransportSocket(); + client_.tsi_socket_ = + std::make_unique(client_.handshaker_factory_, nullptr, + Network::TransportSocketPtr{client_.raw_socket_}, false); + ON_CALL(client_.callbacks_.connection_, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(server_.callbacks_.connection_, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(client_.callbacks_.connection_, id()).WillByDefault(Return(11)); + ON_CALL(server_.callbacks_.connection_, id()).WillByDefault(Return(12)); + ON_CALL(server_.callbacks_, shouldDrainReadBuffer()).WillByDefault(Return(false)); + ON_CALL(*server_.raw_socket_, doWrite(_, _)) + .WillByDefault(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + server_to_client_.move(buffer); + return result; + })); + ON_CALL(*client_.raw_socket_, doRead(_)).WillByDefault(Invoke([&](Buffer::Instance& buffer) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, server_to_client_.length(), false}; + buffer.move(server_to_client_); + return result; + })); + ON_CALL(*server_.raw_socket_, doRead(_)).WillByDefault(Invoke([&](Buffer::Instance& buffer) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), false}; buffer.move(client_to_server_); return result; })); - expectIoResult({Network::PostIoAction::Close, 0UL, true}, - server_.tsi_socket_->doRead(server_.read_buffer_)); -} - -TEST_F(TsiSocketTest, HandshakeValidationFail) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return false; }; - initialize(validator, validator); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); - - doFakeInitHandshake(); - - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("CLIENT_FINISHED"), client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)); - EXPECT_CALL(server_.callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush, _)); - // doRead won't immediately fail, but it will result connection close. - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(0, server_to_client_.length()); -} - -TEST_F(TsiSocketTest, HandshakerCreationFail) { - client_.handshaker_factory_ = - [](Event::Dispatcher&, const Network::Address::InstanceConstSharedPtr&, - const Network::Address::InstanceConstSharedPtr&) { return nullptr; }; - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - - EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)).Times(0); - EXPECT_CALL(client_.callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush, _)); + EXPECT_CALL(*client_.raw_socket_, setTransportSocketCallbacks(_)); + client_.tsi_socket_->setTransportSocketCallbacks(client_.callbacks_); + EXPECT_CALL(*server_.raw_socket_, setTransportSocketCallbacks(_)); + server_.tsi_socket_->setTransportSocketCallbacks(server_.callbacks_); + ON_CALL(dispatcher_, post(_)).WillByDefault(WithArgs<0>(Invoke([](Event::PostCb callback) { + callback(); + }))); + + // On the client side, get the ClientInit and write it to the server. + EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); client_.tsi_socket_->onConnected(); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - EXPECT_EQ("", client_to_server_.toString()); + expectIoResult(client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ClientInit); + // On the server side, read the ClientInit and write the ServerInit and the + // ServerFinished to the client. EXPECT_CALL(*server_.raw_socket_, doRead(_)); - EXPECT_CALL(*server_.raw_socket_, doWrite(_, _)).Times(0); + EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While reading ClientInit."); + EXPECT_EQ(server_.read_buffer_.length(), 0L); + EXPECT_EQ(server_to_client_.toString(), absl::StrCat(ServerInit, ServerFinished)); + + // On the client side, read the ServerInit and the ServerFinished, and write + // the ClientFinished to the server. + EXPECT_CALL(*client_.raw_socket_, doRead(_)).Times(2); + EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + EXPECT_CALL(client_.callbacks_, raiseEvent(Envoy::Network::ConnectionEvent::Connected)); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doRead(client_.read_buffer_), + "While reading ServerInit and ServerFinished."); + EXPECT_EQ(client_.read_buffer_.length(), 0L); + EXPECT_EQ(client_to_server_.toString(), ClientFinished); + + // On the server side, read the ClientFinished. + EXPECT_CALL(*server_.raw_socket_, doRead(_)).Times(2); + EXPECT_CALL(server_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + "While reading ClientFinished."); + EXPECT_EQ(server_.read_buffer_.toString(), ""); + + // Write all of the data except for the last byte. + client_.write_buffer_.add(ApplicationData); + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + EXPECT_GT(buffer.length(), 0); + Network::IoResult result = {Network::PostIoAction::KeepOpen, 1, false}; + client_to_server_.add(buffer.linearize(0), buffer.length() - 1); + return result; + })); expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ("", server_to_client_.toString()); + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While writing application data."); + EXPECT_EQ(client_to_server_.length(), AltsFrameOverhead + ApplicationData.length() - 1); } -TEST_F(TsiSocketTest, HandshakeWithUnusedData) { - initialize(nullptr, nullptr); - - InSequence s; +TEST_P(TsiSocketTest, HandshakeSuccessAndTransferDataWithValidation) { + startFakeHandshakerService(); + auto validator = [](TsiInfo&, std::string&) { return true; }; + initializeSockets(validator, validator); + doHandshakeAndExpectSuccess(); + expectTransferDataFromClientToServer(ApplicationData); +} - doFakeInitHandshake(); - EXPECT_CALL(*client_.raw_socket_, doRead(_)); +TEST_P(TsiSocketTest, HandshakeSuccessAndTransferDataWithValidationFailure) { + startFakeHandshakerService(); + auto validator = [](TsiInfo&, std::string&) { return false; }; + initializeSockets(validator, validator); + // On the client side, get the ClientInit and write it to the server. EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("CLIENT_FINISHED"), client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); - - // Inject unused data - client_to_server_.add(makeFakeTsiFrame(ClientToServerData)); + client_.tsi_socket_->onConnected(); + expectIoResult(client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ClientInit); + // On the server side, read the ClientInit and write the ServerInit and the + // ServerFinished to the client. EXPECT_CALL(*server_.raw_socket_, doRead(_)); EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); - EXPECT_CALL(server_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*server_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED"), server_to_client_.toString()); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While reading ClientInit."); + EXPECT_EQ(server_.read_buffer_.length(), 0L); + EXPECT_EQ(server_to_client_.toString(), absl::StrCat(ServerInit, ServerFinished)); + // On the client side, read the ServerInit and the ServerFinished, and fail + // the validation before writing the ClientFinished to the server. EXPECT_CALL(*client_.raw_socket_, doRead(_)); - EXPECT_CALL(client_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doRead(client_.read_buffer_), + "While reading ServerInit and ServerFinished."); + EXPECT_EQ(client_.read_buffer_.length(), 0L); + EXPECT_EQ(client_to_server_.toString(), ""); } -TEST_F(TsiSocketTest, HandshakeWithUnusedDataAndEndOfStream) { - initialize(nullptr, nullptr); - - InSequence s; +TEST_P(TsiSocketTest, HandshakeSuccessAndFailToUnprotect) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + doHandshakeAndExpectSuccess(); - doFakeInitHandshake(); - EXPECT_CALL(*client_.raw_socket_, doRead(_)); + EXPECT_EQ(server_.read_buffer_.length(), 0); + EXPECT_EQ(client_.read_buffer_.length(), 0); + EXPECT_EQ(client_.tsi_socket_->protocol(), ""); + client_.write_buffer_.add(ApplicationData); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("CLIENT_FINISHED"), client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); + EXPECT_CALL(*server_.raw_socket_, doRead(_)) + .WillOnce(Invoke([&](Envoy::Buffer::Instance& buffer) { + Envoy::Network::IoResult result = {Envoy::Network::PostIoAction::KeepOpen, + client_to_server_.length(), true}; + buffer.move(client_to_server_); + return result; + })); + client_to_server_.drain(client_to_server_.length()); + client_to_server_.add("not-an-alts-frame"); + expectIoResult({Envoy::Network::PostIoAction::Close, 0L, true}, + server_.tsi_socket_->doRead(server_.read_buffer_), + "While the server is reading application data."); + EXPECT_EQ(server_.read_buffer_.toString(), ""); +} - // Inject unused data - client_to_server_.add(makeFakeTsiFrame(ClientToServerData)); +TEST_P(TsiSocketTest, HandshakeSuccessWithUnusedData) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr, + /*have_client_raw_socket_write_default=*/false); - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), true}; - buffer.move(client_to_server_); - return result; - })); + // On the client side, get the ClientInit and write it to the server. + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + client_.tsi_socket_->onConnected(); + expectIoResult(client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ClientInit); + + // On the server side, read the ClientInit and write the ServerInit and the + // ServerFinished to the client. + EXPECT_CALL(*server_.raw_socket_, doRead(_)); EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While reading ClientInit."); + EXPECT_EQ(server_.read_buffer_.length(), 0L); + EXPECT_EQ(server_to_client_.toString(), absl::StrCat(ServerInit, ServerFinished)); + + // On the client side, read the ServerInit and the ServerFinished, and write + // the ClientFinished to the server. + EXPECT_CALL(*client_.raw_socket_, doRead(_)).Times(2); + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + EXPECT_CALL(client_.callbacks_, raiseEvent(Envoy::Network::ConnectionEvent::Connected)); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doRead(client_.read_buffer_), + "While reading ServerInit and ServerFinished."); + EXPECT_EQ(client_.read_buffer_.length(), 0L); + EXPECT_EQ(client_to_server_.toString(), ClientFinished); + + // Write application data before the server reads the ClientFinished. + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + client_.write_buffer_.add(ApplicationData); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); + EXPECT_EQ(client_to_server_.length(), + ClientFinished.length() + AltsFrameOverhead + ApplicationData.length()); + + // On the server side, read the ClientFinished. + EXPECT_CALL(*server_.raw_socket_, doRead(_)).Times(2); EXPECT_CALL(server_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, true}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED"), server_to_client_.toString()); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); - - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - EXPECT_CALL(client_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + "While reading ClientFinished."); + EXPECT_EQ(server_.read_buffer_.toString(), ApplicationData); } -TEST_F(TsiSocketTest, HandshakeWithImmediateReadError) { - initialize(nullptr, nullptr); +TEST_P(TsiSocketTest, HandshakeSuccessWithUnusedDataAndShortWrite) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr, + /*have_client_raw_socket_write_default=*/false); - InSequence s; + // On the client side, get the ClientInit and write it to the server. + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + client_.tsi_socket_->onConnected(); + expectIoResult(client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ClientInit); - EXPECT_CALL(*client_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::Close, server_to_client_.length(), false}; - buffer.move(server_to_client_); + // On the server side, read the ClientInit and write the ServerInit and the + // ServerFinished to the client. + EXPECT_CALL(*server_.raw_socket_, doRead(_)); + EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, "While reading ClientInit."); + EXPECT_EQ(server_.read_buffer_.length(), 0L); + EXPECT_EQ(server_to_client_.toString(), absl::StrCat(ServerInit, ServerFinished)); + + // On the client side, read the ServerInit and the ServerFinished, but do not + // write the ClientFinished to the server. + EXPECT_CALL(*client_.raw_socket_, doRead(_)).Times(2); + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)).WillOnce(Invoke([&](Buffer::Instance&, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, 0L, false}; return result; })); - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)).Times(0); - expectIoResult({Network::PostIoAction::Close, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ("", client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); + EXPECT_CALL(client_.callbacks_, raiseEvent(Envoy::Network::ConnectionEvent::Connected)); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doRead(client_.read_buffer_), + "While reading ServerInit and ServerFinished."); + EXPECT_EQ(client_.read_buffer_.length(), 0L); + EXPECT_EQ(client_to_server_.length(), 0L); + + // Write application data before the server reads the ClientFinished. + EXPECT_CALL(*client_.raw_socket_, doWrite(_, _)) + .Times(2) + .WillRepeatedly(Invoke([&](Buffer::Instance& buffer, bool) { + Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; + client_to_server_.move(buffer); + return result; + })); + client_.write_buffer_.add(ApplicationData); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); + EXPECT_EQ(client_to_server_.length(), + ClientFinished.length() + AltsFrameOverhead + ApplicationData.length()); + + // On the server side, read the ClientFinished. + EXPECT_CALL(*server_.raw_socket_, doRead(_)).Times(2); + EXPECT_CALL(server_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); + expectIoResult(server_.tsi_socket_->doRead(server_.read_buffer_), + {Envoy::Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + "While reading ClientFinished."); + EXPECT_EQ(server_.read_buffer_.toString(), ApplicationData); } -TEST_F(TsiSocketTest, HandshakeWithReadError) { - initialize(nullptr, nullptr); +TEST_P(TsiSocketTest, HandshakeErrorButStreamIsKeptAlive) { + startErrorHandshakerService(/*keep_stream_alive=*/true); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + client_.tsi_socket_->onConnected(); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ""); +} - InSequence s; +TEST_P(TsiSocketTest, HandshakeErrorButStreamIsNotKeptAlive) { + startErrorHandshakerService(/*keep_stream_alive=*/false); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); + client_.tsi_socket_->onConnected(); + expectIoResult({Envoy::Network::PostIoAction::KeepOpen, 0UL, false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, /*end_stream=*/false), + "While writing ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ""); +} - doFakeInitHandshake(); +TEST_P(TsiSocketTest, HandshakeWithImmediateReadError) { + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); EXPECT_CALL(*client_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { Network::IoResult result = {Network::PostIoAction::Close, server_to_client_.length(), false}; @@ -422,55 +782,24 @@ TEST_F(TsiSocketTest, HandshakeWithReadError) { return result; })); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)).Times(0); - EXPECT_CALL(client_.callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush, _)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ("", client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); -} - -TEST_F(TsiSocketTest, HandshakeWithInternalError) { - auto raw_handshaker = tsi_create_fake_handshaker(/* is_client= */ 1); - const tsi_handshaker_vtable* vtable = raw_handshaker->vtable; - tsi_handshaker_vtable mock_vtable = *vtable; - mock_vtable.next = [](tsi_handshaker*, const unsigned char*, size_t, const unsigned char**, - size_t*, tsi_handshaker_result**, tsi_handshaker_on_next_done_cb, void*, - std::string*) { return TSI_INTERNAL_ERROR; }; - raw_handshaker->vtable = &mock_vtable; - - client_.handshaker_factory_ = [&](Event::Dispatcher& dispatcher, - const Network::Address::InstanceConstSharedPtr&, - const Network::Address::InstanceConstSharedPtr&) { - CHandshakerPtr handshaker{raw_handshaker}; - - return std::make_unique(std::move(handshaker), dispatcher); - }; - - initialize(nullptr, nullptr); - - InSequence s; - - EXPECT_CALL(client_.callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush, _)); - // doWrite won't immediately fail, but it will result connection close. - client_.tsi_socket_->onConnected(); - - raw_handshaker->vtable = vtable; + expectIoResult({Network::PostIoAction::Close, 0UL, false}, + client_.tsi_socket_->doRead(client_.read_buffer_), + "While reading after sending ClientInit."); + EXPECT_EQ(client_to_server_.toString(), ""); + EXPECT_EQ(client_.read_buffer_.length(), 0L); } -TEST_F(TsiSocketTest, DoReadEndOfStream) { - initialize(nullptr, nullptr); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); +TEST_P(TsiSocketTest, DoReadEndOfStream) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); + client_.write_buffer_.add(ApplicationData); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); + expectIoResult({Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), true}; @@ -478,90 +807,23 @@ TEST_F(TsiSocketTest, DoReadEndOfStream) { return result; })); EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, true}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); + expectIoResult({Network::PostIoAction::KeepOpen, ApplicationData.size(), true}, + server_.tsi_socket_->doRead(server_.read_buffer_), + "While the server is reading application data."); + EXPECT_EQ(server_.read_buffer_.toString(), ApplicationData); } -TEST_F(TsiSocketTest, DoReadNoData) { - initialize(nullptr, nullptr); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); +TEST_P(TsiSocketTest, DoReadOnceError) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); + client_.write_buffer_.add(ApplicationData); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()); - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 0UL, false}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoReadTwiceError) { - initialize(nullptr, nullptr); - - client_.write_buffer_.add(ClientToServerData); - - InSequence s; - - doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()); - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::Close, 0UL, false}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::Close, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoReadOnceError) { - initialize(nullptr, nullptr); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); - - doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); + expectIoResult({Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { Network::IoResult result = {Network::PostIoAction::Close, client_to_server_.length(), false}; @@ -569,52 +831,24 @@ TEST_F(TsiSocketTest, DoReadOnceError) { return result; })); EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()); - expectIoResult({Network::PostIoAction::Close, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); + expectIoResult({Network::PostIoAction::Close, ApplicationData.size(), false}, + server_.tsi_socket_->doRead(server_.read_buffer_), + "While the server is reading application data."); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); + EXPECT_EQ(server_.read_buffer_.toString(), ApplicationData); } -TEST_F(TsiSocketTest, DoReadDrainBuffer) { - initialize(nullptr, nullptr); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); +TEST_P(TsiSocketTest, DoReadDrainBuffer) { + startFakeHandshakerService(); + initializeSockets(/*server_validator=*/nullptr, /*client_validator=*/nullptr); doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()).WillOnce(Return(true)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoReadDrainBufferTwice) { - initialize(nullptr, nullptr); - - InSequence s; - - client_.write_buffer_.add(ClientToServerData); - - doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); + client_.write_buffer_.add(ApplicationData); EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); + expectIoResult({Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + client_.tsi_socket_->doWrite(client_.write_buffer_, false), + "While the client is writing application data."); EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), false}; @@ -622,305 +856,10 @@ TEST_F(TsiSocketTest, DoReadDrainBufferTwice) { return result; })); EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()).WillOnce(Return(true)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); - - // Client sends data again. - server_.read_buffer_.drain(server_.read_buffer_.length()); - client_.write_buffer_.add(ClientToServerData); - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - EXPECT_CALL(server_.callbacks_, shouldDrainReadBuffer()); - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 0UL, false}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoWriteSmallFrameSize) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - client_.write_buffer_.add(ClientToServerData); - - doHandshakeAndExpectSuccess(); - - EXPECT_EQ(0L, server_.read_buffer_.length()); - EXPECT_EQ(0L, client_.read_buffer_.length()); - - EXPECT_EQ("", client_.tsi_socket_->protocol()); - client_.tsi_socket_->setActualFrameSizeToUse(SmallFrameSize); - // Since we use a small frame size, original data is divided into two parts, - // and written to network in two iterations. - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; - client_to_server_.move(buffer); - return result; - })); - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; - client_to_server_.move(buffer); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_EQ(makeFakeTsiFrame(ClientToServerDataFirstHalf) + - makeFakeTsiFrame(ClientToServerDataSecondHalf), - client_to_server_.toString()); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::Close, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::Close, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoWriteSingleShortWrite) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - client_.write_buffer_.add(ClientToServerData); - - doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - - // Write the whole data except for the last byte. - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length() - 1, false}; - client_to_server_.add(buffer.linearize(0), buffer.length() - 1); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_EQ(makeFakeTsiFrame(ClientToServerData).substr(0, 20), client_to_server_.toString()); - - // TSI frame is invalid, return with Close action. - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::Close, 20UL, true}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::Close, 0UL, true}, - server_.tsi_socket_->doRead(server_.read_buffer_)); -} - -TEST_F(TsiSocketTest, DoWriteMultipleShortWrites) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - client_.write_buffer_.add(ClientToServerData); - - doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - - // Write the whole data except for the last byte. - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length() - 1, false}; - client_to_server_.add(buffer.linearize(0), buffer.length() - 1); - buffer.drain(buffer.length() - 1); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - - EXPECT_EQ(makeFakeTsiFrame(ClientToServerData).substr(0, 20), client_to_server_.toString()); - - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 1, false}; - client_to_server_.add(buffer.linearize(buffer.length() - 1), 1); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame(ClientToServerData), client_to_server_.toString()); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::Close, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::Close, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoWriteMixShortFullWrites) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - client_.write_buffer_.add(ClientToServerData); - - doHandshakeAndExpectSuccess(); - - client_.tsi_socket_->setActualFrameSizeToUse(SmallFrameSize); - - // Short write occurred when writing the first half of data. - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length() - 1, false}; - client_to_server_.add(buffer.linearize(0), buffer.length() - 1); - buffer.drain(buffer.length() - 1); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame(ClientToServerDataFirstHalf).substr(0, 12), - client_to_server_.toString()); - - // In the next write, we first finish the remaining data that has not been - // written in the previous write and then write the second half of data. - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 1, false}; - client_to_server_.move(buffer); - return result; - })); - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length(), false}; - client_to_server_.move(buffer); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doWrite(client_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame(ClientToServerDataFirstHalf) + - makeFakeTsiFrame(ClientToServerDataSecondHalf), - client_to_server_.toString()); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)).WillOnce(Invoke([&](Buffer::Instance& buffer) { - Network::IoResult result = {Network::PostIoAction::Close, client_to_server_.length(), false}; - buffer.move(client_to_server_); - return result; - })); - expectIoResult({Network::PostIoAction::Close, 17UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - EXPECT_EQ(ClientToServerData, server_.read_buffer_.toString()); -} - -TEST_F(TsiSocketTest, DoWriteOutstandingHandshakeData) { - auto validator = [](const tsi_peer&, TsiInfo&, std::string&) { return true; }; - initialize(validator, validator); - - InSequence s; - doFakeInitHandshake(); - - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - EXPECT_CALL(*client_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ(makeFakeTsiFrame("CLIENT_FINISHED"), client_to_server_.toString()); - EXPECT_EQ(0L, client_.read_buffer_.length()); - - EXPECT_CALL(*server_.raw_socket_, doRead(_)); - - // Write the first part of handshake data (14 bytes). - EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, buffer.length() - 5, false}; - server_to_client_.move(buffer, 14); - return result; - })); - EXPECT_CALL(server_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*server_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - server_.tsi_socket_->doRead(server_.read_buffer_)); - - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED").length(), 19); - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED").substr(0, 14), server_to_client_.toString()); - - server_.write_buffer_.add(ServerToClientData); - server_.tsi_socket_->setActualFrameSizeToUse(LargeFrameSize); - - // Write the second part of handshake data (4 bytes). - EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 4, false}; - server_to_client_.move(buffer, 4); - return result; - })); - expectIoResult({Network::PostIoAction::KeepOpen, 0UL, false}, - server_.tsi_socket_->doWrite(server_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED").substr(0, 18), server_to_client_.toString()); - - // Write the last part of handshake data (1 byte) and frame data. - EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - Network::IoResult result = {Network::PostIoAction::KeepOpen, 1, false}; - server_to_client_.move(buffer); - return result; - })); - EXPECT_CALL(*server_.raw_socket_, doWrite(_, false)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - server_.tsi_socket_->doWrite(server_.write_buffer_, false)); - EXPECT_EQ(makeFakeTsiFrame("SERVER_FINISHED") + makeFakeTsiFrame(ServerToClientData), - server_to_client_.toString()); - - // Check client side (handshake completes + receive unused data). - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - EXPECT_CALL(client_.callbacks_, raiseEvent(Network::ConnectionEvent::Connected)); - EXPECT_CALL(*client_.raw_socket_, doRead(_)); - expectIoResult({Network::PostIoAction::KeepOpen, 17UL, false}, - client_.tsi_socket_->doRead(client_.read_buffer_)); - EXPECT_EQ(ServerToClientData, client_.read_buffer_.toString()); -} - -class TsiSocketFactoryTest : public testing::Test { -protected: - void SetUp() override { - auto handshaker_factory = [](Event::Dispatcher& dispatcher, - const Network::Address::InstanceConstSharedPtr&, - const Network::Address::InstanceConstSharedPtr&) { - CHandshakerPtr handshaker{tsi_create_fake_handshaker(/*is_client=*/0)}; - - return std::make_unique(std::move(handshaker), dispatcher); - }; - - socket_factory_ = std::make_unique(handshaker_factory, nullptr); - } - Network::UpstreamTransportSocketFactoryPtr socket_factory_; -}; - -TEST_F(TsiSocketFactoryTest, CreateTransportSocket) { - EXPECT_NE(nullptr, socket_factory_->createTransportSocket(nullptr, nullptr)); -} - -TEST_F(TsiSocketFactoryTest, ImplementsSecureTransport) { - EXPECT_TRUE(socket_factory_->implementsSecureTransport()); -} - -TEST_F(TsiSocketFactoryTest, HashKey) { - std::vector key; - socket_factory_->hashKey(key, nullptr); - EXPECT_EQ(0, key.size()); + expectIoResult({Network::PostIoAction::KeepOpen, ApplicationData.size(), false}, + server_.tsi_socket_->doRead(server_.read_buffer_), + "While the server is reading application data."); + EXPECT_EQ(server_.read_buffer_.toString(), ApplicationData); } } // namespace diff --git a/test/extensions/transport_sockets/http_11_proxy/connect_test.cc b/test/extensions/transport_sockets/http_11_proxy/connect_test.cc index 37b2340235e5..37b63d8817c7 100644 --- a/test/extensions/transport_sockets/http_11_proxy/connect_test.cc +++ b/test/extensions/transport_sockets/http_11_proxy/connect_test.cc @@ -91,7 +91,7 @@ TEST_P(Http11ConnectTest, InjectsHeaderOnlyOnce) { .WillOnce(Invoke([&](Buffer::Instance& buffer) { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length, Api::IoError::none()); })); Buffer::OwnedImpl msg("initial data"); @@ -178,15 +178,13 @@ TEST_P(Http11ConnectTest, ReturnsKeepOpenWhenWriteErrorIsAgain) { InSequence s; EXPECT_CALL(io_handle_, write(BufferStringEqual(connect_data_.toString()))) .WillOnce(Invoke([&](Buffer::Instance&) { - return Api::IoCallUint64Result( - 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)); + return Api::IoCallUint64Result(0, Network::IoSocketError::getIoSocketEagainError()); })); EXPECT_CALL(io_handle_, write(BufferStringEqual(connect_data_.toString()))) .WillOnce(Invoke([&](Buffer::Instance& buffer) { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(length, Api::IoError::none()); })); } @@ -204,8 +202,7 @@ TEST_P(Http11ConnectTest, ReturnsCloseWhenWriteErrorIsNotAgain) { { InSequence s; EXPECT_CALL(io_handle_, write(_)).WillOnce(Invoke([&](Buffer::Instance&) { - return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(EADDRNOTAVAIL), - Network::IoSocketError::deleteIoError)); + return Api::IoCallUint64Result(0, Network::IoSocketError::create(EADDRNOTAVAIL)); })); } @@ -222,15 +219,13 @@ TEST_P(Http11ConnectTest, StipsHeaderOnce) { EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)) .WillOnce(Invoke([&initial_data](void* buffer, size_t, int) { memcpy(buffer, initial_data.data(), initial_data.length()); - return Api::IoCallUint64Result(initial_data.length(), - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(initial_data.length(), Api::IoError::none()); })); absl::optional expected_bytes(connect.length()); EXPECT_CALL(io_handle_, read(_, expected_bytes)) .WillOnce(Invoke([&](Buffer::Instance& buffer, absl::optional) { buffer.add(connect); - return Api::IoCallUint64Result(connect.length(), - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(connect.length(), Api::IoError::none()); })); EXPECT_CALL(*inner_socket_, doRead(_)) .WillOnce(Return(Network::IoResult{Network::PostIoAction::KeepOpen, 1, false})); @@ -247,8 +242,7 @@ TEST_P(Http11ConnectTest, InsufficientData) { EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)) .WillOnce(Invoke([&initial_data](void* buffer, size_t, int) { memcpy(buffer, initial_data.data(), initial_data.length()); - return Api::IoCallUint64Result(initial_data.length(), - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(initial_data.length(), Api::IoError::none()); })); absl::optional expected_bytes(connect.length()); Buffer::OwnedImpl buffer(""); @@ -262,9 +256,8 @@ TEST_P(Http11ConnectTest, PeekFail) { std::string connect("HTTP/1.1 200 OK\r\n\r\n"); std::string initial_data(connect + "follow up data"); EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)) - .WillOnce(Return(ByMove( - Api::IoCallUint64Result({}, Api::IoErrorPtr(new Network::IoSocketError(EADDRNOTAVAIL), - Network::IoSocketError::deleteIoError))))); + .WillOnce(Return( + ByMove(Api::IoCallUint64Result({}, Network::IoSocketError::create(EADDRNOTAVAIL))))); EXPECT_CALL(io_handle_, read(_, _)).Times(0); EXPECT_CALL(*inner_socket_, doRead(_)).Times(0); @@ -282,14 +275,12 @@ TEST_P(Http11ConnectTest, ReadFail) { EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)) .WillOnce(Invoke([&initial_data](void* buffer, size_t, int) { memcpy(buffer, initial_data.data(), initial_data.length()); - return Api::IoCallUint64Result(initial_data.length(), - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(initial_data.length(), Api::IoError::none()); })); absl::optional expected_bytes(connect.length()); EXPECT_CALL(io_handle_, read(_, expected_bytes)) .WillOnce(Return(ByMove(Api::IoCallUint64Result( - connect.length(), Api::IoErrorPtr(new Network::IoSocketError(EADDRNOTAVAIL), - Network::IoSocketError::deleteIoError))))); + connect.length(), Network::IoSocketError::create(EADDRNOTAVAIL))))); EXPECT_CALL(*inner_socket_, doRead(_)).Times(0); Buffer::OwnedImpl buffer(""); @@ -306,13 +297,12 @@ TEST_P(Http11ConnectTest, ShortRead) { EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)) .WillOnce(Invoke([&initial_data](void* buffer, size_t, int) { memcpy(buffer, initial_data.data(), initial_data.length()); - return Api::IoCallUint64Result(initial_data.length(), - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(initial_data.length(), Api::IoError::none()); })); absl::optional expected_bytes(connect.length()); EXPECT_CALL(io_handle_, read(_, expected_bytes)) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - connect.length() - 1, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce( + Return(ByMove(Api::IoCallUint64Result(connect.length() - 1, Api::IoError::none())))); EXPECT_CALL(*inner_socket_, doRead(_)).Times(0); Buffer::OwnedImpl buffer(""); @@ -326,7 +316,7 @@ TEST_P(Http11ConnectTest, LongHeaders) { EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)).WillOnce(Invoke([](void* buffer, size_t, int) { memset(buffer, 0, 2000); - return Api::IoCallUint64Result(2000, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(2000, Api::IoError::none()); })); EXPECT_CALL(io_handle_, read(_, _)).Times(0); EXPECT_CALL(*inner_socket_, doRead(_)).Times(0); @@ -347,8 +337,7 @@ TEST_P(Http11ConnectTest, InvalidResponse) { EXPECT_CALL(io_handle_, recv(_, 2000, MSG_PEEK)) .WillOnce(Invoke([&initial_data](void* buffer, size_t, int) { memcpy(buffer, initial_data.data(), initial_data.length()); - return Api::IoCallUint64Result(initial_data.length(), - Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(initial_data.length(), Api::IoError::none()); })); EXPECT_CALL(*inner_socket_, doRead(_)).Times(0); diff --git a/test/extensions/transport_sockets/internal_upstream/BUILD b/test/extensions/transport_sockets/internal_upstream/BUILD index 9d0ebfca9b77..b262ab6a6b54 100644 --- a/test/extensions/transport_sockets/internal_upstream/BUILD +++ b/test/extensions/transport_sockets/internal_upstream/BUILD @@ -37,18 +37,22 @@ envoy_extension_cc_test( srcs = [ "internal_upstream_integration_test.cc", ], - extension_names = ["envoy.bootstrap.internal_listener"], + extension_names = [ + "envoy.bootstrap.internal_listener", + "envoy.filters.http.set_filter_state", + ], deps = [ "//source/common/network:connection_lib", "//source/common/network:utility_lib", + "//source/common/router:string_accessor_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/bootstrap/internal_listener:config", + "//source/extensions/filters/http/set_filter_state:config", "//source/extensions/filters/network/tcp_proxy:config", "//source/extensions/io_socket/user_space:config", "//source/extensions/transport_sockets/internal_upstream:config", "//source/extensions/transport_sockets/raw_buffer:config", "//test/integration:http_integration_lib", - "//test/integration/filters:header_to_filter_state_lib", "//test/test_common:network_utility_lib", "//test/test_common:resources_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/transport_sockets/internal_upstream/internal_upstream_integration_test.cc b/test/extensions/transport_sockets/internal_upstream/internal_upstream_integration_test.cc index f9a0a12db9cc..9b4276ac5a64 100644 --- a/test/extensions/transport_sockets/internal_upstream/internal_upstream_integration_test.cc +++ b/test/extensions/transport_sockets/internal_upstream/internal_upstream_integration_test.cc @@ -4,6 +4,7 @@ #include "envoy/extensions/transport_sockets/internal_upstream/v3/internal_upstream.pb.h" #include "envoy/network/connection.h" +#include "source/common/router/string_accessor_impl.h" #include "source/extensions/transport_sockets/internal_upstream/config.h" #include "test/integration/http_integration.h" @@ -17,6 +18,27 @@ namespace Envoy { namespace { +class TestObject1Factory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "internal_state"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +class TestObject2Factory : public StreamInfo::FilterState::ObjectFactory { +public: + std::string name() const override { return "internal_state_once"; } + std::unique_ptr + createFromBytes(absl::string_view data) const override { + return std::make_unique(data); + } +}; + +REGISTER_FACTORY(TestObject1Factory, StreamInfo::FilterState::ObjectFactory); +REGISTER_FACTORY(TestObject2Factory, StreamInfo::FilterState::ObjectFactory); + class InternalUpstreamIntegrationTest : public testing::Test, public HttpIntegrationTest { public: InternalUpstreamIntegrationTest() @@ -38,18 +60,24 @@ class InternalUpstreamIntegrationTest : public testing::Test, public HttpIntegra config_helper_.prependFilter(R"EOF( name: header-to-filter-state typed_config: - "@type": type.googleapis.com/test.integration.filters.HeaderToFilterStateFilterConfig - header_name: internal-header - state_name: internal_state - shared: TRANSITIVE + "@type": type.googleapis.com/envoy.extensions.filters.http.set_filter_state.v3.Config + on_request_headers: + - object_key: internal_state + format_string: + text_format_source: + inline_string: "%REQ(internal-header)%" + shared_with_upstream: TRANSITIVE )EOF"); config_helper_.prependFilter(R"EOF( name: header-to-filter-state typed_config: - "@type": type.googleapis.com/test.integration.filters.HeaderToFilterStateFilterConfig - header_name: internal-header-once - state_name: internal_state_once - shared: ONCE + "@type": type.googleapis.com/envoy.extensions.filters.http.set_filter_state.v3.Config + on_request_headers: + - object_key: internal_state_once + format_string: + text_format_source: + inline_string: "%REQ(internal-header-once)%" + shared_with_upstream: ONCE )EOF"); config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { setupBootstrapExtension(bootstrap); diff --git a/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_test.cc b/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_test.cc index 53d1646b50f9..9f1a7c3bd656 100644 --- a/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/transport_sockets/proxy_protocol/proxy_protocol_test.cc @@ -70,7 +70,7 @@ TEST_F(ProxyProtocolTest, InjectesHeaderOnlyOnce) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); auto msg2 = Buffer::OwnedImpl("more data"); @@ -102,7 +102,7 @@ TEST_F(ProxyProtocolTest, BytesProcessedIncludesProxyProtocolHeader) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); auto msg2 = Buffer::OwnedImpl("more data"); @@ -138,15 +138,13 @@ TEST_F(ProxyProtocolTest, ReturnsKeepOpenWhenWriteErrorIsAgain) { InSequence s; EXPECT_CALL(io_handle_, write(BufferStringEqual(expected_buff.toString()))) .WillOnce(Invoke([&](Buffer::Instance&) -> Api::IoCallUint64Result { - return Api::IoCallUint64Result( - 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)); + return {0, Network::IoSocketError::getIoSocketEagainError()}; })); EXPECT_CALL(io_handle_, write(BufferStringEqual(expected_buff.toString()))) .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)) .WillOnce(Return(Network::IoResult{Network::PostIoAction::KeepOpen, msg.length(), false})); @@ -176,9 +174,7 @@ TEST_F(ProxyProtocolTest, ReturnsCloseWhenWriteErrorIsNotAgain) { InSequence s; EXPECT_CALL(io_handle_, write(_)) .WillOnce(Invoke([&](Buffer::Instance&) -> Api::IoCallUint64Result { - return Api::IoCallUint64Result(0, - Api::IoErrorPtr(new Network::IoSocketError(EADDRNOTAVAIL), - Network::IoSocketError::deleteIoError)); + return {0, Network::IoSocketError::create(EADDRNOTAVAIL)}; })); } @@ -203,7 +199,7 @@ TEST_F(ProxyProtocolTest, V1IPV4LocalAddressWhenTransportOptionsAreNull) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -228,7 +224,7 @@ TEST_F(ProxyProtocolTest, V1IPV4LocalAddressesWhenHeaderOptionsAreNull) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = 43; buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {static_cast(length), Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -253,7 +249,7 @@ TEST_F(ProxyProtocolTest, V1IPV6LocalAddressesWhenHeaderOptionsAreNull) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -287,7 +283,7 @@ TEST_F(ProxyProtocolTest, V1IPV4DownstreamAddresses) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -321,7 +317,7 @@ TEST_F(ProxyProtocolTest, V1IPV6DownstreamAddresses) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -346,7 +342,7 @@ TEST_F(ProxyProtocolTest, V2IPV4LocalCommandWhenTransportOptionsAreNull) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -371,7 +367,7 @@ TEST_F(ProxyProtocolTest, V2IPV4LocalCommandWhenHeaderOptionsAreNull) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -405,7 +401,7 @@ TEST_F(ProxyProtocolTest, V2IPV4DownstreamAddresses) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -440,7 +436,7 @@ TEST_F(ProxyProtocolTest, V2IPV6DownstreamAddresses) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -503,7 +499,7 @@ TEST_F(ProxyProtocolTest, V2IPV4DownstreamAddressesAndTLVs) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -543,7 +539,7 @@ TEST_F(ProxyProtocolTest, V2IPV4PassSpecificTLVs) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -581,7 +577,7 @@ TEST_F(ProxyProtocolTest, V2IPV4PassEmptyTLVs) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -649,7 +645,7 @@ TEST_F(ProxyProtocolTest, V2IPV6DownstreamAddressesAndTLVs) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); @@ -687,7 +683,7 @@ TEST_F(ProxyProtocolTest, V2IPV6DownstreamAddressesAndTLVsWithoutPassConfig) { .WillOnce(Invoke([&](Buffer::Instance& buffer) -> Api::IoCallUint64Result { auto length = buffer.length(); buffer.drain(length); - return Api::IoCallUint64Result(length, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return {length, Api::IoError::none()}; })); auto msg = Buffer::OwnedImpl("some data"); EXPECT_CALL(*inner_socket_, doWrite(BufferEqual(&msg), false)); diff --git a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc index fe298d275640..dee7ca351132 100644 --- a/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc +++ b/test/extensions/transport_sockets/starttls/upstream_starttls_integration_test.cc @@ -93,7 +93,7 @@ class StartTlsSwitchFilter : public Network::Filter { Network::ReadFilterCallbacks* read_callbacks_{}; }; - // Helper read filter inserted into downstream filter chain. The filter reacts to + // Helper read filter inserted into downstream network filter chain. The filter reacts to // SwitchViaFilterManager string and signals to the filter manager to signal to the terminal // filter to switch upstream connection to secure mode. struct DownstreamReadFilter : public Network::ReadFilter { @@ -180,8 +180,8 @@ class StartTlsSwitchFilterConfigFactory : public Extensions::NetworkFilters::Com // Inject two filters into downstream connection: first is helper read filter and then // terminal filter. filter_manager.addReadFilter(std::make_shared()); - filter_manager.addReadFilter( - StartTlsSwitchFilter::newInstance(context.clusterManager(), upstream_callbacks_)); + filter_manager.addReadFilter(StartTlsSwitchFilter::newInstance( + context.serverFactoryContext().clusterManager(), upstream_callbacks_)); }; } diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 2324f740dba9..7ba751cbf0f2 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -11,11 +11,15 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_test_library( + name = "ssl_certs_test_lib", + hdrs = ["ssl_certs_test.h"], +) + envoy_cc_test( name = "ssl_socket_test", size = "large", srcs = [ - "ssl_certs_test.h", "ssl_socket_test.cc", ], data = [ @@ -27,6 +31,7 @@ envoy_cc_test( ], external_deps = ["ssl"], deps = [ + ":ssl_certs_test_lib", ":test_private_key_method_provider_test_lib", "//envoy/network:transport_socket_interface", "//source/common/buffer:buffer_lib", @@ -72,13 +77,13 @@ envoy_cc_test( name = "context_impl_test", srcs = [ "context_impl_test.cc", - "ssl_certs_test.h", ], data = [ "//test/extensions/transport_sockets/tls/ocsp/test_data:certs", "//test/extensions/transport_sockets/tls/test_data:certs", ], deps = [ + "ssl_certs_test_lib", ":ssl_test_utils", "//source/common/common:base64_lib", "//source/common/json:json_loader_lib", @@ -132,7 +137,7 @@ envoy_cc_test( envoy_cc_test_library( name = "ssl_test_utils", - srcs = [ + hdrs = [ "ssl_test_utility.h", ], deps = [ diff --git a/test/extensions/transport_sockets/tls/cert_validator/BUILD b/test/extensions/transport_sockets/tls/cert_validator/BUILD index 8dd2de7a9bb6..0a6f835d2c9d 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/BUILD +++ b/test/extensions/transport_sockets/tls/cert_validator/BUILD @@ -61,18 +61,23 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "default_validator_integration_test_lib", + hdrs = ["default_validator_integration_test.h"], +) + envoy_cc_test( name = "cert_validator_integration_test", size = "large", srcs = [ "default_validator_integration_test.cc", - "default_validator_integration_test.h", ], data = [ "//test/config/integration/certs", "//test/extensions/transport_sockets/tls/test_data:certs", ], deps = [ + ":default_validator_integration_test_lib", "//test/integration:http_integration_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD b/test/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD index 084c9a87d1cc..470fae0034f0 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD +++ b/test/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", "envoy_package", ) load( @@ -31,13 +32,15 @@ envoy_extension_cc_test( ], ) +envoy_cc_test_library( + name = "spiffe_validator_integration_test_lib", + hdrs = ["spiffe_validator_integration_test.h"], +) + envoy_extension_cc_test( name = "spiffe_validator_integration_test", size = "large", - srcs = [ - "spiffe_validator_integration_test.cc", - "spiffe_validator_integration_test.h", - ], + srcs = ["spiffe_validator_integration_test.cc"], data = [ "//test/config/integration/certs", "//test/extensions/transport_sockets/tls/test_data:certs", @@ -46,6 +49,7 @@ envoy_extension_cc_test( # Broken until bazel 5.0.0 fix to shorten resulting paths for SymInitialize() failure tags = ["skip_on_windows"], deps = [ + ":spiffe_validator_integration_test_lib", "//source/extensions/transport_sockets/tls/cert_validator/spiffe:config", "//test/integration:http_integration_lib", "//test/test_common:utility_lib", diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index a01f08a050ab..11a441051229 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -98,18 +98,18 @@ TEST_P(SslLibraryCipherSuiteSupport, CipherSuitesNotRemoved) { class SslContextImplTest : public SslCertsTest { public: ABSL_MUST_USE_RESULT Cleanup cleanUpHelper(Envoy::Ssl::ClientContextSharedPtr& context) { - return Cleanup([&manager = manager_, &context]() { + return {[&manager = manager_, &context]() { if (context != nullptr) { manager.removeContext(context); } - }); + }}; } ABSL_MUST_USE_RESULT Cleanup cleanUpHelper(Envoy::Ssl::ServerContextSharedPtr& context) { - return Cleanup([&manager = manager_, &context]() { + return {[&manager = manager_, &context]() { if (context != nullptr) { manager.removeContext(context); } - }); + }}; } void loadConfig(ServerContextConfigImpl& cfg) { Envoy::Ssl::ServerContextSharedPtr server_ctx( @@ -989,7 +989,7 @@ name: "abc.com" )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; envoy::extensions::transport_sockets::tls::v3::TlsCertificate* server_cert = @@ -1149,11 +1149,11 @@ TEST_F(SslServerContextImplTicketTest, StatelessSessionResumptionEnabledWhenKeyI class ClientContextConfigImplTest : public SslCertsTest { public: ABSL_MUST_USE_RESULT Cleanup cleanUpHelper(Envoy::Ssl::ClientContextSharedPtr& context) { - return Cleanup([&manager = manager_, &context]() { + return {[&manager = manager_, &context]() { if (context != nullptr) { manager.removeContext(context); } - }); + }}; } Event::SimulatedTimeSystem time_system_; @@ -1478,7 +1478,7 @@ name: "abc.com" ->Add() ->set_name("abc.com"); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); ClientContextConfigImpl client_context_config(tls_context, factory_context_); const std::string cert_pem = @@ -1514,7 +1514,7 @@ TEST_F(ClientContextConfigImplTest, PasswordProtectedTlsCertificates) { ->Add() ->set_name("abc.com"); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); ClientContextConfigImpl client_context_config(tls_context, factory_context_); const std::string cert_pem = @@ -1554,7 +1554,7 @@ TEST_F(ClientContextConfigImplTest, PasswordProtectedPkcs12) { ->Add() ->set_name("abc.com"); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); ClientContextConfigImpl client_context_config(tls_context, factory_context_); const std::string cert_p12 = @@ -1588,7 +1588,7 @@ TEST_F(ClientContextConfigImplTest, PasswordWrongPkcs12) { ->Add() ->set_name("abc.com"); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); ClientContextConfigImpl client_context_config(tls_context, factory_context_); Stats::IsolatedStoreImpl store; @@ -1616,7 +1616,7 @@ TEST_F(ClientContextConfigImplTest, PasswordNotSuppliedPkcs12) { ->Add() ->set_name("abc.com"); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); ClientContextConfigImpl client_context_config(tls_context, factory_context_); Stats::IsolatedStoreImpl store; @@ -1647,7 +1647,7 @@ TEST_F(ClientContextConfigImplTest, PasswordNotSuppliedTlsCertificates) { ->Add() ->set_name("abc.com"); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); ClientContextConfigImpl client_context_config(tls_context, factory_context_); Stats::IsolatedStoreImpl store; @@ -1670,7 +1670,7 @@ TEST_F(ClientContextConfigImplTest, StaticCertificateValidationContext) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(tls_certificate_yaml), tls_certificate_secret_config); - factory_context_.secretManager().addStaticSecret(tls_certificate_secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(tls_certificate_secret_config).ok()); envoy::extensions::transport_sockets::tls::v3::Secret certificate_validation_context_secret_config; const std::string certificate_validation_context_yaml = R"EOF( @@ -1681,7 +1681,9 @@ TEST_F(ClientContextConfigImplTest, StaticCertificateValidationContext) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(certificate_validation_context_yaml), certificate_validation_context_secret_config); - factory_context_.secretManager().addStaticSecret(certificate_validation_context_secret_config); + EXPECT_TRUE(factory_context_.secretManager() + .addStaticSecret(certificate_validation_context_secret_config) + .ok()); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; tls_context.mutable_common_tls_context() @@ -1715,7 +1717,7 @@ name: "abc.com" TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - factory_context_.secretManager().addStaticSecret(secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(secret_config).ok()); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; tls_context.mutable_common_tls_context() @@ -1742,7 +1744,7 @@ TEST_F(ClientContextConfigImplTest, MissingStaticCertificateValidationContext) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(tls_certificate_yaml), tls_certificate_secret_config); - factory_context_.secretManager().addStaticSecret(tls_certificate_secret_config); + EXPECT_TRUE(factory_context_.secretManager().addStaticSecret(tls_certificate_secret_config).ok()); envoy::extensions::transport_sockets::tls::v3::Secret certificate_validation_context_secret_config; const std::string certificate_validation_context_yaml = R"EOF( @@ -1753,7 +1755,9 @@ TEST_F(ClientContextConfigImplTest, MissingStaticCertificateValidationContext) { )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(certificate_validation_context_yaml), certificate_validation_context_secret_config); - factory_context_.secretManager().addStaticSecret(certificate_validation_context_secret_config); + EXPECT_TRUE(factory_context_.secretManager() + .addStaticSecret(certificate_validation_context_secret_config) + .ok()); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; tls_context.mutable_common_tls_context() diff --git a/test/extensions/transport_sockets/tls/integration/BUILD b/test/extensions/transport_sockets/tls/integration/BUILD index 195733497745..37013e4df2e4 100644 --- a/test/extensions/transport_sockets/tls/integration/BUILD +++ b/test/extensions/transport_sockets/tls/integration/BUILD @@ -10,18 +10,23 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_test_library( + name = "ssl_integration_test_lib", + hdrs = ["ssl_integration_test.h"], +) + envoy_cc_test( name = "ssl_integration_test", size = "large", srcs = [ "ssl_integration_test.cc", - "ssl_integration_test.h", ], data = [ "//test/config/integration/certs", ], deps = [ ":sni_to_header_filter_lib", + ":ssl_integration_test_lib", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", "//source/common/network:connection_lib", @@ -35,6 +40,7 @@ envoy_cc_test( "//test/integration:http_integration_lib", "//test/integration/filters:stream_info_to_headers_filter_lib", "//test/mocks/secret:secret_mocks", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc index 3dd65c3089b8..b7548ba8ace7 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc @@ -25,6 +25,7 @@ #include "test/integration/utility.h" #include "test/test_common/network_utility.h" #include "test/test_common/registry.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "absl/strings/match.h" @@ -463,8 +464,8 @@ TEST_P(SslIntegrationTest, RouterHeaderOnlyRequestAndResponseWithSni) { TEST_P(SslIntegrationTest, AsyncCertValidationSucceeds) { // Config client to use an async cert validator which defer the actual validation by 5ms. - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: @@ -474,7 +475,7 @@ name: "envoy.tls.cert_validator.timed_cert_validator" initialize(); Network::ClientConnectionPtr connection = makeSslClientConnection( - ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config)); + ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config.get())); ConnectionStatusCallbacks callbacks; connection->addConnectionCallbacks(callbacks); connection->connect(); @@ -492,8 +493,8 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(SslIntegrationTest, AsyncCertValidationSucceedsWithLocalAddress) { - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: @@ -508,7 +509,7 @@ name: "envoy.tls.cert_validator.timed_cert_validator" initialize(); Network::Address::InstanceConstSharedPtr address = getSslAddress(version_, lookupPort("http")); auto client_transport_socket_factory_ptr = createClientSslTransportSocketFactory( - ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config), + ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config.get()), *context_manager_, *api_); Network::ClientConnectionPtr connection = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), @@ -545,8 +546,8 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(SslIntegrationTest, AsyncCertValidationAfterTearDown) { - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: @@ -563,7 +564,7 @@ name: "envoy.tls.cert_validator.timed_cert_validator" initialize(); Network::Address::InstanceConstSharedPtr address = getSslAddress(version_, lookupPort("http")); auto client_transport_socket_factory_ptr = createClientSslTransportSocketFactory( - ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config), + ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config.get()), *context_manager_, *api_); Network::ClientConnectionPtr connection = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), @@ -594,8 +595,8 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(SslIntegrationTest, AsyncCertValidationAfterSslShutdown) { - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: @@ -612,7 +613,7 @@ name: "envoy.tls.cert_validator.timed_cert_validator" initialize(); Network::Address::InstanceConstSharedPtr address = getSslAddress(version_, lookupPort("http")); auto client_transport_socket_factory_ptr = createClientSslTransportSocketFactory( - ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config), + ClientSslTransportOptions().setCustomCertValidatorConfig(custom_validator_config.get()), *context_manager_, *api_); Network::ClientConnectionPtr connection = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), @@ -861,6 +862,35 @@ TEST_P(SslCertficateIntegrationTest, ServerEcdsaClientRsaOnly) { // Server has only an ECDSA certificate, client is only RSA capable, leads to a connection fail. // Test the access log. TEST_P(SslCertficateIntegrationTest, ServerEcdsaClientRsaOnlyWithAccessLog) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ssl_transport_failure_reason_format", "true"}}); + useListenerAccessLog("DOWNSTREAM_TRANSPORT_FAILURE_REASON=%DOWNSTREAM_TRANSPORT_FAILURE_REASON% " + "FILTER_CHAIN_NAME=%FILTER_CHAIN_NAME%"); + server_rsa_cert_ = false; + server_ecdsa_cert_ = true; + initialize(); + auto codec_client = + makeRawHttpConnection(makeSslClientConnection(rsaOnlyClientOptions()), absl::nullopt); + EXPECT_FALSE(codec_client->connected()); + + auto log_result = waitForAccessLog(listener_access_log_name_); + if (tls_version_ == envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_3) { + EXPECT_THAT(log_result, + StartsWith("DOWNSTREAM_TRANSPORT_FAILURE_REASON=TLS_error:|268435709:SSL_routines:" + "OPENSSL_internal:NO_COMMON_SIGNATURE_ALGORITHMS:TLS_error_end")); + } else { + EXPECT_THAT(log_result, + StartsWith("DOWNSTREAM_TRANSPORT_FAILURE_REASON=TLS_error:|268435640:" + "SSL_routines:OPENSSL_internal:NO_SHARED_CIPHER:TLS_error_end")); + } +} + +// Server has only an ECDSA certificate, client is only RSA capable, leads to a connection fail. +TEST_P(SslCertficateIntegrationTest, ServerEcdsaClientRsaOnlyWithAccessLogOriginalFormat) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.ssl_transport_failure_reason_format", "false"}}); useListenerAccessLog("DOWNSTREAM_TRANSPORT_FAILURE_REASON=%DOWNSTREAM_TRANSPORT_FAILURE_REASON% " "FILTER_CHAIN_NAME=%FILTER_CHAIN_NAME%"); server_rsa_cert_ = false; diff --git a/test/extensions/transport_sockets/tls/io_handle_bio_test.cc b/test/extensions/transport_sockets/tls/io_handle_bio_test.cc index 36a2625314a2..0f744ae48879 100644 --- a/test/extensions/transport_sockets/tls/io_handle_bio_test.cc +++ b/test/extensions/transport_sockets/tls/io_handle_bio_test.cc @@ -27,9 +27,8 @@ class IoHandleBioTest : public testing::Test { TEST_F(IoHandleBioTest, WriteError) { EXPECT_CALL(io_handle_, writev(_, 1)) - .WillOnce(Return(testing::ByMove( - Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(100), - Network::IoSocketError::deleteIoError))))); + .WillOnce( + Return(testing::ByMove(Api::IoCallUint64Result(0, Network::IoSocketError::create(100))))); EXPECT_EQ(-1, bio_->method->bwrite(bio_, nullptr, 10)); const int err = ERR_get_error(); EXPECT_EQ(ERR_GET_LIB(err), ERR_LIB_SYS); @@ -56,8 +55,7 @@ TEST_F(IoHandleBioTest, TestMiscApis) { EXPECT_EQ(ret, 1); EXPECT_CALL(io_handle_, close()) - .WillOnce(Return(testing::ByMove(Api::IoCallUint64Result{ - 0, Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)}))); + .WillOnce(Return(testing::ByMove(Api::IoCallUint64Result{0, Api::IoError::none()}))); bio_->init = 1; } diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index f30a49954bda..ff73e213ff7e 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -13,6 +13,7 @@ #include "source/common/json/json_loader.h" #include "source/common/network/address_impl.h" #include "source/common/network/listen_socket_impl.h" +#include "source/common/network/tcp_listener_impl.h" #include "source/common/network/transport_socket_options_impl.h" #include "source/common/network/utility.h" #include "source/common/stream_info/stream_info_impl.h" @@ -125,9 +126,7 @@ class TestUtilOptions : public TestUtilOptionsBase { TestUtilOptions(const std::string& client_ctx_yaml, const std::string& server_ctx_yaml, bool expect_success, Network::Address::IpVersion version) : TestUtilOptionsBase(expect_success, version), client_ctx_yaml_(client_ctx_yaml), - server_ctx_yaml_(server_ctx_yaml), expect_no_cert_(false), expect_no_cert_chain_(false), - expect_private_key_method_(false), - expected_server_close_event_(Network::ConnectionEvent::RemoteClose) { + server_ctx_yaml_(server_ctx_yaml) { if (expect_success) { setExpectedServerStats("ssl.handshake"); } else { @@ -305,11 +304,11 @@ class TestUtilOptions : public TestUtilOptionsBase { const std::string client_ctx_yaml_; const std::string server_ctx_yaml_; - bool expect_no_cert_; - bool expect_no_cert_chain_; - bool expect_private_key_method_; + bool expect_no_cert_{false}; + bool expect_no_cert_chain_{false}; + bool expect_private_key_method_{false}; NiceMock runtime_; - Network::ConnectionEvent expected_server_close_event_; + Network::ConnectionEvent expected_server_close_event_{Network::ConnectionEvent::RemoteClose}; std::string expected_sha256_digest_; std::string expected_sha1_digest_; std::vector expected_local_uri_; @@ -329,6 +328,17 @@ class TestUtilOptions : public TestUtilOptionsBase { std::string expected_sni_; }; +Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, + Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state, + Random::RandomGenerator& rng, Event::Dispatcher& dispatcher) { + return std::make_unique( + dispatcher, rng, runtime, socket, cb, listener_config.bindToPort(), + listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); +} + void testUtil(const TestUtilOptions& options) { Event::SimulatedTimeSystem time_system; @@ -368,8 +378,10 @@ void testUtil(const TestUtilOptions& options) { Network::Test::getCanonicalLoopbackAddress(options.version())); Network::MockTcpListenerCallbacks callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher->createListener(socket, callbacks, runtime, listener_config); + createListener(socket, callbacks, runtime, listener_config, overload_state, + server_api->randomGenerator(), *dispatcher); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(options.clientCtxYaml()), @@ -725,8 +737,10 @@ void testUtilV2(const TestUtilOptionsV2& options) { Network::Test::getCanonicalLoopbackAddress(options.version())); NiceMock callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher->createListener(socket, callbacks, runtime, listener_config); + createListener(socket, callbacks, runtime, listener_config, overload_state, + server_api->randomGenerator(), *dispatcher); Stats::TestUtil::TestStore client_stats_store; Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); @@ -964,6 +978,17 @@ class SslSocketTest : public SslCertsTest, const std::string& client_ctx_yaml, bool expect_reuse, const Network::Address::IpVersion version); + Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, + Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state, + Event::Dispatcher& dispatcher) { + return std::make_unique( + dispatcher, api_->randomGenerator(), runtime, std::move(socket), cb, + listener_config.bindToPort(), listener_config.ignoreGlobalConnLimit(), + listener_config.maxConnectionsToAcceptPerSocketEvent(), overload_state); + } + NiceMock runtime_; Event::DispatcherPtr dispatcher_; StreamInfo::StreamInfoImpl stream_info_; @@ -3058,8 +3083,9 @@ TEST_P(SslSocketTest, FlushCloseDuringHandshake) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher_->createListener(socket, callbacks, runtime_, listener_config); + createListener(socket, callbacks, runtime_, listener_config, overload_state, *dispatcher_); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -3116,8 +3142,9 @@ TEST_P(SslSocketTest, HalfClose) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks listener_callbacks; NiceMock listener_config; - Network::ListenerPtr listener = - dispatcher_->createListener(socket, listener_callbacks, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + Network::ListenerPtr listener = createListener(socket, listener_callbacks, runtime_, + listener_config, overload_state, *dispatcher_); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -3200,8 +3227,9 @@ TEST_P(SslSocketTest, ShutdownWithCloseNotify) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks listener_callbacks; NiceMock listener_config; - Network::ListenerPtr listener = - dispatcher_->createListener(socket, listener_callbacks, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + Network::ListenerPtr listener = createListener(socket, listener_callbacks, runtime_, + listener_config, overload_state, *dispatcher_); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -3290,8 +3318,9 @@ TEST_P(SslSocketTest, ShutdownWithoutCloseNotify) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks listener_callbacks; NiceMock listener_config; - Network::ListenerPtr listener = - dispatcher_->createListener(socket, listener_callbacks, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + Network::ListenerPtr listener = createListener(socket, listener_callbacks, runtime_, + listener_config, overload_state, *dispatcher_); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -3396,8 +3425,9 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher_->createListener(socket, callbacks, runtime_, listener_config); + createListener(socket, callbacks, runtime_, listener_config, overload_state, *dispatcher_); const std::string client_ctx_yaml = R"EOF( common_tls_context: @@ -3498,10 +3528,13 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, NiceMock callbacks; Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener1 = - dispatcher->createListener(socket1, callbacks, runtime, listener_config); + createListener(socket1, callbacks, runtime, listener_config, overload_state, + server_api->randomGenerator(), *dispatcher); Network::ListenerPtr listener2 = - dispatcher->createListener(socket2, callbacks, runtime, listener_config); + createListener(socket2, callbacks, runtime, listener_config, overload_state, + server_api->randomGenerator(), *dispatcher); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context); @@ -3618,10 +3651,10 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, EXPECT_EQ(expect_reuse ? 1UL : 0UL, client_stats_store.counter("ssl.session_reused").value()); } -void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml, - const std::string& client_ctx_yaml, - bool expect_support, - const Network::Address::IpVersion ip_version) { +void testSupportForSessionResumption(const std::string& server_ctx_yaml, + const std::string& client_ctx_yaml, bool expect_stateless, + bool expect_stateful, + const Network::Address::IpVersion ip_version) { Event::SimulatedTimeSystem time_system; ContextManagerImpl manager(*time_system); @@ -3643,9 +3676,11 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml Network::Test::getCanonicalLoopbackAddress(ip_version)); NiceMock callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); Network::ListenerPtr listener = - dispatcher->createListener(tcp_socket, callbacks, runtime, listener_config); + createListener(tcp_socket, callbacks, runtime, listener_config, overload_state, + server_api->randomGenerator(), *dispatcher); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context); @@ -3681,11 +3716,17 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml dynamic_cast(server_connection->ssl().get()); SSL* server_ssl_socket = ssl_socket->ssl(); SSL_CTX* server_ssl_context = SSL_get_SSL_CTX(server_ssl_socket); - if (expect_support) { + if (expect_stateless) { EXPECT_EQ(0, (SSL_CTX_get_options(server_ssl_context) & SSL_OP_NO_TICKET)); } else { EXPECT_EQ(SSL_OP_NO_TICKET, (SSL_CTX_get_options(server_ssl_context) & SSL_OP_NO_TICKET)); } + if (expect_stateful) { + EXPECT_EQ(SSL_SESS_CACHE_SERVER, + (SSL_CTX_get_session_cache_mode(server_ssl_context) & SSL_SESS_CACHE_SERVER)); + } else { + EXPECT_EQ(SSL_SESS_CACHE_OFF, SSL_CTX_get_session_cache_mode(server_ssl_context)); + } })); EXPECT_CALL(callbacks, recordConnectionsAcceptedOnSocketEvent(_)); @@ -4144,6 +4185,25 @@ TEST_P(SslSocketTest, TicketSessionResumptionDifferentServerCertDifferentSAN) { version_); } +TEST_P(SslSocketTest, SessionResumptionDisabled) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/unittest_cert.pem" + private_key: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/unittest_key.pem" + disable_stateless_session_resumption: true + disable_stateful_session_resumption: true +)EOF"; + + const std::string client_ctx_yaml = R"EOF( + common_tls_context: + )EOF"; + + testSupportForSessionResumption(server_ctx_yaml, client_ctx_yaml, false, false, version_); +} + TEST_P(SslSocketTest, StatelessSessionResumptionDisabled) { const std::string server_ctx_yaml = R"EOF( common_tls_context: @@ -4159,10 +4219,28 @@ TEST_P(SslSocketTest, StatelessSessionResumptionDisabled) { common_tls_context: )EOF"; - testSupportForStatelessSessionResumption(server_ctx_yaml, client_ctx_yaml, false, version_); + testSupportForSessionResumption(server_ctx_yaml, client_ctx_yaml, false, true, version_); +} + +TEST_P(SslSocketTest, StatefulSessionResumptionDisabled) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/unittest_cert.pem" + private_key: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/unittest_key.pem" + disable_stateful_session_resumption: true +)EOF"; + + const std::string client_ctx_yaml = R"EOF( + common_tls_context: + )EOF"; + + testSupportForSessionResumption(server_ctx_yaml, client_ctx_yaml, true, false, version_); } -TEST_P(SslSocketTest, SatelessSessionResumptionEnabledExplicitly) { +TEST_P(SslSocketTest, SessionResumptionEnabledExplicitly) { const std::string server_ctx_yaml = R"EOF( common_tls_context: tls_certificates: @@ -4171,16 +4249,17 @@ TEST_P(SslSocketTest, SatelessSessionResumptionEnabledExplicitly) { private_key: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/unittest_key.pem" disable_stateless_session_resumption: false + disable_stateful_session_resumption: false )EOF"; const std::string client_ctx_yaml = R"EOF( common_tls_context: )EOF"; - testSupportForStatelessSessionResumption(server_ctx_yaml, client_ctx_yaml, true, version_); + testSupportForSessionResumption(server_ctx_yaml, client_ctx_yaml, true, true, version_); } -TEST_P(SslSocketTest, StatelessSessionResumptionEnabledByDefault) { +TEST_P(SslSocketTest, SessionResumptionEnabledByDefault) { const std::string server_ctx_yaml = R"EOF( common_tls_context: tls_certificates: @@ -4194,7 +4273,7 @@ TEST_P(SslSocketTest, StatelessSessionResumptionEnabledByDefault) { common_tls_context: )EOF"; - testSupportForStatelessSessionResumption(server_ctx_yaml, client_ctx_yaml, true, version_); + testSupportForSessionResumption(server_ctx_yaml, client_ctx_yaml, true, true, version_); } // Test that if two listeners use the same cert and session ticket key, but @@ -4248,10 +4327,11 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher_->createListener(socket, callbacks, runtime_, listener_config); + createListener(socket, callbacks, runtime_, listener_config, overload_state, *dispatcher_); Network::ListenerPtr listener2 = - dispatcher_->createListener(socket2, callbacks, runtime_, listener_config); + createListener(socket2, callbacks, runtime_, listener_config, overload_state, *dispatcher_); const std::string client_ctx_yaml = R"EOF( common_tls_context: tls_certificates: @@ -4372,8 +4452,9 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya NiceMock listener_config; Api::ApiPtr api = Api::createApiForTest(server_stats_store, time_system_); Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher->createListener(socket, callbacks, runtime_, listener_config); + createListener(socket, callbacks, runtime_, listener_config, overload_state, *dispatcher); Network::ConnectionPtr server_connection; Network::MockConnectionCallbacks server_connection_callbacks; @@ -4635,8 +4716,9 @@ TEST_P(SslSocketTest, SslError) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher_->createListener(socket, callbacks, runtime_, listener_config); + createListener(socket, callbacks, runtime_, listener_config, overload_state, *dispatcher_); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -5166,8 +5248,9 @@ TEST_P(SslSocketTest, SetSignatureAlgorithms) { Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks callbacks; NiceMock listener_config; + Server::ThreadLocalOverloadStateOptRef overload_state; Network::ListenerPtr listener = - dispatcher_->createListener(socket, callbacks, runtime_, listener_config); + createListener(socket, callbacks, runtime_, listener_config, overload_state, *dispatcher_); const std::string client_ctx_yaml = R"EOF( common_tls_context: @@ -5869,8 +5952,9 @@ class SslReadBufferLimitTest : public SslSocketTest { socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(version_)); NiceMock listener_config; - listener_ = - dispatcher_->createListener(socket_, listener_callbacks_, runtime_, listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; + listener_ = createListener(socket_, listener_callbacks_, runtime_, listener_config, + overload_state, *dispatcher_); TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml_), upstream_tls_context_); auto client_cfg = diff --git a/test/extensions/transport_sockets/tls/test_data/certs.sh b/test/extensions/transport_sockets/tls/test_data/certs.sh index 75a63ba41605..a63bbddacec9 100755 --- a/test/extensions/transport_sockets/tls/test_data/certs.sh +++ b/test/extensions/transport_sockets/tls/test_data/certs.sh @@ -114,7 +114,7 @@ generate_cert_chain() { ca_name="i$((x - 1))" fi echo "$x: $certname $ca_name" - generate_ca $certname $ca_name + generate_ca "$certname" "$ca_name" done for x in {1..3}; do cat "i${x}_cert.pem" >> test_long_cert_chain.pem diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc index 83e0ce46edae..2145b173b75a 100644 --- a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc @@ -10,7 +10,7 @@ namespace Envoy { namespace Extensions { namespace PrivateKeyMethodProvider { -void TestPrivateKeyConnection::delayed_op() { +void TestPrivateKeyConnection::delayedOp() { const std::chrono::milliseconds timeout_0ms{0}; timer_ = dispatcher_.createTimer([this]() -> void { @@ -84,7 +84,7 @@ static ssl_private_key_result_t ecdsaPrivateKeySign(SSL* ssl, uint8_t* out, size ops->output_.assign(out, out + out_len_unsigned); // Tell SSL socket that the operation is ready to be called again. - ops->delayed_op(); + ops->delayedOp(); return ssl_private_key_retry; } @@ -160,7 +160,7 @@ static ssl_private_key_result_t rsaPrivateKeySign(SSL* ssl, uint8_t* out, size_t } ops->output_.assign(out, out + *out_len); - ops->delayed_op(); + ops->delayedOp(); return ssl_private_key_retry; } @@ -197,7 +197,7 @@ static ssl_private_key_result_t rsaPrivateKeyDecrypt(SSL* ssl, uint8_t* out, siz } ops->output_.assign(out, out + *out_len); - ops->delayed_op(); + ops->delayedOp(); return ssl_private_key_retry; } @@ -359,8 +359,11 @@ TestPrivateKeyMethodProvider::TestPrivateKeyMethodProvider( return; } - std::string private_key = - factory_context.serverFactoryContext().api().fileSystem().fileReadToEnd(private_key_path); + std::string private_key = factory_context.serverFactoryContext() + .api() + .fileSystem() + .fileReadToEnd(private_key_path) + .value(); bssl::UniquePtr bio( BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); bssl::UniquePtr pkey(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr)); diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.h b/test/extensions/transport_sockets/tls/test_private_key_method_provider.h index 133d6a1d0ae0..7de910d4a2db 100644 --- a/test/extensions/transport_sockets/tls/test_private_key_method_provider.h +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.h @@ -44,7 +44,7 @@ class TestPrivateKeyConnection { bssl::UniquePtr pkey, TestPrivateKeyConnectionTestOptions& test_options); EVP_PKEY* getPrivateKey() { return pkey_.get(); } - void delayed_op(); + void delayedOp(); // Store the output data temporarily. std::vector output_; // The complete callback can return other value than "retry" only after diff --git a/test/extensions/upstreams/http/generic/config_test.cc b/test/extensions/upstreams/http/generic/config_test.cc index 3c51f53b782d..b2d9ab160a1c 100644 --- a/test/extensions/upstreams/http/generic/config_test.cc +++ b/test/extensions/upstreams/http/generic/config_test.cc @@ -21,7 +21,7 @@ class GenericGenericConnPoolFactoryTest : public ::testing::Test { protected: NiceMock thread_local_cluster_; - NiceMock route_entry_; + Upstream::ResourcePriority priority_ = Upstream::ResourcePriority::Default; Upstream::HostConstSharedPtr host_; GenericGenericConnPoolFactory factory_; }; @@ -29,26 +29,26 @@ class GenericGenericConnPoolFactoryTest : public ::testing::Test { TEST_F(GenericGenericConnPoolFactoryTest, CreateValidHttpConnPool) { EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_, Router::GenericConnPoolFactory::UpstreamProtocol::HTTP, - route_entry_, Envoy::Http::Protocol::Http2, nullptr)); + priority_, Envoy::Http::Protocol::Http2, nullptr)); } TEST_F(GenericGenericConnPoolFactoryTest, CreateValidTcpConnPool) { EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_, Router::GenericConnPoolFactory::UpstreamProtocol::TCP, - route_entry_, Envoy::Http::Protocol::Http2, nullptr)); + priority_, Envoy::Http::Protocol::Http2, nullptr)); } TEST_F(GenericGenericConnPoolFactoryTest, CreateValidUdpConnPool) { EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_, Router::GenericConnPoolFactory::UpstreamProtocol::UDP, - route_entry_, Envoy::Http::Protocol::Http2, nullptr)); + priority_, Envoy::Http::Protocol::Http2, nullptr)); } TEST_F(GenericGenericConnPoolFactoryTest, InvalidConnPool) { // Passes an invalid UpstreamProtocol and check a nullptr is returned. EXPECT_FALSE(factory_.createGenericConnPool( thread_local_cluster_, static_cast(0xff), - route_entry_, Envoy::Http::Protocol::Http2, nullptr)); + priority_, Envoy::Http::Protocol::Http2, nullptr)); } } // namespace Generic diff --git a/test/extensions/upstreams/http/tcp/upstream_request_test.cc b/test/extensions/upstreams/http/tcp/upstream_request_test.cc index d2745bdc529e..e48d6a4ad7e4 100644 --- a/test/extensions/upstreams/http/tcp/upstream_request_test.cc +++ b/test/extensions/upstreams/http/tcp/upstream_request_test.cc @@ -36,12 +36,12 @@ namespace Tcp { class TcpConnPoolTest : public ::testing::Test { public: TcpConnPoolTest() : host_(std::make_shared>()) { - NiceMock route_entry; + Upstream::ResourcePriority priority = Upstream::ResourcePriority::Default; NiceMock cm; cm.initializeThreadLocalClusters({"fake_cluster"}); EXPECT_CALL(cm.thread_local_cluster_, tcpConnPool(_, _)) .WillOnce(Return(Upstream::TcpPoolData([]() {}, &mock_pool_))); - conn_pool_ = std::make_unique(cm.thread_local_cluster_, route_entry, nullptr); + conn_pool_ = std::make_unique(cm.thread_local_cluster_, priority, nullptr); } std::unique_ptr conn_pool_; @@ -112,15 +112,17 @@ class TcpUpstreamTest : public ::testing::Test { .WillRepeatedly(Return(&request_)); EXPECT_CALL(mock_router_filter_, cluster()).Times(AnyNumber()); EXPECT_CALL(mock_router_filter_, callbacks()).Times(AnyNumber()); - mock_router_filter_.requests_.push_back(std::make_unique( + upstream_request_ = std::make_unique( mock_router_filter_, std::make_unique>(), false, - false)); + false); auto data = std::make_unique>(); - EXPECT_CALL(*data, connection()).Times(AnyNumber()).WillRepeatedly(ReturnRef(connection_)); - tcp_upstream_ = - std::make_unique(mock_router_filter_.requests_.front().get(), std::move(data)); + EXPECT_CALL(*data, connection()).Times(AnyNumber()).WillRepeatedly(ReturnRef(connection())); + tcp_upstream_ = std::make_unique(upstream_request_.get(), std::move(data)); } ~TcpUpstreamTest() override { EXPECT_CALL(mock_router_filter_, config()).Times(AnyNumber()); } + NiceMock& connection() { + return mock_router_filter_.client_connection_; + } protected: TestRequestHeaderMapImpl request_{{":method", "CONNECT"}, @@ -128,20 +130,19 @@ class TcpUpstreamTest : public ::testing::Test { {":protocol", "bytestream"}, {":scheme", "https"}, {":authority", "host"}}; - NiceMock connection_; NiceMock mock_router_filter_; - Envoy::Tcp::ConnectionPool::MockConnectionData* mock_connection_data_; + std::unique_ptr upstream_request_; std::unique_ptr tcp_upstream_; }; TEST_F(TcpUpstreamTest, Basic) { // Swallow the request headers and generate response headers. - EXPECT_CALL(connection_, write(_, false)).Times(0); + EXPECT_CALL(connection(), write(_, false)).Times(0); EXPECT_CALL(mock_router_filter_, onUpstreamHeaders(200, _, _, false)); EXPECT_TRUE(tcp_upstream_->encodeHeaders(request_, false).ok()); // Proxy the data. - EXPECT_CALL(connection_, write(BufferStringEqual("foo"), false)); + EXPECT_CALL(connection(), write(BufferStringEqual("foo"), false)); Buffer::OwnedImpl buffer("foo"); tcp_upstream_->encodeData(buffer, false); @@ -162,7 +163,8 @@ TEST_F(TcpUpstreamTest, Basic) { TEST_F(TcpUpstreamTest, V1Header) { envoy::config::core::v3::ProxyProtocolConfig* proxy_config = - mock_router_filter_.route_.route_entry_.connect_config_->mutable_proxy_protocol_config(); + mock_router_filter_.callbacks_.route_->route_entry_.connect_config_ + ->mutable_proxy_protocol_config(); proxy_config->set_version(envoy::config::core::v3::ProxyProtocolConfig::V1); mock_router_filter_.client_connection_.stream_info_.downstream_connection_info_provider_ ->setRemoteAddress(std::make_shared("1.2.3.4", 5)); @@ -174,18 +176,19 @@ TEST_F(TcpUpstreamTest, V1Header) { *proxy_config, mock_router_filter_.client_connection_, expected_data); // encodeHeaders now results in the proxy proto header being sent. - EXPECT_CALL(connection_, write(BufferEqual(&expected_data), false)); + EXPECT_CALL(connection(), write(BufferEqual(&expected_data), false)); EXPECT_TRUE(tcp_upstream_->encodeHeaders(request_, false).ok()); // Data is proxied as usual. - EXPECT_CALL(connection_, write(BufferStringEqual("foo"), false)); + EXPECT_CALL(connection(), write(BufferStringEqual("foo"), false)); Buffer::OwnedImpl buffer("foo"); tcp_upstream_->encodeData(buffer, false); } TEST_F(TcpUpstreamTest, V2Header) { envoy::config::core::v3::ProxyProtocolConfig* proxy_config = - mock_router_filter_.route_.route_entry_.connect_config_->mutable_proxy_protocol_config(); + mock_router_filter_.callbacks_.route_->route_entry_.connect_config_ + ->mutable_proxy_protocol_config(); proxy_config->set_version(envoy::config::core::v3::ProxyProtocolConfig::V2); mock_router_filter_.client_connection_.stream_info_.downstream_connection_info_provider_ ->setRemoteAddress(std::make_shared("1.2.3.4", 5)); @@ -197,11 +200,11 @@ TEST_F(TcpUpstreamTest, V2Header) { *proxy_config, mock_router_filter_.client_connection_, expected_data); // encodeHeaders now results in the proxy proto header being sent. - EXPECT_CALL(connection_, write(BufferEqual(&expected_data), false)); + EXPECT_CALL(connection(), write(BufferEqual(&expected_data), false)); EXPECT_TRUE(tcp_upstream_->encodeHeaders(request_, false).ok()); // Data is proxied as usual. - EXPECT_CALL(connection_, write(BufferStringEqual("foo"), false)); + EXPECT_CALL(connection(), write(BufferStringEqual("foo"), false)); Buffer::OwnedImpl buffer("foo"); tcp_upstream_->encodeData(buffer, false); } @@ -210,26 +213,26 @@ TEST_F(TcpUpstreamTest, TrailersEndStream) { // Swallow the headers. EXPECT_TRUE(tcp_upstream_->encodeHeaders(request_, false).ok()); - EXPECT_CALL(connection_, write(BufferStringEqual(""), true)); + EXPECT_CALL(connection(), write(BufferStringEqual(""), true)); Envoy::Http::TestRequestTrailerMapImpl trailers{{"foo", "bar"}}; tcp_upstream_->encodeTrailers(trailers); } TEST_F(TcpUpstreamTest, HeaderEndStreamHalfClose) { - EXPECT_CALL(connection_, write(BufferStringEqual(""), true)); + EXPECT_CALL(connection(), write(BufferStringEqual(""), true)); EXPECT_TRUE(tcp_upstream_->encodeHeaders(request_, true).ok()); } TEST_F(TcpUpstreamTest, ReadDisable) { - EXPECT_CALL(connection_, readDisable(true)); + EXPECT_CALL(connection(), readDisable(true)); tcp_upstream_->readDisable(true); - EXPECT_CALL(connection_, readDisable(false)); + EXPECT_CALL(connection(), readDisable(false)); tcp_upstream_->readDisable(false); // Once the connection is closed, don't touch it. - connection_.state_ = Network::Connection::State::Closed; - EXPECT_CALL(connection_, readDisable(_)).Times(0); + connection().state_ = Network::Connection::State::Closed; + EXPECT_CALL(connection(), readDisable(_)).Times(0); tcp_upstream_->readDisable(true); } diff --git a/test/extensions/upstreams/http/udp/BUILD b/test/extensions/upstreams/http/udp/BUILD index e94d6b501009..2d757dd91c2f 100644 --- a/test/extensions/upstreams/http/udp/BUILD +++ b/test/extensions/upstreams/http/udp/BUILD @@ -28,6 +28,7 @@ envoy_cc_test( "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/upstreams/http/udp/config_test.cc b/test/extensions/upstreams/http/udp/config_test.cc index 258940e111bf..c770496b9259 100644 --- a/test/extensions/upstreams/http/udp/config_test.cc +++ b/test/extensions/upstreams/http/udp/config_test.cc @@ -22,7 +22,7 @@ class UdpGenericConnPoolFactoryTest : public ::testing::Test { protected: NiceMock thread_local_cluster_; - NiceMock route_entry_; + Upstream::ResourcePriority priority_ = Upstream::ResourcePriority::Default; Upstream::HostConstSharedPtr host_; UdpGenericConnPoolFactory factory_; }; @@ -32,14 +32,14 @@ TEST_F(UdpGenericConnPoolFactoryTest, CreateValidUdpConnPool) { EXPECT_CALL(thread_local_cluster_.lb_, chooseHost).WillOnce(Return(host)); EXPECT_TRUE(factory_.createGenericConnPool(thread_local_cluster_, Router::GenericConnPoolFactory::UpstreamProtocol::UDP, - route_entry_, Envoy::Http::Protocol::Http2, nullptr)); + priority_, Envoy::Http::Protocol::Http2, nullptr)); } TEST_F(UdpGenericConnPoolFactoryTest, CreateInvalidUdpConnPool) { EXPECT_CALL(thread_local_cluster_.lb_, chooseHost).WillOnce(Return(nullptr)); EXPECT_FALSE(factory_.createGenericConnPool(thread_local_cluster_, Router::GenericConnPoolFactory::UpstreamProtocol::UDP, - route_entry_, Envoy::Http::Protocol::Http2, nullptr)); + priority_, Envoy::Http::Protocol::Http2, nullptr)); } } // namespace Udp diff --git a/test/extensions/upstreams/http/udp/upstream_request_test.cc b/test/extensions/upstreams/http/udp/upstream_request_test.cc index 3f89801321e2..423b21589589 100644 --- a/test/extensions/upstreams/http/udp/upstream_request_test.cc +++ b/test/extensions/upstreams/http/udp/upstream_request_test.cc @@ -2,6 +2,7 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/network/address_impl.h" +#include "source/common/network/socket_option_factory.h" #include "source/common/network/utility.h" #include "source/common/router/config_impl.h" #include "source/common/router/router.h" @@ -16,6 +17,7 @@ #include "test/mocks/router/router_filter_interface.h" #include "test/mocks/server/factory_context.h" #include "test/mocks/server/instance.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -185,6 +187,68 @@ TEST_F(UdpUpstreamTest, SocketConnectError) { EXPECT_FALSE(udp_upstream_->encodeHeaders(connect_udp_headers_, false).ok()); } +class UdpConnPoolTest : public ::testing::Test { +public: + UdpConnPoolTest() { + ON_CALL(*mock_thread_local_cluster_.lb_.host_, address) + .WillByDefault( + Return(Network::Utility::parseInternetAddressAndPortNoThrow("127.0.0.1:80", false))); + udp_conn_pool_ = std::make_unique(mock_thread_local_cluster_, nullptr); + EXPECT_CALL(*mock_thread_local_cluster_.lb_.host_, address).Times(2); + EXPECT_CALL(*mock_thread_local_cluster_.lb_.host_, cluster); + mock_thread_local_cluster_.lb_.host_->cluster_.source_address_ = + Network::Utility::parseInternetAddressAndPortNoThrow("127.0.0.1:10001", false); + } + +protected: + NiceMock mock_thread_local_cluster_; + std::unique_ptr udp_conn_pool_; + Router::MockGenericConnectionPoolCallbacks mock_callback_; +}; + +TEST_F(UdpConnPoolTest, BindToUpstreamLocalAddress) { + EXPECT_CALL(mock_callback_, upstreamToDownstream); + NiceMock downstream_connection_; + EXPECT_CALL(mock_callback_.upstream_to_downstream_, connection) + .WillRepeatedly( + Return(Envoy::OptRef(downstream_connection_))); + EXPECT_CALL(mock_callback_, onPoolReady); + // Mock syscall to make the bind call succeed. + NiceMock mock_os_sys_calls; + Envoy::TestThreadsafeSingletonInjector os_sys_calls( + &mock_os_sys_calls); + EXPECT_CALL(mock_os_sys_calls, bind).WillOnce(Return(Api::SysCallIntResult{0, 0})); + udp_conn_pool_->newStream(&mock_callback_); +} + +TEST_F(UdpConnPoolTest, ApplySocketOptionsFailure) { + Upstream::UpstreamLocalAddress upstream_local_address = { + mock_thread_local_cluster_.lb_.host_->cluster_.source_address_, + Network::SocketOptionFactory::buildIpFreebindOptions()}; + // Return a socket option to make the setsockopt syscall is called. + EXPECT_CALL(*mock_thread_local_cluster_.lb_.host_->cluster_.upstream_local_address_selector_, + getUpstreamLocalAddressImpl) + .WillOnce(Return(upstream_local_address)); + EXPECT_CALL(mock_callback_, onPoolFailure); + // Mock syscall to make the setsockopt call fail. + NiceMock mock_os_sys_calls; + Envoy::TestThreadsafeSingletonInjector os_sys_calls( + &mock_os_sys_calls); + // Use ON_CALL since the applyOptions call fail without calling the setsockopt_ in Windows. + ON_CALL(mock_os_sys_calls, setsockopt_).WillByDefault(Return(-1)); + udp_conn_pool_->newStream(&mock_callback_); +} + +TEST_F(UdpConnPoolTest, BindFailure) { + EXPECT_CALL(mock_callback_, onPoolFailure); + // Mock syscall to make the bind call fail. + NiceMock mock_os_sys_calls; + Envoy::TestThreadsafeSingletonInjector os_sys_calls( + &mock_os_sys_calls); + EXPECT_CALL(mock_os_sys_calls, bind).WillOnce(Return(Api::SysCallIntResult{-1, 0})); + udp_conn_pool_->newStream(&mock_callback_); +} + } // namespace Udp } // namespace Http } // namespace Upstreams diff --git a/test/extensions/upstreams/tcp/generic/config_test.cc b/test/extensions/upstreams/tcp/generic/config_test.cc index 10dcc66b126e..f2c62b449174 100644 --- a/test/extensions/upstreams/tcp/generic/config_test.cc +++ b/test/extensions/upstreams/tcp/generic/config_test.cc @@ -113,6 +113,15 @@ TEST_F(TcpConnPoolTest, Http3Config) { &lb_context_, callbacks_, downstream_stream_info_)); } +TEST(DisableTunnelingObjectFactory, CreateFromBytes) { + auto* factory = Registry::FactoryRegistry::getFactory( + TcpProxy::DisableTunnelingFilterStateKey); + ASSERT_NE(nullptr, factory); + auto object = factory->createFromBytes("true"); + ASSERT_NE(nullptr, object); + EXPECT_EQ(true, dynamic_cast(object.get())->value()); +} + } // namespace Generic } // namespace Tcp } // namespace Upstreams diff --git a/test/fuzz/BUILD b/test/fuzz/BUILD index a2e921ddab32..8a613fd72f6f 100644 --- a/test/fuzz/BUILD +++ b/test/fuzz/BUILD @@ -87,8 +87,8 @@ envoy_cc_test_library( "//source/common/protobuf:message_validator_lib", "//source/common/protobuf:utility_lib_header", "//source/common/protobuf:visitor_lib", - "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", - "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_github_cncf_xds//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/v3:pkg_cc_proto", "@com_google_absl//absl/cleanup", ], ) @@ -101,8 +101,8 @@ envoy_cc_test_library( "//test/fuzz:mutable_visitor_lib", "//test/fuzz:random_lib", "@com_envoyproxy_protoc_gen_validate//validate:cc_validate", - "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto", - "@com_github_cncf_udpa//xds/type/v3:pkg_cc_proto", + "@com_github_cncf_xds//udpa/type/v1:pkg_cc_proto", + "@com_github_cncf_xds//xds/type/v3:pkg_cc_proto", "@com_github_google_libprotobuf_mutator//:libprotobuf_mutator", ], ) diff --git a/test/fuzz/fuzz_runner.cc b/test/fuzz/fuzz_runner.cc index d1b54095bd87..e2f37c94c2eb 100644 --- a/test/fuzz/fuzz_runner.cc +++ b/test/fuzz/fuzz_runner.cc @@ -1,5 +1,7 @@ #include "test/fuzz/fuzz_runner.h" +#include + #include "source/common/common/thread.h" #include "source/common/common/utility.h" #include "source/common/event/libevent.h" @@ -88,5 +90,6 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) { testing::GMOCK_FLAG(verbose) = "error"; testing::InitGoogleMock(argc, *argv); Envoy::Fuzz::Runner::setupEnvironment(1, *argv, spdlog::level::critical); + atexit(Envoy::Fuzz::runCleanupHooks); return 0; } diff --git a/test/fuzz/main.cc b/test/fuzz/main.cc index 07e4952369a6..976006275c2b 100644 --- a/test/fuzz/main.cc +++ b/test/fuzz/main.cc @@ -45,7 +45,7 @@ class FuzzerCorpusTest : public testing::TestWithParam { TEST_P(FuzzerCorpusTest, RunOneCorpusFile) { ENVOY_LOG_MISC(info, "Corpus file: {}", GetParam()); - const std::string buf = api_->fileSystem().fileReadToEnd(GetParam()); + const std::string buf = api_->fileSystem().fileReadToEnd(GetParam()).value(); // Everything from here on is the same as under the fuzzer lib. LLVMFuzzerTestOneInput(reinterpret_cast(buf.c_str()), buf.size()); } diff --git a/test/integration/BUILD b/test/integration/BUILD index 145128ab3692..2cd45af6b497 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1,3 +1,4 @@ +load("@base_pip3//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary") load( "//bazel:envoy_build_system.bzl", @@ -7,6 +8,7 @@ load( "envoy_cc_test_library", "envoy_package", "envoy_proto_library", + "envoy_py_test", "envoy_select_admin_functionality", "envoy_select_enable_http3", "envoy_select_enable_yaml", @@ -306,7 +308,7 @@ envoy_cc_test( envoy_cc_test_library( name = "vhds_lib", - srcs = ["vhds.h"], + hdrs = ["vhds.h"], deps = [ ":http_integration_lib", "//source/common/config:protobuf_link_hacks", @@ -349,7 +351,46 @@ envoy_cc_test_binary( "//source/exe:platform_impl_lib", "//source/extensions/clusters/static:static_cluster_lib", "//source/extensions/clusters/strict_dns:strict_dns_cluster_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/least_request:config", + "//source/extensions/load_balancing_policies/random:config", "//source/extensions/load_balancing_policies/ring_hash:config", + "//source/extensions/load_balancing_policies/round_robin:config", + ], +) + +envoy_py_test( + name = "hotrestart_handoff_test", + size = "medium", + srcs = select({ + "//bazel:disable_hot_restart_or_admin": ["null_test.py"], + "//conditions:default": ["hotrestart_handoff_test.py"], + }), + args = [ + "--envoy-binary=$(location :hotrestart_main)", + "--h3-request=$(location //tools/h3_request)", + "--ca-certs=$(location //test/config/integration/certs:cacert.pem)", + "--ca-key=$(location //test/config/integration/certs:cakey.pem)", + ], + data = [ + ":hotrestart_main", + "//test/config/integration/certs:cacert.pem", + "//test/config/integration/certs:cakey.pem", + "//tools/h3_request", + ], + main = select({ + "//bazel:disable_hot_restart_or_admin": "null_test.py", + "//conditions:default": "hotrestart_handoff_test.py", + }), + # Hot restart does not apply on Windows. + # py_test doesn't do coverage. + tags = [ + "nocoverage", + "skip_on_windows", + ], + deps = [ + requirement("aiohttp"), + requirement("cryptography"), ], ) @@ -492,7 +533,7 @@ envoy_cc_test( srcs = [ "multiplexed_integration_test.cc", ], - shard_count = 8, + shard_count = 10, tags = [ "cpu:3", ], @@ -504,6 +545,8 @@ envoy_cc_test( "//source/extensions/filters/http/buffer:config", "//source/extensions/load_balancing_policies/ring_hash:config", "//test/integration/filters:encode1xx_local_reply_config_lib", + "//test/integration/filters:local_reply_during_decoding_filter_lib", + "//test/integration/filters:metadata_control_filter_lib", "//test/integration/filters:metadata_stop_all_filter_config_lib", "//test/integration/filters:on_local_reply_filter_config_lib", "//test/integration/filters:request_metadata_filter_config_lib", @@ -633,6 +676,8 @@ envoy_cc_test( ], deps = [ ":http_integration_lib", + "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/ring_hash:config", "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", @@ -688,7 +733,16 @@ envoy_cc_test( ]), deps = [ ":http_integration_lib", - "//test/integration/filters:set_response_code_filter_lib", + "//test/integration/filters:set_route_filter_lib", + ], +) + +envoy_cc_test_library( + name = "http_timeout_integration_test_lib", + hdrs = ["http_timeout_integration_test.h"], + deps = [ + "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", ], ) @@ -697,15 +751,14 @@ envoy_cc_test( size = "large", srcs = [ "http_timeout_integration_test.cc", - "http_timeout_integration_test.h", ], tags = [ "cpu:3", ], deps = [ ":http_integration_lib", - "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + ":http_timeout_integration_test_lib", + "//test/test_common:test_runtime_lib", ], ) @@ -750,10 +803,8 @@ envoy_cc_test( envoy_cc_test_library( name = "protocol_integration_test_lib", - srcs = [ - "protocol_integration_test.cc", - "protocol_integration_test.h", - ], + srcs = ["protocol_integration_test.cc"], + hdrs = ["protocol_integration_test.h"], deps = [ ":http_protocol_integration_lib", ":socket_interface_swap_lib", @@ -768,6 +819,7 @@ envoy_cc_test_library( "//test/integration/filters:continue_headers_only_inject_body", "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/integration/filters:invalid_header_filter_lib", + "//test/integration/filters:local_reply_during_decoding_filter_lib", "//test/integration/filters:local_reply_during_encoding_data_filter_lib", "//test/integration/filters:local_reply_during_encoding_filter_lib", "//test/integration/filters:local_reply_with_metadata_filter_lib", @@ -818,6 +870,7 @@ envoy_cc_test( srcs = [ "multiplexed_upstream_integration_test.cc", ], + shard_count = 4, tags = [ "cpu:3", ], @@ -833,14 +886,20 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "integration_admin_test_lib", + hdrs = ["integration_admin_test.h"], + deps = [ + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "integration_admin_test", size = "large", srcs = envoy_select_admin_functionality( - [ - "integration_admin_test.cc", - "integration_admin_test.h", - ], + ["integration_admin_test.cc"], ), tags = [ "cpu:3", @@ -857,7 +916,9 @@ envoy_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", - ], + ] + envoy_select_admin_functionality( + ["integration_admin_test_lib"], + ), ) envoy_proto_library( @@ -867,7 +928,7 @@ envoy_proto_library( envoy_cc_test_library( name = "test_host_predicate_lib", - srcs = [ + hdrs = [ "test_host_predicate.h", "test_host_predicate_config.h", ], @@ -1025,6 +1086,8 @@ envoy_cc_test_library( "//source/common/http/http1:codec_lib", "//source/common/http/http2:codec_lib", "//source/common/http/http3:codec_stats_lib", + "//source/common/listener_manager:active_raw_udp_listener_config", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:filter_lib", "//source/common/network:listen_socket_lib", @@ -1033,8 +1096,6 @@ envoy_cc_test_library( "//source/common/network:utility_lib", "//source/common/runtime:runtime_features_lib", "//source/common/stats:isolated_store_lib", - "//source/extensions/listener_managers/listener_manager:active_raw_udp_listener_config", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/http:header_validator_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", @@ -1050,7 +1111,7 @@ envoy_cc_test_library( "//source/common/quic:quic_server_factory_lib", "@com_github_google_quiche//:quic_test_tools_session_peer_lib", ]) + envoy_select_envoy_mobile_listener([ - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//source/common/listener_manager:listener_manager_lib", ]), ) @@ -1073,7 +1134,11 @@ envoy_cc_test_library( "//source/extensions/clusters/static:static_cluster_lib", "//source/extensions/config_subscription/grpc:grpc_collection_subscription_lib", "//source/extensions/config_subscription/grpc:grpc_subscription_lib", + "//source/extensions/load_balancing_policies/cluster_provided:config", + "//source/extensions/load_balancing_policies/least_request:config", "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/random:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/network/dns_resolver/cares:config", "//source/extensions/request_id/uuid:config", "//source/extensions/transport_sockets/tls:context_config_lib", @@ -1224,6 +1289,7 @@ envoy_cc_test_library( "//source/common/http:headers_lib", "//source/common/http/http1:codec_lib", "//source/common/http/http2:codec_lib", + "//source/common/listener_manager:connection_handler_lib", "//source/common/local_info:local_info_lib", "//source/common/network:filter_lib", "//source/common/network:listen_socket_lib", @@ -1239,7 +1305,6 @@ envoy_cc_test_library( "//source/extensions/health_checkers/grpc:health_checker_lib", "//source/extensions/health_checkers/http:health_checker_lib", "//source/extensions/health_checkers/tcp:health_checker_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:drain_manager_lib", "//source/server:hot_restart_nop_lib", @@ -1263,18 +1328,21 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "integration_test_lib", + hdrs = ["integration_test.h"], +) + envoy_cc_test( name = "integration_test", size = "large", - srcs = [ - "integration_test.cc", - "integration_test.h", - ], + srcs = ["integration_test.cc"], tags = [ "cpu:3", ], deps = [ ":http_integration_lib", + ":integration_test_lib", "//envoy/registry", "//source/common/http:header_map_lib", "//source/common/http:headers_lib", @@ -1330,18 +1398,21 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "websocket_integration_test_lib", + hdrs = ["websocket_integration_test.h"], +) + envoy_cc_test( name = "websocket_integration_test", size = "large", - srcs = [ - "websocket_integration_test.cc", - "websocket_integration_test.h", - ], + srcs = ["websocket_integration_test.cc"], tags = [ "cpu:3", ], deps = [ ":http_protocol_integration_lib", + ":websocket_integration_test_lib", "//source/common/http:header_map_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/filters/http/buffer:config", @@ -1493,7 +1564,9 @@ envoy_cc_test( "//test/test_common:test_runtime_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", - ], + ] + envoy_select_enable_http3([ + "//test/extensions/quic/proof_source:pending_proof_source_factory_impl_lib", + ]), ) envoy_cc_test( @@ -1672,13 +1745,15 @@ envoy_proto_library( srcs = [":tcp_proxy_integration_test.proto"], ) +envoy_cc_test_library( + name = "tcp_proxy_integration_test_lib", + hdrs = ["tcp_proxy_integration_test.h"], +) + envoy_cc_test( name = "tcp_proxy_integration_test", size = "large", - srcs = [ - "tcp_proxy_integration_test.cc", - "tcp_proxy_integration_test.h", - ], + srcs = ["tcp_proxy_integration_test.cc"], data = [ "//test/config/integration/certs", ], @@ -1686,8 +1761,10 @@ envoy_cc_test( "cpu:3", ], deps = [ + ":fake_access_log_lib", ":integration_lib", ":tcp_proxy_integration_proto_cc_proto", + ":tcp_proxy_integration_test_lib", "//source/common/config:api_version_lib", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", @@ -1779,10 +1856,13 @@ envoy_cc_test( deps = [ ":http_integration_lib", ":http_protocol_integration_lib", + "//source/extensions/filters/udp/udp_proxy:config", + "//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:config", "//source/extensions/upstreams/http/udp:config", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/file/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/upstreams/http/udp/v3:pkg_cc_proto", ], ) @@ -1853,12 +1933,17 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "uds_integration_test_lib", + hdrs = ["uds_integration_test.h"], + deps = ["@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto"], +) + envoy_cc_test( name = "uds_integration_test", size = "large", srcs = envoy_select_admin_functionality([ "uds_integration_test.cc", - "uds_integration_test.h", ]), deps = [ ":http_integration_lib", @@ -1868,7 +1953,9 @@ envoy_cc_test( "//source/common/stats:stats_lib", "//test/test_common:environment_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", - ], + ] + envoy_select_admin_functionality([ + "uds_integration_test_lib", + ]), ) envoy_cc_test( @@ -1923,12 +2010,17 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "xfcc_integration_test_lib", + hdrs = ["xfcc_integration_test.h"], + deps = ["@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto"], +) + envoy_cc_test( name = "xfcc_integration_test", size = "large", srcs = envoy_select_admin_functionality([ "xfcc_integration_test.cc", - "xfcc_integration_test.h", ]), data = [ "//test/config/integration/certs", @@ -1945,7 +2037,9 @@ envoy_cc_test( "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", - ], + ] + envoy_select_admin_functionality([ + "xfcc_integration_test_lib", + ]), ) H1_FUZZ_LIB_DEPS = [ @@ -2072,14 +2166,24 @@ envoy_cc_fuzz_test( ], ) +envoy_cc_test_library( + name = "scoped_rds_test_lib", + hdrs = ["scoped_rds.h"], + deps = [ + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "scoped_rds_lib", size = "large", - srcs = [ - "scoped_rds.h", - ], deps = [ ":http_integration_lib", + ":scoped_rds_test_lib", "//source/common/config:api_version_lib", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", @@ -2089,11 +2193,6 @@ envoy_cc_test( "//test/config:v2_link_hacks", "//test/test_common:resources_lib", "//test/test_common:utility_lib", - "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/route/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], ) @@ -2391,6 +2490,21 @@ envoy_cc_test( ], ) +envoy_proto_library( + name = "fake_access_log_proto", + srcs = ["fake_access_log.proto"], +) + +envoy_cc_test_library( + name = "fake_access_log_lib", + hdrs = [ + "fake_access_log.h", + ], + deps = [ + ":fake_access_log_proto_cc_proto", + ], +) + envoy_proto_library( name = "typed_metadata_integration_proto", srcs = ["typed_metadata_integration_test.proto"], diff --git a/test/integration/admin_html/test_server.cc b/test/integration/admin_html/test_server.cc index 2a9b2107d0fc..4a53d7661bb9 100644 --- a/test/integration/admin_html/test_server.cc +++ b/test/integration/admin_html/test_server.cc @@ -18,18 +18,18 @@ namespace { */ Http::Code testCallback(Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, Server::AdminStream& admin_stream) { - Http::Utility::QueryParams query_params = admin_stream.queryParams(); - auto iter = query_params.find("file"); + Http::Utility::QueryParamsMulti query_params = admin_stream.queryParams(); + auto leafSuffix = query_params.getFirstValue("file"); std::string prefix; - if (iter != query_params.end()) { + if (leafSuffix.has_value()) { prefix = "test/integration/admin_html/"; - } else if (iter = query_params.find("src"); iter != query_params.end()) { + } else if (leafSuffix = query_params.getFirstValue("src"); leafSuffix.has_value()) { prefix = "source/server/admin/html/"; } else { response.add("query param 'file' or 'src' missing"); return Http::Code::BadRequest; } - absl::string_view leaf = iter->second; + absl::string_view leaf = leafSuffix.value(); // ".." is not a good thing to allow into the path, even for a test server. if (leaf.find("..") != absl::string_view::npos) { @@ -39,7 +39,11 @@ Http::Code testCallback(Http::ResponseHeaderMap& response_headers, Buffer::Insta Filesystem::InstanceImpl file_system; std::string path = absl::StrCat(prefix, leaf); - TRY_ASSERT_MAIN_THREAD { response.add(file_system.fileReadToEnd(path)); } + TRY_ASSERT_MAIN_THREAD { + auto file_or_error = file_system.fileReadToEnd(path); + THROW_IF_STATUS_NOT_OK(file_or_error, throw); + response.add(file_or_error.value()); + } END_TRY catch (EnvoyException& e) { response.add(e.what()); @@ -61,7 +65,7 @@ class DebugHtmlResourceProvider : public Server::AdminHtmlUtil::ResourceProvider std::string path = absl::StrCat("source/server/admin/html/", resource_name); Filesystem::InstanceImpl file_system; TRY_ASSERT_MAIN_THREAD { - buf = file_system.fileReadToEnd(path); + buf = file_system.fileReadToEnd(path).value(); ENVOY_LOG_MISC(info, "Read {} bytes from {}", buf.size(), path); } END_TRY diff --git a/test/integration/autonomous_upstream.cc b/test/integration/autonomous_upstream.cc index 97b714e29233..8b05a02b5cb7 100644 --- a/test/integration/autonomous_upstream.cc +++ b/test/integration/autonomous_upstream.cc @@ -21,9 +21,12 @@ const char AutonomousStream::RESPONSE_SIZE_BYTES[] = "response_size_bytes"; const char AutonomousStream::RESPONSE_DATA_BLOCKS[] = "response_data_blocks"; const char AutonomousStream::EXPECT_REQUEST_SIZE_BYTES[] = "expect_request_size_bytes"; const char AutonomousStream::RESET_AFTER_REQUEST[] = "reset_after_request"; +const char AutonomousStream::RESET_AFTER_RESPONSE_HEADERS[] = "reset_after_response_headers"; +const char AutonomousStream::RESET_AFTER_RESPONSE_DATA[] = "reset_after_response_data"; const char AutonomousStream::CLOSE_AFTER_RESPONSE[] = "close_after_response"; const char AutonomousStream::NO_TRAILERS[] = "no_trailers"; const char AutonomousStream::NO_END_STREAM[] = "no_end_stream"; +const char AutonomousStream::RESPOND_AFTER_REQUEST_HEADERS[] = "respond_after_request_headers"; AutonomousStream::AutonomousStream(FakeHttpConnection& parent, Http::ResponseEncoder& encoder, AutonomousUpstream& upstream, bool allow_incomplete_streams) @@ -36,10 +39,20 @@ AutonomousStream::~AutonomousStream() { } } +void AutonomousStream::decodeHeaders(Http::RequestHeaderMapSharedPtr&& headers, bool end_stream) { + bool send_response = !headers->get(Http::LowerCaseString(RESPOND_AFTER_REQUEST_HEADERS)).empty(); + FakeStream::decodeHeaders(std::move(headers), end_stream); + + if (send_response) { + absl::MutexLock lock(&lock_); + sendResponse(); + } +} + // By default, automatically send a response when the request is complete. void AutonomousStream::setEndStream(bool end_stream) { FakeStream::setEndStream(end_stream); - if (end_stream) { + if (end_stream && headers_->get(Http::LowerCaseString(RESPOND_AFTER_REQUEST_HEADERS)).empty()) { sendResponse(); } } @@ -76,6 +89,12 @@ void AutonomousStream::sendResponse() { } encodeHeaders(upstream_.responseHeaders(), headers_only_response); + + if (!headers.get_(RESET_AFTER_RESPONSE_HEADERS).empty()) { + encodeResetStream(); + return; + } + if (!headers_only_response) { if (upstream_.responseBody().has_value()) { encodeData(*upstream_.responseBody(), !send_trailers); @@ -83,6 +102,11 @@ void AutonomousStream::sendResponse() { for (int32_t i = 0; i < response_data_blocks; ++i) { encodeData(response_body_length, i == (response_data_blocks - 1) && !send_trailers && end_stream); + + if (!headers.get_(RESET_AFTER_RESPONSE_DATA).empty()) { + encodeResetStream(); + return; + } } } if (send_trailers) { diff --git a/test/integration/autonomous_upstream.h b/test/integration/autonomous_upstream.h index 9ee0c857c4c5..9aeb88a90587 100644 --- a/test/integration/autonomous_upstream.h +++ b/test/integration/autonomous_upstream.h @@ -21,18 +21,26 @@ class AutonomousStream : public FakeStream { // If set, the stream will reset when the request is complete, rather than // sending a response. static const char RESET_AFTER_REQUEST[]; + // If set, the stream will reset when the response headers are sent. + static const char RESET_AFTER_RESPONSE_HEADERS[]; + // If set, the stream will reset after the first data chunk. Note by default + // there is only one data chunk. + static const char RESET_AFTER_RESPONSE_DATA[]; // Prevents upstream from sending trailers. static const char NO_TRAILERS[]; // Prevents upstream from finishing response. static const char NO_END_STREAM[]; // Closes the underlying connection after a given response is sent. static const char CLOSE_AFTER_RESPONSE[]; + // Send the response after the request headers are received. + static const char RESPOND_AFTER_REQUEST_HEADERS[]; AutonomousStream(FakeHttpConnection& parent, Http::ResponseEncoder& encoder, AutonomousUpstream& upstream, bool allow_incomplete_streams); ~AutonomousStream() override; void setEndStream(bool set) ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) override; + void decodeHeaders(Http::RequestHeaderMapSharedPtr&& headers, bool end_stream) override; private: AutonomousUpstream& upstream_; diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 38cd432d54ce..9813d24d2923 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -180,10 +180,11 @@ void BaseIntegrationTest::createUpstream(Network::Address::InstanceConstSharedPt upstream_tls_ ? createUpstreamTlsContext(config) : Network::Test::createRawBufferDownstreamSocketFactory(); if (autonomous_upstream_) { - fake_upstreams_.emplace_back(new AutonomousUpstream(std::move(factory), endpoint, config, - autonomous_allow_incomplete_streams_)); + fake_upstreams_.emplace_back(std::make_unique( + std::move(factory), endpoint, config, autonomous_allow_incomplete_streams_)); } else { - fake_upstreams_.emplace_back(new FakeUpstream(std::move(factory), endpoint, config)); + fake_upstreams_.emplace_back( + std::make_unique(std::move(factory), endpoint, config)); } } @@ -459,11 +460,7 @@ void BaseIntegrationTest::createGeneratedApiTestServer( if (config_helper_.bootstrap().static_resources().listeners_size() > 0 && !defer_listener_finalization_) { - // Wait for listeners to be created before invoking registerTestServerPorts() below, as that - // needs to know about the bound listener ports. - // Using 2x default timeout to cover for slow TLS implementations (no inline asm) on slow - // computers (e.g., Raspberry Pi) that sometimes time out on TLS listeners here. - Event::TestTimeSystem::RealTimeBound bound(2 * TestUtility::DefaultTimeout); + Event::TestTimeSystem::RealTimeBound bound(listeners_bound_timeout_ms_); const char* success = "listener_manager.listener_create_success"; const char* rejected = "listener_manager.lds.update_rejected"; for (Stats::CounterSharedPtr success_counter = test_server->counter(success), diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h index 474789478570..7b16822a4c7f 100644 --- a/test/integration/base_integration_test.h +++ b/test/integration/base_integration_test.h @@ -242,7 +242,7 @@ class BaseIntegrationTest : protected Logger::Loggable { discovery_response.set_version_info(version); discovery_response.set_type_url(type_url); for (const auto& message : messages) { - if (metadata.size() != 0) { + if (!metadata.empty()) { envoy::service::discovery::v3::Resource resource; resource.mutable_resource()->PackFrom(message); resource.set_name(intResourceName(message)); @@ -474,6 +474,13 @@ class BaseIntegrationTest : protected Logger::Loggable { void checkForMissingTagExtractionRules(); + // Sets the timeout to wait for listeners to be created before invoking + // registerTestServerPorts(), as that needs to know about the bound listener ports. + // Needs to be called before invoking createEnvoy() (invoked during initialize()). + void setListenersBoundTimeout(const std::chrono::milliseconds& duration) { + listeners_bound_timeout_ms_ = duration; + } + std::unique_ptr upstream_stats_store_; // Make sure the test server will be torn down after any fake client. @@ -527,6 +534,13 @@ class BaseIntegrationTest : protected Logger::Loggable { spdlog::level::level_enum default_log_level_; + // Timeout to wait for listeners to be created before invoking + // registerTestServerPorts(), as that needs to know about the bound listener ports. + // Using 2x default timeout to cover for slow TLS implementations (no inline asm) on slow + // computers (e.g., Raspberry Pi) that sometimes time out on TLS listeners, or when + // the number of listeners in a test is large. + std::chrono::milliseconds listeners_bound_timeout_ms_{2 * TestUtility::DefaultTimeout}; + // Target number of upstreams. uint32_t fake_upstreams_count_{1}; diff --git a/test/integration/buffer_accounting_integration_test.cc b/test/integration/buffer_accounting_integration_test.cc index 05bb8c1ae3ea..46d5194893d1 100644 --- a/test/integration/buffer_accounting_integration_test.cc +++ b/test/integration/buffer_accounting_integration_test.cc @@ -878,105 +878,6 @@ TEST_P(Http2OverloadManagerIntegrationTest, EXPECT_EQ(smallest_response->headers().getStatusValue(), "200"); } -TEST_P(Http2OverloadManagerIntegrationTest, CanResetStreamIfEnvoyLevelStreamEnded) { - // This test is not applicable if expand_agnostic_stream_lifetime is enabled - // as the gap between lifetimes of the codec level and envoy level stream - // shrinks. - if (Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)) { - return; - } - - useAccessLog("%RESPONSE_CODE%"); - initializeOverloadManagerInBootstrap( - TestUtility::parseYaml(R"EOF( - name: "envoy.overload_actions.reset_high_memory_stream" - triggers: - - name: "envoy.resource_monitors.testonly.fake_resource_monitor" - scaled: - scaling_threshold: 0.90 - saturation_threshold: 0.98 - )EOF")); - initialize(); - - // Set 10MiB receive window for the client. - const int downstream_window_size = 10 * 1024 * 1024; - envoy::config::core::v3::Http2ProtocolOptions http2_options = - ::Envoy::Http2::Utility::initializeAndValidateOptions( - envoy::config::core::v3::Http2ProtocolOptions()); - http2_options.mutable_initial_stream_window_size()->set_value(downstream_window_size); - http2_options.mutable_initial_connection_window_size()->set_value(downstream_window_size); - codec_client_ = makeRawHttpConnection(makeClientConnection(lookupPort("http")), http2_options); - - // Makes us have Envoy's writes to downstream return EAGAIN - write_matcher_->setSourcePort(lookupPort("http")); - write_matcher_->setWriteReturnsEgain(); - - // Send a request - auto encoder_decoder = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{ - {":method", "POST"}, - {":path", "/"}, - {":scheme", "http"}, - {":authority", "host"}, - {"content-length", "10"}, - }); - auto& encoder = encoder_decoder.first; - const std::string data(10, 'a'); - codec_client_->sendData(encoder, data, true); - auto response = std::move(encoder_decoder.second); - - waitForNextUpstreamRequest(); - FakeStreamPtr upstream_request_for_response = std::move(upstream_request_); - - // Send the responses back. It is larger than the downstream's receive window - // size. Thus, the codec will not end the stream, but the Envoy level stream - // should. - upstream_request_for_response->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, - false); - const int response_size = downstream_window_size + 1024; // Slightly over the window size. - upstream_request_for_response->encodeData(response_size, true); - - if (streamBufferAccounting()) { - if (deferProcessingBackedUpStreams()) { - // Wait for an accumulation of data, as we cannot rely on the access log - // output since we're deferring the processing of the stream data. - EXPECT_TRUE(buffer_factory_->waitUntilTotalBufferedExceeds(10 * 10 * 1024)); - - } else { - // Wait for access log to know the Envoy level stream has been deleted. - EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("200")); - } - } - - // Set the pressure so the overload action kills the response if doing stream - // accounting - updateResource(0.95); - test_server_->waitForGaugeEq( - "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 62); - - if (streamBufferAccounting()) { - test_server_->waitForCounterGe("envoy.overload_actions.reset_high_memory_stream.count", 1); - } - - // Reduce resource pressure - updateResource(0.80); - test_server_->waitForGaugeEq( - "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 0); - - // Resume writes to downstream. - write_matcher_->setResumeWrites(); - - if (streamBufferAccounting()) { - EXPECT_TRUE(response->waitForReset()); - EXPECT_TRUE(response->reset()); - } else { - // If we're not doing the accounting, we didn't end up resetting the - // streams. - ASSERT_TRUE(response->waitForEndStream()); - ASSERT_TRUE(response->complete()); - EXPECT_EQ(response->headers().getStatusValue(), "200"); - } -} - class Http2DeferredProcessingIntegrationTest : public Http2BufferWatermarksTest { public: Http2DeferredProcessingIntegrationTest() : registered_tee_factory_(tee_filter_factory_) { diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 1956b21e39c5..aa3fb252c749 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -157,8 +157,10 @@ class CdsIntegrationTest : public Grpc::DeltaSotwDeferredClustersIntegrationPara cluster_creator_; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDeltaDeferredCluster, CdsIntegrationTest, - DELTA_SOTW_GRPC_CLIENT_DEFERRED_CLUSTERS_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P( + IpVersionsClientTypeDeltaDeferredCluster, CdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_DEFERRED_CLUSTERS_INTEGRATION_PARAMS, + Grpc::DeltaSotwDeferredClustersIntegrationParamTest::protocolTestParamsToString); // 1) Envoy starts up with no static clusters (other than the CDS-over-gRPC server). // 2) Envoy is told of a cluster via CDS. @@ -303,8 +305,10 @@ class DeferredCreationClusterStatsTest : public CdsIntegrationTest { } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, DeferredCreationClusterStatsTest, - DELTA_SOTW_GRPC_CLIENT_DEFERRED_CLUSTERS_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P( + IpVersionsClientTypeDelta, DeferredCreationClusterStatsTest, + DELTA_SOTW_GRPC_CLIENT_DEFERRED_CLUSTERS_INTEGRATION_PARAMS, + Grpc::DeltaSotwDeferredClustersIntegrationParamTest::protocolTestParamsToString); // Test that DeferredCreationTrafficStats gets created and updated correctly. TEST_P(DeferredCreationClusterStatsTest, @@ -331,8 +335,9 @@ TEST_P(DeferredCreationClusterStatsTest, test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4); EXPECT_EQ(test_server_->counter("cluster_manager.cluster_added")->value(), 3); // Now the cluster_1 stats are gone, as well as the lazy init wrapper. + test_server_->waitForCounterNonexistent("cluster.cluster_1.upstream_cx_total", + TestUtility::DefaultTimeout); EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized"), nullptr); - EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr); // No cluster_2 traffic stats. EXPECT_EQ(test_server_->gauge("cluster.cluster_2.ClusterTrafficStats.initialized")->value(), 0); @@ -362,14 +367,15 @@ TEST_P(DeferredCreationClusterStatsTest, test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4); EXPECT_EQ(test_server_->counter("cluster_manager.cluster_added")->value(), 3); // Now the cluster_1 stats are gone. - EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr); + test_server_->waitForCounterNonexistent("cluster.cluster_1.upstream_cx_total", + TestUtility::DefaultTimeout); // cluster_2 traffic stats stays. EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total")->value(), 0); } // Test that DeferredCreationTrafficStats with cluster_1 create-remove-create sequence. TEST_P(DeferredCreationClusterStatsTest, - DeferredCreationTrafficStatsWithClusterCreateDeleteRecrete) { + DeferredCreationTrafficStatsWithClusterCreateDeleteRecreate) { initializeDeferredCreationTest(/*enable_deferred_creation_stats=*/true); EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized")->value(), 0); EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr); @@ -387,8 +393,9 @@ TEST_P(DeferredCreationClusterStatsTest, EXPECT_EQ(test_server_->gauge("cluster.cluster_2.ClusterTrafficStats.initialized")->value(), 0); EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total"), nullptr); // Now the cluster_1 stats are gone, as well as the lazy init wrapper. + test_server_->waitForCounterNonexistent("cluster.cluster_1.upstream_cx_total", + TestUtility::DefaultTimeout); EXPECT_EQ(test_server_->gauge("cluster.cluster_1.ClusterTrafficStats.initialized"), nullptr); - EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr); // Now add cluster1 back. updateCluster(); test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4); @@ -405,7 +412,7 @@ TEST_P(DeferredCreationClusterStatsTest, // Test that Non-DeferredCreationTrafficStats with cluster_1 create-remove-create sequence. TEST_P(DeferredCreationClusterStatsTest, - NonDeferredCreationTrafficStatsWithClusterCreateDeleteRecrete) { + NonDeferredCreationTrafficStatsWithClusterCreateDeleteRecreate) { initializeDeferredCreationTest(/*enable_deferred_creation_stats=*/false); EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 0); @@ -420,7 +427,8 @@ TEST_P(DeferredCreationClusterStatsTest, // cluster_2 traffic stats created. EXPECT_EQ(test_server_->counter("cluster.cluster_2.upstream_cx_total")->value(), 0); // Now the cluster_1 stats are gone. - EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total"), nullptr); + test_server_->waitForCounterNonexistent("cluster.cluster_1.upstream_cx_total", + TestUtility::DefaultTimeout); // Now add cluster1 back. updateCluster(); test_server_->waitForCounterGe("cluster_manager.cds.update_success", 4); diff --git a/test/integration/cluster_filter_integration_test.cc b/test/integration/cluster_filter_integration_test.cc index 3aecae9b11be..f9450fc37592 100644 --- a/test/integration/cluster_filter_integration_test.cc +++ b/test/integration/cluster_filter_integration_test.cc @@ -115,8 +115,8 @@ class ClusterFilterIntegrationTestBase std::get<1>(GetParam())); } - // Get the test parameter whether upstream filters are initialized right after the upstream - // connection has been established + // Get the test parameter whether upstream network filters are initialized right after the + // upstream connection has been established bool upstreamFiltersInitializedWhenConnected() const { return std::get<1>(GetParam()); } void initialize() { on_new_connection_called_after_on_write_.store(absl::optional{}); } diff --git a/test/integration/cx_limit_integration_test.cc b/test/integration/cx_limit_integration_test.cc index d2fe7e89b6cc..c998727d01b9 100644 --- a/test/integration/cx_limit_integration_test.cc +++ b/test/integration/cx_limit_integration_test.cc @@ -136,8 +136,13 @@ TEST_P(ConnectionLimitIntegrationTest, TestListenerLimit) { doTest(init_func, "downstream_cx_overflow"); } +// TODO (nezdolik) move this test to overload manager test suite, once runtime key is fully +// deprecated. TEST_P(ConnectionLimitIntegrationTest, TestEmptyGlobalCxRuntimeLimit) { - const std::string log_line = "no configured limit to the number of allowed active connections."; + const std::string log_line = + "There is no configured limit to the number of allowed active downstream connections. " + "Configure a " + "limit in `envoy.resource_monitors.downstream_connections` resource monitor."; EXPECT_LOG_CONTAINS("warn", log_line, { initialize(); }); } diff --git a/test/integration/drain_close_integration_test.cc b/test/integration/drain_close_integration_test.cc index ecfa06efe0f7..5ffaaabb2869 100644 --- a/test/integration/drain_close_integration_test.cc +++ b/test/integration/drain_close_integration_test.cc @@ -167,6 +167,60 @@ TEST_P(DrainCloseIntegrationTest, RepeatedAdminGracefulDrain) { ASSERT_TRUE(waitForPortAvailable(http_port)); } +TEST_P(DrainCloseIntegrationTest, AdminGracefulDrainSkipExit) { + drain_strategy_ = Server::DrainStrategy::Immediate; + drain_time_ = std::chrono::seconds(1); + initialize(); + uint32_t http_port = lookupPort("http"); + codec_client_ = makeHttpConnection(http_port); + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(0); + upstream_request_->encodeHeaders(default_response_headers_, true); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_THAT(response->headers(), Http::HttpStatusIs("200")); + // The request is completed but the connection remains open. + EXPECT_TRUE(codec_client_->connected()); + + // Invoke /drain_listeners with graceful drain + BufferingStreamDecoderPtr admin_response = IntegrationUtil::makeSingleRequest( + lookupPort("admin"), "POST", "/drain_listeners?graceful&skip_exit", "", downstreamProtocol(), + version_); + EXPECT_EQ(admin_response->headers().Status()->value().getStringView(), "200"); + + // Listeners should remain open + EXPECT_EQ(test_server_->counter("listener_manager.listener_stopped")->value(), 0); + + response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(0); + upstream_request_->encodeHeaders(default_response_headers_, true); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_THAT(response->headers(), Http::HttpStatusIs("200")); + + // Connections will terminate on request complete + ASSERT_TRUE(codec_client_->waitForDisconnect()); + if (downstream_protocol_ == Http::CodecType::HTTP2) { + EXPECT_TRUE(codec_client_->sawGoAway()); + } else { + EXPECT_EQ("close", response->headers().getConnectionValue()); + } + + // New connections can still be made. + auto second_codec_client_ = makeRawHttpConnection(makeClientConnection(http_port), absl::nullopt); + EXPECT_TRUE(second_codec_client_->connected()); + + // Invoke /drain_listeners and shut down listeners. + second_codec_client_->rawConnection().close(Network::ConnectionCloseType::NoFlush); + admin_response = IntegrationUtil::makeSingleRequest( + lookupPort("admin"), "POST", "/drain_listeners", "", downstreamProtocol(), version_); + EXPECT_EQ(admin_response->headers().Status()->value().getStringView(), "200"); + + test_server_->waitForCounterEq("listener_manager.listener_stopped", 1); + ASSERT_TRUE(waitForPortAvailable(http_port)); +} + INSTANTIATE_TEST_SUITE_P(Protocols, DrainCloseIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( {Http::CodecType::HTTP1, Http::CodecType::HTTP2}, diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc index f077db74b862..010969397424 100644 --- a/test/integration/eds_integration_test.cc +++ b/test/integration/eds_integration_test.cc @@ -90,6 +90,7 @@ class EdsIntegrationTest uint32_t disable_active_hc_endpoints = 0; absl::optional weighted_priority_health = absl::nullopt; absl::optional overprovisioning_factor = absl::nullopt; + absl::optional drop_overload_numerator = absl::nullopt; }; // We need to supply the endpoints via EDS to provide health status. Use a @@ -109,6 +110,16 @@ class EdsIntegrationTest cluster_load_assignment.mutable_policy()->set_weighted_priority_health( endpoint_setting.weighted_priority_health.value()); } + + if (endpoint_setting.drop_overload_numerator.has_value()) { + auto* drop_overload = cluster_load_assignment.mutable_policy()->add_drop_overloads(); + drop_overload->set_category("test"); + drop_overload->mutable_drop_percentage()->set_denominator( + envoy::type::v3::FractionalPercent::HUNDRED); + drop_overload->mutable_drop_percentage()->set_numerator( + endpoint_setting.drop_overload_numerator.value()); + } + auto* locality_lb_endpoints = cluster_load_assignment.add_endpoints(); for (uint32_t i = 0; i < endpoint_setting.total_endpoints; ++i) { @@ -195,6 +206,27 @@ class EdsIntegrationTest void initializeTest(bool http_active_hc) { initializeTest(http_active_hc, nullptr); } + void dropOverloadTest(uint32_t numerator, const std::string& status) { + autonomous_upstream_ = true; + + initializeTest(false); + EndpointSettingOptions options; + options.total_endpoints = 2; + options.healthy_endpoints = 2; + options.drop_overload_numerator = numerator; + setEndpoints(options); + + // Check deferred. + if (deferred_cluster_creation_) { + test_server_->waitForGaugeEq("thread_local_cluster_manager.worker_0.clusters_inflated", 0); + } + BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest( + lookupPort("http"), "GET", "/cluster_0", "", downstream_protocol_, version_, "foo.com"); + ASSERT_TRUE(response->complete()); + EXPECT_EQ(status, response->headers().getStatusValue()); + cleanupUpstreamAndDownstream(); + } + envoy::type::v3::CodecClientType codec_client_type_{}; const bool deferred_cluster_creation_{}; EdsHelper eds_helper_; @@ -895,5 +927,10 @@ TEST_P(EdsIntegrationTest, DataplaneTrafficAfterEdsUpdateOfInitializedCluster) { } } +// Test EDS cluster DROP_OVERLOAD configuration. +TEST_P(EdsIntegrationTest, DropOverloadTestForEdsClusterNoDrop) { dropOverloadTest(0, "200"); } + +TEST_P(EdsIntegrationTest, DropOverloadTestForEdsClusterAllDrop) { dropOverloadTest(100, "503"); } + } // namespace } // namespace Envoy diff --git a/test/integration/fake_access_log.h b/test/integration/fake_access_log.h new file mode 100644 index 000000000000..de62cedc2c83 --- /dev/null +++ b/test/integration/fake_access_log.h @@ -0,0 +1,57 @@ +#pragma once + +#include "envoy/access_log/access_log_config.h" + +#include "source/common/protobuf/protobuf.h" + +#include "test/integration/fake_access_log.pb.h" + +namespace Envoy { + +using LogSignature = + std::function; + +class FakeAccessLog : public AccessLog::Instance { +public: + FakeAccessLog(LogSignature cb) : log_cb_(cb) {} + + void log(const Formatter::HttpFormatterContext& context, + const StreamInfo::StreamInfo& info) override { + if (log_cb_) { + log_cb_(context, info); + } + } + +private: + LogSignature log_cb_; +}; + +class FakeAccessLogFactory : public AccessLog::AccessLogInstanceFactory { +public: + AccessLog::InstanceSharedPtr + createAccessLogInstance(const Protobuf::Message&, AccessLog::FilterPtr&&, + Server::Configuration::FactoryContext&) override { + std::lock_guard guard(log_callback_lock_); + auto access_log_instance = std::make_shared(log_cb_); + access_log_instances_.push_back(access_log_instance); + return access_log_instance; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new test::integration::accesslog::FakeAccessLog()}; + } + + void setLogCallback(LogSignature callable) { + std::lock_guard guard(log_callback_lock_); + log_cb_ = callable; + } + + std::string name() const override { return "envoy.access_loggers.test"; } + +private: + std::mutex log_callback_lock_; + LogSignature log_cb_{nullptr}; + std::vector access_log_instances_; +}; + +} // namespace Envoy diff --git a/test/integration/fake_access_log.proto b/test/integration/fake_access_log.proto new file mode 100644 index 000000000000..f220e25a7a3d --- /dev/null +++ b/test/integration/fake_access_log.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +package test.integration.accesslog; + +message FakeAccessLog { +} diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index b64ad12d8d5c..bfb987199f25 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -22,7 +22,7 @@ #include "quiche/quic/test_tools/quic_session_peer.h" #endif -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "test/integration/utility.h" #include "test/test_common/network_utility.h" @@ -685,7 +685,7 @@ void FakeUpstream::initializeServer() { dispatcher_->post([this]() -> void { socket_factories_[0]->doFinalPreWorkerInit(); - handler_->addListener(absl::nullopt, listener_, runtime_); + handler_->addListener(absl::nullopt, listener_, runtime_, random_); server_initialized_.setReady(); }); thread_ = api_->threadFactory().createThread([this]() -> void { threadRoutine(); }); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index bf3b3bba0394..0fd9bf07da94 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -45,7 +45,7 @@ #include "source/common/quic/quic_stat_names.h" #endif -#include "source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.h" +#include "source/common/listener_manager/active_raw_udp_listener_config.h" #include "test/mocks/common.h" #include "test/mocks/runtime/mocks.h" @@ -194,15 +194,18 @@ class FakeStream : public Http::RequestDecoder, if (!waitForData(client_dispatcher, 5, timeout)) { return testing::AssertionFailure() << "Timed out waiting for start of gRPC message."; } + int last_body_size = 0; { absl::MutexLock lock(&lock_); + last_body_size = body_.length(); if (!grpc_decoder_.decode(body_, decoded_grpc_frames_)) { return testing::AssertionFailure() << "Couldn't decode gRPC data frame: " << body_.toString(); } } if (decoded_grpc_frames_.empty()) { - if (!waitForData(client_dispatcher, grpc_decoder_.length(), bound.timeLeft())) { + if (!waitForData(client_dispatcher, grpc_decoder_.length() - last_body_size, + bound.timeLeft())) { return testing::AssertionFailure() << "Timed out waiting for end of gRPC message."; } { @@ -853,7 +856,8 @@ class FakeUpstream : Logger::Loggable, }; FakeListener(FakeUpstream& parent, bool is_quic = false) - : parent_(parent), name_("fake_upstream"), init_manager_(nullptr) { + : parent_(parent), name_("fake_upstream"), init_manager_(nullptr), + listener_info_(std::make_shared>()) { if (is_quic) { #if defined(ENVOY_ENABLE_QUIC) udp_listener_config_.listener_factory_ = std::make_unique( @@ -892,12 +896,12 @@ class FakeUpstream : Logger::Loggable, Network::ConnectionBalancer& connectionBalancer(const Network::Address::Instance&) override { return connection_balancer_; } - envoy::config::core::v3::TrafficDirection direction() const override { - return envoy::config::core::v3::UNSPECIFIED; - } const std::vector& accessLogs() const override { return empty_access_logs_; } + const Network::ListenerInfoConstSharedPtr& listenerInfo() const override { + return listener_info_; + } ResourceLimit& openConnections() override { return connection_resource_; } uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } uint32_t maxConnectionsToAcceptPerSocketEvent() const override { @@ -917,6 +921,7 @@ class FakeUpstream : Logger::Loggable, BasicResourceLimitImpl connection_resource_; const std::vector empty_access_logs_; std::unique_ptr init_manager_; + const Network::ListenerInfoConstSharedPtr listener_info_; }; void threadRoutine(); @@ -942,6 +947,7 @@ class FakeUpstream : Logger::Loggable, Network::ConnectionHandlerPtr handler_; std::list new_connections_ ABSL_GUARDED_BY(lock_); testing::NiceMock runtime_; + testing::NiceMock random_; // When a QueuedConnectionWrapper is popped from new_connections_, ownership is transferred to // consumed_connections_. This allows later the Connection destruction (when the FakeUpstream is diff --git a/test/integration/filter_integration_test.cc b/test/integration/filter_integration_test.cc index 312326bef37b..1f7582aed6ca 100644 --- a/test/integration/filter_integration_test.cc +++ b/test/integration/filter_integration_test.cc @@ -73,7 +73,7 @@ TEST_P(FilterIntegrationTest, OnLocalReply) { } // The second two tests validate the filter intercepting local replies, but - // upstream filters don't run on local replies. + // upstream HTTP filters don't run on local replies. if (!testing_downstream_filter_) { return; } @@ -164,6 +164,40 @@ TEST_P(FilterIntegrationTest, AddBodyToResponseAndWaitForIt) { EXPECT_EQ("body", response->body()); } +// Verify that filters can add a body and trailers to a header-only request or +// response. +TEST_P(FilterIntegrationTest, AddBodyAndTrailer) { + prependFilter(R"EOF( + name: add-body-filter + typed_config: + "@type": type.googleapis.com/test.integration.filters.AddBodyFilterConfig + add_trailers_in_encode_decode_header: true + )EOF"); + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(); + EXPECT_EQ("body", upstream_request_->body().toString()); + EXPECT_EQ("dummy_request_trailer_value", + upstream_request_->trailers() + ->get(Http::LowerCaseString("dummy_request_trailer"))[0] + ->value() + .getStringView()); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ("body", response->body()); + EXPECT_EQ("dummy_response_trailer_value", + response->trailers() + ->get(Http::LowerCaseString("dummy_response_trailer"))[0] + ->value() + .getStringView()); +} + TEST_P(FilterIntegrationTest, MissingHeadersLocalReplyDownstreamBytesCount) { useAccessLog("%DOWNSTREAM_WIRE_BYTES_SENT% %DOWNSTREAM_WIRE_BYTES_RECEIVED% " "%DOWNSTREAM_HEADER_BYTES_SENT% %DOWNSTREAM_HEADER_BYTES_RECEIVED%"); @@ -184,15 +218,9 @@ TEST_P(FilterIntegrationTest, MissingHeadersLocalReplyDownstreamBytesCount) { EXPECT_EQ("200", response->headers().getStatusValue()); if (testing_downstream_filter_) { - if (Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)) { - expectDownstreamBytesSentAndReceived(BytesCountExpectation(90, 88, 71, 54), - BytesCountExpectation(40, 58, 40, 58), - BytesCountExpectation(7, 10, 7, 8)); - } else { - expectDownstreamBytesSentAndReceived(BytesCountExpectation(90, 88, 71, 54), - BytesCountExpectation(0, 58, 0, 58), - BytesCountExpectation(7, 10, 7, 8)); - } + expectDownstreamBytesSentAndReceived(BytesCountExpectation(90, 88, 71, 54), + BytesCountExpectation(40, 58, 40, 58), + BytesCountExpectation(7, 10, 7, 8)); } } @@ -305,7 +333,7 @@ TEST_P(FilterIntegrationTest, MissingHeadersLocalReplyWithBodyBytesCount) { EXPECT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); if (testing_downstream_filter_) { - // When testing an upstream filters, we may receive body bytes before we + // When testing an upstream HTTP filters, we may receive body bytes before we // process headers, so don't set expectations. expectDownstreamBytesSentAndReceived(BytesCountExpectation(109, 1152, 90, 81), BytesCountExpectation(0, 58, 0, 58), @@ -1173,8 +1201,15 @@ TEST_P(FilterIntegrationTest, OverflowDecoderBufferFromDecodeTrailersWithContinu codec_client_->sendData(*request_encoder, 1024, false); codec_client_->sendData(*request_encoder, 1024, false); - codec_client_->sendTrailers(*request_encoder, - Http::TestRequestTrailerMapImpl{{"some", "trailer"}}); + if (std::get<0>(GetParam()).http2_implementation == Http2Impl::Oghttp2) { + EXPECT_LOG_NOT_CONTAINS( + "error", "DataFrameSource will send fin, preventing trailers", + codec_client_->sendTrailers(*request_encoder, + Http::TestRequestTrailerMapImpl{{"some", "trailer"}})); + } else { + codec_client_->sendTrailers(*request_encoder, + Http::TestRequestTrailerMapImpl{{"some", "trailer"}}); + } waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index 8d7f58372578..867bca9cef3e 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -487,6 +487,7 @@ envoy_cc_test_library( ":set_response_code_filter_config_proto_cc_proto", "//envoy/http:filter_interface", "//envoy/registry", + "//source/common/http:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", ], @@ -917,27 +918,6 @@ envoy_cc_test_library( ], ) -envoy_proto_library( - name = "header_to_filter_state_proto", - srcs = ["header_to_filter_state.proto"], -) - -envoy_cc_test_library( - name = "header_to_filter_state_lib", - srcs = [ - "header_to_filter_state.cc", - ], - deps = [ - ":header_to_filter_state_proto_cc_proto", - "//envoy/http:filter_interface", - "//envoy/registry", - "//envoy/server:filter_config_interface", - "//source/common/router:string_accessor_lib", - "//source/extensions/filters/http/common:factory_base_lib", - "//source/extensions/filters/http/common:pass_through_filter_lib", - ], -) - envoy_proto_library( name = "test_network_filter_proto", srcs = [":test_network_filter.proto"], diff --git a/test/integration/filters/add_body_filter.cc b/test/integration/filters/add_body_filter.cc index 25883263b195..09cb943ad78d 100644 --- a/test/integration/filters/add_body_filter.cc +++ b/test/integration/filters/add_body_filter.cc @@ -19,13 +19,16 @@ class AddBodyFilterConfig { AddBodyFilterConfig( test::integration::filters::AddBodyFilterConfig::FilterCallback where_to_add_body, uint32_t body_size, - test::integration::filters::AddBodyFilterConfig::FilterCallback where_to_stop_and_buffer) + test::integration::filters::AddBodyFilterConfig::FilterCallback where_to_stop_and_buffer, + bool add_trailers_in_encode_decode_header) : where_to_add_body_(where_to_add_body), body_size_(body_size), - where_to_stop_and_buffer_(where_to_stop_and_buffer) {} + where_to_stop_and_buffer_(where_to_stop_and_buffer), + add_trailers_in_encode_decode_header_(add_trailers_in_encode_decode_header) {} const test::integration::filters::AddBodyFilterConfig::FilterCallback where_to_add_body_; const uint32_t body_size_; const test::integration::filters::AddBodyFilterConfig::FilterCallback where_to_stop_and_buffer_; + bool add_trailers_in_encode_decode_header_; }; // A test filter that adds body data to a request/response without body payload. @@ -40,6 +43,11 @@ class AddBodyStreamFilter : public Http::PassThroughFilter { Buffer::OwnedImpl body("body"); headers.setContentLength(body.length()); decoder_callbacks_->addDecodedData(body, false); + if (config_->add_trailers_in_encode_decode_header_) { + auto& trailers = decoder_callbacks_->addDecodedTrailers(); + trailers.addCopy(Http::LowerCaseString("dummy_request_trailer"), + "dummy_request_trailer_value"); + } } else { headers.removeContentLength(); } @@ -103,6 +111,11 @@ class AddBodyStreamFilter : public Http::PassThroughFilter { Buffer::OwnedImpl body("body"); headers.setContentLength(body.length()); encoder_callbacks_->addEncodedData(body, false); + if (config_->add_trailers_in_encode_decode_header_) { + auto& trailers = encoder_callbacks_->addEncodedTrailers(); + trailers.addCopy(Http::LowerCaseString("dummy_response_trailer"), + "dummy_response_trailer_value"); + } } } else if (config_->where_to_add_body_ == test::integration::filters::AddBodyFilterConfig::ENCODE_HEADERS) { @@ -124,12 +137,13 @@ class AddBodyFilterFactory : public Extensions::HttpFilters::Common::DualFactory AddBodyFilterFactory() : DualFactoryBase("add-body-filter") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const test::integration::filters::AddBodyFilterConfig& proto_config, const std::string&, DualInfo, Server::Configuration::ServerFactoryContext&) override { auto filter_config = std::make_shared( proto_config.where_to_add_body(), proto_config.body_size(), - proto_config.where_to_stop_and_buffer()); + proto_config.where_to_stop_and_buffer(), + proto_config.add_trailers_in_encode_decode_header()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; diff --git a/test/integration/filters/add_body_filter.proto b/test/integration/filters/add_body_filter.proto index 0c14be97659d..59f09fc810a9 100644 --- a/test/integration/filters/add_body_filter.proto +++ b/test/integration/filters/add_body_filter.proto @@ -15,4 +15,5 @@ message AddBodyFilterConfig { FilterCallback where_to_add_body = 1; uint32 body_size = 2; FilterCallback where_to_stop_and_buffer = 3; + bool add_trailers_in_encode_decode_header = 4; } diff --git a/test/integration/filters/add_header_filter.cc b/test/integration/filters/add_header_filter.cc index ea505a99b7c1..098cf0c5af52 100644 --- a/test/integration/filters/add_header_filter.cc +++ b/test/integration/filters/add_header_filter.cc @@ -26,8 +26,8 @@ class AddHeaderFilter : public Http::PassThroughFilter { class AddHeaderFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpDualFilterConfig { public: AddHeaderFilterConfig() : EmptyHttpDualFilterConfig("add-header-filter") {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::AddHeaderFilter>()); }; @@ -68,13 +68,13 @@ class AddConfigurableHeaderFilterFactory return std::make_unique(); } - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message& config, const std::string&, Server::Configuration::UpstreamFactoryContext& context) override { const auto& proto_config = MessageUtil::downcastAndValidate( - config, context.getServerFactoryContext().messageValidationVisitor()); + config, context.serverFactoryContext().messageValidationVisitor()); return [proto_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared( diff --git a/test/integration/filters/add_trailers_filter.cc b/test/integration/filters/add_trailers_filter.cc index 94d53aabc600..23ea7d644e7d 100644 --- a/test/integration/filters/add_trailers_filter.cc +++ b/test/integration/filters/add_trailers_filter.cc @@ -36,8 +36,8 @@ class AddTrailersStreamFilterConfig public: AddTrailersStreamFilterConfig() : EmptyHttpDualFilterConfig("add-trailers-filter") {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::AddTrailersStreamFilter>()); }; diff --git a/test/integration/filters/backpressure_filter.cc b/test/integration/filters/backpressure_filter.cc index b45ec9991736..2a07a189c968 100644 --- a/test/integration/filters/backpressure_filter.cc +++ b/test/integration/filters/backpressure_filter.cc @@ -41,8 +41,8 @@ class BackpressureConfig : public Extensions::HttpFilters::Common::EmptyHttpFilt public: BackpressureConfig() : EmptyHttpFilterConfig("backpressure-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::BackpressureFilter>()); }; diff --git a/test/integration/filters/buffer_continue_filter.cc b/test/integration/filters/buffer_continue_filter.cc index 9e36458fe7d2..39707da0324a 100644 --- a/test/integration/filters/buffer_continue_filter.cc +++ b/test/integration/filters/buffer_continue_filter.cc @@ -59,8 +59,8 @@ class BufferContinueFilterConfig public: BufferContinueFilterConfig() : EmptyHttpDualFilterConfig("buffer-continue-filter") {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::BufferContinueStreamFilter>()); }; diff --git a/test/integration/filters/clear_route_cache_filter.cc b/test/integration/filters/clear_route_cache_filter.cc index 3ed8986610db..a4a68ec4a9b9 100644 --- a/test/integration/filters/clear_route_cache_filter.cc +++ b/test/integration/filters/clear_route_cache_filter.cc @@ -23,8 +23,8 @@ class ClearRouteCacheFilterConfig : public Extensions::HttpFilters::Common::Empt public: ClearRouteCacheFilterConfig() : EmptyHttpFilterConfig("clear-route-cache") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ClearRouteCacheFilter>()); }; diff --git a/test/integration/filters/common.h b/test/integration/filters/common.h index 00d13dbc878d..131eef791b72 100644 --- a/test/integration/filters/common.h +++ b/test/integration/filters/common.h @@ -15,8 +15,8 @@ class SimpleFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpDual public: SimpleFilterConfig() : EmptyHttpDualFilterConfig(T::name) {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared()); }; diff --git a/test/integration/filters/crash_filter.cc b/test/integration/filters/crash_filter.cc index 6b1c5f93810c..0a5c6d528134 100644 --- a/test/integration/filters/crash_filter.cc +++ b/test/integration/filters/crash_filter.cc @@ -80,7 +80,7 @@ class CrashFilterFactory : public Extensions::HttpFilters::Common::DualFactoryBa CrashFilterFactory() : DualFactoryBase("crash-filter") {} private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + absl::StatusOr createFilterFactoryFromProtoTyped( const test::integration::filters::CrashFilterConfig& proto_config, const std::string&, DualInfo, Server::Configuration::ServerFactoryContext&) override { auto filter_config = std::make_shared( diff --git a/test/integration/filters/eds_ready_filter.cc b/test/integration/filters/eds_ready_filter.cc index ed075324fa91..057d6af9faa7 100644 --- a/test/integration/filters/eds_ready_filter.cc +++ b/test/integration/filters/eds_ready_filter.cc @@ -57,11 +57,11 @@ class EdsReadyFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpFi public: EdsReadyFilterConfig() : EmptyHttpFilterConfig("eds-ready-filter") {} - Http::FilterFactoryCb + absl::StatusOr createFilter(const std::string&, Server::Configuration::FactoryContext& factory_context) override { return [&factory_context](Http::FilterChainFactoryCallbacks& callbacks) { - const Stats::Scope& scope = factory_context.api().rootScope(); + const Stats::Scope& scope = factory_context.serverFactoryContext().api().rootScope(); Stats::SymbolTable& symbol_table = factory_context.scope().symbolTable(); callbacks.addStreamFilter(std::make_shared(scope, symbol_table)); }; diff --git a/test/integration/filters/encode1xx_local_reply_filter.cc b/test/integration/filters/encode1xx_local_reply_filter.cc index 33d704b0dae0..fe9e36dddfd0 100644 --- a/test/integration/filters/encode1xx_local_reply_filter.cc +++ b/test/integration/filters/encode1xx_local_reply_filter.cc @@ -26,8 +26,8 @@ class Encode1xxLocalReplyFilterConfig public: Encode1xxLocalReplyFilterConfig() : EmptyHttpFilterConfig("encode1xx-local-reply-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared()); }; diff --git a/test/integration/filters/encoder_decoder_buffer_filter.cc b/test/integration/filters/encoder_decoder_buffer_filter.cc index c526e2fe2a8e..e196dc906475 100644 --- a/test/integration/filters/encoder_decoder_buffer_filter.cc +++ b/test/integration/filters/encoder_decoder_buffer_filter.cc @@ -39,8 +39,8 @@ class EncoderDecoderBufferFilterConfig public: EncoderDecoderBufferFilterConfig() : EmptyHttpDualFilterConfig("encoder-decoder-buffer-filter") {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::EncoderDecoderBufferStreamFilter>()); }; diff --git a/test/integration/filters/header_to_filter_state.cc b/test/integration/filters/header_to_filter_state.cc deleted file mode 100644 index bbe0559e5a67..000000000000 --- a/test/integration/filters/header_to_filter_state.cc +++ /dev/null @@ -1,74 +0,0 @@ -#include - -#include "envoy/http/filter.h" -#include "envoy/registry/registry.h" -#include "envoy/server/filter_config.h" - -#include "source/common/router/string_accessor_impl.h" -#include "source/extensions/filters/http/common/factory_base.h" -#include "source/extensions/filters/http/common/pass_through_filter.h" - -#include "test/integration/filters/header_to_filter_state.pb.h" -#include "test/integration/filters/header_to_filter_state.pb.validate.h" - -namespace Envoy { - -// This filter extracts a string header from "header" and -// save it into FilterState as name "state" as read-only Router::StringAccessor. -class HeaderToFilterStateFilter : public Http::PassThroughDecoderFilter { -public: - HeaderToFilterStateFilter(const std::string& header, const std::string& state, bool read_only, - test::integration::filters::SharingConfig shared) - : header_(header), state_(state), read_only_(read_only), shared_(shared) {} - - Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, bool) override { - const auto entry = headers.get(header_); - if (!entry.empty()) { - auto shared = StreamInfo::StreamSharingMayImpactPooling::None; - switch (shared_) { - case test::integration::filters::SharingConfig::ONCE: - shared = StreamInfo::StreamSharingMayImpactPooling::SharedWithUpstreamConnectionOnce; - break; - case test::integration::filters::SharingConfig::TRANSITIVE: - shared = StreamInfo::StreamSharingMayImpactPooling::SharedWithUpstreamConnection; - break; - default: - break; - } - decoder_callbacks_->streamInfo().filterState()->setData( - state_, std::make_unique(entry[0]->value().getStringView()), - read_only_ ? StreamInfo::FilterState::StateType::ReadOnly - : StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::FilterChain, shared); - } - return Http::FilterHeadersStatus::Continue; - } - -private: - const Http::LowerCaseString header_; - const std::string state_; - const bool read_only_; - const test::integration::filters::SharingConfig shared_; -}; - -class HeaderToFilterStateFilterFactory - : public Extensions::HttpFilters::Common::FactoryBase< - test::integration::filters::HeaderToFilterStateFilterConfig> { -public: - HeaderToFilterStateFilterFactory() : FactoryBase("header-to-filter-state") {} - -private: - Http::FilterFactoryCb createFilterFactoryFromProtoTyped( - const test::integration::filters::HeaderToFilterStateFilterConfig& proto_config, - const std::string&, Server::Configuration::FactoryContext&) override { - return [=](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamDecoderFilter(std::make_shared( - proto_config.header_name(), proto_config.state_name(), proto_config.read_only(), - proto_config.shared())); - }; - } -}; - -REGISTER_FACTORY(HeaderToFilterStateFilterFactory, - Server::Configuration::NamedHttpFilterConfigFactory); -} // namespace Envoy diff --git a/test/integration/filters/header_to_filter_state.proto b/test/integration/filters/header_to_filter_state.proto deleted file mode 100644 index 84672e271372..000000000000 --- a/test/integration/filters/header_to_filter_state.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; - -package test.integration.filters; - -enum SharingConfig { - NONE = 0; - ONCE = 1; - TRANSITIVE = 2; -}; - -message HeaderToFilterStateFilterConfig { - string header_name = 1; - string state_name = 2; - bool read_only = 3; - SharingConfig shared = 4; -} diff --git a/test/integration/filters/header_to_proxy_filter.cc b/test/integration/filters/header_to_proxy_filter.cc index 8dd9ba16f424..cb1213ce4a48 100644 --- a/test/integration/filters/header_to_proxy_filter.cc +++ b/test/integration/filters/header_to_proxy_filter.cc @@ -38,8 +38,8 @@ class HeaderToProxyFilterConfig : public Extensions::HttpFilters::Common::EmptyH public: HeaderToProxyFilterConfig() : EmptyHttpFilterConfig("header-to-proxy-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::HeaderToProxyFilter>()); }; diff --git a/test/integration/filters/invalid_header_filter.cc b/test/integration/filters/invalid_header_filter.cc index db7e14104daf..cff2a0ea346d 100644 --- a/test/integration/filters/invalid_header_filter.cc +++ b/test/integration/filters/invalid_header_filter.cc @@ -30,7 +30,8 @@ class InvalidHeaderFilter : public Http::PassThroughFilter { // Insert invalid characters by bypassing assert(valid()) checks, which would not run // in release mode headers.addCopy(Http::LowerCaseString("x-foo"), "hello x-oops: yes"); - absl::string_view value = headers.getByKey("x-foo").value(); + absl::string_view value = + headers.get(Http::LowerCaseString("x-foo"))[0]->value().getStringView(); char* data = const_cast(value.data()); data[5] = '\r'; data[6] = '\n'; @@ -40,7 +41,8 @@ class InvalidHeaderFilter : public Http::PassThroughFilter { // Insert invalid characters by bypassing assert(valid()) checks, which would not run // in release mode headers.addCopy(Http::LowerCaseString("x-foo"), "hello GET /evil HTTP/1.1"); - absl::string_view value = headers.getByKey("x-foo").value(); + absl::string_view value = + headers.get(Http::LowerCaseString("x-foo"))[0]->value().getStringView(); char* data = const_cast(value.data()); data[5] = '\r'; data[6] = '\n'; diff --git a/test/integration/filters/listener_typed_metadata_filter.cc b/test/integration/filters/listener_typed_metadata_filter.cc index 3d797e2a18e6..8150d3615337 100644 --- a/test/integration/filters/listener_typed_metadata_filter.cc +++ b/test/integration/filters/listener_typed_metadata_filter.cc @@ -58,11 +58,11 @@ class ListenerTypedMetadataFilterFactory ListenerTypedMetadataFilterFactory() : EmptyHttpFilterConfig(std::string(kFilterName)) {} private: - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext& context) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext& context) override { // Main assertions to ensure the metadata from the listener was parsed correctly. - const auto& typed_metadata = context.listenerTypedMetadata(); + const auto& typed_metadata = context.listenerInfo().typedMetadata(); const Baz* value = typed_metadata.get(std::string(kMetadataKey)); EXPECT_NE(value, nullptr); EXPECT_EQ(value->item_, kExpectedMetadataValue); diff --git a/test/integration/filters/local_reply_during_decoding_filter.cc b/test/integration/filters/local_reply_during_decoding_filter.cc index 69d822e8dcca..c01cd9f09401 100644 --- a/test/integration/filters/local_reply_during_decoding_filter.cc +++ b/test/integration/filters/local_reply_during_decoding_filter.cc @@ -15,11 +15,31 @@ class LocalReplyDuringDecode : public Http::PassThroughFilter { public: constexpr static char name[] = "local-reply-during-decode"; - Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap&, bool) override { + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& request_headers, bool) override { + auto result = request_headers.get(Http::LowerCaseString("skip-local-reply")); + if (!result.empty() && result[0]->value() == "true") { + local_reply_skipped_ = true; + return Http::FilterHeadersStatus::Continue; + } decoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, "", nullptr, absl::nullopt, ""); return Http::FilterHeadersStatus::StopIteration; } + + // Due to the above local reply, this method should never be invoked in tests. + Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override { + ASSERT(local_reply_skipped_); + return Http::FilterDataStatus::Continue; + } + + // Due to the above local reply, this method should never be invoked in tests. + Http::FilterMetadataStatus decodeMetadata(Http::MetadataMap&) override { + ASSERT(local_reply_skipped_); + return Http::FilterMetadataStatus::Continue; + } + +private: + bool local_reply_skipped_ = false; }; constexpr char LocalReplyDuringDecode::name[]; diff --git a/test/integration/filters/modify_buffer_filter.cc b/test/integration/filters/modify_buffer_filter.cc index 6649eb7501e2..f8fe671355ea 100644 --- a/test/integration/filters/modify_buffer_filter.cc +++ b/test/integration/filters/modify_buffer_filter.cc @@ -47,8 +47,8 @@ class ModifyBufferFilterConfig : public Extensions::HttpFilters::Common::EmptyHt public: ModifyBufferFilterConfig() : EmptyHttpFilterConfig("modify-buffer-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ModifyBufferStreamFilter>()); }; diff --git a/test/integration/filters/on_local_reply_filter.cc b/test/integration/filters/on_local_reply_filter.cc index 7d81e3123e60..40f7d6be34cf 100644 --- a/test/integration/filters/on_local_reply_filter.cc +++ b/test/integration/filters/on_local_reply_filter.cc @@ -47,8 +47,8 @@ class OnLocalReplyFilter : public Http::PassThroughFilter { class OnLocalReplyFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpDualFilterConfig { public: OnLocalReplyFilterConfig() : EmptyHttpDualFilterConfig("on-local-reply-filter") {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::OnLocalReplyFilter>()); }; diff --git a/test/integration/filters/pause_filter.cc b/test/integration/filters/pause_filter.cc index de6a79d94dc2..d0606bc7ea34 100644 --- a/test/integration/filters/pause_filter.cc +++ b/test/integration/filters/pause_filter.cc @@ -66,8 +66,8 @@ class TestPauseFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpF public: TestPauseFilterConfig() : EmptyHttpFilterConfig("pause-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [&](Http::FilterChainFactoryCallbacks& callbacks) -> void { // ABSL_GUARDED_BY insists the lock be held when the guarded variables are passed by // reference. diff --git a/test/integration/filters/pause_filter_for_quic.cc b/test/integration/filters/pause_filter_for_quic.cc index 9f24f8e8a8d6..525e97bf672f 100644 --- a/test/integration/filters/pause_filter_for_quic.cc +++ b/test/integration/filters/pause_filter_for_quic.cc @@ -67,8 +67,8 @@ class TestPauseFilterConfigForQuic : public Extensions::HttpFilters::Common::Emp public: TestPauseFilterConfigForQuic() : EmptyHttpFilterConfig("pause-filter-for-quic") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [&](Http::FilterChainFactoryCallbacks& callbacks) -> void { // ABSL_GUARDED_BY insists the lock be held when the guarded variables are passed by // reference. diff --git a/test/integration/filters/process_context_filter.cc b/test/integration/filters/process_context_filter.cc index 74c9322bc978..6fb6e941c516 100644 --- a/test/integration/filters/process_context_filter.cc +++ b/test/integration/filters/process_context_filter.cc @@ -40,12 +40,12 @@ class ProcessContextFilterConfig : public Extensions::HttpFilters::Common::Empty public: ProcessContextFilterConfig() : EmptyHttpFilterConfig("process-context-filter") {} - Http::FilterFactoryCb + absl::StatusOr createFilter(const std::string&, Server::Configuration::FactoryContext& factory_context) override { return [&factory_context](Http::FilterChainFactoryCallbacks& callbacks) { - callbacks.addStreamFilter( - std::make_shared(*factory_context.processContext())); + callbacks.addStreamFilter(std::make_shared( + *factory_context.serverFactoryContext().processContext())); }; } }; diff --git a/test/integration/filters/random_pause_filter.cc b/test/integration/filters/random_pause_filter.cc index 943b809f217f..5c899d9fc4d2 100644 --- a/test/integration/filters/random_pause_filter.cc +++ b/test/integration/filters/random_pause_filter.cc @@ -49,8 +49,8 @@ class RandomPauseFilterConfig : public Extensions::HttpFilters::Common::EmptyHtt public: RandomPauseFilterConfig() : EmptyHttpFilterConfig("random-pause-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [&](Http::FilterChainFactoryCallbacks& callbacks) -> void { absl::WriterMutexLock m(&rand_lock_); if (rng_ == nullptr) { diff --git a/test/integration/filters/repick_cluster_filter.cc b/test/integration/filters/repick_cluster_filter.cc index a20b471909d6..09a001b73398 100644 --- a/test/integration/filters/repick_cluster_filter.cc +++ b/test/integration/filters/repick_cluster_filter.cc @@ -30,8 +30,8 @@ class RepickClusterFilterConfig : public Extensions::HttpFilters::Common::EmptyH public: RepickClusterFilterConfig() : EmptyHttpFilterConfig("repick-cluster-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter( std::make_shared<::Envoy::RepickClusterFilter::RepickClusterFilter>()); diff --git a/test/integration/filters/request_metadata_filter.cc b/test/integration/filters/request_metadata_filter.cc index 2f56f1659233..7a1a095e66f6 100644 --- a/test/integration/filters/request_metadata_filter.cc +++ b/test/integration/filters/request_metadata_filter.cc @@ -52,8 +52,8 @@ class AddRequestMetadataStreamFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpFilterConfig { public: AddRequestMetadataStreamFilterConfig() : EmptyHttpFilterConfig("request-metadata-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::RequestMetadataStreamFilter>()); }; diff --git a/test/integration/filters/reset_idle_timer_filter.cc b/test/integration/filters/reset_idle_timer_filter.cc index d1f7430b5525..d1ecedb06b3d 100644 --- a/test/integration/filters/reset_idle_timer_filter.cc +++ b/test/integration/filters/reset_idle_timer_filter.cc @@ -23,8 +23,8 @@ class ResetIdleTimerFilterConfig : public Extensions::HttpFilters::Common::Empty public: ResetIdleTimerFilterConfig() : EmptyHttpFilterConfig("reset-idle-timer-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ResetIdleTimerFilter>()); }; diff --git a/test/integration/filters/reset_stream_filter.cc b/test/integration/filters/reset_stream_filter.cc index ef2e9edef189..982acfe1a2d3 100644 --- a/test/integration/filters/reset_stream_filter.cc +++ b/test/integration/filters/reset_stream_filter.cc @@ -22,8 +22,8 @@ class ResetFilter : public Http::PassThroughFilter { class ResetFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpDualFilterConfig { public: ResetFilterConfig() : EmptyHttpDualFilterConfig("reset-stream-filter") {} - Http::FilterFactoryCb createDualFilter(const std::string&, - Server::Configuration::ServerFactoryContext&) override { + absl::StatusOr + createDualFilter(const std::string&, Server::Configuration::ServerFactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ResetFilter>()); }; diff --git a/test/integration/filters/response_metadata_filter.cc b/test/integration/filters/response_metadata_filter.cc index 21fac60760f2..63917ac27675 100644 --- a/test/integration/filters/response_metadata_filter.cc +++ b/test/integration/filters/response_metadata_filter.cc @@ -77,8 +77,8 @@ class AddMetadataStreamFilterConfig public: AddMetadataStreamFilterConfig() : EmptyHttpFilterConfig("response-metadata-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override { + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ResponseMetadataStreamFilter>()); }; diff --git a/test/integration/filters/set_response_code_filter.cc b/test/integration/filters/set_response_code_filter.cc index f2cc3dc8e633..f5c26b3e6bb9 100644 --- a/test/integration/filters/set_response_code_filter.cc +++ b/test/integration/filters/set_response_code_filter.cc @@ -3,6 +3,7 @@ #include "envoy/http/filter.h" #include "envoy/registry/registry.h" +#include "source/common/http/utility.h" #include "source/extensions/filters/http/common/factory_base.h" #include "source/extensions/filters/http/common/pass_through_filter.h" @@ -17,7 +18,21 @@ namespace Envoy { class SetResponseCodeFilterConfig { public: SetResponseCodeFilterConfig(const std::string& prefix, uint32_t code, const std::string& body, - Server::Configuration::FactoryContextBase& context) + Server::Configuration::CommonFactoryContext& context) + : prefix_(prefix), code_(code), body_(body), tls_slot_(context.threadLocal()) {} + + const std::string prefix_; + const uint32_t code_; + const std::string body_; + // Allocate a slot to validate that it is destroyed on a main thread only. + ThreadLocal::TypedSlot<> tls_slot_; +}; + +class SetResponseCodeFilterRouteSpecificConfig : public Envoy::Router::RouteSpecificFilterConfig { +public: + SetResponseCodeFilterRouteSpecificConfig(const std::string& prefix, uint32_t code, + const std::string& body, + Server::Configuration::CommonFactoryContext& context) : prefix_(prefix), code_(code), body_(body), tls_slot_(context.threadLocal()) {} const std::string prefix_; @@ -32,9 +47,26 @@ class SetResponseCodeFilter : public Http::PassThroughFilter { SetResponseCodeFilter(std::shared_ptr config) : config_(config) {} Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, bool) override { - if (absl::StartsWith(headers.Path()->value().getStringView(), config_->prefix_)) { - decoder_callbacks_->sendLocalReply(static_cast(config_->code_), config_->body_, - nullptr, absl::nullopt, ""); + const auto* per_route_config = Envoy::Http::Utility::resolveMostSpecificPerFilterConfig< + SetResponseCodeFilterRouteSpecificConfig>(decoder_callbacks_); + + std::string prefix; + uint32_t code; + std::string body; + // Route level config takes precedence over filter level config, if present. + if (per_route_config != nullptr) { + prefix = per_route_config->prefix_; + code = per_route_config->code_; + body = per_route_config->body_; + } else { + prefix = config_->prefix_; + code = config_->code_; + body = config_->body_; + } + + if (absl::StartsWith(headers.Path()->value().getStringView(), prefix)) { + decoder_callbacks_->sendLocalReply(static_cast(code), body, nullptr, + absl::nullopt, ""); return Http::FilterHeadersStatus::StopIteration; } return Http::FilterHeadersStatus::Continue; @@ -44,8 +76,10 @@ class SetResponseCodeFilter : public Http::PassThroughFilter { const std::shared_ptr config_; }; -class SetResponseCodeFilterFactory : public Extensions::HttpFilters::Common::FactoryBase< - test::integration::filters::SetResponseCodeFilterConfig> { +class SetResponseCodeFilterFactory + : public Extensions::HttpFilters::Common::FactoryBase< + test::integration::filters::SetResponseCodeFilterConfig, + test::integration::filters::SetResponseCodePerRouteFilterConfig> { public: SetResponseCodeFilterFactory() : FactoryBase("set-response-code-filter") {} @@ -54,7 +88,8 @@ class SetResponseCodeFilterFactory : public Extensions::HttpFilters::Common::Fac const test::integration::filters::SetResponseCodeFilterConfig& proto_config, const std::string&, Server::Configuration::FactoryContext& context) override { auto filter_config = std::make_shared( - proto_config.prefix(), proto_config.code(), proto_config.body(), context); + proto_config.prefix(), proto_config.code(), proto_config.body(), + context.serverFactoryContext()); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared(filter_config)); }; @@ -68,6 +103,14 @@ class SetResponseCodeFilterFactory : public Extensions::HttpFilters::Common::Fac callbacks.addStreamFilter(std::make_shared(filter_config)); }; } + + Router::RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfigTyped( + const test::integration::filters::SetResponseCodePerRouteFilterConfig& proto_config, + Server::Configuration::ServerFactoryContext& context, + ProtobufMessage::ValidationVisitor&) override { + return std::make_shared( + proto_config.prefix(), proto_config.code(), proto_config.body(), context); + } }; REGISTER_FACTORY(SetResponseCodeFilterFactory, Server::Configuration::NamedHttpFilterConfigFactory); diff --git a/test/integration/filters/set_response_code_filter_config.proto b/test/integration/filters/set_response_code_filter_config.proto index 09765c970d01..d9ca4298101f 100644 --- a/test/integration/filters/set_response_code_filter_config.proto +++ b/test/integration/filters/set_response_code_filter_config.proto @@ -9,3 +9,10 @@ message SetResponseCodeFilterConfig { uint32 code = 2 [(validate.rules).uint32 = {lt: 600 gte: 200}]; string body = 3; } + +// Per route config overrides filter config, if present. +message SetResponseCodePerRouteFilterConfig { + string prefix = 1; + uint32 code = 2 [(validate.rules).uint32 = {lt: 600 gte: 200}]; + string body = 3; +} diff --git a/test/integration/filters/tee_filter.cc b/test/integration/filters/tee_filter.cc index f7111b7ba990..ade0be04e85c 100644 --- a/test/integration/filters/tee_filter.cc +++ b/test/integration/filters/tee_filter.cc @@ -48,8 +48,8 @@ class StreamTeeFilter : public Http::PassThroughFilter, public StreamTee { } }; -Http::FilterFactoryCb StreamTeeFilterConfig::createFilter(const std::string&, - Server::Configuration::FactoryContext&) { +absl::StatusOr +StreamTeeFilterConfig::createFilter(const std::string&, Server::Configuration::FactoryContext&) { return [this](Http::FilterChainFactoryCallbacks& callbacks) -> void { auto filter = std::make_shared(); // TODO(kbaichoo): support multiple connections. diff --git a/test/integration/filters/tee_filter.h b/test/integration/filters/tee_filter.h index f4a08e86f3d2..3a1121d8629c 100644 --- a/test/integration/filters/tee_filter.h +++ b/test/integration/filters/tee_filter.h @@ -41,8 +41,8 @@ class StreamTeeFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpF public: StreamTeeFilterConfig() : EmptyHttpFilterConfig("stream-tee-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, - Server::Configuration::FactoryContext&) override; + absl::StatusOr + createFilter(const std::string&, Server::Configuration::FactoryContext&) override; bool inspectStreamTee(uint32_t stream_id, std::function inspector); bool setEncodeDataCallback(uint32_t stream_id, std::function void { filter_manager.addReadFilter(std::make_shared( - config, context.scope(), context.clusterManager())); + config, context.scope(), context.serverFactoryContext().clusterManager())); }; } }; diff --git a/test/integration/filters/test_network_filter.cc b/test/integration/filters/test_network_filter.cc index 725a0dd1b1bf..cf10310e66ec 100644 --- a/test/integration/filters/test_network_filter.cc +++ b/test/integration/filters/test_network_filter.cc @@ -150,7 +150,7 @@ class TestDrainerUpstreamNetworkFilterConfigFactory Server::Configuration::UpstreamFactoryContext& context) override { const auto& config = MessageUtil::downcastAndValidate< const test::integration::filters::TestDrainerUpstreamNetworkFilterConfig&>( - proto_config, context.getServerFactoryContext().messageValidationVisitor()); + proto_config, context.serverFactoryContext().messageValidationVisitor()); return [config](Network::FilterManager& filter_manager) -> void { filter_manager.addWriteFilter(std::make_shared(config)); }; diff --git a/test/integration/header_integration_test.cc b/test/integration/header_integration_test.cc index b9905ff94054..20c31de71483 100644 --- a/test/integration/header_integration_test.cc +++ b/test/integration/header_integration_test.cc @@ -1341,8 +1341,9 @@ TEST_P(HeaderIntegrationTest, PathWithEscapedSlashesRedirected) { }); } -// Validates TE header is forwarded if it contains a supported value +// Validates legacy TE handling: TE header is forwarded if it contains a supported value TEST_P(HeaderIntegrationTest, TestTeHeaderPassthrough) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.sanitize_te", "false"); initializeFilter(HeaderMode::Append, false); performRequest( Http::TestRequestHeaderMapImpl{ @@ -1375,8 +1376,9 @@ TEST_P(HeaderIntegrationTest, TestTeHeaderPassthrough) { }); } -// Validates TE header is stripped if it contains an unsupported value +// Validates legacy TE handling: that TE header stripped if it contains an unsupported value. TEST_P(HeaderIntegrationTest, TestTeHeaderSanitized) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.sanitize_te", "false"); initializeFilter(HeaderMode::Append, false); performRequest( Http::TestRequestHeaderMapImpl{ diff --git a/test/integration/hotrestart_handoff_test.py b/test/integration/hotrestart_handoff_test.py new file mode 100644 index 000000000000..7d975bd7dda7 --- /dev/null +++ b/test/integration/hotrestart_handoff_test.py @@ -0,0 +1,505 @@ +"""Tests the behavior of connection handoff between instances during hot restart. + +Specifically, tests that: +1. TCP connections opened before hot restart begins continue to function during drain. +2. TCP connections opened after hot restart begins while the old instance is still running + go to the new instance. +TODO(ravenblack): perform the same tests for QUIC connections once they will work as expected. +""" + +import abc +import argparse +import asyncio +from functools import cached_property +import logging +import os +import pathlib +import random +import sys +import tempfile +from typing import Awaitable +import unittest +from datetime import datetime, timedelta +from aiohttp import client_exceptions, web, ClientSession + + +def random_loopback_host(): + """Returns a randomized loopback IP. + This can be used to reduce the chance of port conflicts when tests are + running in parallel.""" + return f"127.{random.randrange(0,256)}.{random.randrange(0,256)}.{random.randrange(1, 255)}" + + +# This is a timeout that must be long enough that the hot restarted +# instance can reliably be fully started up within this many seconds, or the +# test will be flaky. 3 seconds is enough on a not-busy host with a non-tsan +# non-coverage build; 10 seconds should be enough to be not flaky in most +# configurations. +# +# Unfortunately, because the test is verifying the behavior of a connection +# during drain, the first connection must last for the full tolerance duration, +# so increasing this value increases the duration of the test. For this +# reason we want to keep it as low as possible without causing flaky failure. +# +# Ideally this would be adjusted (3x) for tsan and coverage runs, but making that +# possible for python is outside the scope of this test, so we're stuck using the +# 3x value for all tests. +STARTUP_TOLERANCE_SECONDS = 10 + +# We send multiple requests in parallel and require them all to function correctly +# - this makes it so if something is flaky we're more likely to encounter it, and +# also tests that there's not an "only one" success situation. +PARALLEL_REQUESTS = 5 + +UPSTREAM_SLOW_PORT = 54321 +UPSTREAM_FAST_PORT = 54322 +UPSTREAM_HOST = random_loopback_host() +ENVOY_HOST = UPSTREAM_HOST +ENVOY_PORT = 54323 +ENVOY_ADMIN_PORT = 54324 +# Append process ID to the socket path to minimize chances of +# conflict. We can't use TEST_TMPDIR for this because it makes +# the socket path too long. +SOCKET_PATH = f"@envoy_domain_socket_{os.getpid()}" +SOCKET_MODE = 0 + +# This log config makes logs interleave with other test output, which +# is useful since with all the async operations it can be hard to figure +# out what's happening. +log = logging.getLogger() +log.level = logging.INFO +_stream_handler = logging.StreamHandler(sys.stdout) +log.addHandler(_stream_handler) + + +class Upstream: + # This class runs a server which takes an http request to + # path=/ and responds with "start\n" [three second pause] "end\n". + # This allows us to test that during hot restart an already-opened + # connection will persist. + # If initialized with True it will instead respond with + # "fast instance" immediately. + def __init__(self, fast_version=False): + self.port = UPSTREAM_FAST_PORT if fast_version else UPSTREAM_SLOW_PORT + self.app = web.Application() + self.app.add_routes([ + web.get("/", self.fast_response) if fast_version else web.get("/", self.slow_response), + ]) + + async def start(self): + self.runner = web.AppRunner(self.app, handle_signals=False) + await self.runner.setup() + site = web.TCPSite(self.runner, host=UPSTREAM_HOST, port=self.port) + await site.start() + + async def stop(self): + await self.runner.shutdown() + await self.runner.cleanup() + log.debug("runner cleaned up") + + async def fast_response(self, request): + return web.Response( + status=200, + reason="OK", + headers={"content-type": "text/plain"}, + body="fast instance", + ) + + async def slow_response(self, request): + log.debug("slow request received") + response = web.StreamResponse( + status=200, reason="OK", headers={"content-type": "text/plain"}) + await response.prepare(request) + await response.write(b"start\n") + await asyncio.sleep(STARTUP_TOLERANCE_SECONDS + 0.5) + await response.write(b"end\n") + await response.write_eof() + return response + + +class LineGenerator: + + @cached_property + def _queue(self) -> asyncio.Queue[str]: + return asyncio.Queue() + + @cached_property + def _task(self): + return asyncio.create_task(self.generator()) + + @abc.abstractmethod + async def generator(self) -> None: + raise NotImplementedError + + def __init__(self): + self._task + + async def join(self) -> int: + await self._task + return self._queue.qsize() + + async def line(self) -> str: + line = await self._queue.get() + self._queue.task_done() + return line + + +class Http3RequestLineGenerator(LineGenerator): + + def __init__(self, url): + self._url = url + super().__init__() + + async def generator(self) -> None: + proc = await asyncio.create_subprocess_exec( + IntegrationTest.h3_request, + f"--ca-certs={IntegrationTest.ca_certs}", + self._url, + stdout=asyncio.subprocess.PIPE, + ) + async for line in proc.stdout: + await self._queue.put(line) + await proc.wait() + + +class HttpRequestLineGenerator(LineGenerator): + + def __init__(self, url): + self._url = url + super().__init__() + + async def generator(self) -> None: + # Separate session per request is against aiohttp idioms, but is + # intentional here because the point of the test is verifying + # where connections go - reusing a connection would do the wrong thing. + async with ClientSession() as session: + async with session.get(self._url) as response: + async for line in response.content: + await self._queue.put(line) + + +async def _full_http3_request_task(url: str) -> str: + proc = await asyncio.create_subprocess_exec( + IntegrationTest.h3_request, + f"--ca-certs={IntegrationTest.ca_certs}", + url, + stdout=asyncio.subprocess.PIPE, + ) + (stdout, _) = await proc.communicate() + await proc.wait() + return stdout.decode("utf-8") + + +def _full_http3_request(url: str) -> Awaitable[str]: + return asyncio.create_task(_full_http3_request_task(url)) + + +async def _full_http_request_task(url: str) -> str: + # Separate session per request is against aiohttp idioms, but is + # intentional here because the point of the test is verifying + # where connections go - reusing a connection would do the wrong thing. + async with ClientSession() as session: + async with session.get(url) as response: + return await response.text() + + +def _full_http_request(url: str) -> Awaitable[str]: + return asyncio.create_task(_full_http_request_task(url)) + + +def filter_chains(codec_type: str = "AUTO") -> str: + return f""" + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: {codec_type} + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: some_service + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router +""" + + +def _make_envoy_config_yaml(upstream_port: int, file_path: pathlib.Path): + file_path.write_text( + f""" +admin: + address: + socket_address: + address: {ENVOY_HOST} + port_value: {ENVOY_ADMIN_PORT} + +static_resources: + listeners: + - name: listener_quic + address: + socket_address: + protocol: UDP + address: {ENVOY_HOST} + port_value: {ENVOY_PORT} +{filter_chains("HTTP3")} + transport_socket: + name: "envoy.transport_sockets.quic" + typed_config: + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicDownstreamTransport" + downstream_tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{IntegrationTest.server_cert}" + private_key: + filename: "{IntegrationTest.server_key}" + udp_listener_config: + quic_options: {"{}"} + downstream_socket_config: + prefer_gro: true + - name: listener_http + address: + socket_address: + address: {ENVOY_HOST} + port_value: {ENVOY_PORT} +{filter_chains()} + clusters: + - name: some_service + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {UPSTREAM_HOST} + port_value: {upstream_port} +""") + + +async def _wait_for_envoy_epoch(i: int): + """Load the admin/server_info page until restart_epoch is i, or timeout""" + expected_substring = f'"restart_epoch": {i}' + deadline = datetime.now() + timedelta(seconds=STARTUP_TOLERANCE_SECONDS) + response = "admin port not responding within timeout" + while datetime.now() < deadline: + try: + response = await _full_http_request( + f"http://{ENVOY_HOST}:{ENVOY_ADMIN_PORT}/server_info") + if expected_substring in response: + return + except client_exceptions.ClientConnectorError: + pass + await asyncio.sleep(0.2) + # Envoy instance with expected restart_epoch should have started up + assert expected_substring in response, f"server_info={response}" + + +class IntegrationTest(unittest.IsolatedAsyncioTestCase): + server_cert: pathlib.Path + server_key: pathlib.Path + ca_certs: pathlib.Path + h3_request: pathlib.Path + envoy_binary: pathlib.Path + + async def asyncSetUp(self) -> None: + print(os.environ) + tmpdir = os.environ["TEST_TMPDIR"] + self.slow_config_path = pathlib.Path(tmpdir, "slow_config.yaml") + self.fast_config_path = pathlib.Path(tmpdir, "fast_config.yaml") + self.base_id_path = pathlib.Path(tmpdir, "base_id.txt") + _make_envoy_config_yaml(upstream_port=UPSTREAM_SLOW_PORT, file_path=self.slow_config_path) + _make_envoy_config_yaml(upstream_port=UPSTREAM_FAST_PORT, file_path=self.fast_config_path) + self.base_envoy_args = [ + IntegrationTest.envoy_binary, + "--socket-path", + SOCKET_PATH, + "--socket-mode", + str(SOCKET_MODE), + ] + log.info(f"starting upstreams on https://{ENVOY_HOST}:{ENVOY_PORT}/") + await super().asyncSetUp() + self.slow_upstream = Upstream() + await self.slow_upstream.start() + self.fast_upstream = Upstream(True) + await self.fast_upstream.start() + + async def asyncTearDown(self) -> None: + await self.slow_upstream.stop() + await self.fast_upstream.stop() + return await super().asyncTearDown() + + async def test_connection_handoffs(self) -> None: + log.info("starting envoy") + envoy_process_1 = await asyncio.create_subprocess_exec( + *self.base_envoy_args, + "--restart-epoch", + "0", + "--use-dynamic-base-id", + "--base-id-path", + self.base_id_path, + "-c", + self.slow_config_path, + ) + log.info(f"cert path = {IntegrationTest.server_cert}") + log.info("waiting for envoy ready") + await _wait_for_envoy_epoch(0) + log.info("making requests") + request_url = f"http://{ENVOY_HOST}:{ENVOY_PORT}/" + srequest_url = f"https://{ENVOY_HOST}:{ENVOY_PORT}/" + slow_responses = [ + HttpRequestLineGenerator(request_url) for i in range(PARALLEL_REQUESTS) + ] + [Http3RequestLineGenerator(srequest_url) for i in range(PARALLEL_REQUESTS)] + log.info("waiting for responses to begin") + for response in slow_responses: + self.assertEqual(await response.line(), b"start\n") + base_id = int(self.base_id_path.read_text()) + log.info(f"starting envoy hot restart for base id {base_id}") + envoy_process_2 = await asyncio.create_subprocess_exec( + *self.base_envoy_args, + "--restart-epoch", + "1", + "--parent-shutdown-time-s", + str(STARTUP_TOLERANCE_SECONDS + 1), + "--base-id", + str(base_id), + "-c", + self.fast_config_path, + ) + log.info("waiting for new envoy instance to begin") + await _wait_for_envoy_epoch(1) + log.info("sending request to fast upstream") + fast_responses = [_full_http_request(request_url) for i in range(PARALLEL_REQUESTS) + ] + [_full_http3_request(srequest_url) for i in range(PARALLEL_REQUESTS)] + for response in fast_responses: + self.assertEqual( + await response, + "fast instance", + "new requests after hot restart begins should go to new cluster", + ) + + # Now wait for the slow request to complete, and make sure it still gets the + # response from the old instance. + log.info("waiting for completion of original slow request") + t1 = datetime.now() + for response in slow_responses: + self.assertEqual(await response.line(), b"end\n") + t2 = datetime.now() + self.assertGreater( + (t2 - t1).total_seconds(), + 0.5, + "slow request should be incomplete when the test waits for it, otherwise the test is not necessarily validating during-drain behavior", + ) + for response in slow_responses: + self.assertEquals(await response.join(), 0) + log.info("waiting for parent instance to terminate") + await envoy_process_1.wait() + log.info("sending second request to fast upstream") + fast_responses = [_full_http_request(request_url) for i in range(PARALLEL_REQUESTS) + ] + [_full_http3_request(srequest_url) for i in range(PARALLEL_REQUESTS)] + for response in fast_responses: + self.assertEqual( + await response, + "fast instance", + "new requests after old instance terminates should go to new cluster", + ) + log.info("shutting child instance down") + envoy_process_2.terminate() + await envoy_process_2.wait() + + +def generate_server_cert( + ca_key_path: pathlib.Path, + ca_cert_path: pathlib.Path) -> "tuple[pathlib.Path, pathlib.Path]": + """Generates a temporary key and cert pem file and returns the paths. + + This is necessary because the http3 client validates that the server + certificate matches the host of the request, and our host is an + arbitrary randomized 127.x.y.z IP address to reduce the likelihood + of port collisions during testing. We therefore must use a generated + certificate that really matches the host IP. + """ + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization + from ipaddress import ip_address + + with open(ca_key_path, "rb") as ca_key_file: + ca_key = serialization.load_pem_private_key( + ca_key_file.read(), + password=None, + ) + with open(ca_cert_path, "rb") as ca_cert_file: + ca_cert = x509.load_pem_x509_certificate(ca_cert_file.read()) + + key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend(), + ) + + hostname = "testhost" + name = x509.Name([x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, hostname)]) + alt_names = [x509.DNSName(hostname)] + alt_names.append(x509.IPAddress(ip_address(ENVOY_HOST))) + san = x509.SubjectAlternativeName(alt_names) + basic_constraints = x509.BasicConstraints(ca=True, path_length=0) + now = datetime.utcnow() + cert = ( + x509.CertificateBuilder() # Comment to keep linter from uglifying! + .subject_name(name).issuer_name(ca_cert.subject).public_key(key.public_key()).serial_number( + 1).not_valid_before(now).not_valid_after(now + timedelta(days=30)).add_extension( + basic_constraints, + False).add_extension(san, False).sign(ca_key, hashes.SHA256(), default_backend())) + cert_pem = cert.public_bytes(encoding=serialization.Encoding.PEM) + key_pem = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + cert_file = tempfile.NamedTemporaryFile( + suffix="_key.pem", delete=False, dir=os.environ["TEST_TMPDIR"]) + cert_file.write(cert_pem) + cert_file.close() + key_file = tempfile.NamedTemporaryFile( + suffix="_cert.pem", delete=False, dir=os.environ["TEST_TMPDIR"]) + key_file.write(key_pem) + key_file.close() + return key_file.name, cert_file.name + + +def main(): + parser = argparse.ArgumentParser(description="Hot restart handoff test") + parser.add_argument("--envoy-binary", type=str, required=True) + parser.add_argument("--h3-request", type=str, required=True) + parser.add_argument("--ca-certs", type=str, required=True) + parser.add_argument("--ca-key", type=str, required=True) + # unittest also parses some args, so we strip out the ones we're using + # and leave the rest for unittest to consume. + (args, sys.argv[1:]) = parser.parse_known_args() + (IntegrationTest.server_key, + IntegrationTest.server_cert) = generate_server_cert(args.ca_key, args.ca_certs) + IntegrationTest.ca_certs = args.ca_certs + IntegrationTest.h3_request = args.h3_request + IntegrationTest.envoy_binary = args.envoy_binary + + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/test/integration/hotrestart_test.sh b/test/integration/hotrestart_test.sh index be637c356585..2725cd471096 100755 --- a/test/integration/hotrestart_test.sh +++ b/test/integration/hotrestart_test.sh @@ -97,7 +97,7 @@ echo "Hot restart test using dynamic base id" TEST_INDEX=0 function run_testsuite() { local BASE_ID BASE_ID_PATH HOT_RESTART_JSON="$1" - local SOCKET_PATH=@envoy_domain_socket + local SOCKET_PATH=@envoy_domain_socket_$$ local SOCKET_MODE=0 if [ -n "$2" ] && [ -n "$3" ] then diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 4ae499b7a06d..4bfebe84cfd2 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -29,7 +29,7 @@ #ifdef ENVOY_ENABLE_QUIC #include "source/common/quic/client_connection_factory_impl.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" #endif #include "source/extensions/transport_sockets/tls/context_config_impl.h" @@ -266,7 +266,8 @@ IntegrationCodecClientPtr HttpIntegrationTest::makeHttpConnection(uint32_t port) IntegrationCodecClientPtr HttpIntegrationTest::makeRawHttpConnection( Network::ClientConnectionPtr&& conn, - absl::optional http2_options) { + absl::optional http2_options, + bool wait_till_connected) { std::shared_ptr cluster{new NiceMock()}; cluster->max_response_headers_count_ = 200; if (!http2_options.has_value()) { @@ -295,7 +296,8 @@ IntegrationCodecClientPtr HttpIntegrationTest::makeRawHttpConnection( // This call may fail in QUICHE because of INVALID_VERSION. QUIC connection doesn't support // in-connection version negotiation. auto codec = std::make_unique(*dispatcher_, random_, std::move(conn), - host_description, downstream_protocol_); + host_description, downstream_protocol_, + wait_till_connected); if (downstream_protocol_ == Http::CodecType::HTTP3 && codec->disconnected()) { // Connection may get closed during version negotiation or handshake. // TODO(#8479) QUIC connection doesn't support in-connection version negotiationPropagate diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index f0d1020e46bd..3cb15cda441f 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -156,9 +156,10 @@ class HttpIntegrationTest : public BaseIntegrationTest { IntegrationCodecClientPtr makeHttpConnection(uint32_t port); // Makes a http connection object without checking its connected state. - virtual IntegrationCodecClientPtr makeRawHttpConnection( - Network::ClientConnectionPtr&& conn, - absl::optional http2_options); + virtual IntegrationCodecClientPtr + makeRawHttpConnection(Network::ClientConnectionPtr&& conn, + absl::optional http2_options, + bool wait_till_connected = true); // Makes a downstream network connection object based on client codec version. Network::ClientConnectionPtr makeClientConnectionWithOptions( uint32_t port, const Network::ConnectionSocket::OptionsSharedPtr& options) override; diff --git a/test/integration/http_protocol_integration.h b/test/integration/http_protocol_integration.h index 25c96660a997..5988e1ecc061 100644 --- a/test/integration/http_protocol_integration.h +++ b/test/integration/http_protocol_integration.h @@ -17,6 +17,8 @@ struct HttpProtocolTestParams { bool use_universal_header_validator; }; +absl::string_view http2ImplementationToString(Http2Impl impl); + // Allows easy testing of Envoy code for HTTP/HTTP2 upstream/downstream. // // Usage: @@ -68,10 +70,11 @@ class HttpProtocolIntegrationTest : public testing::TestWithParam& p); HttpProtocolIntegrationTest() - : HttpIntegrationTest( - GetParam().downstream_protocol, GetParam().version, - ConfigHelper::httpProxyConfig(/*downstream_is_quic=*/GetParam().downstream_protocol == - Http::CodecType::HTTP3)), + : HttpProtocolIntegrationTest(ConfigHelper::httpProxyConfig( + /*downstream_is_quic=*/GetParam().downstream_protocol == Http::CodecType::HTTP3)) {} + + HttpProtocolIntegrationTest(const std::string config) + : HttpIntegrationTest(GetParam().downstream_protocol, GetParam().version, config), use_universal_header_validator_(GetParam().use_universal_header_validator) { setupHttp1ImplOverrides(GetParam().http1_implementation); setupHttp2ImplOverrides(GetParam().http2_implementation); diff --git a/test/integration/http_timeout_integration_test.cc b/test/integration/http_timeout_integration_test.cc index 50c6726395c3..74830d697d03 100644 --- a/test/integration/http_timeout_integration_test.cc +++ b/test/integration/http_timeout_integration_test.cc @@ -1,5 +1,7 @@ #include "test/integration/http_timeout_integration_test.h" +#include "test/test_common/test_runtime.h" + #include "gtest/gtest.h" namespace Envoy { @@ -16,6 +18,10 @@ TEST_P(HttpTimeoutIntegrationTest, GlobalTimeout) { config_helper_.addConfigModifier(configureProxyStatus()); initialize(); + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.proxy_status_upstream_request_timeout", "true"}}); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); auto encoder_decoder = codec_client_->startRequest( Http::TestRequestHeaderMapImpl{{":method", "POST"}, @@ -49,7 +55,7 @@ TEST_P(HttpTimeoutIntegrationTest, GlobalTimeout) { EXPECT_TRUE(response->complete()); EXPECT_EQ("504", response->headers().getStatusValue()); EXPECT_EQ(response->headers().getProxyStatusValue(), - "envoy; error=connection_timeout; details=\"response_timeout; UT\""); + "envoy; error=http_response_timeout; details=\"response_timeout; UT\""); } // Testing that `x-envoy-expected-timeout-ms` header, set by egress envoy, is respected by ingress diff --git a/test/integration/http_typed_per_filter_config_test.cc b/test/integration/http_typed_per_filter_config_test.cc index d6a07c91b89f..6c9ceaf46883 100644 --- a/test/integration/http_typed_per_filter_config_test.cc +++ b/test/integration/http_typed_per_filter_config_test.cc @@ -1,6 +1,6 @@ #include "envoy/config/route/v3/route_components.pb.h" -#include "test/integration/filters/set_response_code_filter_config.pb.h" +#include "test/integration/filters/set_route_filter_config.pb.h" #include "test/integration/http_integration.h" #include "gtest/gtest.h" @@ -18,21 +18,20 @@ TEST_F(HTTPTypedPerFilterConfigTest, RejectUnsupportedTypedPerFilterConfig) { config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) { - test::integration::filters::SetResponseCodeFilterConfig response_code; - response_code.set_code(403); + test::integration::filters::SetRouteFilterConfig set_route_config; auto* virtual_host = hcm.mutable_route_config()->mutable_virtual_hosts(0); auto* config = virtual_host->mutable_typed_per_filter_config(); - (*config)["set-response-code-filter"].PackFrom(response_code); + (*config)["set-route-filter"].PackFrom(set_route_config); auto* filter = hcm.mutable_http_filters()->Add(); - filter->set_name("set-response-code-filter"); - filter->mutable_typed_config()->PackFrom(response_code); + filter->set_name("set-route-filter"); + filter->mutable_typed_config()->PackFrom(set_route_config); // keep router the last auto size = hcm.http_filters_size(); hcm.mutable_http_filters()->SwapElements(size - 2, size - 1); }); - EXPECT_DEATH(initialize(), "The filter set-response-code-filter doesn't support virtual host or " + EXPECT_DEATH(initialize(), "The filter set-route-filter doesn't support virtual host or " "route specific configurations"); } @@ -49,28 +48,6 @@ TEST_F(HTTPTypedPerFilterConfigTest, RejectUnknownHttpFilterInTypedPerFilterConf "'google.protobuf.Struct'"); } -TEST_F(HTTPTypedPerFilterConfigTest, HcmIgnoreUnknownOptionalHttpFilterInTypedPerFilterConfig) { - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - config_helper_.addRuntimeOverride( - "envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"); - - config_helper_.addConfigModifier( - [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& - hcm) { - auto* virtual_host = hcm.mutable_route_config()->mutable_virtual_hosts(0); - auto* config = virtual_host->mutable_typed_per_filter_config(); - (*config)["filter.unknown"].PackFrom(Envoy::ProtobufWkt::Struct()); - - auto* filter = hcm.mutable_http_filters()->Add(); - filter->set_name("filter.unknown"); - filter->set_is_optional(true); - // keep router the last - auto size = hcm.http_filters_size(); - hcm.mutable_http_filters()->SwapElements(size - 2, size - 1); - }); - initialize(); -} - TEST_F(HTTPTypedPerFilterConfigTest, IgnoreUnknownOptionalHttpFilterInTypedPerFilterConfig) { config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& diff --git a/test/integration/integration_stream_decoder.cc b/test/integration/integration_stream_decoder.cc index bb7551aba23d..d5c3c914b403 100644 --- a/test/integration/integration_stream_decoder.cc +++ b/test/integration/integration_stream_decoder.cc @@ -24,6 +24,10 @@ namespace Envoy { IntegrationStreamDecoder::IntegrationStreamDecoder(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} +IntegrationStreamDecoder::~IntegrationStreamDecoder() { + ENVOY_LOG_MISC(trace, "Destroying IntegrationStreamDecoder"); +} + void IntegrationStreamDecoder::waitFor1xxHeaders() { if (!continue_headers_.get()) { waiting_for_continue_headers_ = true; diff --git a/test/integration/integration_stream_decoder.h b/test/integration/integration_stream_decoder.h index e4e5b3c85b20..080b5cbbf915 100644 --- a/test/integration/integration_stream_decoder.h +++ b/test/integration/integration_stream_decoder.h @@ -24,6 +24,7 @@ namespace Envoy { class IntegrationStreamDecoder : public Http::ResponseDecoder, public Http::StreamCallbacks { public: IntegrationStreamDecoder(Event::Dispatcher& dispatcher); + ~IntegrationStreamDecoder(); const std::string& body() { return body_; } bool complete() { return saw_end_stream_; } diff --git a/test/integration/integration_tcp_client.h b/test/integration/integration_tcp_client.h index a79c8b32b445..5552c928d5f9 100644 --- a/test/integration/integration_tcp_client.h +++ b/test/integration/integration_tcp_client.h @@ -49,6 +49,7 @@ class IntegrationTcpClient { bool connected() const { return !disconnected_; } // clear up to the `count` number of bytes of received data void clearData(size_t count = std::string::npos) { payload_reader_->clearData(count); } + Network::Connection* connection() const { return connection_.get(); } private: struct ConnectionCallbacks : public Network::ConnectionCallbacks { diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index b37cdd34b415..702c6bf6d9ed 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -1078,13 +1078,6 @@ TEST_P(IntegrationTest, MissingDelimiter) { } TEST_P(IntegrationTest, InvalidCharacterInFirstline) { -#ifndef ENVOY_ENABLE_UHV - if (http1_implementation_ == Http1ParserImpl::BalsaParser) { - // BalsaParser allows custom methods if UHV is enabled. - return; - } -#endif - initialize(); std::string response; sendRawHttpAndWaitForResponse(lookupPort("http"), "GE(T / HTTP/1.1\r\nHost: host\r\n\r\n", @@ -1600,6 +1593,7 @@ TEST_P(IntegrationTest, AbsolutePathWithoutPort) { // Ensure that connect behaves the same with allow_absolute_url enabled and without TEST_P(IntegrationTest, Connect) { + setListenersBoundTimeout(3 * TestUtility::DefaultTimeout); const std::string& request = "CONNECT www.somewhere.com:80 HTTP/1.1\r\n\r\n"; config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { // Clone the whole listener. diff --git a/test/integration/listener_extension_discovery_integration_test.cc b/test/integration/listener_extension_discovery_integration_test.cc index 29e284b6764b..acdba1d6bf5c 100644 --- a/test/integration/listener_extension_discovery_integration_test.cc +++ b/test/integration/listener_extension_discovery_integration_test.cc @@ -774,7 +774,7 @@ TEST_P(ListenerExtensionDiscoveryIntegrationTest, TwoSubscriptionsConfigDumpWith #include "quiche/quic/core/deterministic_connection_id_generator.h" #include "source/common/quic/client_connection_factory_impl.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_server_transport_socket_factory.h" #include "test/integration/utility.h" namespace Envoy { diff --git a/test/integration/load_balancers/custom_lb_policy.h b/test/integration/load_balancers/custom_lb_policy.h index 0fd5d87ba589..57be537f90d0 100644 --- a/test/integration/load_balancers/custom_lb_policy.h +++ b/test/integration/load_balancers/custom_lb_policy.h @@ -59,6 +59,11 @@ class ThreadAwareLbImpl : public Upstream::ThreadAwareLoadBalancer { class CustomLbFactory : public Upstream::TypedLoadBalancerFactoryBase< ::test::integration::custom_lb::CustomLbConfig> { public: + class EmptyLoadBalancerConfig : public Upstream::LoadBalancerConfig { + public: + EmptyLoadBalancerConfig() = default; + }; + CustomLbFactory() : TypedLoadBalancerFactoryBase("envoy.load_balancers.custom_lb") {} Upstream::ThreadAwareLoadBalancerPtr create(OptRef, @@ -67,6 +72,11 @@ class CustomLbFactory : public Upstream::TypedLoadBalancerFactoryBase< Random::RandomGenerator&, TimeSource&) override { return std::make_unique(); } + + Upstream::LoadBalancerConfigPtr loadConfig(const Protobuf::Message&, + ProtobufMessage::ValidationVisitor&) override { + return std::make_unique(); + } }; } // namespace Envoy diff --git a/test/integration/load_stats_integration_test.cc b/test/integration/load_stats_integration_test.cc index 4dd11637cc98..668cdb0fcf52 100644 --- a/test/integration/load_stats_integration_test.cc +++ b/test/integration/load_stats_integration_test.cc @@ -191,6 +191,20 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, cluster_stats->set_total_dropped_requests(cluster_stats->total_dropped_requests() + local_cluster_stats.total_dropped_requests()); + if (local_cluster_stats.dropped_requests().size() > 0) { + const uint64_t local_drop_count = local_cluster_stats.dropped_requests(0).dropped_count(); + if (local_drop_count > 0) { + envoy::config::endpoint::v3::ClusterStats::DroppedRequests* drop_request; + if (cluster_stats->dropped_requests().size() > 0) { + drop_request = cluster_stats->mutable_dropped_requests(0); + drop_request->set_dropped_count(drop_request->dropped_count() + local_drop_count); + } else { + drop_request = cluster_stats->add_dropped_requests(); + drop_request->set_dropped_count(local_drop_count); + } + drop_request->set_category("drop_overload"); + } + } for (int i = 0; i < local_cluster_stats.upstream_locality_stats_size(); ++i) { const auto& local_upstream_locality_stats = local_cluster_stats.upstream_locality_stats(i); @@ -243,7 +257,7 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, ABSL_MUST_USE_RESULT AssertionResult waitForLoadStatsRequest(const std::vector& expected_locality_stats, - uint64_t dropped = 0) { + uint64_t dropped = 0, bool drop_overload_test = false) { Event::TestTimeSystem::RealTimeBound bound(TestUtility::DefaultTimeout); Protobuf::RepeatedPtrField expected_cluster_stats; if (!expected_locality_stats.empty() || dropped != 0) { @@ -253,6 +267,11 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, cluster_stats->set_cluster_service_name("service_name_0"); if (dropped > 0) { cluster_stats->set_total_dropped_requests(dropped); + if (drop_overload_test) { + auto* drop_request = cluster_stats->add_dropped_requests(); + drop_request->set_category("drop_overload"); + drop_request->set_dropped_count(dropped); + } } std::copy( expected_locality_stats.begin(), expected_locality_stats.end(), @@ -368,6 +387,19 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, cleanupUpstreamAndDownstream(); } + void updateDropOverloadConfig() { + envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment; + cluster_load_assignment.set_cluster_name("service_name_0"); + // Config drop_overload to drop 100% requests. + auto* policy = cluster_load_assignment.mutable_policy(); + auto* drop_overload = policy->add_drop_overloads(); + drop_overload->set_category("drop_overload"); + auto* drop_percentage = drop_overload->mutable_drop_percentage(); + drop_percentage->set_numerator(100); + drop_percentage->set_denominator(envoy::type::v3::FractionalPercent::HUNDRED); + eds_helper_.setEdsAndWait({cluster_load_assignment}, *test_server_); + } + static constexpr uint32_t upstream_endpoints_ = 5; IntegrationStreamDecoderPtr response_; @@ -646,5 +678,30 @@ TEST_P(LoadStatsIntegrationTest, Dropped) { cleanupLoadStatsConnection(); } +// Validate the load reports for dropped requests due to drop_overload make sense. +TEST_P(LoadStatsIntegrationTest, DropOverloadDropped) { + initialize(); + waitForLoadStatsStream(); + ASSERT_TRUE(waitForLoadStatsRequest({})); + loadstats_stream_->startGrpcStream(); + updateClusterLoadAssignment({{0}}, {}, {}, {}); + updateDropOverloadConfig(); + + requestLoadStatsResponse({"cluster_0"}); + initiateClientConnection(); + ASSERT_TRUE(response_->waitForEndStream()); + ASSERT_TRUE(response_->complete()); + EXPECT_EQ("503", response_->headers().getStatusValue()); + cleanupUpstreamAndDownstream(); + + ASSERT_TRUE(waitForLoadStatsRequest({}, 1, true)); + + EXPECT_EQ(1, test_server_->counter("load_reporter.requests")->value()); + EXPECT_LE(2, test_server_->counter("load_reporter.responses")->value()); + EXPECT_EQ(0, test_server_->counter("load_reporter.errors")->value()); + + cleanupLoadStatsConnection(); +} + } // namespace } // namespace Envoy diff --git a/test/integration/multiplexed_integration_test.cc b/test/integration/multiplexed_integration_test.cc index b678090bec3e..71573f90a5e7 100644 --- a/test/integration/multiplexed_integration_test.cc +++ b/test/integration/multiplexed_integration_test.cc @@ -1,4 +1,5 @@ #include +#include #include #include @@ -25,6 +26,7 @@ #include "test/mocks/http/mocks.h" #include "test/test_common/network_utility.h" #include "test/test_common/printers.h" +#include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -92,6 +94,15 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, MultiplexedIntegrationTest, {Http::CodecType::HTTP1})), HttpProtocolIntegrationTest::protocolTestParamsToString); +class MultiplexedIntegrationTestWithSimulatedTime : public Event::TestUsingSimulatedTime, + public MultiplexedIntegrationTest {}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, MultiplexedIntegrationTestWithSimulatedTime, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( + {Http::CodecType::HTTP2, Http::CodecType::HTTP3}, + {Http::CodecType::HTTP1})), + HttpProtocolIntegrationTest::protocolTestParamsToString); + TEST_P(MultiplexedIntegrationTest, RouterRequestAndResponseWithBodyNoBuffer) { testRouterRequestAndResponseWithBody(1024, 512, false, false); } @@ -1076,6 +1087,109 @@ TEST_P(MultiplexedIntegrationTest, GoAway) { EXPECT_EQ("200", response->headers().getStatusValue()); } +// TODO(rch): Add a unit test which covers internal redirect handling. +TEST_P(MultiplexedIntegrationTestWithSimulatedTime, GoAwayAfterTooManyResets) { + EXCLUDE_DOWNSTREAM_HTTP3; // Need to wait for the server to reset the stream + // before opening new one. + config_helper_.addRuntimeOverride("envoy.restart_features.send_goaway_for_premature_rst_streams", + "true"); + const int total_streams = 100; + config_helper_.addRuntimeOverride("overload.premature_reset_total_stream_count", + absl::StrCat(total_streams)); + autonomous_upstream_ = true; + autonomous_allow_incomplete_streams_ = true; + initialize(); + + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, {":path", "/healthcheck"}, {":scheme", "http"}, {":authority", "host"}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + + for (int i = 0; i < total_streams; ++i) { + // Send and wait + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + codec_client_->sendData(*request_encoder_, 0, true); + auto response = std::move(encoder_decoder.second); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + } + for (int i = 0; i < total_streams; ++i) { + // Send and reset + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + codec_client_->sendReset(*request_encoder_); + ASSERT_TRUE(response->waitForReset()); + } + + // Envoy should disconnect client due to premature reset check + ASSERT_TRUE(codec_client_->waitForDisconnect()); + test_server_->waitForCounterEq("http.config_test.downstream_rq_rx_reset", total_streams); + test_server_->waitForCounterEq("http.config_test.downstream_rq_too_many_premature_resets", 1); +} + +TEST_P(MultiplexedIntegrationTestWithSimulatedTime, GoAwayQuicklyAfterTooManyResets) { + EXCLUDE_DOWNSTREAM_HTTP3; // Need to wait for the server to reset the stream + // before opening new one. + config_helper_.addRuntimeOverride("envoy.restart_features.send_goaway_for_premature_rst_streams", + "true"); + const int total_streams = 100; + config_helper_.addRuntimeOverride("overload.premature_reset_total_stream_count", + absl::StrCat(total_streams)); + const int num_reset_streams = total_streams / 2; + initialize(); + + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, {":path", "/healthcheck"}, {":scheme", "http"}, {":authority", "host"}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + for (int i = 0; i < num_reset_streams; i++) { + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + codec_client_->sendReset(*request_encoder_); + ASSERT_TRUE(response->waitForReset()); + } + + // Envoy should disconnect client due to premature reset check + ASSERT_TRUE(codec_client_->waitForDisconnect()); + test_server_->waitForCounterEq("http.config_test.downstream_rq_rx_reset", num_reset_streams); + test_server_->waitForCounterEq("http.config_test.downstream_rq_too_many_premature_resets", 1); +} + +TEST_P(MultiplexedIntegrationTestWithSimulatedTime, DontGoAwayAfterTooManyResetsForLongStreams) { + EXCLUDE_DOWNSTREAM_HTTP3; // Need to wait for the server to reset the stream + // before opening new one. + config_helper_.addRuntimeOverride("envoy.restart_features.send_goaway_for_premature_rst_streams", + "true"); + const int total_streams = 100; + const int stream_lifetime_seconds = 2; + config_helper_.addRuntimeOverride("overload.premature_reset_total_stream_count", + absl::StrCat(total_streams)); + + config_helper_.addRuntimeOverride("overload.premature_reset_min_stream_lifetime_seconds", + absl::StrCat(stream_lifetime_seconds)); + + initialize(); + + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, {":path", "/healthcheck"}, {":scheme", "http"}, {":authority", "host"}}; + codec_client_ = makeHttpConnection(lookupPort("http")); + + std::string request_counter = "http.config_test.downstream_rq_total"; + std::string reset_counter = "http.config_test.downstream_rq_rx_reset"; + for (int i = 0; i < total_streams * 2; ++i) { + auto encoder_decoder = codec_client_->startRequest(headers); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + test_server_->waitForCounterEq(request_counter, i + 1); + timeSystem().advanceTimeWait(std::chrono::seconds(2 * stream_lifetime_seconds)); + codec_client_->sendReset(*request_encoder_); + ASSERT_TRUE(response->waitForReset()); + test_server_->waitForCounterEq(reset_counter, i + 1); + } +} + TEST_P(MultiplexedIntegrationTest, Trailers) { testTrailers(1024, 2048, false, false); } TEST_P(MultiplexedIntegrationTest, TrailersGiantBody) { @@ -1803,27 +1917,52 @@ TEST_P(MultiplexedRingHashIntegrationTest, CookieRoutingWithCookieWithTtlSet) { struct FrameIntegrationTestParam { Network::Address::IpVersion ip_version; + Http2Impl http2_implementation; }; std::string frameIntegrationTestParamToString(const testing::TestParamInfo& params) { - return TestUtility::ipVersionToString(params.param.ip_version); + return absl::StrCat(TestUtility::ipVersionToString(params.param.ip_version), "_", + http2ImplementationToString(params.param.http2_implementation)); } class Http2FrameIntegrationTest : public testing::TestWithParam, public Http2RawFrameIntegrationTest { public: - Http2FrameIntegrationTest() : Http2RawFrameIntegrationTest(GetParam().ip_version) {} + Http2FrameIntegrationTest() : Http2RawFrameIntegrationTest(GetParam().ip_version) { + setupHttp2ImplOverrides(GetParam().http2_implementation); + } static std::vector testParams() { std::vector v; for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { - v.push_back({ip_version}); + v.push_back({ip_version, Http2Impl::Nghttp2}); + v.push_back({ip_version, Http2Impl::Oghttp2}); } return v; } }; +TEST_P(Http2FrameIntegrationTest, MaxConcurrentStreamsIsRespected) { + const int kTotalRequests = 101; + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.mutable_http2_protocol_options()->mutable_max_concurrent_streams()->set_value(100); + }); + beginSession(); + + std::string buffer; + for (int i = 0; i < kTotalRequests; ++i) { + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(i), "a", "/"); + absl::StrAppend(&buffer, std::string(request)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + tcp_client_->waitForDisconnect(); + test_server_->waitForCounterGe("http.config_test.downstream_cx_destroy_local", 1); +} + // Regression test. TEST_P(Http2FrameIntegrationTest, SetDetailsTwice) { autonomous_upstream_ = true; @@ -2020,11 +2159,6 @@ TEST_P(Http2FrameIntegrationTest, AccessLogOfWireBytesIfResponseSizeGreaterThanW // Check access log if the agnostic stream lifetime is not extended. // It should have access logged since it has received the entire response. int hcm_logged_wire_bytes_sent, hcm_logged_wire_header_bytes_sent; - if (!Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)) { - auto access_log_values = stoiAccessLogString(waitForAccessLog(access_log_name_)); - hcm_logged_wire_bytes_sent = access_log_values[0]; - hcm_logged_wire_header_bytes_sent = access_log_values[1]; - } // Grant the sender (Envoy) additional window so it can finish sending the // stream. @@ -2042,13 +2176,11 @@ TEST_P(Http2FrameIntegrationTest, AccessLogOfWireBytesIfResponseSizeGreaterThanW EXPECT_EQ(accumulator.bodyWireBytesReceivedDiscountingHeaders(), accumulator.bodyWireBytesReceivedGivenPayloadAndFrames()); - if (Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)) { - // Access logs are only available now due to the expanded agnostic stream - // lifetime. - auto access_log_values = stoiAccessLogString(waitForAccessLog(access_log_name_)); - hcm_logged_wire_bytes_sent = access_log_values[0]; - hcm_logged_wire_header_bytes_sent = access_log_values[1]; - } + // Access logs are only available now due to the expanded agnostic stream + // lifetime. + auto access_log_values = stoiAccessLogString(waitForAccessLog(access_log_name_)); + hcm_logged_wire_bytes_sent = access_log_values[0]; + hcm_logged_wire_header_bytes_sent = access_log_values[1]; EXPECT_EQ(accumulator.stream_wire_header_bytes_recieved_, hcm_logged_wire_header_bytes_sent); EXPECT_EQ(accumulator.stream_wire_bytes_recieved_, hcm_logged_wire_bytes_sent) << "Received " << accumulator.stream_wire_bytes_recieved_ @@ -2058,6 +2190,415 @@ TEST_P(Http2FrameIntegrationTest, AccessLogOfWireBytesIfResponseSizeGreaterThanW tcp_client_->close(); } +TEST_P(Http2FrameIntegrationTest, HostDifferentFromAuthority) { + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(request_idx), + "one.example.com", "/path", {{"host", "two.example.com"}}); + sendFrame(request); + + waitForNextUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().getHostValue(), "one.example.com"); + upstream_request_->encodeHeaders(default_response_headers_, true); + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, HostSameAsAuthority) { + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(request_idx), + "one.example.com", "/path", {{"host", "one.example.com"}}); + sendFrame(request); + + waitForNextUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().getHostValue(), "one.example.com"); + upstream_request_->encodeHeaders(default_response_headers_, true); + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, HostConcatenatedWithAuthorityWithOverride) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.http2_discard_host_header", "false"); + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(request_idx), + "one.example.com", "/path", {{"host", "two.example.com"}}); + sendFrame(request); + + waitForNextUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().getHostValue(), "one.example.com,two.example.com"); + upstream_request_->encodeHeaders(default_response_headers_, true); + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + tcp_client_->close(); +} + +// All HTTP/2 static headers must be before non-static headers. +// Verify that codecs validate this. +TEST_P(Http2FrameIntegrationTest, HostBeforeAuthorityIsRejected) { +#ifdef ENVOY_ENABLE_UHV + // TODO(yanavlasov): fix this check for oghttp2 in UHV mode. + if (GetParam().http2_implementation == Http2Impl::Oghttp2) { + return; + } +#endif + beginSession(); + + Http2Frame request = Http2Frame::makeEmptyHeadersFrame(Http2Frame::makeClientStreamId(0), + Http2Frame::HeadersFlags::EndHeaders); + request.appendStaticHeader(Http2Frame::StaticHeaderIndex::MethodPost); + request.appendStaticHeader(Http2Frame::StaticHeaderIndex::SchemeHttps); + request.appendHeaderWithoutIndexing(Http2Frame::StaticHeaderIndex::Path, "/path"); + // Add the `host` header before `:authority` + request.appendHeaderWithoutIndexing({"host", "two.example.com"}); + request.appendHeaderWithoutIndexing(Http2Frame::StaticHeaderIndex::Authority, "one.example.com"); + request.adjustPayloadSize(); + + sendFrame(request); + + // By default codec treats stream errors as protocol errors and closes the connection. + tcp_client_->waitForDisconnect(); + tcp_client_->close(); + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_protocol_error")->value()); +} + +TEST_P(Http2FrameIntegrationTest, MultipleHeaderOnlyRequests) { + const int kRequestsSentPerIOCycle = 20; + autonomous_upstream_ = true; + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + } + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, MultipleRequests) { + const int kRequestsSentPerIOCycle = 20; + autonomous_upstream_ = true; + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = + Http2Frame::makePostRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto data = Http2Frame::makeDataFrame(Http2Frame::makeClientStreamId(i), "a", + Http2Frame::DataFlags::EndStream); + absl::StrAppend(&buffer, std::string(data)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + } + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, MultipleRequestsWithMetadata) { + // Allow metadata usage. + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); + ConfigHelper::HttpProtocolOptions protocol_options; + protocol_options.mutable_explicit_http_config() + ->mutable_http2_protocol_options() + ->set_allow_metadata(true); + ConfigHelper::setProtocolOptions(*bootstrap.mutable_static_resources()->mutable_clusters(0), + protocol_options); + }); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + config_helper_.prependFilter(R"EOF( + name: metadata-control-filter + )EOF"); + + const int kRequestsSentPerIOCycle = 20; + autonomous_upstream_ = true; + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = + Http2Frame::makePostRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + Http::MetadataMap metadata_map{{"should_continue", absl::StrCat(i)}}; + auto metadata = Http2Frame::makeMetadataFrameFromMetadataMap( + Http2Frame::makeClientStreamId(i), metadata_map, Http2Frame::MetadataFlags::EndMetadata); + absl::StrAppend(&buffer, std::string(metadata)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto data = Http2Frame::makeDataFrame(Http2Frame::makeClientStreamId(i), "", + Http2Frame::DataFlags::EndStream); + absl::StrAppend(&buffer, std::string(data)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + } + tcp_client_->close(); +} + +// Validate the request completion during processing of deferred list works. +TEST_P(Http2FrameIntegrationTest, MultipleRequestsDecodeHeadersEndsRequest) { + const int kRequestsSentPerIOCycle = 20; + // The local-reply-during-decode will call sendLocalReply, completing them + // when processing headers. This will cause the ConnectionManagerImpl::ActiveRequest + // object to be removed from the streams_ list during the onDeferredRequestProcessing call. + config_helper_.addFilter("{ name: local-reply-during-decode }"); + // Process more than 1 deferred request at a time to validate the removal of elements from + // the list does not break reverse iteration. + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "3"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = + Http2Frame::makePostRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto data = Http2Frame::makeDataFrame(Http2Frame::makeClientStreamId(i), "a", + Http2Frame::DataFlags::EndStream); + absl::StrAppend(&buffer, std::string(data)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + + // The local-reply-during-decode filter sends 500 status to the client + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::InternalServerError, frame.responseStatus()); + } + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, MultipleRequestsWithTrailers) { + const int kRequestsSentPerIOCycle = 20; + autonomous_upstream_ = true; + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = + Http2Frame::makePostRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto data = Http2Frame::makeDataFrame(Http2Frame::makeClientStreamId(i), "a"); + absl::StrAppend(&buffer, std::string(data)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto trailers = Http2Frame::makeEmptyHeadersFrame( + Http2Frame::makeClientStreamId(i), + static_cast(Http::Http2::orFlags( + Http2Frame::HeadersFlags::EndStream, Http2Frame::HeadersFlags::EndHeaders))); + trailers.appendHeaderWithoutIndexing({"k", "v"}); + trailers.adjustPayloadSize(); + absl::StrAppend(&buffer, std::string(trailers)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()); + } + tcp_client_->close(); +} + +// Validate the request completion during processing of headers in the deferred requests, +// is ok, when deferred data and trailers are also present. +TEST_P(Http2FrameIntegrationTest, MultipleRequestsWithTrailersDecodeHeadersEndsRequest) { + const int kRequestsSentPerIOCycle = 20; + autonomous_upstream_ = true; + config_helper_.addFilter("{ name: local-reply-during-decode }"); + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "6"); + beginSession(); + + std::string buffer; + // Make every 4th request to be reset by the local-reply-during-decode filter, this will give a + // good distribution of removed requests from the deferred sequence. + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = Http2Frame::makePostRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, + {"no_trailers", "1"}, + {"skip-local-reply", i % 4 ? "true" : "false"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto data = Http2Frame::makeDataFrame(Http2Frame::makeClientStreamId(i), "a"); + absl::StrAppend(&buffer, std::string(data)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto trailers = Http2Frame::makeEmptyHeadersFrame( + Http2Frame::makeClientStreamId(i), + static_cast(Http::Http2::orFlags( + Http2Frame::HeadersFlags::EndStream, Http2Frame::HeadersFlags::EndHeaders))); + trailers.appendHeaderWithoutIndexing({"k", "v"}); + trailers.adjustPayloadSize(); + absl::StrAppend(&buffer, std::string(trailers)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); + uint32_t stream_id = frame.streamId(); + // Client stream indexes are multiples of 2 starting at 1 + if ((stream_id / 2) % 4) { + EXPECT_EQ(Http2Frame::ResponseStatus::Ok, frame.responseStatus()) + << " for stream=" << stream_id; + } else { + EXPECT_EQ(Http2Frame::ResponseStatus::InternalServerError, frame.responseStatus()) + << " for stream=" << stream_id; + } + } + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, MultipleHeaderOnlyRequestsFollowedByReset) { + // This number of requests stays below premature reset detection. + const int kRequestsSentPerIOCycle = 20; + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto reset = Http2Frame::makeResetStreamFrame(Http2Frame::makeClientStreamId(i), + Http2Frame::ErrorCode::Cancel); + absl::StrAppend(&buffer, std::string(reset)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + test_server_->waitForCounterEq("http.config_test.downstream_rq_rx_reset", + kRequestsSentPerIOCycle); + // Client should remain connected + ASSERT_TRUE(tcp_client_->connected()); + tcp_client_->close(); +} + +// This test depends on an another patch with premature resets +TEST_P(Http2FrameIntegrationTest, ResettingDeferredRequestsTriggersPrematureResetCheck) { + const int kRequestsSentPerIOCycle = 20; + // Set premature stream count to twice the number of streams we are about to send. + config_helper_.addRuntimeOverride("overload.premature_reset_total_stream_count", "40"); + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(i), "a", "/", + {{"response_data_blocks", "0"}, {"no_trailers", "1"}}); + absl::StrAppend(&buffer, std::string(request)); + } + + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto reset = Http2Frame::makeResetStreamFrame(Http2Frame::makeClientStreamId(i), + Http2Frame::ErrorCode::Cancel); + absl::StrAppend(&buffer, std::string(reset)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + // Envoy should close the client connection due to too many premature resets + tcp_client_->waitForDisconnect(); + test_server_->waitForCounterEq("http.config_test.downstream_rq_too_many_premature_resets", 1); + tcp_client_->close(); +} + +TEST_P(Http2FrameIntegrationTest, CloseConnectionWithDeferredStreams) { + // Use large number of requests to ensure close is detected while there are + // still some deferred streams. + const int kRequestsSentPerIOCycle = 20000; + config_helper_.addRuntimeOverride("http.max_requests_per_io_cycle", "1"); + // Ensure premature reset detection does not get in the way + config_helper_.addRuntimeOverride("overload.premature_reset_total_stream_count", "1001"); + // Disable the request timeout, iouring may failed the test due to request timeout. + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.mutable_route_config() + ->mutable_virtual_hosts(0) + ->mutable_routes(0) + ->mutable_route() + ->mutable_timeout() + ->set_seconds(0); + }); + beginSession(); + + std::string buffer; + for (int i = 0; i < kRequestsSentPerIOCycle; ++i) { + auto request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(i), "a", "/"); + absl::StrAppend(&buffer, std::string(request)); + } + + ASSERT_TRUE(tcp_client_->write(buffer, false, false)); + ASSERT_TRUE(tcp_client_->connected()); + // Drop the downstream connection + tcp_client_->close(); + // Test that Envoy can clean-up deferred streams + // Make the timeout longer to accommodate non optimized builds + test_server_->waitForCounterEq("http.config_test.downstream_rq_rx_reset", kRequestsSentPerIOCycle, + TestUtility::DefaultTimeout * 3); +} + INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FrameIntegrationTest, testing::ValuesIn(Http2FrameIntegrationTest::testParams()), frameIntegrationTestParamToString); @@ -2182,13 +2723,7 @@ TEST_P(Http2MetadataIntegrationTest, UpstreamMetadataAfterEndStream) { } TEST_P(MultiplexedIntegrationTest, InvalidTrailers) { -#ifdef ENVOY_ENABLE_UHV - if (GetParam().http2_implementation == Http2Impl::Oghttp2 && - downstreamProtocol() == Http::CodecType::HTTP2) { - return; - } -#endif - + disable_client_header_validation_ = true; autonomous_allow_incomplete_streams_ = true; useAccessLog("%RESPONSE_CODE_DETAILS%"); autonomous_upstream_ = true; @@ -2254,6 +2789,7 @@ TEST_P(MultiplexedIntegrationTest, InconsistentContentLength) { // reset the request. TEST_P(MultiplexedIntegrationTest, Reset101SwitchProtocolResponse) { #ifdef ENVOY_ENABLE_UHV + // TODO(#29071): reject responses with 101 in H/2 and H/3 UHV if (GetParam().http2_implementation == Http2Impl::Oghttp2 && downstreamProtocol() == Http::CodecType::HTTP2) { return; diff --git a/test/integration/null_test.py b/test/integration/null_test.py new file mode 100644 index 000000000000..f40f735dedb9 --- /dev/null +++ b/test/integration/null_test.py @@ -0,0 +1,5 @@ +# py_test targets with empty srcs provoke a build error, so this file is +# necessary for hotrestart_handoff_test to be possible to disable based on config. + +if __name__ == '__main__': + pass diff --git a/test/integration/overload_integration_test.cc b/test/integration/overload_integration_test.cc index f621b89f658a..1232968546dc 100644 --- a/test/integration/overload_integration_test.cc +++ b/test/integration/overload_integration_test.cc @@ -260,6 +260,62 @@ TEST_P(OverloadScaledTimerIntegrationTest, CloseIdleHttpConnections) { codec_client_->close(); } +TEST_P(OverloadScaledTimerIntegrationTest, HTTP3CloseIdleHttpConnectionsDuringHandshake) { + if (downstreamProtocol() != Http::CodecClient::Type::HTTP3) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({{"envoy.reloadable_features.quic_fix_filter_manager_uaf", "true"}}); + + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* proof_source_config = bootstrap.mutable_static_resources() + ->mutable_listeners(0) + ->mutable_udp_listener_config() + ->mutable_quic_options() + ->mutable_proof_source_config(); + proof_source_config->set_name("envoy.quic.proof_source.pending_signing"); + proof_source_config->mutable_typed_config(); + }); + initializeOverloadManager( + TestUtility::parseYaml(R"EOF( + timer_scale_factors: + - timer: HTTP_DOWNSTREAM_CONNECTION_IDLE + min_timeout: 3s + )EOF")); + + // Set the load so the timer is reduced but not to the minimum value. + updateResource(0.8); + test_server_->waitForGaugeGe("overload.envoy.overload_actions.reduce_timeouts.scale_percent", 50); + // Create an HTTP connection without finishing the handshake. + codec_client_ = makeRawHttpConnection(makeClientConnection((lookupPort("http"))), absl::nullopt, + /*wait_till_connected=*/false); + EXPECT_FALSE(codec_client_->connected()); + + // Advancing past the minimum time shouldn't close the connection. + timeSystem().advanceTimeWait(std::chrono::seconds(3)); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_FALSE(codec_client_->connected()); + EXPECT_FALSE(codec_client_->disconnected()); + + // Increase load more so that the timer is reduced to the minimum. + updateResource(0.9); + test_server_->waitForGaugeEq("overload.envoy.overload_actions.reduce_timeouts.scale_percent", + 100); + + // Create another HTTP connection without finishing handshake. + IntegrationCodecClientPtr codec_client2 = makeRawHttpConnection( + makeClientConnection((lookupPort("http"))), absl::nullopt, /*wait_till_connected=*/false); + EXPECT_FALSE(codec_client2->connected()); + // Advancing past the minimum time and wait for the proxy to notice and close both connections. + timeSystem().advanceTimeWait(std::chrono::seconds(3)); + test_server_->waitForCounterGe("http.config_test.downstream_cx_idle_timeout", 2); + ASSERT_TRUE(codec_client_->waitForDisconnect()); + EXPECT_FALSE(codec_client_->sawGoAway()); + EXPECT_FALSE(codec_client2->connected()); + ASSERT_TRUE(codec_client2->waitForDisconnect()); + EXPECT_FALSE(codec_client2->sawGoAway()); +} + TEST_P(OverloadScaledTimerIntegrationTest, CloseIdleHttpStream) { initializeOverloadManager( TestUtility::parseYaml(R"EOF( diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index a2402bafe07f..0a58b55ce8df 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -201,15 +201,9 @@ TEST_P(DownstreamProtocolIntegrationTest, RouterRedirectHttpRequest) { EXPECT_EQ("301", response->headers().getStatusValue()); EXPECT_EQ("https://www.redirect.com/foo", response->headers().get(Http::Headers::get().Location)[0]->value().getStringView()); - if (Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)) { - expectDownstreamBytesSentAndReceived(BytesCountExpectation(145, 45, 111, 23), - BytesCountExpectation(69, 30, 69, 30), - BytesCountExpectation(0, 30, 0, 30)); - } else { - expectDownstreamBytesSentAndReceived(BytesCountExpectation(145, 45, 111, 23), - BytesCountExpectation(0, 30, 0, 30), - BytesCountExpectation(0, 30, 0, 30)); - } + expectDownstreamBytesSentAndReceived(BytesCountExpectation(145, 45, 111, 23), + BytesCountExpectation(69, 30, 69, 30), + BytesCountExpectation(0, 30, 0, 30)); } else { // All QUIC requests use https, and should not be redirected. (Even those sent with http scheme // will be overridden to https by HCM.) @@ -294,7 +288,7 @@ TEST_P(ProtocolIntegrationTest, AddBodyToRequestAndWaitForIt) { EXPECT_EQ("200", response->headers().getStatusValue()); } -TEST_P(ProtocolIntegrationTest, RouterOnlyTracing) { +TEST_P(ProtocolIntegrationTest, DEPRECATED_FEATURE_TEST(RouterOnlyTracing)) { config_helper_.addConfigModifier( [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { @@ -717,15 +711,9 @@ TEST_P(DownstreamProtocolIntegrationTest, MissingHeadersLocalReplyDownstreamByte ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); - if (Runtime::runtimeFeatureEnabled(Runtime::expand_agnostic_stream_lifetime)) { - expectDownstreamBytesSentAndReceived(BytesCountExpectation(90, 88, 71, 54), - BytesCountExpectation(40, 58, 40, 58), - BytesCountExpectation(7, 10, 7, 8)); - } else { - expectDownstreamBytesSentAndReceived(BytesCountExpectation(90, 88, 71, 54), - BytesCountExpectation(0, 58, 0, 58), - BytesCountExpectation(7, 10, 7, 8)); - } + expectDownstreamBytesSentAndReceived(BytesCountExpectation(90, 88, 71, 54), + BytesCountExpectation(40, 58, 40, 58), + BytesCountExpectation(7, 10, 7, 8)); } TEST_P(DownstreamProtocolIntegrationTest, MissingHeadersLocalReplyUpstreamBytesCount) { @@ -796,6 +784,29 @@ TEST_P(DownstreamProtocolIntegrationTest, MissingHeadersLocalReplyWithBodyBytesC BytesCountExpectation(7, 10, 7, 8)); } +TEST_P(DownstreamProtocolIntegrationTest, TeSanitization) { + if (downstreamProtocol() != Http::CodecType::HTTP1) { + return; + } + + autonomous_upstream_ = true; + config_helper_.addRuntimeOverride("envoy.reloadable_features.sanitize_te", "true"); + + default_request_headers_.setTE("gzip"); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + auto upstream_headers = + reinterpret_cast(fake_upstreams_[0].get())->lastRequestHeaders(); + EXPECT_TRUE(upstream_headers != nullptr); + EXPECT_EQ("", upstream_headers->getTEValue()); +} + // Regression test for https://github.com/envoyproxy/envoy/issues/10270 TEST_P(ProtocolIntegrationTest, LongHeaderValueWithSpaces) { // Header with at least 20kb of spaces surrounded by non-whitespace characters to ensure that @@ -1107,6 +1118,7 @@ TEST_P(ProtocolIntegrationTest, RetryStreamingCancelDueToBufferOverflow) { EXPECT_TRUE(response->complete()); EXPECT_EQ("507", response->headers().getStatusValue()); test_server_->waitForCounterEq("cluster.cluster_0.retry_or_shadow_abandoned", 1); + cleanupUpstreamAndDownstream(); } // Tests that the x-envoy-attempt-count header is properly set on the upstream request and the @@ -2975,6 +2987,10 @@ TEST_P(DownstreamProtocolIntegrationTest, ConnectIsBlocked) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_EQ("404", response->headers().getStatusValue()); EXPECT_TRUE(response->complete()); + // Wait to process STOP_SENDING on the client for quic. + if (downstreamProtocol() == Http::CodecType::HTTP3) { + EXPECT_TRUE(response->waitForReset()); + } } // Make sure that with override_stream_error_on_invalid_http_message true, CONNECT @@ -3096,6 +3112,50 @@ TEST_P(ProtocolIntegrationTest, ContinueAllFromDecodeMetadata) { cleanupUpstreamAndDownstream(); } +TEST_P(DownstreamProtocolIntegrationTest, ContinueAllFromDecodeMetadataFollowedByLocalReply) { + if (downstream_protocol_ != Http::CodecType::HTTP2 || + upstreamProtocol() != Http::CodecType::HTTP2) { + GTEST_SKIP() << "Metadata is not enabled for non HTTP2 protocols."; + } + + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + RELEASE_ASSERT(bootstrap.mutable_static_resources()->clusters_size() >= 1, ""); + ConfigHelper::HttpProtocolOptions protocol_options; + protocol_options.mutable_explicit_http_config() + ->mutable_http2_protocol_options() + ->set_allow_metadata(true); + ConfigHelper::setProtocolOptions(*bootstrap.mutable_static_resources()->mutable_clusters(0), + protocol_options); + }); + config_helper_.prependFilter(R"EOF( + name: local-reply-during-decode + )EOF"); + config_helper_.prependFilter(R"EOF( + name: metadata-control-filter + )EOF"); + autonomous_upstream_ = false; + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = codec_client_->startRequest(default_request_headers_); + Http::RequestEncoder& encoder = encoder_decoder.first; + IntegrationStreamDecoderPtr& decoder = encoder_decoder.second; + // Send some data; this should be buffered. + codec_client_->sendData(encoder, "abc", false); + // Allow the headers through. + Http::MetadataMap metadata; + metadata["should_continue"] = "true"; + codec_client_->sendMetadata(encoder, metadata); + + ASSERT_TRUE(decoder->waitForEndStream(std::chrono::milliseconds(500))); + EXPECT_EQ("500", decoder->headers().getStatusValue()); + cleanupUpstreamAndDownstream(); +} + TEST_P(ProtocolIntegrationTest, ContinueAllFromEncodeMetadata) { if (upstreamProtocol() != Http::CodecType::HTTP2 || downstream_protocol_ != Http::CodecType::HTTP2) { @@ -3412,6 +3472,7 @@ TEST_P(DownstreamProtocolIntegrationTest, OverflowDecoderBufferFromDecodeData) { ASSERT_TRUE(response->waitForEndStream()); EXPECT_TRUE(response->complete()); EXPECT_EQ("413", response->headers().getStatusValue()); + cleanupUpstreamAndDownstream(); } // Verify that when a filter decodeData callback overflows request buffer in filter manager the @@ -4032,10 +4093,6 @@ TEST_P(DownstreamProtocolIntegrationTest, ValidateUpstreamHeaders) { } TEST_P(ProtocolIntegrationTest, ValidateUpstreamMixedCaseHeaders) { - if (use_universal_header_validator_) { - // UHV does not support this case so far. - return; - } if (upstreamProtocol() != Http::CodecType::HTTP1) { autonomous_allow_incomplete_streams_ = true; autonomous_upstream_ = true; @@ -4280,9 +4337,9 @@ TEST_P(DownstreamProtocolIntegrationTest, HandleDownstreamSocketFail) { waitForNextUpstreamRequest(); // Makes us have Envoy's writes to downstream return EBADF - Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance(); + Api::IoErrorPtr ebadf = Network::IoSocketError::getIoSocketEbadfError(); socket_swap.write_matcher_->setSourcePort(lookupPort("http")); - socket_swap.write_matcher_->setWriteOverride(ebadf); + socket_swap.write_matcher_->setWriteOverride(std::move(ebadf)); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); if (downstreamProtocol() == Http::CodecType::HTTP3) { @@ -4297,7 +4354,7 @@ TEST_P(DownstreamProtocolIntegrationTest, HandleDownstreamSocketFail) { } else { ASSERT_TRUE(codec_client_->waitForDisconnect()); } - socket_swap.write_matcher_->setWriteOverride(nullptr); + socket_swap.write_matcher_->setWriteOverride(Api::IoError::none()); // Shut down the server and upstreams before os_calls goes out of scope to avoid syscalls // during its removal. test_server_.reset(); @@ -4325,9 +4382,9 @@ TEST_P(ProtocolIntegrationTest, HandleUpstreamSocketFail) { RELEASE_ASSERT(result, result.message()); // Makes us have Envoy's writes to upstream return EBADF - Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance(); + Api::IoErrorPtr ebadf = Network::IoSocketError::getIoSocketEbadfError(); socket_swap.write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); - socket_swap.write_matcher_->setWriteOverride(ebadf); + socket_swap.write_matcher_->setWriteOverride(std::move(ebadf)); Buffer::OwnedImpl data("HTTP body content goes here"); codec_client_->sendData(*downstream_request, data, true); @@ -4337,7 +4394,7 @@ TEST_P(ProtocolIntegrationTest, HandleUpstreamSocketFail) { HasSubstr("upstream_reset_before_response_started{connection_termination}")); EXPECT_TRUE(response->complete()); EXPECT_EQ("503", response->headers().getStatusValue()); - socket_swap.write_matcher_->setWriteOverride(nullptr); + socket_swap.write_matcher_->setWriteOverride(Api::IoError::none()); // Shut down the server before os_calls goes out of scope to avoid syscalls // during its removal. test_server_.reset(); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 641bc37e7804..7d442edf53f7 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -18,7 +18,7 @@ #include "source/common/quic/envoy_quic_packet_writer.h" #include "source/common/quic/envoy_quic_proof_verifier.h" #include "source/common/quic/envoy_quic_utils.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" #include "source/extensions/transport_sockets/tls/context_config_impl.h" #include "test/common/config/dummy_config.pb.h" @@ -36,7 +36,6 @@ #include "test/test_common/utility.h" #include "quiche/quic/core/crypto/quic_client_session_cache.h" -#include "quiche/quic/core/http/quic_client_push_promise_index.h" #include "quiche/quic/core/quic_utils.h" #include "quiche/quic/test_tools/quic_sent_packet_manager_peer.h" #include "quiche/quic/test_tools/quic_session_peer.h" @@ -92,6 +91,8 @@ class TestEnvoyQuicClientConnection : public EnvoyQuicClientConnection { return AssertionFailure() << "Timed out waiting for path response\n"; } } + waiting_for_path_response_ = false; + saw_path_response_ = false; return AssertionSuccess(); } @@ -133,12 +134,42 @@ class TestEnvoyQuicClientConnection : public EnvoyQuicClientConnection { return EnvoyQuicClientConnection::OnHandshakeDoneFrame(frame); } + AssertionResult waitForNewCid(std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) { + bool timer_fired = false; + if (!saw_new_cid_) { + Event::TimerPtr timer(dispatcher_.createTimer([this, &timer_fired]() -> void { + timer_fired = true; + dispatcher_.exit(); + })); + timer->enableTimer(timeout); + waiting_for_new_cid_ = true; + dispatcher_.run(Event::Dispatcher::RunType::Block); + if (timer_fired) { + return AssertionFailure() << "Timed out waiting for new cid\n"; + } + } + waiting_for_new_cid_ = false; + return AssertionSuccess(); + } + + bool OnNewConnectionIdFrame(const quic::QuicNewConnectionIdFrame& frame) override { + bool ret = EnvoyQuicClientConnection::OnNewConnectionIdFrame(frame); + saw_new_cid_ = true; + if (waiting_for_new_cid_) { + dispatcher_.exit(); + } + saw_new_cid_ = false; + return ret; + } + private: Event::Dispatcher& dispatcher_; bool saw_path_response_{false}; bool saw_handshake_done_{false}; + bool saw_new_cid_{false}; bool waiting_for_path_response_{false}; bool waiting_for_handshake_done_{false}; + bool waiting_for_new_cid_{false}; bool validation_failure_on_path_response_{false}; }; @@ -189,7 +220,7 @@ class QuicHttpIntegrationTestBase : public HttpIntegrationTest { (host.empty() ? transport_socket_factory_->clientContextConfig()->serverNameIndication() : host), static_cast(port), false}, - transport_socket_factory_->getCryptoConfig(), &push_promise_index_, *dispatcher_, + transport_socket_factory_->getCryptoConfig(), *dispatcher_, // Use smaller window than the default one to have test coverage of client codec buffer // exceeding high watermark. /*send_buffer_limit=*/2 * Http2::Utility::OptionsLimits::MIN_INITIAL_STREAM_WINDOW_SIZE, @@ -198,12 +229,13 @@ class QuicHttpIntegrationTestBase : public HttpIntegrationTest { return session; } - IntegrationCodecClientPtr makeRawHttpConnection( - Network::ClientConnectionPtr&& conn, - absl::optional http2_options) override { + IntegrationCodecClientPtr + makeRawHttpConnection(Network::ClientConnectionPtr&& conn, + absl::optional http2_options, + bool wait_till_connected = true) override { ENVOY_LOG(debug, "Creating a new client {}", conn->connectionInfoProvider().localAddress()->asStringView()); - return makeRawHttp3Connection(std::move(conn), http2_options, true); + return makeRawHttp3Connection(std::move(conn), http2_options, wait_till_connected); } // Create Http3 codec client with the option not to wait for 1-RTT key establishment. @@ -360,7 +392,6 @@ class QuicHttpIntegrationTestBase : public HttpIntegrationTest { } protected: - quic::QuicClientPushPromiseIndex push_promise_index_; quic::ParsedQuicVersionVector supported_versions_; EnvoyQuicConnectionHelper conn_helper_; EnvoyQuicAlarmFactory alarm_factory_; @@ -716,6 +747,11 @@ TEST_P(QuicHttpIntegrationTest, PortMigration) { initialize(); uint32_t old_port = lookupPort("http"); codec_client_ = makeHttpConnection(old_port); + + // Verify the default path degrading timeout is set correctly. + EXPECT_EQ(4u, quic::test::QuicSentPacketManagerPeer::GetNumPtosForPathDegrading( + &quic_connection_->sent_packet_manager())); + auto encoder_decoder = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, {":path", "/test/long/url"}, @@ -793,12 +829,20 @@ TEST_P(QuicHttpIntegrationTest, PortMigrationOnPathDegrading) { codec_client_->sendData(*request_encoder_, 1024u, false); ASSERT_TRUE(quic_connection_->waitForHandshakeDone()); - auto old_self_addr = quic_connection_->self_address(); - EXPECT_CALL(*option, setOption(_, _)).Times(3u); + + for (uint8_t i = 0; i < 5; i++) { + auto old_self_addr = quic_connection_->self_address(); + EXPECT_CALL(*option, setOption(_, _)).Times(3u); + quic_connection_->OnPathDegradingDetected(); + ASSERT_TRUE(quic_connection_->waitForPathResponse()); + auto self_addr = quic_connection_->self_address(); + EXPECT_NE(old_self_addr, self_addr); + ASSERT_TRUE(quic_connection_->waitForNewCid()); + } + + // port migration is disabled once socket switch limit is reached. + EXPECT_CALL(*option, setOption(_, _)).Times(0); quic_connection_->OnPathDegradingDetected(); - ASSERT_TRUE(quic_connection_->waitForPathResponse()); - auto self_addr = quic_connection_->self_address(); - EXPECT_NE(old_self_addr, self_addr); // Send the rest data. codec_client_->sendData(*request_encoder_, 1024u, true); @@ -1068,30 +1112,30 @@ TEST_P(QuicHttpIntegrationTest, NoStreams) { TEST_P(QuicHttpIntegrationTest, AsyncCertVerificationSucceeds) { // Config the client to defer cert validation by 5ms. - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: "@type": type.googleapis.com/test.common.config.DummyConfig )EOF"), *custom_validator_config); - ssl_client_option_.setCustomCertValidatorConfig(custom_validator_config); + ssl_client_option_.setCustomCertValidatorConfig(custom_validator_config.get()); initialize(); codec_client_ = makeRawHttpConnection(makeClientConnection(lookupPort("http")), absl::nullopt); EXPECT_TRUE(codec_client_->connected()); } TEST_P(QuicHttpIntegrationTest, AsyncCertVerificationAfterDisconnect) { - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: "@type": type.googleapis.com/test.common.config.DummyConfig )EOF"), *custom_validator_config); - ssl_client_option_.setCustomCertValidatorConfig(custom_validator_config); + ssl_client_option_.setCustomCertValidatorConfig(custom_validator_config.get()); // Change the configured cert validation to defer 1s. auto* cert_validator_factory = @@ -1122,15 +1166,15 @@ name: "envoy.tls.cert_validator.timed_cert_validator" } TEST_P(QuicHttpIntegrationTest, AsyncCertVerificationAfterTearDown) { - envoy::config::core::v3::TypedExtensionConfig* custom_validator_config = - new envoy::config::core::v3::TypedExtensionConfig(); + auto custom_validator_config = std::make_unique( + envoy::config::core::v3::TypedExtensionConfig()); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( name: "envoy.tls.cert_validator.timed_cert_validator" typed_config: "@type": type.googleapis.com/test.common.config.DummyConfig )EOF"), *custom_validator_config); - ssl_client_option_.setCustomCertValidatorConfig(custom_validator_config); + ssl_client_option_.setCustomCertValidatorConfig(custom_validator_config.get()); // Change the configured cert validation to defer 1s. auto cert_validator_factory = Registry::FactoryRegistry:: @@ -1176,6 +1220,8 @@ TEST_P(QuicHttpIntegrationTest, MultipleNetworkFilters) { } TEST_P(QuicHttpIntegrationTest, DeferredLogging) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", + "true"); useAccessLog( "%PROTOCOL%,%ROUNDTRIP_DURATION%,%REQUEST_DURATION%,%RESPONSE_DURATION%,%RESPONSE_" "CODE%,%BYTES_RECEIVED%,%ROUTE_NAME%,%VIRTUAL_CLUSTER_NAME%,%RESPONSE_CODE_DETAILS%,%" @@ -1243,6 +1289,8 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingDisabled) { } TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithReset) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", + "true"); useAccessLog( "%PROTOCOL%,%ROUNDTRIP_DURATION%,%REQUEST_DURATION%,%RESPONSE_DURATION%,%RESPONSE_" "CODE%,%BYTES_RECEIVED%,%ROUTE_NAME%,%VIRTUAL_CLUSTER_NAME%,%RESPONSE_CODE_DETAILS%,%" @@ -1271,6 +1319,8 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithReset) { } TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithQuicReset) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", + "true"); useAccessLog( "%PROTOCOL%,%ROUNDTRIP_DURATION%,%REQUEST_DURATION%,%RESPONSE_DURATION%,%RESPONSE_" "CODE%,%BYTES_RECEIVED%,%ROUTE_NAME%,%VIRTUAL_CLUSTER_NAME%,%RESPONSE_CODE_DETAILS%,%" @@ -1340,6 +1390,8 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithEnvoyReset) { } TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithInternalRedirect) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", + "true"); useAccessLog( "%PROTOCOL%,%ROUNDTRIP_DURATION%,%REQUEST_DURATION%,%RESPONSE_DURATION%,%RESPONSE_" "CODE%,%BYTES_RECEIVED%,%ROUTE_NAME%,%VIRTUAL_CLUSTER_NAME%,%RESPONSE_CODE_DETAILS%,%" @@ -1418,6 +1470,8 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithInternalRedirect) { } TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithRetransmission) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", + "true"); useAccessLog("%BYTES_RETRANSMITTED%,%PACKETS_RETRANSMITTED%"); initialize(); @@ -1432,9 +1486,9 @@ TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithRetransmission) { SocketInterfaceSwap socket_swap(downstreamProtocol() == Http::CodecType::HTTP3 ? Network::Socket::Type::Datagram : Network::Socket::Type::Stream); - Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance(); + Api::IoErrorPtr ebadf = Network::IoSocketError::getIoSocketEbadfError(); socket_swap.write_matcher_->setDestinationPort(lookupPort("http")); - socket_swap.write_matcher_->setWriteOverride(ebadf); + socket_swap.write_matcher_->setWriteOverride(std::move(ebadf)); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); timeSystem().advanceTimeWait(std::chrono::seconds(TSAN_TIMEOUT_FACTOR)); } diff --git a/test/integration/redirect_integration_test.cc b/test/integration/redirect_integration_test.cc index b51d94cfd4dc..db289c321775 100644 --- a/test/integration/redirect_integration_test.cc +++ b/test/integration/redirect_integration_test.cc @@ -11,6 +11,7 @@ namespace { constexpr char HandleThreeHopLocationFormat[] = "http://handle.internal.redirect.max.three.hop/path{}"; constexpr char kTestHeaderKey[] = "test-header"; +constexpr char kTestHeaderKeyToClear[] = "test-header-to-clear"; } // namespace class RedirectIntegrationTest : public HttpProtocolIntegrationTest { @@ -20,6 +21,8 @@ class RedirectIntegrationTest : public HttpProtocolIntegrationTest { setMaxRequestHeadersCount(100); envoy::config::route::v3::RetryPolicy retry_policy; + // If there is a need for another virtual host, it's recommended + // to add it at the end, so the numbering remains stable. auto pass_through = config_helper_.createVirtualHost("pass.through.internal.redirect"); config_helper_.addVirtualHost(pass_through); @@ -48,6 +51,14 @@ class RedirectIntegrationTest : public HttpProtocolIntegrationTest { ->set_inline_string(EMPTY_STRING); config_helper_.addVirtualHost(handle_by_direct_response); + auto handle_with_copy = config_helper_.createVirtualHost("handle.internal.redirect.and.copy"); + handle_with_copy.mutable_routes(0)->set_name("preserve_response_headers"); + auto policy = + handle_with_copy.mutable_routes(0)->mutable_route()->mutable_internal_redirect_policy(); + policy->add_response_headers_to_copy(kTestHeaderKey); + policy->add_response_headers_to_copy(kTestHeaderKeyToClear); + config_helper_.addVirtualHost(handle_with_copy); + HttpProtocolIntegrationTest::initialize(); } @@ -831,6 +842,60 @@ TEST_P(RedirectIntegrationTest, InternalRedirectHandledByDirectResponse) { EXPECT_THAT(waitForAccessLog(access_log_name_, 1), HasSubstr("204 direct_response -")); } +TEST_P(RedirectIntegrationTest, PreserveOrClearResponseHeaders) { + useAccessLog("%RESPONSE_FLAGS% %RESPONSE_CODE% %RESPONSE_CODE_DETAILS% %RESP(test-header)% " + "%RESP(test-header-to-clear)%"); + // Validate that header sanitization is only called once. + config_helper_.addConfigModifier([](envoy::extensions::filters::network::http_connection_manager:: + v3::HttpConnectionManager& hcm) { + hcm.mutable_delayed_close_timeout()->set_seconds(0); + hcm.set_via("via_value"); + hcm.mutable_common_http_protocol_options()->mutable_max_requests_per_connection()->set_value(1); + }); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + default_request_headers_.setHost("handle.internal.redirect.and.copy"); + default_request_headers_.setCopy(Http::LowerCaseString(kTestHeaderKeyToClear), "to_clear"); + IntegrationStreamDecoderPtr response = + codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(redirect_response_, true); + EXPECT_THAT(waitForAccessLog(access_log_name_, 0), + HasSubstr("302 internal_redirect test-header-value")); + + waitForNextUpstreamRequest(); + ASSERT(upstream_request_->headers().EnvoyOriginalUrl() != nullptr); + EXPECT_EQ("http://handle.internal.redirect.and.copy/test/long/url", + upstream_request_->headers().getEnvoyOriginalUrlValue()); + EXPECT_EQ("/new/url", upstream_request_->headers().getPathValue()); + EXPECT_EQ("authority2", upstream_request_->headers().getHostValue()); + EXPECT_EQ("via_value", upstream_request_->headers().getViaValue()); + EXPECT_EQ(0, + upstream_request_->headers().get(Http::LowerCaseString(kTestHeaderKeyToClear)).size()); + EXPECT_NE(0, upstream_request_->headers().get(Http::LowerCaseString(kTestHeaderKey)).size()); + EXPECT_EQ("test-header-value", upstream_request_->headers() + .get(Http::LowerCaseString(kTestHeaderKey))[0] + ->value() + .getStringView()); + + upstream_request_->encodeHeaders(default_response_headers_, true); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.upstream_internal_redirect_succeeded_total") + ->value()); + // 302 was never returned downstream + EXPECT_EQ(0, test_server_->counter("http.config_test.downstream_rq_3xx")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_rq_2xx")->value()); + // No test header + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), HasSubstr("200 via_upstream -")); +} + INSTANTIATE_TEST_SUITE_P(Protocols, RedirectIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( {Http::CodecType::HTTP1, Http::CodecType::HTTP2, diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 118725693f2b..2ecfe70f5839 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -434,67 +434,6 @@ route_configuration_name: {} cleanupUpstreamAndDownstream(); } -TEST_P(ScopedRdsIntegrationTest, IgnoreUnknownOptionalHttpFilterInPerFilterTypedConfig) { - constexpr absl::string_view scope_tmpl = R"EOF( -name: {} -route_configuration_name: {} -key: - fragments: - - string_key: {} -)EOF"; - const std::string scope_route = fmt::format(scope_tmpl, "foo_scope1", "foo_route", "foo-route"); - - constexpr absl::string_view route_config_tmpl = R"EOF( - name: {} - virtual_hosts: - - name: integration - domains: ["*"] - routes: - - match: {{ prefix: "/" }} - route: {{ cluster: {} }} - typed_per_filter_config: - filter.unknown: - "@type": type.googleapis.com/google.protobuf.Struct -)EOF"; - - on_server_init_function_ = [&]() { - createScopedRdsStream(); - sendSrdsResponse({scope_route}, {scope_route}, {}, "1"); - createRdsStream("foo_route"); - // CreateRdsStream waits for connection which is fired by RDS subscription. - sendRdsResponse(fmt::format(route_config_tmpl, "foo_route", "cluster_0"), "1"); - }; - - // TODO(wbpcode): This test should be removed once the deprecated flag is removed. - config_helper_.addRuntimeOverride( - "envoy.reloadable_features.ignore_optional_option_from_hcm_for_route_config", "false"); - - config_helper_.addConfigModifier( - [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& - http_connection_manager) { - auto* filter = http_connection_manager.mutable_http_filters()->Add(); - filter->set_name("filter.unknown"); - filter->set_is_optional(true); - // keep router the last - auto size = http_connection_manager.http_filters_size(); - http_connection_manager.mutable_http_filters()->SwapElements(size - 2, size - 1); - }); - - initialize(); - registerTestServerPorts({"http"}); - - test_server_->waitForCounterGe("http.config_test.rds.foo_route.update_attempt", 1); - sendRequestAndVerifyResponse( - Http::TestRequestHeaderMapImpl{{":method", "GET"}, - {":path", "/meh"}, - {":authority", "host"}, - {":scheme", "http"}, - {"Addr", "x-foo-key=foo-route"}}, - 456, Http::TestResponseHeaderMapImpl{{":status", "200"}, {"service", "bluh"}}, 123, - /*cluster_0*/ 0); - cleanupUpstreamAndDownstream(); -} - TEST_P(ScopedRdsIntegrationTest, RejectKeyConflictInDeltaUpdate) { if (!isDelta()) { return; diff --git a/test/integration/sds_generic_secret_integration_test.cc b/test/integration/sds_generic_secret_integration_test.cc index 19daab6d0866..2d1b3aed08a9 100644 --- a/test/integration/sds_generic_secret_integration_test.cc +++ b/test/integration/sds_generic_secret_integration_test.cc @@ -67,11 +67,12 @@ class SdsGenericSecretTestFilterConfig grpc_service->mutable_envoy_grpc()->set_cluster_name("sds_cluster"); } - Http::FilterFactoryCb + absl::StatusOr createFilter(const std::string&, Server::Configuration::FactoryContext& factory_context) override { auto secret_provider = - factory_context.clusterManager() + factory_context.serverFactoryContext() + .clusterManager() .clusterManagerFactory() .secretManager() .findOrCreateGenericSecretProvider(config_source_, "encryption_key", @@ -80,7 +81,7 @@ class SdsGenericSecretTestFilterConfig return [&factory_context, secret_provider](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared<::Envoy::SdsGenericSecretTestFilter>( - factory_context.api(), secret_provider)); + factory_context.serverFactoryContext().api(), secret_provider)); }; } diff --git a/test/integration/server.cc b/test/integration/server.cc index 3a19040a98b5..91f491595a7c 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -12,6 +12,7 @@ #include "source/common/stats/thread_local_store.h" #include "source/common/thread_local/thread_local_impl.h" #include "source/server/hot_restart_nop_impl.h" +#include "source/server/instance_impl.h" #include "source/server/options_impl.h" #include "source/server/process_context_impl.h" @@ -232,11 +233,11 @@ void IntegrationTestServerImpl::createAndRunEnvoyServer( if (process_object.has_value()) { process_context = std::make_unique(process_object->get()); } - Server::InstanceImpl server(init_manager, options, time_system, local_address, hooks, restarter, - stat_store, access_log_lock, component_factory, - std::move(random_generator), tls, Thread::threadFactoryForTest(), - Filesystem::fileSystemForTest(), std::move(process_context), - watermark_factory); + Server::InstanceImpl server(init_manager, options, time_system, hooks, restarter, stat_store, + access_log_lock, std::move(random_generator), tls, + Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), + std::move(process_context), watermark_factory); + server.initialize(local_address, component_factory); // This is technically thread unsafe (assigning to a shared_ptr accessed // across threads), but because we synchronize below through serverReady(), the only // consumer on the main test thread in ~IntegrationTestServerImpl will not race. diff --git a/test/integration/server.h b/test/integration/server.h index 78b8e7c8b091..0a61fa821f04 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -360,6 +360,7 @@ class TestIsolatedStoreImpl : public StoreRoot { void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override{}; void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override{}; + const Stats::TagVector& fixedTags() override { CONSTRUCT_ON_FIRST_USE(Stats::TagVector); } // Stats::StoreRoot void addSink(Sink&) override {} @@ -463,6 +464,24 @@ class IntegrationTestServer : public Logger::Loggable, notifyingStatsAllocator().waitForCounterExists(name); } + void waitForCounterNonexistent(const std::string& name, + std::chrono::milliseconds timeout) override { + Event::TestTimeSystem::RealTimeBound bound(timeout); + while (TestUtility::findCounter(statStore(), name) != nullptr) { + time_system_.advanceTimeWait(std::chrono::milliseconds(10)); + ASSERT_FALSE(!bound.withinBound()) + << "timed out waiting for counter " << name << " to not exist."; + } + } + + void waitForProactiveOverloadResourceUsageEq( + const Server::OverloadProactiveResourceName resource_name, int64_t value, + Event::Dispatcher& dispatcher, + std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) { + ASSERT_TRUE(TestUtility::waitForProactiveOverloadResourceUsageEq( + overloadState(), resource_name, value, time_system_, dispatcher, timeout)); + } + // TODO(#17956): Add Gauge type to NotifyingAllocator and adopt it in this method. void waitForGaugeDestroyed(const std::string& name) override { ASSERT_TRUE(TestUtility::waitForGaugeDestroyed(statStore(), name, time_system_)); @@ -523,6 +542,7 @@ class IntegrationTestServer : public Logger::Loggable, // Should not be called until createAndRunEnvoyServer() is called. virtual Server::Instance& server() PURE; virtual Stats::Store& statStore() PURE; + virtual Server::ThreadLocalOverloadState& overloadState() PURE; virtual Network::Address::InstanceConstSharedPtr adminAddress() PURE; virtual Stats::NotifyingAllocatorImpl& notifyingStatsAllocator() PURE; void useAdminInterfaceToQuit(bool use) { use_admin_interface_to_quit_ = use; } @@ -594,6 +614,11 @@ class IntegrationTestServerImpl : public IntegrationTestServer { RELEASE_ASSERT(stat_store_ != nullptr, ""); return *stat_store_; } + Server::ThreadLocalOverloadState& overloadState() override { + RELEASE_ASSERT(server_ != nullptr, ""); + return server_->overloadManager().getThreadLocalOverloadState(); + } + Network::Address::InstanceConstSharedPtr adminAddress() override { return admin_address_; } Stats::NotifyingAllocatorImpl& notifyingStatsAllocator() override { diff --git a/test/integration/server_stats.h b/test/integration/server_stats.h index 04384db7d8be..45d08d3bdbb2 100644 --- a/test/integration/server_stats.h +++ b/test/integration/server_stats.h @@ -38,6 +38,14 @@ class IntegrationTestServerStats { */ virtual void waitForCounterExists(const std::string& name) PURE; + /** + * Wait for a counter to not exist. + * @param name counter name. + * @param timeout amount of time to wait for the counter to not exist. + */ + virtual void waitForCounterNonexistent(const std::string& name, + std::chrono::milliseconds timeout) PURE; + /** * Wait until a histogram has samples. * @param name histogram name. diff --git a/test/integration/shadow_policy_integration_test.cc b/test/integration/shadow_policy_integration_test.cc index 4f4c1b8855d7..3da3883b6d0c 100644 --- a/test/integration/shadow_policy_integration_test.cc +++ b/test/integration/shadow_policy_integration_test.cc @@ -29,7 +29,7 @@ class ShadowPolicyIntegrationTest } // Adds a mirror policy that routes to cluster_header or cluster_name, in that order. Additionally - // optionally registers an upstream filter on the cluster specified by + // optionally registers an upstream HTTP filter on the cluster specified by // cluster_with_custom_filter_. void initialConfigSetup(const std::string& cluster_name, const std::string& cluster_header) { config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { @@ -510,6 +510,11 @@ TEST_P(ShadowPolicyIntegrationTest, MainRequestOverBufferLimit) { GTEST_SKIP() << "Not applicable for non-streaming shadows."; } autonomous_upstream_ = true; + if (Runtime::runtimeFeatureEnabled(Runtime::defer_processing_backedup_streams)) { + // With deferred processing, a local reply is triggered so the upstream + // stream will be incomplete. + autonomous_allow_incomplete_streams_ = true; + } cluster_with_custom_filter_ = 0; filter_name_ = "encoder-decoder-buffer-filter"; initialConfigSetup("cluster_1", ""); @@ -537,7 +542,13 @@ TEST_P(ShadowPolicyIntegrationTest, MainRequestOverBufferLimit) { EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_cx_total")->value(), 1); EXPECT_EQ(test_server_->counter("cluster.cluster_1.upstream_cx_total")->value(), 1); - test_server_->waitForCounterEq("cluster.cluster_1.upstream_rq_completed", 1); + if (Runtime::runtimeFeatureEnabled(Runtime::defer_processing_backedup_streams)) { + // With deferred processing, the encoder-decoder-buffer-filter will + // buffer too much data triggering a local reply. + test_server_->waitForCounterEq("http.config_test.downstream_rq_4xx", 1); + } else { + test_server_->waitForCounterEq("cluster.cluster_1.upstream_rq_completed", 1); + } } TEST_P(ShadowPolicyIntegrationTest, ShadowRequestOverBufferLimit) { @@ -783,7 +794,7 @@ TEST_P(ShadowPolicyIntegrationTest, RequestMirrorPolicyWithCluster) { EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_cx_total")->value(), 1); } -// Test request mirroring / shadowing with upstream filters in the router. +// Test request mirroring / shadowing with upstream HTTP filters in the router. TEST_P(ShadowPolicyIntegrationTest, RequestMirrorPolicyWithRouterUpstreamFilters) { initialConfigSetup("cluster_1", ""); config_helper_.addConfigModifier( @@ -811,7 +822,7 @@ TEST_P(ShadowPolicyIntegrationTest, ClusterFilterOverridesRouterFilter) { cluster_with_custom_filter_ = 0; filter_name_ = "add-body-filter"; - // router filter upstream filter adds header: + // router filter upstream HTTP filter adds header: config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& hcm) -> void { diff --git a/test/integration/socket_interface_swap.cc b/test/integration/socket_interface_swap.cc index e4488de81cff..b441ebd09a16 100644 --- a/test/integration/socket_interface_swap.cc +++ b/test/integration/socket_interface_swap.cc @@ -2,8 +2,6 @@ namespace Envoy { -void preserveIoError(Api::IoError*) {} - SocketInterfaceSwap::SocketInterfaceSwap(Network::Socket::Type socket_type) : write_matcher_(std::make_shared(socket_type)) { Envoy::Network::SocketInterfaceSingleton::clear(); @@ -15,15 +13,14 @@ SocketInterfaceSwap::SocketInterfaceSwap(Network::Socket::Type socket_type) // TODO(yanavlasov): refactor into separate method after CVE is public. if (slices == nullptr && size == 0) { // This is connect override check - Network::IoSocketError* error_override = - write_matcher->returnConnectOverride(io_handle); + Api::IoErrorPtr error_override = write_matcher->returnConnectOverride(io_handle); if (error_override) { - return Api::IoCallUint64Result(0, Api::IoErrorPtr(error_override, preserveIoError)); + return Api::IoCallUint64Result(0, std::move(error_override)); } } else { - Network::IoSocketError* error_override = write_matcher->returnOverride(io_handle); + Api::IoErrorPtr error_override = write_matcher->returnOverride(io_handle); if (error_override) { - return Api::IoCallUint64Result(0, Api::IoErrorPtr(error_override, preserveIoError)); + return Api::IoCallUint64Result(0, std::move(error_override)); } } return absl::nullopt; diff --git a/test/integration/socket_interface_swap.h b/test/integration/socket_interface_swap.h index 929c7e06c409..87127e4c71c2 100644 --- a/test/integration/socket_interface_swap.h +++ b/test/integration/socket_interface_swap.h @@ -16,7 +16,7 @@ class SocketInterfaceSwap { struct IoHandleMatcher { explicit IoHandleMatcher(Network::Socket::Type type) : socket_type_(type) {} - Network::IoSocketError* returnOverride(Envoy::Network::TestIoSocketHandle* io_handle) { + Api::IoErrorPtr returnOverride(Envoy::Network::TestIoSocketHandle* io_handle) { absl::MutexLock lock(&mutex_); if (socket_type_ == io_handle->getSocketType() && error_ && (io_handle->localAddress()->ip()->port() == src_port_ || @@ -24,19 +24,21 @@ class SocketInterfaceSwap { ASSERT(matched_iohandle_ == nullptr || matched_iohandle_ == io_handle, "Matched multiple io_handles, expected at most one to match."); matched_iohandle_ = io_handle; - return error_; + return (error_->getSystemErrorCode() == EAGAIN) + ? Envoy::Network::IoSocketError::getIoSocketEagainError() + : Envoy::Network::IoSocketError::create(error_->getSystemErrorCode()); } - return nullptr; + return Api::IoError::none(); } - Network::IoSocketError* returnConnectOverride(Envoy::Network::TestIoSocketHandle* io_handle) { + Api::IoErrorPtr returnConnectOverride(Envoy::Network::TestIoSocketHandle* io_handle) { absl::MutexLock lock(&mutex_); if (block_connect_ && socket_type_ == io_handle->getSocketType() && (io_handle->localAddress()->ip()->port() == src_port_ || (dst_port_ && io_handle->peerAddress()->ip()->port() == dst_port_))) { - return Network::IoSocketError::getIoSocketEagainInstance(); + return Network::IoSocketError::getIoSocketEagainError(); } - return nullptr; + return Api::IoError::none(); } // Source port to match. The port specified should be associated with a listener. @@ -54,14 +56,14 @@ class SocketInterfaceSwap { } void setWriteReturnsEgain() { - setWriteOverride(Network::IoSocketError::getIoSocketEagainInstance()); + setWriteOverride(Network::IoSocketError::getIoSocketEagainError()); } // The caller is responsible for memory management. - void setWriteOverride(Network::IoSocketError* error) { + void setWriteOverride(Api::IoErrorPtr error) { absl::WriterMutexLock lock(&mutex_); ASSERT(src_port_ != 0 || dst_port_ != 0); - error_ = error; + error_ = std::move(error); } void setConnectBlock(bool block) { @@ -76,7 +78,7 @@ class SocketInterfaceSwap { mutable absl::Mutex mutex_; uint32_t src_port_ ABSL_GUARDED_BY(mutex_) = 0; uint32_t dst_port_ ABSL_GUARDED_BY(mutex_) = 0; - Network::IoSocketError* error_ ABSL_GUARDED_BY(mutex_) = nullptr; + Api::IoErrorPtr error_ ABSL_GUARDED_BY(mutex_) = Api::IoError::none(); Network::TestIoSocketHandle* matched_iohandle_{}; Network::Socket::Type socket_type_; bool block_connect_ ABSL_GUARDED_BY(mutex_) = false; diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index 6727632d99f1..1b226d79f9f7 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -85,8 +85,8 @@ void initializeUpstreamTlsContextConfig( tls_context.set_sni(options.sni_); } if (options.custom_validator_config_) { - common_context->mutable_validation_context()->set_allocated_custom_validator_config( - options.custom_validator_config_); + *common_context->mutable_validation_context()->mutable_custom_validator_config() = + *options.custom_validator_config_; } common_context->mutable_tls_params()->set_tls_minimum_protocol_version(options.tls_version_); diff --git a/test/integration/ssl_utility.h b/test/integration/ssl_utility.h index f8ffc8f5614e..3187b3b0e2ad 100644 --- a/test/integration/ssl_utility.h +++ b/test/integration/ssl_utility.h @@ -75,6 +75,7 @@ struct ClientSslTransportOptions { envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLS_AUTO}; bool use_expired_spiffe_cert_{false}; bool client_with_intermediate_cert_{false}; + // It is owned by the caller that invokes `setCustomCertValidatorConfig()`. envoy::config::core::v3::TypedExtensionConfig* custom_validator_config_{nullptr}; }; diff --git a/test/integration/tcp_conn_pool_integration_test.cc b/test/integration/tcp_conn_pool_integration_test.cc index dc1ee5ed0b86..5f62237d1581 100644 --- a/test/integration/tcp_conn_pool_integration_test.cc +++ b/test/integration/tcp_conn_pool_integration_test.cc @@ -90,7 +90,8 @@ class TestFilterConfigFactory : public Server::Configuration::NamedNetworkFilter createFilterFactoryFromProto(const Protobuf::Message&, Server::Configuration::FactoryContext& context) override { return [&context](Network::FilterManager& filter_manager) -> void { - filter_manager.addReadFilter(std::make_shared(context.clusterManager())); + filter_manager.addReadFilter( + std::make_shared(context.serverFactoryContext().clusterManager())); }; } diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index f88cf9a62fe9..90ad072b819a 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -15,6 +15,7 @@ #include "source/extensions/filters/network/common/factory_base.h" #include "source/extensions/transport_sockets/tls/context_manager_impl.h" +#include "test/integration/fake_access_log.h" #include "test/integration/ssl_utility.h" #include "test/integration/tcp_proxy_integration_test.pb.h" #include "test/integration/tcp_proxy_integration_test.pb.validate.h" @@ -25,7 +26,6 @@ using testing::_; using testing::AtLeast; -using testing::HasSubstr; using testing::Invoke; using testing::MatchesRegex; using testing::NiceMock; @@ -993,6 +993,46 @@ TEST_P(TcpProxyIntegrationTest, TestCloseOnHealthFailure) { ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); } +TEST_P(TcpProxyIntegrationTest, RecordsUpstreamConnectionTimeLatency) { + FakeAccessLogFactory factory; + factory.setLogCallback([](const Formatter::HttpFormatterContext&, + const StreamInfo::StreamInfo& stream_info) { + EXPECT_TRUE( + stream_info.upstreamInfo()->upstreamTiming().connectionPoolCallbackLatency().has_value()); + }); + + Registry::InjectFactory factory_register(factory); + + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* filter_chain = + bootstrap.mutable_static_resources()->mutable_listeners(0)->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + + auto* access_log = tcp_proxy_config.add_access_log(); + access_log->set_name("testaccesslog"); + test::integration::accesslog::FakeAccessLog access_log_config; + access_log->mutable_typed_config()->PackFrom(access_log_config); + config_blob->PackFrom(tcp_proxy_config); + }); + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + ASSERT_TRUE(tcp_client->write("ping", true)); + ASSERT_TRUE(fake_upstream_connection->write("pong", true)); + + tcp_client->waitForHalfClose(); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); +} + class TcpProxyMetadataMatchIntegrationTest : public TcpProxyIntegrationTest { public: TcpProxyMetadataMatchIntegrationTest(uint32_t tcp_proxy_filter_index = 0) @@ -1355,8 +1395,8 @@ TEST_P(TcpProxyDynamicMetadataMatchIntegrationTest, DynamicMetadataMatch) { expectEndpointToMatchRoute([](IntegrationTcpClient& tcp_client) -> std::string { // Break the write into two; validate that the first is received before sending the second. This - // validates that a downstream filter can use this functionality, even if it can't make a - // decision after the first `onData()`. + // validates that a downstream network filter can use this functionality, even if it can't make + // a decision after the first `onData()`. EXPECT_TRUE(tcp_client.write("p", false)); tcp_client.waitForData("p"); tcp_client.clearData(); diff --git a/test/integration/typed_metadata_integration_test.cc b/test/integration/typed_metadata_integration_test.cc index e5c8ce2dc2dc..a9d2000eee4a 100644 --- a/test/integration/typed_metadata_integration_test.cc +++ b/test/integration/typed_metadata_integration_test.cc @@ -45,31 +45,20 @@ TEST_P(ListenerTypedMetadataIntegrationTest, Hello) { class MockAccessLog : public AccessLog::Instance { public: - MOCK_METHOD(void, log, - (const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo&, - AccessLog::AccessLogType)); + MOCK_METHOD(void, log, (const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&)); }; -class TestAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +class TestAccessLogFactory : public AccessLog::AccessLogInstanceFactory { public: - AccessLog::InstanceSharedPtr createAccessLogInstance( - const Protobuf::Message&, AccessLog::FilterPtr&&, - Server::Configuration::ListenerAccessLogFactoryContext& context) override { - // Check that expected listener metadata is present - EXPECT_EQ(1, context.listenerMetadata().typed_filter_metadata().size()); - const auto iter = - context.listenerMetadata().typed_filter_metadata().find("test.listener.typed.metadata"); - EXPECT_NE(iter, context.listenerMetadata().typed_filter_metadata().end()); - return std::make_shared>(); - } - AccessLog::InstanceSharedPtr createAccessLogInstance(const Protobuf::Message&, AccessLog::FilterPtr&&, - Server::Configuration::CommonFactoryContext&) override { - // This method should never be called in this test - ASSERT(false); - return nullptr; + Server::Configuration::FactoryContext& context) override { + // Check that expected listener metadata is present + EXPECT_EQ(1, context.listenerInfo().metadata().typed_filter_metadata().size()); + const auto iter = context.listenerInfo().metadata().typed_filter_metadata().find( + "test.listener.typed.metadata"); + EXPECT_NE(iter, context.listenerInfo().metadata().typed_filter_metadata().end()); + return std::make_shared>(); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { @@ -82,8 +71,7 @@ class TestAccessLogFactory : public Server::Configuration::AccessLogInstanceFact // Validate that access logger gets the right context with access to listener metadata TEST_P(ListenerTypedMetadataIntegrationTest, ListenerMetadataPlumbingToAccessLog) { TestAccessLogFactory factory; - Registry::InjectFactory factory_register( - factory); + Registry::InjectFactory factory_register(factory); // Add some typed metadata to the listener. ProtobufWkt::StringValue value; diff --git a/test/integration/udp_tunneling_integration_test.cc b/test/integration/udp_tunneling_integration_test.cc index e657f5fd065b..cf8fa2b38276 100644 --- a/test/integration/udp_tunneling_integration_test.cc +++ b/test/integration/udp_tunneling_integration_test.cc @@ -3,6 +3,7 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/core/v3/proxy_protocol.pb.h" #include "envoy/extensions/access_loggers/file/v3/file.pb.h" +#include "envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.pb.h" #include "envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.pb.h" #include "test/integration/http_integration.h" @@ -329,5 +330,675 @@ TEST_P(ForwardingConnectUdpIntegrationTest, DoNotForwardNonConnectUdp) { cleanupUpstreamAndDownstream(); } +// Tunneling downstream UDP over an upstream HTTP CONNECT tunnel. +class UdpTunnelingIntegrationTest : public HttpProtocolIntegrationTest { +public: + UdpTunnelingIntegrationTest() + : HttpProtocolIntegrationTest(ConfigHelper::baseUdpListenerConfig()) {} + + struct BufferOptions { + uint32_t max_buffered_datagrams_; + uint32_t max_buffered_bytes_; + }; + + struct TestConfig { + std::string proxy_host_; + std::string target_host_; + uint32_t max_connect_attempts_; + uint32_t default_target_port_; + bool use_post_; + std::string post_path_; + absl::optional buffer_options_; + absl::optional idle_timeout_; + std::string session_access_log_config_ = ""; + std::string access_log_options_ = ""; + bool propagate_response_headers_ = false; + bool propagate_response_trailers_ = false; + }; + + void setup(const TestConfig& config) { + config_ = config; + + std::string filter_config = + fmt::format(R"EOF( +name: udp_proxy +typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig + stat_prefix: foo + matcher: + on_no_match: + action: + name: route + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route + cluster: cluster_0 + session_filters: + - name: http_capsule + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3.FilterConfig + tunneling_config: + proxy_host: {} + target_host: {} + default_target_port: {} + retry_options: + max_connect_attempts: {} + propagate_response_headers: {} + propagate_response_trailers: {} +)EOF", + config.proxy_host_, config.target_host_, config.default_target_port_, + config.max_connect_attempts_, config.propagate_response_headers_, + config.propagate_response_trailers_); + + if (config.buffer_options_.has_value()) { + filter_config += fmt::format(R"EOF( + buffer_options: + max_buffered_datagrams: {} + max_buffered_bytes: {} +)EOF", + config.buffer_options_.value().max_buffered_datagrams_, + config.buffer_options_.value().max_buffered_bytes_); + } + + if (config.use_post_) { + filter_config += fmt::format(R"EOF( + use_post: true + post_path: {} +)EOF", + config.post_path_); + } + + if (config.idle_timeout_.has_value()) { + filter_config += fmt::format(R"EOF( + idle_timeout: {} +)EOF", + config.idle_timeout_.value()); + } + + filter_config += config.session_access_log_config_ + config.access_log_options_; + + config_helper_.renameListener("udp_proxy"); + config_helper_.addConfigModifier( + [filter_config](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter = listener->add_listener_filters(); + TestUtility::loadFromYaml(filter_config, *filter); + }); + + HttpIntegrationTest::initialize(); + + const uint32_t port = lookupPort("udp_proxy"); + listener_address_ = Network::Utility::resolveUrl( + fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + + client_ = std::make_unique(version_); + }; + + const std::string encapsulate(std::string datagram) { + uint8_t capsule_length = datagram.length() + 1; + return absl::HexStringToBytes(absl::StrCat("00", Hex::encode(&capsule_length, 1), "00")) + + datagram; + } + + const std::string expectedCapsules(std::vector raw_datagrams) { + std::string expected_capsules = ""; + for (std::string datagram : raw_datagrams) { + expected_capsules += encapsulate(datagram); + } + + return expected_capsules; + } + + void expectPostRequestHeaders(const Http::RequestHeaderMap& headers) { + EXPECT_EQ(headers.getMethodValue(), "POST"); + EXPECT_EQ(headers.getPathValue(), config_.post_path_); + } + + void expectConnectRequestHeaders(const Http::RequestHeaderMap& original_headers) { + Http::TestRequestHeaderMapImpl headers(original_headers); + + // For connect-udp case, the server codec will transform the H2 headers to H1. For test + // convenience, transforming the H1 headers back to H2. + Http::Utility::transformUpgradeRequestFromH1toH2(headers); + std::string expected_path = absl::StrCat("/.well-known/masque/udp/", config_.target_host_, "/", + config_.default_target_port_, "/"); + + EXPECT_EQ(headers.getMethodValue(), "CONNECT"); + EXPECT_EQ(headers.getPathValue(), expected_path); + } + + void expectRequestHeaders(const Http::RequestHeaderMap& headers) { + if (config_.use_post_) { + expectPostRequestHeaders(headers); + } else { + expectConnectRequestHeaders(headers); + } + } + + void establishConnection(const std::string initial_datagram, int tunnels_count = 1) { + // Initial datagram will create a session and a tunnel request. + client_->write(initial_datagram, *listener_address_); + + if (!fake_upstream_connection_) { + ASSERT_TRUE( + fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + } + + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); + expectRequestHeaders(upstream_request_->headers()); + + // Send upgrade headers downstream, fully establishing the connection. + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, + {"capsule-protocol", "?1"}}; + upstream_request_->encodeHeaders(response_headers, false); + + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_success", tunnels_count); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 1); + } + + void sendCapsuleDownstream(const std::string datagram, bool end_stream = false) { + // Send data from upstream to downstream. + upstream_request_->encodeData(encapsulate(datagram), end_stream); + Network::UdpRecvData response_datagram; + client_->recv(response_datagram); + EXPECT_EQ(datagram, response_datagram.buffer_->toString()); + } + + TestConfig config_; + Network::Address::InstanceConstSharedPtr listener_address_; + std::unique_ptr client_; +}; + +TEST_P(UdpTunnelingIntegrationTest, BasicFlowWithBuffering) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", + BufferOptions{1, 30}, absl::nullopt}; + setup(config); + + const std::string datagram1 = "hello"; + establishConnection(datagram1); + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram1}))); + + const std::string datagram2 = "hello2"; + client_->write(datagram2, *listener_address_); + ASSERT_TRUE( + upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram1, datagram2}))); + + sendCapsuleDownstream("response1", false); + sendCapsuleDownstream("response2", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, BasicFlowNoBuffering) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", absl::nullopt, absl::nullopt}; + setup(config); + + establishConnection("hello"); + // Since there's no buffering, the first datagram is dropped. Send another datagram and expect + // that it's the only datagram received upstream. + const std::string datagram2 = "hello2"; + client_->write(datagram2, *listener_address_); + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram2}))); + + sendCapsuleDownstream("response", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, BasicFlowWithPost) { + TestConfig config{"host.com", "target.com", 1, 30, true, "/post/path", + BufferOptions{1, 30}, absl::nullopt}; + setup(config); + + const std::string datagram1 = "hello"; + establishConnection(datagram1); + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram1}))); + sendCapsuleDownstream("response", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, TwoConsecutiveDownstreamSessions) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", + BufferOptions{1, 30}, absl::nullopt}; + setup(config); + + establishConnection("hello1"); + sendCapsuleDownstream("response2", true); // Will end first session. + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + establishConnection("hello2", 2); // Will create another session. + sendCapsuleDownstream("response2", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, IdleTimeout) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", BufferOptions{1, 30}, "0.5s"}; + setup(config); + + const std::string datagram = "hello"; + establishConnection(datagram); + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram}))); + + sendCapsuleDownstream("response1", false); + test_server_->waitForCounterEq("udp.foo.idle_timeout", 1); + ASSERT_TRUE(upstream_request_->waitForReset()); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, BufferOverflowDueToCapacity) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", + BufferOptions{1, 30}, absl::nullopt}; + setup(config); + + // Send two datagrams before the upstream is established. Since the buffer capacity is 1 datagram, + // we expect the second one to be dropped. + client_->write("hello1", *listener_address_); + client_->write("hello2", *listener_address_); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_buffer_overflow", 1); + + // "hello3" will drop because it's sent before the tunnel is established, and the buffer is full. + establishConnection("hello3"); + // Wait for the buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({"hello1"}))); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_buffer_overflow", 2); + + sendCapsuleDownstream("response", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, BufferOverflowDueToSize) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", BufferOptions{100, 15}, + absl::nullopt}; + setup(config); + + // Send two datagrams before the upstream is established. Since the buffer capacity is 6 bytes, + // we expect the second one to be dropped. + client_->write("hello1", *listener_address_); + client_->write("hello2", *listener_address_); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_buffer_overflow", 1); + + // "hello3" will drop because it's sent before the tunnel is established, and the buffer is full. + establishConnection("hello3"); + // Wait for the buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({"hello1"}))); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_buffer_overflow", 2); + + sendCapsuleDownstream("response", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, ConnectionReuse) { + TestConfig config{"host.com", "target.com", 1, 30, false, "", BufferOptions{100, 300}, + absl::nullopt}; + setup(config); + + // Establish connection for first session. + establishConnection("hello_1"); + + // Establish connection for second session. + Network::Test::UdpSyncPeer client2(version_); + client2.write("hello_2", *listener_address_); + + FakeStreamPtr upstream_request2; + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request2)); + ASSERT_TRUE(upstream_request2->waitForHeadersComplete()); + expectRequestHeaders(upstream_request2->headers()); + + // Send upgrade headers downstream, fully establishing the connection. + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, {"capsule-protocol", "?1"}}; + upstream_request2->encodeHeaders(response_headers, false); + + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_success", 2); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 2); + + // Wait for buffered datagram for each stream. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({"hello_1"}))); + ASSERT_TRUE(upstream_request2->waitForData(*dispatcher_, expectedCapsules({"hello_2"}))); + + // Send capsule from upstream over the first stream, and close it. + sendCapsuleDownstream("response_1", true); + // First stream is closed so we expect active sessions to decrease to 1. + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 1); + + // Send capsule from upstream over the second stream, and close it. + upstream_request2->encodeData(encapsulate("response_2"), true); + Network::UdpRecvData response_datagram; + client2.recv(response_datagram); + EXPECT_EQ("response_2", response_datagram.buffer_->toString()); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, FailureOnBadResponseHeaders) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%UPSTREAM_REQUEST_ATTEMPT_COUNT% %RESPONSE_FLAGS%\n" +)EOF", + access_log_filename); + + const TestConfig config{"host.com", + "target.com", + 1, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config}; + setup(config); + + // Initial datagram will create a session and a tunnel request. + client_->write("hello", *listener_address_); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); + expectRequestHeaders(upstream_request_->headers()); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "404"}}; + upstream_request_->encodeHeaders(response_headers, true); + + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_connect_attempts_exceeded", 1); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_failure", 1); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_success", 0); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + + EXPECT_THAT(waitForAccessLog(access_log_filename), testing::HasSubstr("1 UF,URX")); +} + +TEST_P(UdpTunnelingIntegrationTest, ConnectionAttemptRetry) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%UPSTREAM_REQUEST_ATTEMPT_COUNT% %RESPONSE_FLAGS%\n" +)EOF", + access_log_filename); + + const TestConfig config{"host.com", + "target.com", + 2, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config}; + setup(config); + + // Initial datagram will create a session and a tunnel request. + client_->write("hello", *listener_address_); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); + expectRequestHeaders(upstream_request_->headers()); + + Http::TestResponseHeaderMapImpl fail_response_headers{{":status", "404"}}; + upstream_request_->encodeHeaders(fail_response_headers, true); + + test_server_->waitForCounterEq("cluster.cluster_0.upstream_rq_retry", 1); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 1); + + // The request is retried, expect new downstream headers + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); + expectRequestHeaders(upstream_request_->headers()); + + // Send upgrade headers downstream, fully establishing the connection. + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}, {"capsule-protocol", "?1"}}; + upstream_request_->encodeHeaders(response_headers, false); + + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_success", 1); + sendCapsuleDownstream("response", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + + EXPECT_THAT(waitForAccessLog(access_log_filename), testing::HasSubstr("2 UF")); +} + +TEST_P(UdpTunnelingIntegrationTest, PropagateValidResponseHeaders) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%FILTER_STATE(envoy.udp_proxy.propagate_response_headers:TYPED)%\n" +)EOF", + access_log_filename); + + const TestConfig config{"host.com", + "target.com", + 1, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config, + "", + true, + false}; + setup(config); + + const std::string datagram = "hello"; + establishConnection(datagram); + + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram}))); + + sendCapsuleDownstream("response", true); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + + // Verify response header value is in the access log. + EXPECT_THAT(waitForAccessLog(access_log_filename), testing::HasSubstr("capsule-protocol")); +} + +TEST_P(UdpTunnelingIntegrationTest, PropagateInvalidResponseHeaders) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%FILTER_STATE(envoy.udp_proxy.propagate_response_headers:TYPED)%\n" +)EOF", + access_log_filename); + + const TestConfig config{"host.com", + "target.com", + 1, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config, + "", + true, + false}; + setup(config); + + client_->write("hello", *listener_address_); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); + expectRequestHeaders(upstream_request_->headers()); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "404"}}; + upstream_request_->encodeHeaders(response_headers, true); + + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_connect_attempts_exceeded", 1); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_failure", 1); + test_server_->waitForCounterEq("cluster.cluster_0.udp.sess_tunnel_success", 0); + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + + // Verify response header value is in the access log. + EXPECT_THAT(waitForAccessLog(access_log_filename), testing::HasSubstr("404")); +} + +TEST_P(UdpTunnelingIntegrationTest, PropagateResponseTrailers) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%FILTER_STATE(envoy.udp_proxy.propagate_response_trailers:TYPED)%\n" +)EOF", + access_log_filename); + + const TestConfig config{"host.com", + "target.com", + 1, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config, + "", + false, + true}; + setup(config); + + const std::string datagram = "hello"; + establishConnection(datagram); + + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram}))); + sendCapsuleDownstream("response", false); + + const std::string trailer_value = "test-trailer-value"; + Http::TestResponseTrailerMapImpl response_trailers{{"test-trailer-name", trailer_value}}; + upstream_request_->encodeTrailers(response_trailers); + + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); + + // Verify response trailer value is in the access log. + EXPECT_THAT(waitForAccessLog(access_log_filename), testing::HasSubstr(trailer_value)); +} + +TEST_P(UdpTunnelingIntegrationTest, FlushAccessLogOnTunnelConnected) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%ACCESS_LOG_TYPE%\n" +)EOF", + access_log_filename); + + const std::string access_log_options = R"EOF( + access_log_options: + flush_access_log_on_tunnel_connected: true +)EOF"; + + const TestConfig config{"host.com", + "target.com", + 1, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config, + access_log_options}; + setup(config); + + const std::string datagram = "hello"; + establishConnection(datagram); + EXPECT_THAT( + waitForAccessLog(access_log_filename), + testing::HasSubstr(AccessLogType_Name(AccessLog::AccessLogType::UdpTunnelUpstreamConnected))); + + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram}))); + sendCapsuleDownstream("response", true); + + test_server_->waitForGaugeEq("udp.foo.downstream_sess_active", 0); +} + +TEST_P(UdpTunnelingIntegrationTest, FlushAccessLogPeriodically) { + const std::string access_log_filename = + TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + + const std::string session_access_log_config = fmt::format(R"EOF( + access_log: + - name: envoy.access_loggers.file + typed_config: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {} + log_format: + text_format_source: + inline_string: "%ACCESS_LOG_TYPE%\n" +)EOF", + access_log_filename); + const std::string access_log_options = R"EOF( + access_log_options: + access_log_flush_interval: 0.5s +)EOF"; + + const TestConfig config{"host.com", + "target.com", + 1, + 30, + false, + "", + BufferOptions{1, 30}, + absl::nullopt, + session_access_log_config, + access_log_options}; + setup(config); + + const std::string datagram = "hello"; + establishConnection(datagram); + // Wait for buffered datagram. + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, expectedCapsules({datagram}))); + + sendCapsuleDownstream("response", false); + EXPECT_THAT(waitForAccessLog(access_log_filename), + testing::HasSubstr(AccessLogType_Name(AccessLog::AccessLogType::UdpPeriodic))); +} + +INSTANTIATE_TEST_SUITE_P(IpAndHttpVersions, UdpTunnelingIntegrationTest, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( + {Http::CodecType::HTTP2}, {Http::CodecType::HTTP2})), + HttpProtocolIntegrationTest::protocolTestParamsToString); + } // namespace } // namespace Envoy diff --git a/test/integration/upstream_access_log_integration_test.cc b/test/integration/upstream_access_log_integration_test.cc index c58f481fdfb5..2421578e8869 100644 --- a/test/integration/upstream_access_log_integration_test.cc +++ b/test/integration/upstream_access_log_integration_test.cc @@ -116,8 +116,8 @@ INSTANTIATE_TEST_SUITE_P(Params, UpstreamAccessLogTest, TestUtility::ipTestParamsToString); /* - * Verifies that the Http Router's `upstream_log` correctly reflects the upstream filter state data - * when the access log format has `UPSTREAM_FILTER_STATE` specifier. + * Verifies that the Http Router's `upstream_log` correctly reflects the upstream HTTP filter state + * data when the access log format has `UPSTREAM_FILTER_STATE` specifier. */ TEST_P(UpstreamAccessLogTest, UpstreamFilterState) { auto log_file = TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); diff --git a/test/integration/upstream_http_filter_integration_test.cc b/test/integration/upstream_http_filter_integration_test.cc index 60feec21d85d..b7662f6f1278 100644 --- a/test/integration/upstream_http_filter_integration_test.cc +++ b/test/integration/upstream_http_filter_integration_test.cc @@ -489,7 +489,8 @@ class DynamicRouterOrClusterFiltersIntegrationTest public UpstreamHttpExtensionDiscoveryIntegrationTestBase { public: DynamicRouterOrClusterFiltersIntegrationTest() - : UpstreamHttpExtensionDiscoveryIntegrationTestBase(ipVersion(), std::get<1>(GetParam())) {} + : UpstreamHttpExtensionDiscoveryIntegrationTestBase( + DynamicRouterOrClusterFiltersIntegrationTest::ipVersion(), std::get<1>(GetParam())) {} Network::Address::IpVersion ipVersion() const override { return std::get<0>(std::get<0>(GetParam())); @@ -741,7 +742,8 @@ class DynamicRouterAndClusterFiltersIntegrationTest public UpstreamHttpExtensionDiscoveryIntegrationTestBase { public: DynamicRouterAndClusterFiltersIntegrationTest() - : UpstreamHttpExtensionDiscoveryIntegrationTestBase(ipVersion(), false) {} + : UpstreamHttpExtensionDiscoveryIntegrationTestBase( + DynamicRouterAndClusterFiltersIntegrationTest::ipVersion(), false) {} Network::Address::IpVersion ipVersion() const override { return std::get<0>(GetParam()); } Grpc::ClientType clientType() const override { return std::get<1>(GetParam()); } diff --git a/test/integration/upstreams/BUILD b/test/integration/upstreams/BUILD index 238f2fa3d4f4..15246549e521 100644 --- a/test/integration/upstreams/BUILD +++ b/test/integration/upstreams/BUILD @@ -10,7 +10,7 @@ envoy_package() envoy_cc_library( name = "per_host_upstream_config", - srcs = [ + hdrs = [ "per_host_upstream_config.h", ], deps = [ diff --git a/test/integration/upstreams/per_host_upstream_config.h b/test/integration/upstreams/per_host_upstream_config.h index c19569ac1c5f..d5881885e25f 100644 --- a/test/integration/upstreams/per_host_upstream_config.h +++ b/test/integration/upstreams/per_host_upstream_config.h @@ -71,10 +71,10 @@ class PerHostHttpUpstream : public Extensions::Upstreams::Http::Http::HttpUpstre class PerHostHttpConnPool : public Extensions::Upstreams::Http::Http::HttpConnPool { public: PerHostHttpConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) - : HttpConnPool(thread_local_cluster, route_entry, downstream_protocol, ctx) {} + : HttpConnPool(thread_local_cluster, priority, downstream_protocol, ctx) {} void onPoolReady(Envoy::Http::RequestEncoder& callbacks_encoder, Upstream::HostDescriptionConstSharedPtr host, StreamInfo::StreamInfo& info, @@ -97,7 +97,7 @@ class PerHostGenericConnPoolFactory : public Router::GenericConnPoolFactory { Router::GenericConnPoolPtr createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, Router::GenericConnPoolFactory::UpstreamProtocol upstream_protocol, - const Router::RouteEntry& route_entry, + Upstream::ResourcePriority priority, absl::optional downstream_protocol, Upstream::LoadBalancerContext* ctx) const override { if (upstream_protocol != UpstreamProtocol::HTTP) { @@ -105,7 +105,7 @@ class PerHostGenericConnPoolFactory : public Router::GenericConnPoolFactory { return nullptr; } auto upstream_http_conn_pool = std::make_unique( - thread_local_cluster, route_entry, downstream_protocol, ctx); + thread_local_cluster, priority, downstream_protocol, ctx); return (upstream_http_conn_pool->valid() ? std::move(upstream_http_conn_pool) : nullptr); } diff --git a/test/integration/utility.cc b/test/integration/utility.cc index 69d4efc965cf..6ca68e0f333d 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -26,7 +26,7 @@ #ifdef ENVOY_ENABLE_QUIC #include "source/common/quic/client_connection_factory_impl.h" -#include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/quic_client_transport_socket_factory.h" #include "quiche/quic/core/deterministic_connection_id_generator.h" #endif diff --git a/test/main.cc b/test/main.cc index eae6c3fc4f68..f097bf0ea5da 100644 --- a/test/main.cc +++ b/test/main.cc @@ -31,5 +31,5 @@ int main(int argc, char** argv) { // v4 and v6 addresses is desired. This feature is in progress and will be rolled out to all tests // in upcoming PRs. Envoy::TestEnvironment::setEnvVar("ENVOY_IP_TEST_VERSIONS", "all", 0); - return Envoy::TestRunner::RunTests(argc, argv); + return Envoy::TestRunner::runTests(argc, argv); } diff --git a/test/mocks/access_log/mocks.h b/test/mocks/access_log/mocks.h index c4d5ea6837d1..66b2afd37d5d 100644 --- a/test/mocks/access_log/mocks.h +++ b/test/mocks/access_log/mocks.h @@ -28,10 +28,7 @@ class MockFilter : public Filter { // AccessLog::Filter MOCK_METHOD(bool, evaluate, - (const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, - const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, AccessLogType access_log_type), - (const)); + (const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&), (const)); }; class MockAccessLogManager : public AccessLogManager { @@ -53,11 +50,7 @@ class MockInstance : public Instance { ~MockInstance() override; // AccessLog::Instance - MOCK_METHOD(void, log, - (const Http::RequestHeaderMap* request_headers, - const Http::ResponseHeaderMap* response_headers, - const Http::ResponseTrailerMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, AccessLogType access_log_type)); + MOCK_METHOD(void, log, (const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo&)); }; } // namespace AccessLog diff --git a/test/mocks/api/mocks.cc b/test/mocks/api/mocks.cc index 4a8a8bc5408c..585ee9200499 100644 --- a/test/mocks/api/mocks.cc +++ b/test/mocks/api/mocks.cc @@ -1,5 +1,7 @@ #include "mocks.h" +#include + #include "source/common/common/assert.h" #include "source/common/common/lock_guard.h" @@ -66,7 +68,9 @@ SysCallIntResult MockOsSysCalls::setsockopt(os_fd_t sockfd, int level, int optna } if (optlen >= sizeof(int)) { - boolsockopts_[SockOptKey(sockfd, level, optname)] = !!*reinterpret_cast(optval); + int val = 0; + memcpy(&val, optval, sizeof(int)); + boolsockopts_[SockOptKey(sockfd, level, optname)] = (val != 0); } return SysCallIntResult{0, 0}; }; diff --git a/test/mocks/buffer/mocks.cc b/test/mocks/buffer/mocks.cc index ea99399b3332..56fb48e0d0fe 100644 --- a/test/mocks/buffer/mocks.cc +++ b/test/mocks/buffer/mocks.cc @@ -21,7 +21,8 @@ MockBufferBase::MockBufferBase(std::function, std::fu ASSERT(0); // This constructor is not supported for OwnedImpl. } -template <> MockBufferBase::MockBufferBase() {} +// NOLINTNEXTLINE(modernize-use-equals-default) +template <> MockBufferBase::MockBufferBase(){}; MockBufferFactory::MockBufferFactory() = default; MockBufferFactory::~MockBufferFactory() = default; diff --git a/test/mocks/config/eds_resources_cache.cc b/test/mocks/config/eds_resources_cache.cc index 1f11831228d5..aaa825a81ae3 100644 --- a/test/mocks/config/eds_resources_cache.cc +++ b/test/mocks/config/eds_resources_cache.cc @@ -7,7 +7,6 @@ namespace Config { using testing::_; using testing::Return; -using testing::Invoke; MockEdsResourcesCache::MockEdsResourcesCache() { ON_CALL(*this, getResource(_, _)).WillByDefault(Return(absl::nullopt)); } diff --git a/test/mocks/event/mocks.cc b/test/mocks/event/mocks.cc index 16468bf697cb..e6a0b436f671 100644 --- a/test/mocks/event/mocks.cc +++ b/test/mocks/event/mocks.cc @@ -76,12 +76,23 @@ MockSchedulableCallback::~MockSchedulableCallback() { } } +MockSchedulableCallback::MockSchedulableCallback(MockDispatcher* dispatcher, + std::function callback, + testing::MockFunction* destroy_cb) + : dispatcher_(dispatcher), callback_(callback), destroy_cb_(destroy_cb) { + ON_CALL(*this, scheduleCallbackCurrentIteration()).WillByDefault(Assign(&enabled_, true)); + ON_CALL(*this, scheduleCallbackNextIteration()).WillByDefault(Assign(&enabled_, true)); + ON_CALL(*this, cancel()).WillByDefault(Assign(&enabled_, false)); + ON_CALL(*this, enabled()).WillByDefault(ReturnPointee(&enabled_)); +} + MockSchedulableCallback::MockSchedulableCallback(MockDispatcher* dispatcher, testing::MockFunction* destroy_cb) : dispatcher_(dispatcher), destroy_cb_(destroy_cb) { EXPECT_CALL(*dispatcher, createSchedulableCallback_(_)) .WillOnce(DoAll(SaveArg<0>(&callback_), Return(this))) .RetiresOnSaturation(); + ON_CALL(*this, scheduleCallbackCurrentIteration()).WillByDefault(Assign(&enabled_, true)); ON_CALL(*this, scheduleCallbackNextIteration()).WillByDefault(Assign(&enabled_, true)); ON_CALL(*this, cancel()).WillByDefault(Assign(&enabled_, false)); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 4ca4a2276a43..c363c1fe57fd 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -70,18 +70,6 @@ class MockDispatcher : public Dispatcher { return Filesystem::WatcherPtr{createFilesystemWatcher_()}; } - Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime, - const Network::ListenerConfig& listener_config) override { - return Network::ListenerPtr{createListener_(std::move(socket), cb, runtime, listener_config)}; - } - - Network::UdpListenerPtr - createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, - const envoy::config::core::v3::UdpSocketConfig& config) override { - return Network::UdpListenerPtr{createUdpListener_(socket, cb, config)}; - } - Event::TimerPtr createTimer(Event::TimerCb cb) override { auto timer = Event::TimerPtr{createTimer_(cb)}; // Assert that the timer is not null to avoid confusing test failures down the line. @@ -140,12 +128,6 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD(FileEvent*, createFileEvent_, (os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events)); MOCK_METHOD(Filesystem::Watcher*, createFilesystemWatcher_, ()); - MOCK_METHOD(Network::Listener*, createListener_, - (Network::SocketSharedPtr && socket, Network::TcpListenerCallbacks& cb, - Runtime::Loader& runtime, const Network::ListenerConfig& listener_config)); - MOCK_METHOD(Network::UdpListener*, createUdpListener_, - (Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, - const envoy::config::core::v3::UdpSocketConfig& config)); MOCK_METHOD(Timer*, createTimer_, (Event::TimerCb cb)); MOCK_METHOD(Timer*, createScaledTimer_, (ScaledTimerMinimum minimum, Event::TimerCb cb)); MOCK_METHOD(Timer*, createScaledTypedTimer_, (ScaledTimerType timer_type, Event::TimerCb cb)); @@ -230,6 +212,8 @@ class MockSchedulableCallback : public SchedulableCallback { public: MockSchedulableCallback(MockDispatcher* dispatcher, testing::MockFunction* destroy_cb = nullptr); + MockSchedulableCallback(MockDispatcher* dispatcher, std::function callback, + testing::MockFunction* destroy_cb = nullptr); ~MockSchedulableCallback() override; void invokeCallback() { diff --git a/test/mocks/event/wrapped_dispatcher.h b/test/mocks/event/wrapped_dispatcher.h index 90d9bbc12341..ff5b93a80cfe 100644 --- a/test/mocks/event/wrapped_dispatcher.h +++ b/test/mocks/event/wrapped_dispatcher.h @@ -60,18 +60,6 @@ class WrappedDispatcher : public Dispatcher { return impl_.createFilesystemWatcher(); } - Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, Runtime::Loader& runtime, - const Network::ListenerConfig& listener_config) override { - return impl_.createListener(std::move(socket), cb, runtime, listener_config); - } - - Network::UdpListenerPtr - createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, - const envoy::config::core::v3::UdpSocketConfig& config) override { - return impl_.createUdpListener(std::move(socket), cb, config); - } - TimerPtr createTimer(TimerCb cb) override { return impl_.createTimer(std::move(cb)); } TimerPtr createScaledTimer(ScaledTimerMinimum minimum, TimerCb cb) override { return impl_.createScaledTimer(minimum, std::move(cb)); diff --git a/test/mocks/filesystem/mocks.h b/test/mocks/filesystem/mocks.h index 7f1d61b578da..524cefd269f3 100644 --- a/test/mocks/filesystem/mocks.h +++ b/test/mocks/filesystem/mocks.h @@ -63,8 +63,8 @@ class MockInstance : public Instance { MOCK_METHOD(bool, fileExists, (const std::string&)); MOCK_METHOD(bool, directoryExists, (const std::string&)); MOCK_METHOD(ssize_t, fileSize, (const std::string&)); - MOCK_METHOD(std::string, fileReadToEnd, (const std::string&)); - MOCK_METHOD(PathSplitResult, splitPathFromFilename, (absl::string_view)); + MOCK_METHOD(absl::StatusOr, fileReadToEnd, (const std::string&)); + MOCK_METHOD(absl::StatusOr, splitPathFromFilename, (absl::string_view)); MOCK_METHOD(bool, illegalPath, (const std::string&)); MOCK_METHOD(Api::IoCallResult, stat, (absl::string_view)); MOCK_METHOD(Api::IoCallBoolResult, createPath, (absl::string_view)); diff --git a/test/mocks/grpc/mocks.h b/test/mocks/grpc/mocks.h index 2fdf85b34062..920ccb7aa7d7 100644 --- a/test/mocks/grpc/mocks.h +++ b/test/mocks/grpc/mocks.h @@ -46,7 +46,7 @@ template using ResponseTypePtr = std::unique_ptr class MockAsyncRequestCallbacks : public AsyncRequestCallbacks { public: - void onSuccess(ResponseTypePtr&& response, Tracing::Span& span) { + void onSuccess(ResponseTypePtr&& response, Tracing::Span& span) override { onSuccess_(*response, span); } @@ -59,13 +59,15 @@ class MockAsyncRequestCallbacks : public AsyncRequestCallbacks { template class MockAsyncStreamCallbacks : public AsyncStreamCallbacks { public: - void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&& metadata) { + void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&& metadata) override { onReceiveInitialMetadata_(*metadata); } - void onReceiveMessage(ResponseTypePtr&& message) { onReceiveMessage_(*message); } + void onReceiveMessage(ResponseTypePtr&& message) override { + onReceiveMessage_(*message); + } - void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&& metadata) { + void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&& metadata) override { onReceiveTrailingMetadata_(*metadata); } diff --git a/test/mocks/http/mocks.cc b/test/mocks/http/mocks.cc index 819dfd55484c..e39c1971c73f 100644 --- a/test/mocks/http/mocks.cc +++ b/test/mocks/http/mocks.cc @@ -97,7 +97,8 @@ MockStreamDecoderFilterCallbacks::MockStreamDecoderFilterCallbacks() { })); ON_CALL(*this, routeConfig()) .WillByDefault(Return(absl::optional())); - ON_CALL(*this, upstreamOverrideHost()).WillByDefault(Return(absl::optional())); + ON_CALL(*this, upstreamOverrideHost()) + .WillByDefault(Return(absl::optional())); ON_CALL(*this, mostSpecificPerFilterConfig()) .WillByDefault(Invoke([this]() -> const Router::RouteSpecificFilterConfig* { @@ -227,11 +228,11 @@ MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() = default; namespace Http { IsSubsetOfHeadersMatcher IsSubsetOfHeaders(const HeaderMap& expected_headers) { - return IsSubsetOfHeadersMatcher(expected_headers); + return {expected_headers}; } IsSupersetOfHeadersMatcher IsSupersetOfHeaders(const HeaderMap& expected_headers) { - return IsSupersetOfHeadersMatcher(expected_headers); + return {expected_headers}; } MockReceivedSettings::MockReceivedSettings() { diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 7a26fb2fb2ed..8b84f9f0e7e8 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -269,6 +269,11 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD(OptRef, downstreamCallbacks, ()); MOCK_METHOD(OptRef, upstreamCallbacks, ()); MOCK_METHOD(absl::string_view, filterConfigName, (), (const override)); + MOCK_METHOD(RequestHeaderMapOptRef, requestHeaders, ()); + MOCK_METHOD(RequestTrailerMapOptRef, requestTrailers, ()); + MOCK_METHOD(ResponseHeaderMapOptRef, informationalHeaders, ()); + MOCK_METHOD(ResponseHeaderMapOptRef, responseHeaders, ()); + MOCK_METHOD(ResponseTrailerMapOptRef, responseTrailers, ()); // Http::StreamDecoderFilterCallbacks // NOLINTNEXTLINE(readability-identifier-naming) @@ -278,15 +283,12 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, absl::string_view details); void encode1xxHeaders(ResponseHeaderMapPtr&& headers) override { encode1xxHeaders_(*headers); } - MOCK_METHOD(ResponseHeaderMapOptRef, informationalHeaders, (), (const)); void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) override { stream_info_.setResponseCodeDetails(details); encodeHeaders_(*headers, end_stream); } - MOCK_METHOD(ResponseHeaderMapOptRef, responseHeaders, (), (const)); void encodeTrailers(ResponseTrailerMapPtr&& trailers) override { encodeTrailers_(*trailers); } - MOCK_METHOD(ResponseTrailerMapOptRef, responseTrailers, (), (const)); void encodeMetadata(MetadataMapPtr&& metadata_map) override { encodeMetadata_(std::move(metadata_map)); } @@ -315,8 +317,9 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, const absl::optional grpc_status, absl::string_view details)); MOCK_METHOD(Buffer::BufferMemoryAccountSharedPtr, account, (), (const)); - MOCK_METHOD(void, setUpstreamOverrideHost, (absl::string_view host)); - MOCK_METHOD(absl::optional, upstreamOverrideHost, (), (const)); + MOCK_METHOD(void, setUpstreamOverrideHost, (Upstream::LoadBalancerContext::OverrideHost)); + MOCK_METHOD(absl::optional, upstreamOverrideHost, (), + (const)); Buffer::InstancePtr buffer_; std::list callbacks_{}; @@ -361,6 +364,11 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, MOCK_METHOD(OptRef, downstreamCallbacks, ()); MOCK_METHOD(OptRef, upstreamCallbacks, ()); MOCK_METHOD(absl::string_view, filterConfigName, (), (const override)); + MOCK_METHOD(RequestHeaderMapOptRef, requestHeaders, ()); + MOCK_METHOD(RequestTrailerMapOptRef, requestTrailers, ()); + MOCK_METHOD(ResponseHeaderMapOptRef, informationalHeaders, ()); + MOCK_METHOD(ResponseHeaderMapOptRef, responseHeaders, ()); + MOCK_METHOD(ResponseTrailerMapOptRef, responseTrailers, ()); // Http::StreamEncoderFilterCallbacks MOCK_METHOD(void, addEncodedData, (Buffer::Instance & data, bool streaming)); diff --git a/test/mocks/http/stateful_session.h b/test/mocks/http/stateful_session.h index c89ea4156e85..73eb4b65d661 100644 --- a/test/mocks/http/stateful_session.h +++ b/test/mocks/http/stateful_session.h @@ -20,6 +20,7 @@ class MockSessionStateFactory : public Http::SessionStateFactory { MockSessionStateFactory(); MOCK_METHOD(Http::SessionStatePtr, create, (const Http::RequestHeaderMap& headers), (const)); + MOCK_METHOD(bool, isStrict, (), (const)); }; class MockSessionStateFactoryConfig : public Http::SessionStateFactoryConfig { @@ -31,7 +32,7 @@ class MockSessionStateFactoryConfig : public Http::SessionStateFactoryConfig { } MOCK_METHOD(SessionStateFactorySharedPtr, createSessionStateFactory, - (const Protobuf::Message&, Server::Configuration::CommonFactoryContext&)); + (const Protobuf::Message&, Server::Configuration::GenericFactoryContext&)); std::string name() const override { return "envoy.http.stateful_session.mock"; } }; diff --git a/test/mocks/io/mocks.h b/test/mocks/io/mocks.h index fec078a79420..0eabe5a2b9c7 100644 --- a/test/mocks/io/mocks.h +++ b/test/mocks/io/mocks.h @@ -35,8 +35,15 @@ class MockIoUring : public IoUring { class MockIoUringSocket : public IoUringSocket { public: - MOCK_METHOD(IoUringWorker&, getIoUringWorker, (), (const)); MOCK_METHOD(os_fd_t, fd, (), (const)); + MOCK_METHOD(void, close, (bool, IoUringSocketOnClosedCb)); + MOCK_METHOD(void, shutdown, (int32_t how)); + MOCK_METHOD(void, enableRead, ()); + MOCK_METHOD(void, disableRead, ()); + MOCK_METHOD(void, enableCloseEvent, (bool enable)); + MOCK_METHOD(void, connect, (const Network::Address::InstanceConstSharedPtr& address)); + MOCK_METHOD(void, write, (Buffer::Instance & data)); + MOCK_METHOD(uint64_t, write, (const Buffer::RawSlice* slices, uint64_t num_slice)); MOCK_METHOD(void, onAccept, (Request * req, int32_t result, bool injected)); MOCK_METHOD(void, onConnect, (Request * req, int32_t result, bool injected)); MOCK_METHOD(void, onRead, (Request * req, int32_t result, bool injected)); @@ -45,6 +52,28 @@ class MockIoUringSocket : public IoUringSocket { MOCK_METHOD(void, onCancel, (Request * req, int32_t result, bool injected)); MOCK_METHOD(void, onShutdown, (Request * req, int32_t result, bool injected)); MOCK_METHOD(void, injectCompletion, (Request::RequestType type)); + MOCK_METHOD(IoUringSocketStatus, getStatus, (), (const)); + MOCK_METHOD(IoUringWorker&, getIoUringWorker, (), (const)); + MOCK_METHOD(const OptRef&, getReadParam, (), (const)); + MOCK_METHOD(const OptRef&, getWriteParam, (), (const)); + MOCK_METHOD(void, setFileReadyCb, (Event::FileReadyCb cb)); +}; + +class MockIoUringWorker : public IoUringWorker { +public: + MOCK_METHOD(IoUringSocket&, addServerSocket, + (os_fd_t fd, Event::FileReadyCb cb, bool enable_close_event)); + MOCK_METHOD(IoUringSocket&, addServerSocket, + (os_fd_t fd, Buffer::Instance& read_buf, Event::FileReadyCb cb, + bool enable_close_event)); + MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); + MOCK_METHOD(Request*, submitReadRequest, (IoUringSocket & socket)); + MOCK_METHOD(Request*, submitWriteRequest, + (IoUringSocket & socket, const Buffer::RawSliceVector& slices)); + MOCK_METHOD(Request*, submitCloseRequest, (IoUringSocket & socket)); + MOCK_METHOD(Request*, submitCancelRequest, (IoUringSocket & socket, Request* request_to_cancel)); + MOCK_METHOD(Request*, submitShutdownRequest, (IoUringSocket & socket, int how)); + MOCK_METHOD(uint32_t, getNumOfSockets, (), (const)); }; } // namespace Io diff --git a/test/mocks/network/BUILD b/test/mocks/network/BUILD index 499cb30456e7..82b5ef8c79f1 100644 --- a/test/mocks/network/BUILD +++ b/test/mocks/network/BUILD @@ -58,6 +58,7 @@ envoy_cc_mock( "//envoy/network:transport_socket_interface", "//envoy/server:listener_manager_interface", "//source/common/network:address_lib", + "//source/common/network:listener_lib", "//source/common/network:socket_interface_lib", "//source/common/network:utility_lib", "//source/common/network/dns_resolver:dns_factory_util_lib", diff --git a/test/mocks/network/mocks.cc b/test/mocks/network/mocks.cc index e2b8cd9267ba..1525608d43e8 100644 --- a/test/mocks/network/mocks.cc +++ b/test/mocks/network/mocks.cc @@ -33,7 +33,8 @@ MockUdpListenerConfig::MockUdpListenerConfig(uint32_t concurrency) MockUdpListenerConfig::~MockUdpListenerConfig() = default; MockListenerConfig::MockListenerConfig() - : socket_(std::make_shared>()) { + : socket_(std::make_shared>()), + listener_info_(std::make_shared>()) { socket_factories_.emplace_back(std::make_unique()); ON_CALL(*this, filterChainFactory()).WillByDefault(ReturnRef(filter_chain_factory_)); ON_CALL(*this, listenSocketFactories()).WillByDefault(ReturnRef(socket_factories_)); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 8714d9a2881a..283bb4377df7 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -466,6 +466,14 @@ class MockUdpListenerConfig : public UdpListenerConfig { envoy::config::listener::v3::UdpListenerConfig config_; }; +class MockListenerInfo : public ListenerInfo { +public: + MOCK_METHOD(const envoy::config::core::v3::Metadata&, metadata, (), (const)); + MOCK_METHOD(const Envoy::Config::TypedMetadata&, typedMetadata, (), (const)); + MOCK_METHOD(envoy::config::core::v3::TrafficDirection, direction, (), (const)); + MOCK_METHOD(bool, isQuic, (), (const)); +}; + class MockListenerConfig : public ListenerConfig { public: MockListenerConfig(); @@ -491,17 +499,16 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD(Init::Manager&, initManager, ()); MOCK_METHOD(bool, ignoreGlobalConnLimit, (), (const)); - envoy::config::core::v3::TrafficDirection direction() const override { - return envoy::config::core::v3::UNSPECIFIED; - } - const std::vector& accessLogs() const override { return empty_access_logs_; } + const ListenerInfoConstSharedPtr& listenerInfo() const override { return listener_info_; } + testing::NiceMock filter_chain_factory_; std::vector socket_factories_; SocketSharedPtr socket_; + ListenerInfoConstSharedPtr listener_info_; Stats::IsolatedStoreImpl store_; std::string name_; const std::vector empty_access_logs_; @@ -529,7 +536,7 @@ class MockConnectionHandler : public virtual ConnectionHandler { MOCK_METHOD(void, decNumConnections, ()); MOCK_METHOD(void, addListener, (absl::optional overridden_listener, ListenerConfig& config, - Runtime::Loader& runtime)); + Runtime::Loader& runtime, Random::RandomGenerator& random)); MOCK_METHOD(void, removeListeners, (uint64_t listener_tag)); MOCK_METHOD(void, removeFilterChains, (uint64_t listener_tag, const std::list& filter_chains, diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index f0b17e19e3b8..862b99b7848e 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -74,6 +74,8 @@ MockShadowWriter::~MockShadowWriter() = default; MockVirtualHost::MockVirtualHost() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, rateLimitPolicy()).WillByDefault(ReturnRef(rate_limit_policy_)); + ON_CALL(*this, metadata()).WillByDefault(ReturnRef(metadata_)); + ON_CALL(*this, typedMetadata()).WillByDefault(ReturnRef(typed_metadata_)); } MockVirtualHost::~MockVirtualHost() = default; @@ -129,6 +131,8 @@ MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, internalOnlyHeaders()).WillByDefault(ReturnRef(internal_only_headers_)); ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, usesVhds()).WillByDefault(Return(false)); + ON_CALL(*this, metadata()).WillByDefault(ReturnRef(metadata_)); + ON_CALL(*this, typedMetadata()).WillByDefault(ReturnRef(typed_metadata_)); } MockConfig::~MockConfig() = default; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index e7e3e7af36bb..8a2128d6096f 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -42,6 +42,17 @@ namespace Envoy { namespace Router { using ::testing::NiceMock; +struct MockRouteMetadataObj : public Envoy::Config::TypedMetadata::Object {}; +class MockRouteMetadata : public Envoy::Config::TypedMetadata { +public: + const Envoy::Config::TypedMetadata::Object* getData(const std::string&) const override { + return &object_; + } + +private: + MockRouteMetadataObj object_; +}; + class MockDirectResponseEntry : public DirectResponseEntry { public: MockDirectResponseEntry(); @@ -159,6 +170,7 @@ class MockInternalRedirectPolicy : public InternalRedirectPolicy { MOCK_METHOD(std::vector, predicates, (), (const)); MOCK_METHOD(uint32_t, maxInternalRedirects, (), (const)); MOCK_METHOD(bool, isCrossSchemeRedirectAllowed, (), (const)); + MOCK_METHOD(const std::vector&, responseHeadersToCopy, (), (const)); }; class MockInternalRedirectPredicate : public InternalRedirectPredicate { @@ -313,6 +325,8 @@ class MockVirtualHost : public VirtualHost { MOCK_METHOD(void, traversePerFilterConfig, (const std::string&, std::function), (const)); + MOCK_METHOD(const envoy::config::core::v3::Metadata&, metadata, (), (const)); + MOCK_METHOD(const Envoy::Config::TypedMetadata&, typedMetadata, (), (const)); Stats::StatName statName() const override { stat_name_ = std::make_unique(name(), *symbol_table_); @@ -324,6 +338,8 @@ class MockVirtualHost : public VirtualHost { mutable std::unique_ptr stat_name_; testing::NiceMock rate_limit_policy_; TestCorsPolicy cors_policy_; + envoy::config::core::v3::Metadata metadata_; + MockRouteMetadata typed_metadata_; }; class MockHashPolicy : public Http::HashPolicy { @@ -486,16 +502,6 @@ class MockRouteTracing : public RouteTracing { class MockRoute : public Route { public: - struct MockRouteMetadataObj : public Envoy::Config::TypedMetadata::Object {}; - class MockRouteMetadata : public Envoy::Config::TypedMetadata { - public: - const Envoy::Config::TypedMetadata::Object* getData(const std::string&) const override { - return &object_; - } - - private: - MockRouteMetadataObj object_; - }; MockRoute(); ~MockRoute() override; @@ -504,7 +510,7 @@ class MockRoute : public Route { MOCK_METHOD(const RouteEntry*, routeEntry, (), (const)); MOCK_METHOD(const Decorator*, decorator, (), (const)); MOCK_METHOD(const RouteTracing*, tracingConfig, (), (const)); - MOCK_METHOD(bool, filterDisabled, (absl::string_view), (const)); + MOCK_METHOD(absl::optional, filterDisabled, (absl::string_view), (const)); MOCK_METHOD(const RouteSpecificFilterConfig*, perFilterConfig, (const std::string&), (const)); MOCK_METHOD(const RouteSpecificFilterConfig*, mostSpecificPerFilterConfig, (const std::string&), (const)); @@ -543,10 +549,14 @@ class MockConfig : public Config { MOCK_METHOD(bool, usesVhds, (), (const)); MOCK_METHOD(bool, mostSpecificHeaderMutationsWins, (), (const)); MOCK_METHOD(uint32_t, maxDirectResponseBodySizeBytes, (), (const)); + MOCK_METHOD(const envoy::config::core::v3::Metadata&, metadata, (), (const)); + MOCK_METHOD(const Envoy::Config::TypedMetadata&, typedMetadata, (), (const)); std::shared_ptr route_; std::list internal_only_headers_; std::string name_{"fake_config"}; + envoy::config::core::v3::Metadata metadata_; + MockRouteMetadata typed_metadata_; }; class MockRouteConfigProvider : public RouteConfigProvider { @@ -573,12 +583,10 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { MOCK_METHOD(RouteConfigProviderSharedPtr, createRdsRouteConfigProvider, (const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, Init::Manager& init_manager)); MOCK_METHOD(RouteConfigProviderPtr, createStaticRouteConfigProvider, (const envoy::config::route::v3::RouteConfiguration& route_config, - const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator)); }; diff --git a/test/mocks/router/router_filter_interface.cc b/test/mocks/router/router_filter_interface.cc index 0448b442f19b..13bd92403abf 100644 --- a/test/mocks/router/router_filter_interface.cc +++ b/test/mocks/router/router_filter_interface.cc @@ -17,14 +17,11 @@ MockRouterFilterInterface::MockRouterFilterInterface() ON_CALL(*this, callbacks()).WillByDefault(Return(&callbacks_)); ON_CALL(*this, config()).WillByDefault(ReturnRef(config_)); ON_CALL(*this, cluster()).WillByDefault(Return(cluster_info_)); - ON_CALL(*this, upstreamRequests()).WillByDefault(ReturnRef(requests_)); EXPECT_CALL(callbacks_.dispatcher_, pushTrackedObject(_)).Times(AnyNumber()); EXPECT_CALL(callbacks_.dispatcher_, popTrackedObject(_)).Times(AnyNumber()); - ON_CALL(*this, route()).WillByDefault(Return(&route_)); ON_CALL(callbacks_, connection()) .WillByDefault(Return(OptRef{client_connection_})); - ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); - route_.route_entry_.connect_config_.emplace(RouteEntry::ConnectConfig()); + callbacks_.route_->route_entry_.connect_config_.emplace(RouteEntry::ConnectConfig()); } MockRouterFilterInterface::~MockRouterFilterInterface() = default; diff --git a/test/mocks/router/router_filter_interface.h b/test/mocks/router/router_filter_interface.h index 9badaebfdff8..2e8d217406a0 100644 --- a/test/mocks/router/router_filter_interface.h +++ b/test/mocks/router/router_filter_interface.h @@ -30,7 +30,8 @@ class MockRouterFilterInterface : public RouterFilterInterface { MOCK_METHOD(void, onUpstreamReset, (Envoy::Http::StreamResetReason reset_reason, absl::string_view transport_failure, UpstreamRequest& upstream_request)); - MOCK_METHOD(void, onUpstreamHostSelected, (Upstream::HostDescriptionConstSharedPtr host)); + MOCK_METHOD(void, onUpstreamHostSelected, + (Upstream::HostDescriptionConstSharedPtr host, bool success)); MOCK_METHOD(void, onPerTryTimeout, (UpstreamRequest & upstream_request)); MOCK_METHOD(void, onPerTryIdleTimeout, (UpstreamRequest & upstream_request)); MOCK_METHOD(void, onStreamMaxDurationReached, (UpstreamRequest & upstream_request)); @@ -38,26 +39,16 @@ class MockRouterFilterInterface : public RouterFilterInterface { MOCK_METHOD(Envoy::Http::StreamDecoderFilterCallbacks*, callbacks, ()); MOCK_METHOD(Upstream::ClusterInfoConstSharedPtr, cluster, ()); MOCK_METHOD(FilterConfig&, config, ()); - MOCK_METHOD(FilterUtility::TimeoutData, timeout, ()); + MOCK_METHOD(TimeoutData, timeout, ()); MOCK_METHOD(absl::optional, dynamicMaxStreamDuration, (), (const)); MOCK_METHOD(Envoy::Http::RequestHeaderMap*, downstreamHeaders, ()); MOCK_METHOD(Envoy::Http::RequestTrailerMap*, downstreamTrailers, ()); MOCK_METHOD(bool, downstreamResponseStarted, (), (const)); MOCK_METHOD(bool, downstreamEndStream, (), (const)); MOCK_METHOD(uint32_t, attemptCount, (), (const)); - MOCK_METHOD(const VirtualCluster*, requestVcluster, (), (const)); - MOCK_METHOD(const Route*, route, (), (const)); - MOCK_METHOD(const std::list&, upstreamRequests, (), (const)); - MOCK_METHOD(const UpstreamRequest*, finalUpstreamRequest, (), (const)); - MOCK_METHOD(TimeSource&, timeSource, ()); - - const RouteStatsContextOptRef routeStatsContext() const override { - return RouteStatsContextOptRef(); - } NiceMock callbacks_; - NiceMock route_; - NiceMock client_connection_; + NiceMock client_connection_; envoy::extensions::filters::http::router::v3::Router router_proto; NiceMock context_; @@ -65,8 +56,6 @@ class MockRouterFilterInterface : public RouterFilterInterface { Stats::StatNamePool pool_; FilterConfig config_; std::shared_ptr cluster_info_; - std::list requests_; - Event::GlobalTimeSystem time_system_; }; } // namespace Router diff --git a/test/mocks/secret/mocks.h b/test/mocks/secret/mocks.h index 91c431719610..e87cf0a95cee 100644 --- a/test/mocks/secret/mocks.h +++ b/test/mocks/secret/mocks.h @@ -18,7 +18,7 @@ class MockSecretManager : public SecretManager { MockSecretManager(); ~MockSecretManager() override; - MOCK_METHOD(void, addStaticSecret, + MOCK_METHOD(absl::Status, addStaticSecret, (const envoy::extensions::transport_sockets::tls::v3::Secret& secret)); MOCK_METHOD(TlsCertificateConfigProviderSharedPtr, findStaticTlsCertificateProvider, (const std::string& name), (const)); diff --git a/test/mocks/server/admin.h b/test/mocks/server/admin.h index 5d99e87e344a..c7308d6a044e 100644 --- a/test/mocks/server/admin.h +++ b/test/mocks/server/admin.h @@ -31,11 +31,9 @@ class MockAdmin : public Admin { MOCK_METHOD(Network::Socket&, socket, ()); MOCK_METHOD(ConfigTracker&, getConfigTracker, ()); MOCK_METHOD(void, startHttpListener, - (const std::list& access_logs, - const std::string& address_out_path, + (std::list access_logs, Network::Address::InstanceConstSharedPtr address, - const Network::Socket::OptionsSharedPtr& socket_options, - Stats::ScopeSharedPtr&& listener_scope)); + Network::Socket::OptionsSharedPtr socket_options)); MOCK_METHOD(Http::Code, request, (absl::string_view path_and_query, absl::string_view method, Http::ResponseHeaderMap& response_headers, std::string& body)); diff --git a/test/mocks/server/admin_stream.h b/test/mocks/server/admin_stream.h index f4d52d277e3b..6dab6b45e704 100644 --- a/test/mocks/server/admin_stream.h +++ b/test/mocks/server/admin_stream.h @@ -26,7 +26,7 @@ class MockAdminStream : public AdminStream { MOCK_METHOD(NiceMock&, getDecoderFilterCallbacks, (), (const)); MOCK_METHOD(Http::Http1StreamEncoderOptionsOptRef, http1StreamEncoderOptions, ()); - MOCK_METHOD(Http::Utility::QueryParams, queryParams, (), (const)); + MOCK_METHOD(Http::Utility::QueryParamsMulti, queryParams, (), (const)); }; /** @@ -46,7 +46,7 @@ class StrictMockAdminStream : public AdminStream { MOCK_METHOD(::testing::StrictMock&, getDecoderFilterCallbacks, (), (const)); MOCK_METHOD(Http::Http1StreamEncoderOptionsOptRef, http1StreamEncoderOptions, ()); - MOCK_METHOD(Http::Utility::QueryParams, queryParams, (), (const)); + MOCK_METHOD(Http::Utility::QueryParamsMulti, queryParams, (), (const)); }; } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/factory_context.cc b/test/mocks/server/factory_context.cc index a9fa3f0a8c20..945df5ef2fbd 100644 --- a/test/mocks/server/factory_context.cc +++ b/test/mocks/server/factory_context.cc @@ -14,41 +14,23 @@ namespace Configuration { using ::testing::Return; using ::testing::ReturnRef; -MockFactoryContext::MockFactoryContext() - : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), - grpc_context_(scope_.symbolTable()), http_context_(scope_.symbolTable()), - router_context_(scope_.symbolTable()) { - ON_CALL(*this, getServerFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(*this, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); - ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); - ON_CALL(*this, getTransportSocketFactoryContext()) - .WillByDefault(ReturnRef(transport_socket_factory_context_)); +MockFactoryContext::MockFactoryContext() { + ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); - ON_CALL(*this, localInfo()).WillByDefault(ReturnRef(local_info_)); - ON_CALL(*this, runtime()).WillByDefault(ReturnRef(runtime_loader_)); ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); - ON_CALL(*this, serverScope()).WillByDefault(ReturnRef(scope_)); - ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(*singleton_manager_)); - ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); - ON_CALL(*this, admin()).WillByDefault(Return(OptRef{admin_})); - ON_CALL(*this, listenerScope()).WillByDefault(ReturnRef(*listener_store_.rootScope())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); - ON_CALL(*this, overloadManager()).WillByDefault(ReturnRef(overload_manager_)); - ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); + + ON_CALL(*this, getTransportSocketFactoryContext()) + .WillByDefault(ReturnRef(transport_socket_factory_context_)); + ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); + ON_CALL(*this, listenerScope()).WillByDefault(ReturnRef(*listener_store_.rootScope())); } MockFactoryContext::~MockFactoryContext() = default; MockUpstreamFactoryContext::MockUpstreamFactoryContext() { - ON_CALL(*this, getServerFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); + ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); } diff --git a/test/mocks/server/factory_context.h b/test/mocks/server/factory_context.h index 756160cb1c0c..594afca75f48 100644 --- a/test/mocks/server/factory_context.h +++ b/test/mocks/server/factory_context.h @@ -21,74 +21,33 @@ class MockFactoryContext : public virtual ListenerFactoryContext { MockFactoryContext(); ~MockFactoryContext() override; - MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, (), (const)); - MOCK_METHOD(TransportSocketFactoryContext&, getTransportSocketFactoryContext, (), (const)); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); - MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(bool, healthCheckFailed, ()); + // Server::Configuration::GenericFactoryContext + MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, (), (const)); MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(OverloadManager&, overloadManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); - MOCK_METHOD(Stats::Scope&, listenerScope, ()); - MOCK_METHOD(bool, isQuicListener, (), (const)); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(const envoy::config::core::v3::Metadata&, listenerMetadata, (), (const)); - MOCK_METHOD(const Envoy::Config::TypedMetadata&, listenerTypedMetadata, (), (const)); - MOCK_METHOD(envoy::config::core::v3::TrafficDirection, direction, (), (const)); - MOCK_METHOD(TimeSource&, timeSource, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, (), (const)); - MOCK_METHOD(const Network::ListenerConfig&, listenerConfig, (), (const)); - - Event::TestTimeSystem& timeSystem() { return time_system_; } - Grpc::Context& grpcContext() override { return grpc_context_; } - Http::Context& httpContext() override { return http_context_; } - Router::Context& routerContext() override { return router_context_; } - MOCK_METHOD(ProcessContextOptRef, processContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); + // Server::Configuration::FactoryContext + MOCK_METHOD(TransportSocketFactoryContext&, getTransportSocketFactoryContext, (), (const)); + MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); + MOCK_METHOD(Stats::Scope&, listenerScope, ()); + MOCK_METHOD(const Network::ListenerInfo&, listenerInfo, (), (const)); testing::NiceMock server_factory_context_; - testing::NiceMock access_log_manager_; - testing::NiceMock cluster_manager_; testing::NiceMock transport_socket_factory_context_; - testing::NiceMock dispatcher_; - testing::NiceMock drain_manager_; testing::NiceMock init_manager_; - testing::NiceMock lifecycle_notifier_; - testing::NiceMock local_info_; - testing::NiceMock runtime_loader_; testing::NiceMock store_; Stats::Scope& scope_{*store_.rootScope()}; - testing::NiceMock thread_local_; - testing::NiceMock options_; - Singleton::ManagerPtr singleton_manager_; - testing::NiceMock admin_; Stats::IsolatedStoreImpl listener_store_; Stats::Scope& listener_scope_{*listener_store_.rootScope()}; - Event::GlobalTimeSystem time_system_; - testing::NiceMock validation_context_; - testing::NiceMock overload_manager_; - Grpc::ContextImpl grpc_context_; - Http::ContextImpl http_context_; - Router::ContextImpl router_context_; - testing::NiceMock api_; + testing::NiceMock drain_manager_; }; class MockUpstreamFactoryContext : public UpstreamFactoryContext { public: MockUpstreamFactoryContext(); - MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, (), (const)); + MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, (), (const)); MOCK_METHOD(Init::Manager&, initManager, ()); MOCK_METHOD(Stats::Scope&, scope, ()); testing::NiceMock init_manager_; diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index 14fc610f3bb6..a8561d534b9c 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -18,6 +18,7 @@ class MockInstance : public Instance { Secret::SecretManager& secretManager() override { return *(secret_manager_); } MOCK_METHOD(OptRef, admin, ()); + MOCK_METHOD(void, run, ()); MOCK_METHOD(Api::Api&, api, ()); MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); MOCK_METHOD(const Upstream::ClusterManager&, clusterManager, (), (const)); diff --git a/test/mocks/server/listener_factory_context.cc b/test/mocks/server/listener_factory_context.cc index a1bf9b854050..811c3e6b4e13 100644 --- a/test/mocks/server/listener_factory_context.cc +++ b/test/mocks/server/listener_factory_context.cc @@ -11,36 +11,16 @@ namespace Envoy { namespace Server { namespace Configuration { -using ::testing::Return; using ::testing::ReturnRef; -MockListenerFactoryContext::MockListenerFactoryContext() - : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), - grpc_context_(scope_.symbolTable()), http_context_(scope_.symbolTable()), - router_context_(scope_.symbolTable()) { - ON_CALL(*this, getServerFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); - ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); - ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); - ON_CALL(*this, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); +MockListenerFactoryContext::MockListenerFactoryContext() { + ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); - ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); - ON_CALL(*this, localInfo()).WillByDefault(ReturnRef(local_info_)); - ON_CALL(*this, random()).WillByDefault(ReturnRef(random_)); - ON_CALL(*this, runtime()).WillByDefault(ReturnRef(runtime_loader_)); ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); - ON_CALL(*this, serverScope()).WillByDefault(ReturnRef(scope_)); - ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(*singleton_manager_)); - ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); - ON_CALL(*this, admin()).WillByDefault(Return(OptRef{admin_})); ON_CALL(*this, listenerScope()).WillByDefault(ReturnRef(*listener_scope_.rootScope())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); - ON_CALL(*this, timeSource()).WillByDefault(ReturnRef(time_system_)); - ON_CALL(*this, overloadManager()).WillByDefault(ReturnRef(overload_manager_)); - ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); - ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); } MockListenerFactoryContext::~MockListenerFactoryContext() = default; diff --git a/test/mocks/server/listener_factory_context.h b/test/mocks/server/listener_factory_context.h index 5341b517d1ba..10fbe1e21705 100644 --- a/test/mocks/server/listener_factory_context.h +++ b/test/mocks/server/listener_factory_context.h @@ -15,73 +15,30 @@ namespace Envoy { namespace Server { namespace Configuration { + class MockListenerFactoryContext : public ListenerFactoryContext { public: MockListenerFactoryContext(); ~MockListenerFactoryContext() override; - const Network::ListenerConfig& listenerConfig() const override { return listener_config_; } - MOCK_METHOD(const Network::ListenerConfig&, listenerConfig_, (), (const)); - MOCK_METHOD(ServerFactoryContext&, getServerFactoryContext, (), (const)); + MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, (), (const)); MOCK_METHOD(TransportSocketFactoryContext&, getTransportSocketFactoryContext, (), (const)); - MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, ()); - MOCK_METHOD(Upstream::ClusterManager&, clusterManager, ()); - MOCK_METHOD(Event::Dispatcher&, mainThreadDispatcher, ()); - MOCK_METHOD(const Server::Options&, options, ()); MOCK_METHOD(const Network::DrainDecision&, drainDecision, ()); - MOCK_METHOD(bool, healthCheckFailed, ()); MOCK_METHOD(Init::Manager&, initManager, ()); - MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); - MOCK_METHOD(Envoy::Random::RandomGenerator&, random, ()); - MOCK_METHOD(Envoy::Runtime::Loader&, runtime, ()); MOCK_METHOD(Stats::Scope&, scope, ()); - MOCK_METHOD(Stats::Scope&, serverScope, ()); - MOCK_METHOD(Singleton::Manager&, singletonManager, ()); - MOCK_METHOD(OverloadManager&, overloadManager, ()); - MOCK_METHOD(ThreadLocal::Instance&, threadLocal, ()); - MOCK_METHOD(OptRef, admin, ()); MOCK_METHOD(Stats::Scope&, listenerScope, ()); - MOCK_METHOD(bool, isQuicListener, (), (const)); - MOCK_METHOD(const LocalInfo::LocalInfo&, localInfo, (), (const)); - MOCK_METHOD(const envoy::config::core::v3::Metadata&, listenerMetadata, (), (const)); - MOCK_METHOD(const Envoy::Config::TypedMetadata&, listenerTypedMetadata, (), (const)); MOCK_METHOD(envoy::config::core::v3::TrafficDirection, direction, (), (const)); - MOCK_METHOD(TimeSource&, timeSource, ()); - Event::TestTimeSystem& timeSystem() { return time_system_; } - Grpc::Context& grpcContext() override { return grpc_context_; } - Http::Context& httpContext() override { return http_context_; } - Router::Context& routerContext() override { return router_context_; } - MOCK_METHOD(ProcessContextOptRef, processContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); - MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); - MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, (), (const)); + MOCK_METHOD(const Network::ListenerInfo&, listenerInfo, (), (const)); testing::NiceMock server_factory_context_; - testing::NiceMock access_log_manager_; - testing::NiceMock cluster_manager_; - testing::NiceMock dispatcher_; testing::NiceMock drain_manager_; testing::NiceMock init_manager_; - testing::NiceMock lifecycle_notifier_; - testing::NiceMock local_info_; - testing::NiceMock random_; - testing::NiceMock runtime_loader_; testing::NiceMock store_; Stats::Scope& scope_{*store_.rootScope()}; - testing::NiceMock thread_local_; - Singleton::ManagerPtr singleton_manager_; - testing::NiceMock admin_; Stats::IsolatedStoreImpl listener_scope_; - Event::GlobalTimeSystem time_system_; - testing::NiceMock validation_context_; - testing::NiceMock overload_manager_; - Grpc::ContextImpl grpc_context_; - Http::ContextImpl http_context_; - Router::ContextImpl router_context_; - testing::NiceMock api_; - - Network::MockListenerConfig listener_config_; }; + } // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/listener_manager.h b/test/mocks/server/listener_manager.h index 7285ff78eaa1..2d35d1a1df5d 100644 --- a/test/mocks/server/listener_manager.h +++ b/test/mocks/server/listener_manager.h @@ -21,7 +21,7 @@ class MockListenerManager : public ListenerManager { (ListenerState state)); MOCK_METHOD(uint64_t, numConnections, (), (const)); MOCK_METHOD(bool, removeListener, (const std::string& listener_name)); - MOCK_METHOD(void, startWorkers, (GuardDog & guard_dog, std::function callback)); + MOCK_METHOD(void, startWorkers, (OptRef guard_dog, std::function callback)); MOCK_METHOD(void, stopListeners, (StopListenersType listeners_type, const Network::ExtraShutdownListenerOptions& options)); diff --git a/test/mocks/server/overload_manager.cc b/test/mocks/server/overload_manager.cc index 9ca2c36a8755..738000730487 100644 --- a/test/mocks/server/overload_manager.cc +++ b/test/mocks/server/overload_manager.cc @@ -19,6 +19,8 @@ MockThreadLocalOverloadState::MockThreadLocalOverloadState() ON_CALL(*this, tryAllocateResource).WillByDefault(Return(true)); ON_CALL(*this, tryDeallocateResource).WillByDefault(Return(true)); ON_CALL(*this, isResourceMonitorEnabled).WillByDefault(Return(false)); + ON_CALL(*this, getProactiveResourceMonitorForTest) + .WillByDefault(Return(makeOptRefFromPtr(nullptr))); } MockOverloadManager::MockOverloadManager() { diff --git a/test/mocks/server/overload_manager.h b/test/mocks/server/overload_manager.h index 1456b6a11556..0f7e0605c5b9 100644 --- a/test/mocks/server/overload_manager.h +++ b/test/mocks/server/overload_manager.h @@ -17,6 +17,8 @@ class MockThreadLocalOverloadState : public ThreadLocalOverloadState { MOCK_METHOD(bool, tryAllocateResource, (OverloadProactiveResourceName, int64_t)); MOCK_METHOD(bool, tryDeallocateResource, (OverloadProactiveResourceName, int64_t)); MOCK_METHOD(bool, isResourceMonitorEnabled, (OverloadProactiveResourceName)); + MOCK_METHOD(ProactiveResourceMonitorOptRef, getProactiveResourceMonitorForTest, + (OverloadProactiveResourceName)); private: const OverloadActionState disabled_state_; @@ -35,6 +37,7 @@ class MockOverloadManager : public OverloadManager { MOCK_METHOD(Event::ScaledRangeTimerManagerFactory, scaledTimerFactory, (), (override)); MOCK_METHOD(ThreadLocalOverloadState&, getThreadLocalOverloadState, ()); MOCK_METHOD(LoadShedPoint*, getLoadShedPoint, (absl::string_view)); + MOCK_METHOD(void, stop, ()); testing::NiceMock overload_state_; }; diff --git a/test/mocks/server/server_factory_context.cc b/test/mocks/server/server_factory_context.cc index 0acd1b671908..6de698d5c0a2 100644 --- a/test/mocks/server/server_factory_context.cc +++ b/test/mocks/server/server_factory_context.cc @@ -9,7 +9,8 @@ using ::testing::ReturnRef; MockServerFactoryContext::MockServerFactoryContext() : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), - grpc_context_(store_.symbolTable()), router_context_(store_.symbolTable()) { + http_context_(store_.symbolTable()), grpc_context_(store_.symbolTable()), + router_context_(store_.symbolTable()) { ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); ON_CALL(*this, mainThreadDispatcher()).WillByDefault(ReturnRef(dispatcher_)); ON_CALL(*this, drainDecision()).WillByDefault(ReturnRef(drain_manager_)); @@ -32,12 +33,23 @@ MockServerFactoryContext::MockServerFactoryContext() ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); ON_CALL(*this, lifecycleNotifier()).WillByDefault(ReturnRef(lifecycle_notifier_)); ON_CALL(*this, options()).WillByDefault(ReturnRef(options_)); + ON_CALL(*this, overloadManager()).WillByDefault(ReturnRef(overload_manager_)); } MockServerFactoryContext::~MockServerFactoryContext() = default; MockStatsConfig::MockStatsConfig() = default; MockStatsConfig::~MockStatsConfig() = default; +MockGenericFactoryContext::~MockGenericFactoryContext() = default; + +MockGenericFactoryContext::MockGenericFactoryContext() { + ON_CALL(*this, serverFactoryContext()).WillByDefault(ReturnRef(server_factory_context_)); + ON_CALL(*this, scope()).WillByDefault(ReturnRef(*store_.rootScope())); + ON_CALL(*this, initManager()).WillByDefault(ReturnRef(init_manager_)); + ON_CALL(*this, messageValidationVisitor()) + .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); +} + } // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/test/mocks/server/server_factory_context.h b/test/mocks/server/server_factory_context.h index d7bd11f4719f..9c8d754227aa 100644 --- a/test/mocks/server/server_factory_context.h +++ b/test/mocks/server/server_factory_context.h @@ -71,14 +71,18 @@ class MockServerFactoryContext : public virtual ServerFactoryContext { MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); MOCK_METHOD(Api::Api&, api, ()); + Http::Context& httpContext() override { return http_context_; } Grpc::Context& grpcContext() override { return grpc_context_; } Router::Context& routerContext() override { return router_context_; } + MOCK_METHOD(ProcessContextOptRef, processContext, ()); envoy::config::bootstrap::v3::Bootstrap& bootstrap() override { return bootstrap_; } MOCK_METHOD(Server::DrainManager&, drainManager, ()); MOCK_METHOD(Init::Manager&, initManager, ()); MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); MOCK_METHOD(StatsConfig&, statsConfig, (), ()); MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); + MOCK_METHOD(OverloadManager&, overloadManager, ()); + MOCK_METHOD(bool, healthCheckFailed, (), (const)); testing::NiceMock cluster_manager_; testing::NiceMock dispatcher_; @@ -97,12 +101,29 @@ class MockServerFactoryContext : public virtual ServerFactoryContext { testing::NiceMock admin_; Event::GlobalTimeSystem time_system_; testing::NiceMock api_; + testing::NiceMock overload_manager_; + Http::ContextImpl http_context_; Grpc::ContextImpl grpc_context_; Router::ContextImpl router_context_; envoy::config::bootstrap::v3::Bootstrap bootstrap_; testing::NiceMock options_; }; +class MockGenericFactoryContext : public GenericFactoryContext { +public: + MockGenericFactoryContext(); + ~MockGenericFactoryContext() override; + + MOCK_METHOD(ServerFactoryContext&, serverFactoryContext, (), (const)); + MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, (), (const)); + MOCK_METHOD(Stats::Scope&, scope, ()); + MOCK_METHOD(Init::Manager&, initManager, ()); + + NiceMock server_factory_context_; + testing::NiceMock store_; + testing::NiceMock init_manager_; +}; + // Stateless mock ServerFactoryContext for cases where it needs to be used concurrently in different // threads. Global state in the MockServerFactoryContext causes thread safety issues in this case. class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { @@ -126,14 +147,18 @@ class StatelessMockServerFactoryContext : public virtual ServerFactoryContext { MOCK_METHOD(ProtobufMessage::ValidationContext&, messageValidationContext, ()); MOCK_METHOD(ProtobufMessage::ValidationVisitor&, messageValidationVisitor, ()); MOCK_METHOD(Api::Api&, api, ()); + MOCK_METHOD(Http::Context&, httpContext, ()); MOCK_METHOD(Grpc::Context&, grpcContext, ()); MOCK_METHOD(Router::Context&, routerContext, ()); + MOCK_METHOD(ProcessContextOptRef, processContext, ()); MOCK_METHOD(envoy::config::bootstrap::v3::Bootstrap&, bootstrap, ()); MOCK_METHOD(Server::DrainManager&, drainManager, ()); MOCK_METHOD(Init::Manager&, initManager, ()); MOCK_METHOD(ServerLifecycleNotifier&, lifecycleNotifier, ()); MOCK_METHOD(StatsConfig&, statsConfig, (), ()); MOCK_METHOD(AccessLog::AccessLogManager&, accessLogManager, (), ()); + MOCK_METHOD(OverloadManager&, overloadManager, ()); + MOCK_METHOD(bool, healthCheckFailed, (), (const)); }; } // namespace Configuration diff --git a/test/mocks/server/worker.cc b/test/mocks/server/worker.cc index 6482b4b40880..2990be7ed5cd 100644 --- a/test/mocks/server/worker.cc +++ b/test/mocks/server/worker.cc @@ -12,15 +12,16 @@ using ::testing::_; using ::testing::Invoke; MockWorker::MockWorker() { - ON_CALL(*this, addListener(_, _, _, _)) - .WillByDefault(Invoke([this](absl::optional overridden_listener, - Network::ListenerConfig& config, - AddListenerCompletion completion, Runtime::Loader&) -> void { - UNREFERENCED_PARAMETER(overridden_listener); - config.listenSocketFactories()[0]->getListenSocket(0); - EXPECT_EQ(nullptr, add_listener_completion_); - add_listener_completion_ = completion; - })); + ON_CALL(*this, addListener(_, _, _, _, _)) + .WillByDefault( + Invoke([this](absl::optional overridden_listener, + Network::ListenerConfig& config, AddListenerCompletion completion, + Runtime::Loader&, Random::RandomGenerator&) -> void { + UNREFERENCED_PARAMETER(overridden_listener); + config.listenSocketFactories()[0]->getListenSocket(0); + EXPECT_EQ(nullptr, add_listener_completion_); + add_listener_completion_ = completion; + })); ON_CALL(*this, removeListener(_, _)) .WillByDefault( @@ -46,7 +47,8 @@ MockWorker::MockWorker() { })); ON_CALL(*this, start(_, _)) - .WillByDefault(Invoke([](GuardDog&, const std::function& cb) -> void { cb(); })); + .WillByDefault( + Invoke([](OptRef, const std::function& cb) -> void { cb(); })); } MockWorker::~MockWorker() = default; diff --git a/test/mocks/server/worker.h b/test/mocks/server/worker.h index b5977838f154..a52da20a81d5 100644 --- a/test/mocks/server/worker.h +++ b/test/mocks/server/worker.h @@ -33,11 +33,11 @@ class MockWorker : public Worker { // Server::Worker MOCK_METHOD(void, addListener, (absl::optional overridden_listener, Network::ListenerConfig& listener, - AddListenerCompletion completion, Runtime::Loader&)); + AddListenerCompletion completion, Runtime::Loader&, Random::RandomGenerator&)); MOCK_METHOD(uint64_t, numConnections, (), (const)); MOCK_METHOD(void, removeListener, (Network::ListenerConfig & listener, std::function completion)); - MOCK_METHOD(void, start, (GuardDog & guard_dog, const std::function& cb)); + MOCK_METHOD(void, start, (OptRef guard_dog, const std::function& cb)); MOCK_METHOD(void, initializeStats, (Stats::Scope & scope)); MOCK_METHOD(void, stop, ()); MOCK_METHOD(void, stopListener, diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index fdcac0aa565e..a7e85351b41e 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -142,6 +142,7 @@ class MockServerContextConfig : public ServerContextConfig { MOCK_METHOD(OcspStaplePolicy, ocspStaplePolicy, (), (const)); MOCK_METHOD(const std::vector&, sessionTicketKeys, (), (const)); MOCK_METHOD(bool, disableStatelessSessionResumption, (), (const)); + MOCK_METHOD(bool, disableStatefulSessionResumption, (), (const)); MOCK_METHOD(const Network::Address::IpList&, tlsKeyLogLocal, (), (const)); MOCK_METHOD(const Network::Address::IpList&, tlsKeyLogRemote, (), (const)); MOCK_METHOD(const std::string&, tlsKeyLogPath, (), (const)); diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index d0108548aaa8..d755cdff8a44 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -68,6 +68,8 @@ MockMetricSnapshot::MockMetricSnapshot() { ON_CALL(*this, counters()).WillByDefault(ReturnRef(counters_)); ON_CALL(*this, gauges()).WillByDefault(ReturnRef(gauges_)); ON_CALL(*this, histograms()).WillByDefault(ReturnRef(histograms_)); + ON_CALL(*this, hostCounters()).WillByDefault(ReturnRef(host_counters_)); + ON_CALL(*this, hostGauges()).WillByDefault(ReturnRef(host_gauges_)); ON_CALL(*this, snapshotTime()).WillByDefault(Return(snapshot_time_)); } diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index 008caa3d9f9f..2b0d25435ca9 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -254,12 +254,16 @@ class MockMetricSnapshot : public MetricSnapshot { MOCK_METHOD(const std::vector>&, gauges, ()); MOCK_METHOD(const std::vector>&, histograms, ()); MOCK_METHOD(const std::vector>&, textReadouts, ()); + MOCK_METHOD(const std::vector&, hostCounters, ()); + MOCK_METHOD(const std::vector&, hostGauges, ()); MOCK_METHOD(SystemTime, snapshotTime, (), (const)); std::vector counters_; std::vector> gauges_; std::vector> histograms_; std::vector> text_readouts_; + std::vector host_counters_; + std::vector host_gauges_; SystemTime snapshot_time_; }; @@ -356,6 +360,7 @@ class MockIsolatedStatsStore : public TestUtil::TestStore { ~MockIsolatedStatsStore() override; MOCK_METHOD(void, deliverHistogramToSinks, (const Histogram& histogram, uint64_t value)); + MOCK_METHOD(const TagVector&, fixedTags, ()); }; class MockStatsMatcher : public StatsMatcher { diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index 710848e59b8a..5b92c27f06f2 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -100,6 +100,7 @@ MockStreamInfo::MockStreamInfo() })); ON_CALL(*this, startTime()).WillByDefault(ReturnPointee(&start_time_)); ON_CALL(*this, startTimeMonotonic()).WillByDefault(ReturnPointee(&start_time_monotonic_)); + ON_CALL(*this, timeSource()).WillByDefault(ReturnPointee(&ts_)); ON_CALL(*this, currentDuration()).WillByDefault(ReturnPointee(&end_time_)); ON_CALL(*this, requestComplete()).WillByDefault(ReturnPointee(&end_time_)); ON_CALL(*this, onRequestComplete()).WillByDefault(Invoke([this]() { diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index c22c35debdba..b83a40d7e69c 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -75,6 +75,7 @@ class MockStreamInfo : public StreamInfo { MOCK_METHOD(void, onUpstreamHostSelected, (Upstream::HostDescriptionConstSharedPtr host)); MOCK_METHOD(SystemTime, startTime, (), (const)); MOCK_METHOD(MonotonicTime, startTimeMonotonic, (), (const)); + MOCK_METHOD(TimeSource&, timeSource, (), (const)); MOCK_METHOD(void, setUpstreamInfo, (std::shared_ptr)); MOCK_METHOD(std::shared_ptr, upstreamInfo, ()); MOCK_METHOD(OptRef, upstreamInfo, (), (const)); diff --git a/test/mocks/thread_local/mocks.cc b/test/mocks/thread_local/mocks.cc index 18a8cfb30bae..b6b37e285a83 100644 --- a/test/mocks/thread_local/mocks.cc +++ b/test/mocks/thread_local/mocks.cc @@ -5,6 +5,7 @@ using testing::_; using testing::Invoke; +using testing::ReturnRef; namespace Envoy { namespace ThreadLocal { @@ -15,7 +16,7 @@ MockInstance::MockInstance() { ON_CALL(*this, runOnAllThreads(_, _)) .WillByDefault(Invoke(this, &MockInstance::runOnAllThreads2)); ON_CALL(*this, shutdownThread()).WillByDefault(Invoke(this, &MockInstance::shutdownThread_)); - ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); + ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(*dispatcher_ptr_)); } MockInstance::~MockInstance() { shutdownThread_(); } diff --git a/test/mocks/thread_local/mocks.h b/test/mocks/thread_local/mocks.h index 72427bf2a10e..36ccd578023b 100644 --- a/test/mocks/thread_local/mocks.h +++ b/test/mocks/thread_local/mocks.h @@ -35,6 +35,8 @@ class MockInstance : public Instance { main_callback(); } + void setDispatcher(Event::Dispatcher* dispatcher) { dispatcher_ptr_ = dispatcher; } + void shutdownThread_() { shutdown_ = true; // Reverse order which is same as the production code. @@ -82,7 +84,7 @@ class MockInstance : public Instance { if (parent_.defer_data_) { parent_.deferred_data_[index_] = cb; } else { - parent_.data_[index_] = cb(parent_.dispatcher_); + parent_.data_[index_] = cb(*parent_.dispatcher_ptr_); } } @@ -93,13 +95,14 @@ class MockInstance : public Instance { void call() { for (unsigned i = 0; i < deferred_data_.size(); i++) { - data_[i] = deferred_data_[i](dispatcher_); + data_[i] = deferred_data_[i](*dispatcher_ptr_); } deferred_data_.clear(); } uint32_t current_slot_{}; testing::NiceMock dispatcher_; + Event::Dispatcher* dispatcher_ptr_ = &dispatcher_; std::vector data_; std::vector deferred_data_; bool defer_data_{}; diff --git a/test/mocks/tracing/mocks.cc b/test/mocks/tracing/mocks.cc index ac6f15701439..22987c53152d 100644 --- a/test/mocks/tracing/mocks.cc +++ b/test/mocks/tracing/mocks.cc @@ -4,6 +4,7 @@ #include "gtest/gtest.h" using testing::Return; +using testing::ReturnPointee; namespace Envoy { namespace Tracing { @@ -12,10 +13,11 @@ MockSpan::MockSpan() = default; MockSpan::~MockSpan() = default; MockConfig::MockConfig() { - ON_CALL(*this, operationName()).WillByDefault(Return(operation_name_)); + ON_CALL(*this, operationName()).WillByDefault(ReturnPointee(&operation_name_)); ON_CALL(*this, customTags()).WillByDefault(Return(&custom_tags_)); - ON_CALL(*this, verbose()).WillByDefault(Return(verbose_)); + ON_CALL(*this, verbose()).WillByDefault(ReturnPointee(&verbose_)); ON_CALL(*this, maxPathTagLength()).WillByDefault(Return(uint32_t(256))); + ON_CALL(*this, spawnUpstreamSpan()).WillByDefault(ReturnPointee(&spawn_upstream_span_)); } MockConfig::~MockConfig() = default; diff --git a/test/mocks/tracing/mocks.h b/test/mocks/tracing/mocks.h index 73e170248bb9..665dd4563366 100644 --- a/test/mocks/tracing/mocks.h +++ b/test/mocks/tracing/mocks.h @@ -21,10 +21,12 @@ class MockConfig : public Config { MOCK_METHOD(const CustomTagMap*, customTags, (), (const)); MOCK_METHOD(bool, verbose, (), (const)); MOCK_METHOD(uint32_t, maxPathTagLength, (), (const)); + MOCK_METHOD(bool, spawnUpstreamSpan, (), (const)); OperationName operation_name_{OperationName::Ingress}; CustomTagMap custom_tags_; bool verbose_{false}; + bool spawn_upstream_span_{false}; }; class MockSpan : public Span { @@ -60,14 +62,13 @@ class MockTracer : public Tracer { SpanPtr startSpan(const Config& config, TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, - const Tracing::Decision tracing_decision) override { + Tracing::Decision tracing_decision) override { return SpanPtr{startSpan_(config, trace_context, stream_info, tracing_decision)}; } MOCK_METHOD(Span*, startSpan_, (const Config& config, TraceContext& trace_context, - const StreamInfo::StreamInfo& stream_info, - const Tracing::Decision tracing_decision)); + const StreamInfo::StreamInfo& stream_info, Tracing::Decision tracing_decision)); }; class MockDriver : public Driver { @@ -77,7 +78,7 @@ class MockDriver : public Driver { SpanPtr startSpan(const Config& config, TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision) override { + Tracing::Decision tracing_decision) override { return SpanPtr{ startSpan_(config, trace_context, stream_info, operation_name, tracing_decision)}; } @@ -85,7 +86,7 @@ class MockDriver : public Driver { MOCK_METHOD(Span*, startSpan_, (const Config& config, TraceContext& trace_context, const StreamInfo::StreamInfo& stream_info, const std::string& operation_name, - const Tracing::Decision tracing_decision)); + Tracing::Decision tracing_decision)); }; class MockTracerManager : public TracerManager { diff --git a/test/mocks/upstream/cluster.cc b/test/mocks/upstream/cluster.cc index d0c297506490..39c2bd4e7759 100644 --- a/test/mocks/upstream/cluster.cc +++ b/test/mocks/upstream/cluster.cc @@ -15,6 +15,10 @@ MockCluster::MockCluster() { EXPECT_EQ(nullptr, initialize_callback_); initialize_callback_ = callback; })); + ON_CALL(*this, dropOverload()).WillByDefault(Return(drop_overload_)); + ON_CALL(*this, setDropOverload(_)).WillByDefault(Invoke([this](UnitFloat drop_overload) -> void { + drop_overload_ = drop_overload; + })); } MockCluster::~MockCluster() = default; diff --git a/test/mocks/upstream/cluster.h b/test/mocks/upstream/cluster.h index 0fc2b867f2db..f2fe2bad9970 100644 --- a/test/mocks/upstream/cluster.h +++ b/test/mocks/upstream/cluster.h @@ -25,10 +25,13 @@ class MockCluster : public Cluster { MOCK_METHOD(InitializePhase, initializePhase, (), (const)); MOCK_METHOD(PrioritySet&, prioritySet, ()); MOCK_METHOD(const PrioritySet&, prioritySet, (), (const)); + MOCK_METHOD(UnitFloat, dropOverload, (), (const)); + MOCK_METHOD(void, setDropOverload, (UnitFloat)); std::shared_ptr info_{new ::testing::NiceMock()}; std::function initialize_callback_; Network::Address::InstanceConstSharedPtr source_address_; + UnitFloat drop_overload_{0}; }; } // namespace Upstream } // namespace Envoy diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 84a030b8cfbb..4ca2bd73d1ff 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -85,7 +85,8 @@ class MockUpstreamLocalAddressSelector : public UpstreamLocalAddressSelector { class MockUpstreamLocalAddressSelectorFactory : public UpstreamLocalAddressSelectorFactory { public: - MOCK_METHOD(UpstreamLocalAddressSelectorConstSharedPtr, createLocalAddressSelector, + MOCK_METHOD(absl::StatusOr, + createLocalAddressSelector, (std::vector<::Envoy::Upstream::UpstreamLocalAddress> upstream_local_addresses, absl::optional cluster_name), (const)); @@ -173,6 +174,7 @@ class MockClusterInfo : public ClusterInfo { MOCK_METHOD(ClusterLoadReportStats&, loadReportStats, (), (const)); MOCK_METHOD(ClusterRequestResponseSizeStatsOptRef, requestResponseSizeStats, (), (const)); MOCK_METHOD(ClusterTimeoutBudgetStatsOptRef, timeoutBudgetStats, (), (const)); + MOCK_METHOD(bool, perEndpointStatsEnabled, (), (const)); MOCK_METHOD(UpstreamLocalAddressSelectorConstSharedPtr, getUpstreamLocalAddressSelector, (), (const)); MOCK_METHOD(const LoadBalancerSubsetInfo&, lbSubsetInfo, (), (const)); diff --git a/test/mocks/upstream/cluster_manager.h b/test/mocks/upstream/cluster_manager.h index e2500ee1dda6..1f67e8a85d09 100644 --- a/test/mocks/upstream/cluster_manager.h +++ b/test/mocks/upstream/cluster_manager.h @@ -40,7 +40,7 @@ class MockClusterManager : public ClusterManager { const std::string& version_info)); MOCK_METHOD(void, setPrimaryClustersInitializedCb, (PrimaryClustersReadyCallback)); MOCK_METHOD(void, setInitializedCb, (InitializationCompleteCallback)); - MOCK_METHOD(void, initializeSecondaryClusters, + MOCK_METHOD(absl::Status, initializeSecondaryClusters, (const envoy::config::bootstrap::v3::Bootstrap& bootstrap)); MOCK_METHOD(ClusterInfoMaps, clusters, (), (const)); @@ -80,7 +80,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD(void, drainConnections, (const std::string& cluster, DrainConnectionsHostPredicate predicate)); MOCK_METHOD(void, drainConnections, (DrainConnectionsHostPredicate predicate)); - MOCK_METHOD(void, checkActiveStaticCluster, (const std::string& cluster)); + MOCK_METHOD(absl::Status, checkActiveStaticCluster, (const std::string& cluster)); MOCK_METHOD(OdCdsApiHandlePtr, allocateOdCdsApi, (const envoy::config::core::v3::ConfigSource& odcds_config, OptRef odcds_resources_locator, diff --git a/test/mocks/upstream/host.cc b/test/mocks/upstream/host.cc index c1b96c6ddeca..ed5ca615d6f0 100644 --- a/test/mocks/upstream/host.cc +++ b/test/mocks/upstream/host.cc @@ -50,6 +50,9 @@ MockHostDescription::MockHostDescription() MockHostDescription::~MockHostDescription() = default; +MockHostLight::MockHostLight() = default; +MockHostLight::~MockHostLight() = default; + MockHost::MockHost() : socket_factory_(new testing::NiceMock) { ON_CALL(*this, cluster()).WillByDefault(ReturnRef(cluster_)); ON_CALL(*this, outlierDetector()).WillByDefault(ReturnRef(outlier_detector_)); diff --git a/test/mocks/upstream/host.h b/test/mocks/upstream/host.h index 8ebfc4f9610d..2564c439eb45 100644 --- a/test/mocks/upstream/host.h +++ b/test/mocks/upstream/host.h @@ -123,16 +123,16 @@ class MockHostDescription : public HostDescription { mutable std::unique_ptr locality_zone_stat_name_; }; -class MockHost : public Host { +class MockHostLight : public Host { public: + MockHostLight(); + ~MockHostLight() override; + struct MockCreateConnectionData { Network::ClientConnection* connection_{}; HostDescriptionConstSharedPtr host_description_{}; }; - MockHost(); - ~MockHost() override; - CreateConnectionData createConnection(Event::Dispatcher& dispatcher, const Network::ConnectionSocket::OptionsSharedPtr& options, @@ -161,12 +161,6 @@ class MockHost : public Host { setLastHcPassTime_(last_hc_pass_time); } - Stats::StatName localityZoneStatName() const override { - locality_zone_stat_name_ = - std::make_unique(locality().zone(), *symbol_table_); - return locality_zone_stat_name_->statName(); - } - bool disableActiveHealthCheck() const override { return disable_active_health_check_; } void setDisableActiveHealthCheck(bool disable_active_health_check) override { disable_active_health_check_ = disable_active_health_check; @@ -216,11 +210,26 @@ class MockHost : public Host { MOCK_METHOD(HostHandlePtr, acquireHandle, (), (const)); MOCK_METHOD(const envoy::config::core::v3::Locality&, locality, (), (const)); + MOCK_METHOD(Stats::StatName, localityZoneStatName, (), (const)); MOCK_METHOD(uint32_t, priority, (), (const)); MOCK_METHOD(void, priority, (uint32_t)); MOCK_METHOD(bool, warmed, (), (const)); MOCK_METHOD(absl::optional, lastHcPassTime, (), (const)); + bool disable_active_health_check_ = false; +}; + +class MockHost : public MockHostLight { +public: + MockHost(); + ~MockHost() override; + + Stats::StatName localityZoneStatName() const override { + locality_zone_stat_name_ = + std::make_unique(locality().zone(), *symbol_table_); + return locality_zone_stat_name_->statName(); + } + testing::NiceMock cluster_; Network::UpstreamTransportSocketFactoryPtr socket_factory_; testing::NiceMock outlier_detector_; @@ -228,7 +237,6 @@ class MockHost : public Host { LoadMetricStatsImpl load_metric_stats_; mutable Stats::TestUtil::TestSymbolTable symbol_table_; mutable std::unique_ptr locality_zone_stat_name_; - bool disable_active_health_check_ = false; }; } // namespace Upstream diff --git a/test/mocks/upstream/thread_local_cluster.cc b/test/mocks/upstream/thread_local_cluster.cc index 5842f04d8807..2e4951d46143 100644 --- a/test/mocks/upstream/thread_local_cluster.cc +++ b/test/mocks/upstream/thread_local_cluster.cc @@ -18,6 +18,10 @@ MockThreadLocalCluster::MockThreadLocalCluster() { ON_CALL(*this, tcpConnPool(_, _)) .WillByDefault(Return(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_))); ON_CALL(*this, httpAsyncClient()).WillByDefault(ReturnRef(async_client_)); + ON_CALL(*this, dropOverload()).WillByDefault(Return(cluster_.drop_overload_)); + ON_CALL(*this, setDropOverload(_)).WillByDefault(Invoke([this](UnitFloat drop_overload) -> void { + cluster_.drop_overload_ = drop_overload; + })); } MockThreadLocalCluster::~MockThreadLocalCluster() = default; diff --git a/test/mocks/upstream/thread_local_cluster.h b/test/mocks/upstream/thread_local_cluster.h index bd445fe75da7..0efb100c20bf 100644 --- a/test/mocks/upstream/thread_local_cluster.h +++ b/test/mocks/upstream/thread_local_cluster.h @@ -39,6 +39,8 @@ class MockThreadLocalCluster : public ThreadLocalCluster { MOCK_METHOD(Http::AsyncClient&, httpAsyncClient, ()); MOCK_METHOD(Tcp::AsyncTcpClientPtr, tcpAsyncClient, (LoadBalancerContext * context, Tcp::AsyncTcpClientOptionsConstSharedPtr options)); + MOCK_METHOD(UnitFloat, dropOverload, (), (const)); + MOCK_METHOD(void, setDropOverload, (UnitFloat)); NiceMock cluster_; NiceMock lb_; diff --git a/test/mocks/upstream/typed_load_balancer_factory.h b/test/mocks/upstream/typed_load_balancer_factory.h index a2ec9a45e18d..5efc47d47c3b 100644 --- a/test/mocks/upstream/typed_load_balancer_factory.h +++ b/test/mocks/upstream/typed_load_balancer_factory.h @@ -11,6 +11,11 @@ namespace Envoy { namespace Upstream { class MockTypedLoadBalancerFactory : public TypedLoadBalancerFactory { public: + class EmptyLoadBalancerConfig : public LoadBalancerConfig { + public: + EmptyLoadBalancerConfig() = default; + }; + MockTypedLoadBalancerFactory(); ~MockTypedLoadBalancerFactory() override; @@ -21,9 +26,9 @@ class MockTypedLoadBalancerFactory : public TypedLoadBalancerFactory { const PrioritySet& priority_set, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source)); - LoadBalancerConfigPtr loadConfig(ProtobufTypes::MessagePtr config, + LoadBalancerConfigPtr loadConfig(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override { - return std::make_unique(std::move(config)); + return std::make_unique(); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 7cd000c5ca6d..ae83388bef9e 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -4,25 +4,25 @@ # for existing directories with low coverage. declare -a KNOWN_LOW_COVERAGE=( "source/common:96.2" -"source/common/api:84.5" -"source/common/api/posix:81.8" -"source/common/config:95.4" -"source/common/crypto:88.1" -"source/common/event:95.1" # Emulated edge events guards don't report LCOV +"source/common/api:84.5" # flaky due to posix: be careful adjusting +"source/common/api/posix:83.8" # flaky (accept failover non-deterministic): be careful adjusting +"source/common/config:95.3" +"source/common/crypto:95.5" +"source/common/event:95.0" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" # FileReadToEndNotReadable fails in some env; createPath can't test all failure branches. "source/common/http/http2:95.2" -"source/common/json:94.1" +"source/common/json:94.6" "source/common/matcher:94.6" "source/common/network:94.4" # Flaky, `activateFileEvents`, `startSecureTransport` and `ioctl`, listener_socket do not always report LCOV "source/common/network/dns_resolver:91.4" # A few lines of MacOS code not tested in linux scripts. Tested in MacOS scripts -"source/common/protobuf:96.5" -"source/common/quic:93.3" -"source/common/secret:95.0" +"source/common/protobuf:96.4" +"source/common/quic:93.4" +"source/common/secret:95.1" "source/common/signal:87.2" # Death tests don't report LCOV -"source/common/tcp:94.1" +"source/common/tcp:94.5" "source/common/thread:0.0" # Death tests don't report LCOV "source/common/watchdog:58.6" # Death tests don't report LCOV -"source/exe:90.7" +"source/exe:91.4" "source/extensions/access_loggers/wasm:93.5" "source/extensions/clusters/common:91.5" # This can be increased again once `#24903` lands "source/extensions/common:93.0" #flaky: be careful adjusting @@ -31,12 +31,11 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/common/wasm/ext:92.0" "source/extensions/filters/common/fault:94.5" "source/extensions/filters/common/rbac:90.5" -"source/extensions/filters/http/cache:93.4" -"source/extensions/filters/http/grpc_json_transcoder:93.7" # TODO(#28232) +"source/extensions/filters/http/cache:94.0" +"source/extensions/filters/http/grpc_json_transcoder:93.8" # TODO(#28232) "source/extensions/filters/http/ip_tagging:88.0" "source/extensions/filters/http/kill_request:91.7" # Death tests don't report LCOV -"source/extensions/filters/http/wasm:1.9" -"source/extensions/filters/listener/original_dst:87.9" +"source/extensions/filters/http/wasm:1.8" "source/extensions/filters/listener/original_src:92.1" "source/extensions/filters/network/common:96.4" "source/extensions/filters/network/mongo_proxy:96.0" @@ -47,7 +46,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/rate_limit_descriptors/expr:95.0" "source/extensions/stat_sinks/graphite_statsd:78.6" # Death tests don't report LCOV "source/extensions/stat_sinks/statsd:80.8" # Death tests don't report LCOV -"source/extensions/tracers:95.8" +"source/extensions/tracers:96.1" "source/extensions/tracers/common:73.8" "source/extensions/tracers/common/ot:71.8" "source/extensions/tracers/opencensus:93.2" @@ -60,10 +59,11 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/wasm_runtime/wasmtime:0.0" # Not enabled in coverage build "source/extensions/wasm_runtime/wavm:0.0" # Not enabled in coverage build "source/extensions/watchdog:83.3" # Death tests within extensions -"source/extensions/listener_managers/validation_listener_manager:70.0" +"source/extensions/listener_managers:70.5" +"source/extensions/listener_managers/validation_listener_manager:70.5" "source/extensions/watchdog/profile_action:83.3" -"source/server:90.7" # flaky: be careful adjusting. See https://github.com/envoyproxy/envoy/issues/15239 -"source/server/config_validation:88.4" +"source/server:91.0" # flaky: be careful adjusting. See https://github.com/envoyproxy/envoy/issues/15239 +"source/server/config_validation:89.2" "source/extensions/health_checkers:96.0" "source/extensions/health_checkers/http:93.9" "source/extensions/health_checkers/grpc:92.0" diff --git a/test/proto/BUILD b/test/proto/BUILD index 3c295a373707..9a69572077a5 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -65,7 +65,7 @@ envoy_proto_library( name = "sensitive_proto", srcs = [":sensitive.proto"], deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg", - "@com_github_cncf_udpa//xds/type/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + "@com_github_cncf_xds//xds/type/v3:pkg", ], ) diff --git a/test/server/BUILD b/test/server/BUILD index 62060da63fa5..9cceee1474dc 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -21,7 +21,7 @@ envoy_cc_test( srcs = ["api_listener_test.cc"], deps = [ ":utility_lib", - "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//source/common/listener_manager:listener_manager_lib", "//source/extensions/request_id/uuid:config", "//test/mocks/network:network_mocks", "//test/mocks/server:instance_mocks", @@ -57,6 +57,7 @@ envoy_cc_test( "//source/extensions/access_loggers/file:config", "//source/extensions/clusters/static:static_cluster_lib", "//source/extensions/load_balancing_policies/ring_hash:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/load_balancing_policies/subset:config", "//source/extensions/stat_sinks/statsd:config", "//source/extensions/transport_sockets/raw_buffer:config", @@ -64,6 +65,7 @@ envoy_cc_test( "//test/common/upstream:utility_lib", "//test/mocks:common_lib", "//test/mocks/network:network_mocks", + "//test/mocks/server:factory_context_mocks", "//test/mocks/server:instance_mocks", "//test/test_common:environment_lib", "//test/test_common:utility_lib", @@ -79,12 +81,12 @@ envoy_cc_test( deps = [ "//source/common/common:utility_lib", "//source/common/config:utility_lib", + "//source/common/listener_manager:active_raw_udp_listener_config", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:udp_packet_writer_handler_lib", "//source/common/stats:stats_lib", - "//source/extensions/listener_managers/listener_manager:active_raw_udp_listener_config", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/access_log:access_log_mocks", "//test/mocks/api:api_mocks", "//test/mocks/network:network_mocks", @@ -102,10 +104,10 @@ envoy_cc_test( deps = [ "//source/common/common:utility_lib", "//source/common/config:utility_lib", + "//source/common/listener_manager:connection_handler_lib", "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", "//source/common/stats:stats_lib", - "//source/extensions/listener_managers/listener_manager:connection_handler_lib", "//test/mocks/network:io_handle_mocks", "//test/mocks/network:network_mocks", "//test/test_common:network_utility_lib", @@ -147,10 +149,20 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "hot_restart_udp_forwarding_test_helper", + hdrs = ["hot_restart_udp_forwarding_test_helper.h"], + deps = [ + "//source/server:hot_restart_lib", + ], +) + envoy_cc_test( name = "hot_restart_impl_test", srcs = envoy_select_hot_restart(["hot_restart_impl_test.cc"]), deps = [ + ":hot_restart_udp_forwarding_test_helper", + ":utility_lib", "//source/common/api:os_sys_calls_lib", "//source/common/stats:stats_lib", "//source/server:hot_restart_lib", @@ -160,10 +172,27 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "hot_restarting_child_test", + srcs = envoy_select_hot_restart(["hot_restarting_child_test.cc"]), + deps = [ + ":hot_restart_udp_forwarding_test_helper", + ":utility_lib", + "//source/common/api:os_sys_calls_lib", + "//source/common/stats:stats_lib", + "//source/server:hot_restart_lib", + "//source/server:hot_restarting_child", + "//test/mocks/network:network_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) + envoy_cc_test( name = "hot_restarting_parent_test", srcs = envoy_select_hot_restart(["hot_restarting_parent_test.cc"]), deps = [ + ":utility_lib", "//source/common/stats:stats_lib", "//source/server:hot_restart_lib", "//source/server:hot_restarting_child", @@ -383,6 +412,7 @@ envoy_cc_benchmark_binary( "//envoy/stats:stats_interface", "//source/common/stats:thread_local_store_lib", "//source/server:server_lib", + "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:simulated_time_system_lib", ], ) diff --git a/test/server/active_tcp_listener_test.cc b/test/server/active_tcp_listener_test.cc index 9a033b85190b..e3a9844fb136 100644 --- a/test/server/active_tcp_listener_test.cc +++ b/test/server/active_tcp_listener_test.cc @@ -4,11 +4,11 @@ #include "envoy/network/listener.h" #include "envoy/stats/scope.h" +#include "source/common/listener_manager/active_tcp_listener.h" #include "source/common/network/address_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/raw_buffer_socket.h" #include "source/common/network/utility.h" -#include "source/extensions/listener_managers/listener_manager/active_tcp_listener.h" #include "test/mocks/common.h" #include "test/mocks/network/io_handle.h" @@ -39,6 +39,16 @@ class MockTcpConnectionHandler : public Network::TcpConnectionHandler, (uint64_t listener_tag, const Network::Address::Instance&)); MOCK_METHOD(Network::BalancedConnectionHandlerOptRef, getBalancedHandlerByAddress, (const Network::Address::Instance& address)); + MOCK_METHOD(Network::Listener*, createListener_, + (Network::SocketSharedPtr && socket, Network::TcpListenerCallbacks& cb, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state)); + MOCK_METHOD(Network::ListenerPtr, createListener, + (Network::SocketSharedPtr && socket, Network::TcpListenerCallbacks& cb, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state)); }; class ActiveTcpListenerTest : public testing::Test, protected Logger::Loggable { @@ -143,14 +153,12 @@ TEST_F(ActiveTcpListenerTest, ListenerFilterWithInspectData) { generic_active_listener_->onAcceptWorker(std::move(generic_accepted_socket_), false, true); EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - inspect_size_ / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(inspect_size_ / 2, Api::IoError::none())))); // the filter is looking for more data. EXPECT_CALL(*filter_, onData(_)).WillOnce(Return(Network::FilterStatus::StopIteration)); file_event_callback(Event::FileReadyType::Read); EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove( - Api::IoCallUint64Result(inspect_size_, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(inspect_size_, Api::IoError::none())))); // the filter get enough data, then return Network::FilterStatus::Continue EXPECT_CALL(*filter_, onData(_)).WillOnce(Return(Network::FilterStatus::Continue)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); @@ -179,14 +187,12 @@ TEST_F(ActiveTcpListenerTest, ListenerFilterWithInspectDataFailedWithPeek) { generic_active_listener_->onAcceptWorker(std::move(generic_accepted_socket_), false, true); EXPECT_CALL(io_handle_, close) - .WillOnce(Return( - ByMove(Api::IoCallUint64Result(0, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(0, Api::IoError::none())))); // peek data failed. EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove( - Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(SOCKET_ERROR_INTR), - Network::IoSocketError::deleteIoError))))); + .WillOnce(Return( + ByMove(Api::IoCallUint64Result(0, Network::IoSocketError::create(SOCKET_ERROR_INTR))))); file_event_callback(Event::FileReadyType::Read); EXPECT_EQ(generic_active_listener_->stats_.downstream_listener_filter_error_.value(), 1); @@ -254,15 +260,15 @@ TEST_F(ActiveTcpListenerTest, ListenerFilterWithInspectDataMultipleFilters) { EXPECT_CALL(io_handle_, recv) .WillOnce([&](void*, size_t size, int) { EXPECT_EQ(128, size); - return Api::IoCallUint64Result(128, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(128, Api::IoError::none()); }) .WillOnce([&](void*, size_t size, int) { EXPECT_EQ(512, size); - return Api::IoCallUint64Result(512, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(512, Api::IoError::none()); }) .WillOnce([&](void*, size_t size, int) { EXPECT_EQ(512, size); - return Api::IoCallUint64Result(512, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(512, Api::IoError::none()); }); EXPECT_CALL(*inspect_data_filter1, onData(_)).WillOnce(Return(Network::FilterStatus::Continue)); @@ -338,15 +344,15 @@ TEST_F(ActiveTcpListenerTest, ListenerFilterWithInspectDataMultipleFilters2) { EXPECT_CALL(io_handle_, recv) .WillOnce([&](void*, size_t size, int) { EXPECT_EQ(128, size); - return Api::IoCallUint64Result(128, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(128, Api::IoError::none()); }) .WillOnce([&](void*, size_t size, int) { EXPECT_EQ(512, size); - return Api::IoCallUint64Result(512, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(512, Api::IoError::none()); }) .WillOnce([&](void*, size_t size, int) { EXPECT_EQ(512, size); - return Api::IoCallUint64Result(512, Api::IoErrorPtr(nullptr, [](Api::IoError*) {})); + return Api::IoCallUint64Result(512, Api::IoError::none()); }); EXPECT_CALL(*no_inspect_data_filter, onAccept(_)) @@ -400,19 +406,16 @@ TEST_F(ActiveTcpListenerTest, ListenerFilterWithClose) { generic_active_listener_->onAcceptWorker(std::move(generic_accepted_socket_), false, true); EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - inspect_size_ / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(inspect_size_ / 2, Api::IoError::none())))); // the filter is looking for more data EXPECT_CALL(*filter_, onData(_)).WillOnce(Return(Network::FilterStatus::StopIteration)); file_event_callback(Event::FileReadyType::Read); EXPECT_CALL(io_handle_, recv) - .WillOnce(Return( - ByMove(Api::IoCallUint64Result(0, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(0, Api::IoError::none())))); EXPECT_CALL(io_handle_, close) - .WillOnce(Return( - ByMove(Api::IoCallUint64Result(0, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(0, Api::IoError::none())))); // emit the read event file_event_callback(Event::FileReadyType::Read); EXPECT_EQ(generic_active_listener_->stats_.downstream_listener_filter_remote_close_.value(), 1); @@ -434,8 +437,7 @@ TEST_F(ActiveTcpListenerTest, ListenerFilterCloseSockets) { .WillOnce(SaveArg<1>(&file_event_callback)); EXPECT_CALL(io_handle_, activateFileEvents(Event::FileReadyType::Read)); EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - inspect_size_ / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(inspect_size_ / 2, Api::IoError::none())))); // the filter is looking for more data EXPECT_CALL(*filter_, onData(_)).WillOnce(Invoke([&is_open]() { is_open = false; @@ -469,8 +471,7 @@ TEST_F(ActiveTcpListenerTest, PopulateSNIWhenActiveTcpSocketTimeout) { generic_active_listener_->onAcceptWorker(std::move(generic_accepted_socket_), false, true); EXPECT_CALL(io_handle_, recv) - .WillOnce(Return(ByMove(Api::IoCallUint64Result( - inspect_size_ / 2, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(inspect_size_ / 2, Api::IoError::none())))); // the filter is looking for more data. EXPECT_CALL(*filter_, onData(_)).WillOnce(Return(Network::FilterStatus::StopIteration)); @@ -478,9 +479,12 @@ TEST_F(ActiveTcpListenerTest, PopulateSNIWhenActiveTcpSocketTimeout) { // get the ActiveTcpSocket pointer before unlink() removed from the link-list. ActiveTcpSocket* tcp_socket = generic_active_listener_->sockets().front().get(); + + // Senseless calls to make the coverage CI happy. + const_cast(tcp_socket)->dynamicMetadata(); + // trigger the onTimeout event manually, since the timer is fake. generic_active_listener_->sockets().front()->onTimeout(); - EXPECT_EQ(server_name, tcp_socket->streamInfo()->downstreamAddressProvider().requestedServerName()); } diff --git a/test/server/active_udp_listener_test.cc b/test/server/active_udp_listener_test.cc index be5abf476e68..5ff9ac73e197 100644 --- a/test/server/active_udp_listener_test.cc +++ b/test/server/active_udp_listener_test.cc @@ -194,37 +194,6 @@ TEST_P(ActiveUdpListenerTest, MultipleFiltersOnReceiveErrorStopIteration) { active_listener_->onReceiveError(Api::IoError::IoErrorCode::UnknownError); } -TEST_P(ActiveUdpListenerTest, UdpListenerWorkerRouterTest) { - uint32_t concurrency = 2; - setup(concurrency); - - uint64_t listener_tag = 1; - EXPECT_CALL(listener_config_, listenerTag()).WillOnce(Return(listener_tag)); - active_listener_->destination_ = 1; - - EXPECT_CALL(listener_config_, filterChainFactory()); - auto another_udp_listener = new NiceMock(); - EXPECT_CALL(*another_udp_listener, dispatcher()).WillRepeatedly(ReturnRef(dispatcher_)); - EXPECT_CALL(dispatcher_, createUdpListener_(_, _, _)).WillOnce(Return(another_udp_listener)); -#ifndef NDEBUG - EXPECT_CALL(dispatcher_, isThreadSafe()).WillOnce(Return(false)); -#endif - auto another_active_listener = std::make_unique( - 1, concurrency, conn_handler_, listen_socket_, dispatcher_, listener_config_); - - EXPECT_CALL(conn_handler_, getUdpListenerCallbacks(_, _)) - .WillOnce(Invoke([&](uint64_t tag, const Network::Address::Instance& address) { - EXPECT_EQ(listener_tag, tag); - EXPECT_EQ(*listen_socket_->connectionInfoProvider().localAddress(), address); - return std::reference_wrapper(*another_active_listener); - })); - - Network::UdpRecvData data; - active_listener_->onData(std::move(data)); - - EXPECT_CALL(*another_udp_listener, onDestroy()); -} - } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/admin/BUILD b/test/server/admin/BUILD index 24bfa08c6d57..8e7c6ecce89f 100644 --- a/test/server/admin/BUILD +++ b/test/server/admin/BUILD @@ -21,11 +21,22 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "admin_factory_context_test", + srcs = ["admin_factory_context_test.cc"], + deps = [ + "//source/common/listener_manager:listener_info_lib", + "//source/server/admin:admin_factory_context", + "//test/mocks/server:instance_mocks", + ], +) + envoy_cc_test_library( name = "admin_instance_lib", srcs = envoy_select_admin_functionality(["admin_instance.cc"]), hdrs = envoy_select_admin_functionality(["admin_instance.h"]), deps = [ + "//source/server:configuration_lib", "//source/server/admin:admin_lib", "//test/mocks/runtime:runtime_mocks", "//test/mocks/server:instance_mocks", @@ -88,6 +99,7 @@ envoy_cc_test( "//test/mocks/server:admin_stream_mocks", "//test/test_common:logging_lib", "//test/test_common:real_threads_test_helper_lib", + "//test/test_common:stats_utility_lib", "//test/test_common:utility_lib", ], ) @@ -135,6 +147,8 @@ envoy_cc_test( "//test/mocks/event:event_mocks", "//test/mocks/stats:stats_mocks", "//test/mocks/thread_local:thread_local_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:stats_utility_lib", "//test/test_common:utility_lib", ], ) @@ -148,6 +162,7 @@ envoy_cc_benchmark_binary( "//source/common/stats:thread_local_store_lib", "//source/server/admin:admin_lib", "//test/common/stats:real_thread_test_base", + "//test/mocks/upstream:cluster_manager_mocks", ], ) @@ -172,6 +187,8 @@ envoy_cc_test( "//source/common/stats:tag_producer_lib", "//source/common/stats:thread_local_store_lib", "//source/server/admin:prometheus_stats_lib", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:stats_utility_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/server/admin/admin_factory_context_test.cc b/test/server/admin/admin_factory_context_test.cc new file mode 100644 index 000000000000..4a0bf5d9f459 --- /dev/null +++ b/test/server/admin/admin_factory_context_test.cc @@ -0,0 +1,33 @@ +#include "source/common/listener_manager/listener_info_impl.h" +#include "source/server/admin/admin_factory_context.h" + +#include "test/mocks/server/instance.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { +namespace { + +TEST(AdminFactoryContextTest, AdminFactoryContextTest) { + testing::NiceMock server; + const auto listener_info = std::make_shared(); + + AdminFactoryContext context(server, listener_info); + + context.serverFactoryContext(); + context.getTransportSocketFactoryContext(); + context.scope(); + context.listenerScope(); + context.listenerInfo().isQuic(); + context.listenerInfo().metadata(); + context.listenerInfo().typedMetadata(); + context.listenerInfo().direction(); + context.messageValidationVisitor(); + context.initManager(); + context.drainDecision(); +} + +} // namespace +} // namespace Server +} // namespace Envoy diff --git a/test/server/admin/admin_html_util_test.cc b/test/server/admin/admin_html_util_test.cc index 06ec9799e78a..d0fa724e539f 100644 --- a/test/server/admin/admin_html_util_test.cc +++ b/test/server/admin/admin_html_util_test.cc @@ -27,7 +27,7 @@ const Admin::UrlHandler handler() { class AdminHtmlUtilTest : public testing::Test { protected: Buffer::OwnedImpl response_; - Http::Utility::QueryParams query_params_; + Http::Utility::QueryParamsMulti query_params_; }; TEST_F(AdminHtmlUtilTest, RenderTableBegin) { @@ -54,8 +54,8 @@ TEST_F(AdminHtmlUtilTest, RenderRenderEndpointTableRowNoQuery) { } TEST_F(AdminHtmlUtilTest, RenderRenderEndpointTableRowWithQuery) { - query_params_["boolparam"] = "on"; - query_params_["strparam"] = "str value"; + query_params_.overwrite("boolparam", "on"); + query_params_.overwrite("strparam", "str value"); AdminHtmlUtil::renderEndpointTableRow(response_, handler(), query_params_, 31415, false, false); std::string out = response_.toString(); EXPECT_THAT(out, diff --git a/test/server/admin/admin_instance.cc b/test/server/admin/admin_instance.cc index e11fe39d83a4..7b18c448e18f 100644 --- a/test/server/admin/admin_instance.cc +++ b/test/server/admin/admin_instance.cc @@ -2,13 +2,13 @@ #include "source/common/access_log/access_log_impl.h" #include "source/extensions/access_loggers/common/file_access_log_impl.h" +#include "source/server/configuration_impl.h" namespace Envoy { namespace Server { AdminInstanceTest::AdminInstanceTest() - : address_out_path_(TestEnvironment::temporaryPath("admin.address")), - cpu_profile_path_(TestEnvironment::temporaryPath("envoy.prof")), + : cpu_profile_path_(TestEnvironment::temporaryPath("envoy.prof")), admin_(cpu_profile_path_, server_, false), request_headers_{{":path", "/"}}, admin_filter_(admin_.createRequestFunction()) { std::list access_logs; @@ -16,9 +16,9 @@ AdminInstanceTest::AdminInstanceTest() access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); - admin_.startHttpListener(access_logs, address_out_path_, - Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, - listener_scope_.createScope("listener.admin.")); + server_.options_.admin_address_path_ = TestEnvironment::temporaryPath("admin.address"); + admin_.startHttpListener(access_logs, Network::Test::getCanonicalLoopbackAddress(GetParam()), + nullptr); EXPECT_EQ(std::chrono::milliseconds(100), admin_.drainTimeout()); admin_.tracingStats().random_sampling_.inc(); EXPECT_TRUE(admin_.setCurrentClientCertDetails().empty()); diff --git a/test/server/admin/admin_instance.h b/test/server/admin/admin_instance.h index b087b926e218..544d9b347194 100644 --- a/test/server/admin/admin_instance.h +++ b/test/server/admin/admin_instance.h @@ -29,7 +29,6 @@ class AdminInstanceTest : public testing::TestWithParam server_; Stats::IsolatedStoreImpl listener_scope_; diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index 4963a9654077..2c3d81a07423 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -16,6 +16,7 @@ #include "source/common/upstream/upstream_impl.h" #include "source/extensions/access_loggers/common/file_access_log_impl.h" #include "source/server/admin/stats_request.h" +#include "source/server/configuration_impl.h" #include "test/server/admin/admin_instance.h" #include "test/test_common/logging.h" @@ -57,32 +58,38 @@ TEST_P(AdminInstanceTest, Getters) { envoy::extensions::filters::network::http_connection_manager::v3:: HttpConnectionManager::KEEP_UNCHANGED); EXPECT_NE(nullptr, admin_.scopedRouteConfigProvider()); +#ifdef ENVOY_ENABLE_UHV + // In UHV mode there is always UHV factory + EXPECT_NE(nullptr, admin_.makeHeaderValidator(Http::Protocol::Http11)); +#else + // In non UHV mode, header validator can not be created + EXPECT_EQ(nullptr, admin_.makeHeaderValidator(Http::Protocol::Http11)); +#endif } TEST_P(AdminInstanceTest, WriteAddressToFile) { - std::ifstream address_file(address_out_path_); + std::ifstream address_file(server_.options_.admin_address_path_); std::string address_from_file; std::getline(address_file, address_from_file); EXPECT_EQ(admin_.socket().connectionInfoProvider().localAddress()->asString(), address_from_file); } TEST_P(AdminInstanceTest, AdminAddress) { - const std::string address_out_path = TestEnvironment::temporaryPath("admin.address"); + server_.options_.admin_address_path_ = TestEnvironment::temporaryPath("admin.address"); AdminImpl admin_address_out_path(cpu_profile_path_, server_, false); std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); - EXPECT_LOG_CONTAINS("info", "admin address:", - admin_address_out_path.startHttpListener( - access_logs, address_out_path, - Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, - listener_scope_.createScope("listener.admin."))); + EXPECT_LOG_CONTAINS( + "info", "admin address:", + admin_address_out_path.startHttpListener( + access_logs, Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr)); } TEST_P(AdminInstanceTest, AdminBadAddressOutPath) { - const std::string bad_path = + server_.options_.admin_address_path_ = TestEnvironment::temporaryPath("some/unlikely/bad/path/admin.address"); AdminImpl admin_bad_address_out_path(cpu_profile_path_, server_, false); std::list access_logs; @@ -91,11 +98,12 @@ TEST_P(AdminInstanceTest, AdminBadAddressOutPath) { file_info, {}, Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(), server_.accessLogManager())); EXPECT_LOG_CONTAINS( - "critical", "cannot open admin address output file " + bad_path + " for writing.", + "critical", + "cannot open admin address output file " + server_.options_.admin_address_path_ + + " for writing.", admin_bad_address_out_path.startHttpListener( - access_logs, bad_path, Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, - listener_scope_.createScope("listener.admin."))); - EXPECT_FALSE(std::ifstream(bad_path)); + access_logs, Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr)); + EXPECT_FALSE(std::ifstream(server_.options_.admin_address_path_)); } TEST_P(AdminInstanceTest, CustomHandler) { @@ -144,6 +152,7 @@ TEST_P(AdminInstanceTest, Help) { enable: enables the CPU profiler; One of (y, n) /drain_listeners (POST): drain listeners graceful: When draining listeners, enter a graceful drain period prior to closing listeners. This behaviour and duration is configurable via server options or CLI + skip_exit: When draining listeners, do not exit after the drain period. This must be used with graceful inboundonly: Drains all inbound listeners. traffic_direction field in envoy_v3_api_msg_config.listener.v3.Listener is used to determine whether a listener is inbound or outbound. /healthcheck/fail (POST): cause the server to fail health checks /healthcheck/ok (POST): cause the server to pass health checks @@ -298,8 +307,8 @@ class AdminTestingPeer { AdminImpl::NullScopedRouteConfigProvider& scopedRouteConfigProvider() { return scoped_route_config_provider_; } - AdminImpl::NullOverloadManager& overloadManager() { return admin_.null_overload_manager_; } - AdminImpl::NullOverloadManager::OverloadState& overloadState() { return overload_state_; } + NullOverloadManager& overloadManager() { return admin_.null_overload_manager_; } + NullOverloadManager::OverloadState& overloadState() { return overload_state_; } AdminImpl::AdminListenSocketFactory& socketFactory() { return socket_factory_; } AdminImpl::AdminListener& listener() { return listener_; } @@ -308,10 +317,10 @@ class AdminTestingPeer { Server::Instance& server_; AdminImpl::NullRouteConfigProvider route_config_provider_{server_.timeSource()}; AdminImpl::NullScopedRouteConfigProvider scoped_route_config_provider_{server_.timeSource()}; - AdminImpl::NullOverloadManager::OverloadState overload_state_{server_.dispatcher()}; + NullOverloadManager::OverloadState overload_state_{server_.dispatcher(), false}; AdminImpl::AdminListenSocketFactory socket_factory_{nullptr}; Stats::ScopeSharedPtr listener_scope_; - AdminImpl::AdminListener listener_{admin_, std::move(listener_scope_)}; + AdminImpl::AdminListener listener_{admin_, *listener_scope_}; }; // Covers override methods for admin-specific implementations of classes in @@ -340,6 +349,7 @@ TEST_P(AdminInstanceTest, Overrides) { peer.overloadState().tryAllocateResource(overload_name, 0); peer.overloadState().tryDeallocateResource(overload_name, 0); peer.overloadState().isResourceMonitorEnabled(overload_name); + peer.overloadState().getProactiveResourceMonitorForTest(overload_name); peer.overloadManager().scaledTimerFactory(); @@ -349,7 +359,7 @@ TEST_P(AdminInstanceTest, Overrides) { peer.listener().name(); peer.listener().udpListenerConfig(); - peer.listener().direction(); + peer.listener().listenerInfo()->direction(); peer.listener().tcpBacklogSize(); } diff --git a/test/server/admin/config_dump_handler_test.cc b/test/server/admin/config_dump_handler_test.cc index 8d2dc83b6330..1ab84cffa333 100644 --- a/test/server/admin/config_dump_handler_test.cc +++ b/test/server/admin/config_dump_handler_test.cc @@ -137,11 +137,14 @@ TEST_P(AdminInstanceTest, ConfigDumpWithEndpoint) { addHostInfo(*host, hostname, "tcp://1.2.3.4:80", locality, hostname_for_healthcheck, "tcp://1.2.3.5:90", 5, 6); + // Adding drop_overload config. + ON_CALL(cluster, dropOverload()).WillByDefault(Return(UnitFloat(0.00035))); Buffer::OwnedImpl response; Http::TestResponseHeaderMapImpl header_map; EXPECT_EQ(Http::Code::OK, getCallback("/config_dump?include_eds", header_map, response)); std::string output = response.toString(); + const std::string expected_json = R"EOF({ "configs": [ { @@ -178,6 +181,15 @@ TEST_P(AdminInstanceTest, ConfigDumpWithEndpoint) { } ], "policy": { + "drop_overloads": [ + { + "category": "drop_overload", + "drop_percentage": { + "numerator": 350, + "denominator": "MILLION" + } + } + ], "overprovisioning_factor": 140 } } diff --git a/test/server/admin/logs_handler_test.cc b/test/server/admin/logs_handler_test.cc index 1bc8be70925d..e3226a3c9bd7 100644 --- a/test/server/admin/logs_handler_test.cc +++ b/test/server/admin/logs_handler_test.cc @@ -45,10 +45,11 @@ TEST_P(AdminInstanceTest, LogLevelSetting) { // Test multiple log levels with invalid logger name const std::string file_not_exists = "xxxxxxxxxx_not_exists_xxxxxxxxxxx"; query = fmt::format("/logging?paths={}:warning,{}:warning", __FILE__, file_not_exists); - EXPECT_EQ(Http::Code::BadRequest, postCallback(query, header_map, response)); - FINE_GRAIN_LOG(trace, "After post 3: level should not change due to invalid logger name!"); - EXPECT_THAT(response.toString(), HasSubstr("error: unknown logger name\n")); - EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::info); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(trace, + "After post 3: level should be changed if there is a match with an OK response."); + EXPECT_THAT(response.toString(), HasSubstr("active loggers:\n")); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::warn); EXPECT_THAT(getFineGrainLogContext().getFineGrainLogEntry(file_not_exists), IsNull()); // Test multiple log levels at once @@ -70,5 +71,89 @@ TEST_P(AdminInstanceTest, LogLevelSetting) { EXPECT_EQ(Http::Code::OK, postCallback("/logging?level=warning&paths=", header_map, response)); } +TEST_P(AdminInstanceTest, LogLevelFineGrainGlobSupport) { + Http::TestResponseHeaderMapImpl header_map; + Buffer::OwnedImpl response; + + // Enable fine grain logger right now. + Logger::Context::enableFineGrainLogger(); + postCallback("/logging", header_map, response); + FINE_GRAIN_LOG(error, response.toString()); + + EXPECT_EQ(Http::Code::OK, postCallback("/logging?level=trace", header_map, response)); + FINE_GRAIN_LOG(warn, "After post /logging?level=trace, all level is trace now!"); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::trace); + + std::string query = fmt::format("/logging?{}=info", "logs_handler_test"); + postCallback(query, header_map, response); + FINE_GRAIN_LOG(info, "After post {}, level for this file is info now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::info); + + // Test multiple log levels at once + const std::string file_one = "admin/logs_handler_test_one.cc"; + const std::string file_two = "admin/logs_handler_test_two.cc"; + std::atomic logger_one; + std::atomic logger_two; + getFineGrainLogContext().initFineGrainLogger(file_one, logger_one); + getFineGrainLogContext().initFineGrainLogger(file_two, logger_two); + query = fmt::format("/logging?{}=critical", "logs_handle*"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(critical, "After post {}, level for this file is critical now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), + spdlog::level::critical); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), + spdlog::level::critical); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), + spdlog::level::critical); + + query = fmt::format("/logging?paths={}:warning", "admin/*"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(trace, "After post {}, level for this file is trace (the default) now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::trace); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), spdlog::level::warn); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), spdlog::level::warn); + + query = fmt::format("/logging?paths={}:info", "admin/logs_handler_test????.cc"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(trace, "After post {}, level for this file is still trace!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::trace); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), spdlog::level::info); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), spdlog::level::info); + + query = fmt::format("/logging?paths={}:warning", "*admin/logs_handler_test*"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(warn, "After post {}, level for this file is warn now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::warn); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), spdlog::level::warn); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), spdlog::level::warn); + + // Only first glob match takes effect. + query = + fmt::format("/logging?paths={}:warning,{}:info", "logs_handler_test*", "logs_handler_test*"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(warn, "After post {}, level for this file is warn now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::warn); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), spdlog::level::warn); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), spdlog::level::warn); + + // The first glob or base-name match takes effect. + query = fmt::format("/logging?paths={}:warning,{}:info", "logs_handler_test_one", + "logs_handler_test*"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(info, "After post {}, level for this file is info now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), spdlog::level::info); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), spdlog::level::warn); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), spdlog::level::info); + + // The level of unmatched loggers will be the default. + query = fmt::format("/logging?paths={}:critical", "logs_handler_test"); + EXPECT_EQ(Http::Code::OK, postCallback(query, header_map, response)); + FINE_GRAIN_LOG(info, "After post {}, level for this file is info now!", query); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(__FILE__)->level(), + spdlog::level::critical); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_one)->level(), spdlog::level::trace); + EXPECT_EQ(getFineGrainLogContext().getFineGrainLogEntry(file_two)->level(), spdlog::level::trace); +} + } // namespace Server } // namespace Envoy diff --git a/test/server/admin/prometheus_stats_test.cc b/test/server/admin/prometheus_stats_test.cc index 18a5b3ab83e2..30d8c3b877c0 100644 --- a/test/server/admin/prometheus_stats_test.cc +++ b/test/server/admin/prometheus_stats_test.cc @@ -8,6 +8,8 @@ #include "source/server/admin/prometheus_stats.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/stats_utility.h" #include "test/test_common/utility.h" using testing::NiceMock; @@ -42,7 +44,9 @@ class HistogramWrapper { class PrometheusStatsFormatterTest : public testing::Test { protected: - PrometheusStatsFormatterTest() : alloc_(*symbol_table_), pool_(*symbol_table_) {} + PrometheusStatsFormatterTest() + : alloc_(*symbol_table_), pool_(*symbol_table_), + endpoints_helper_(std::make_unique()) {} ~PrometheusStatsFormatterTest() override { clearStorage(); } @@ -71,6 +75,16 @@ class PrometheusStatsFormatterTest : public testing::Test { textReadouts_.push_back(textReadout); } + void addClusterEndpoints(const std::string& cluster_name, uint32_t num_hosts, + const Stats::TagVector& tags) { + auto& cluster = endpoints_helper_->makeCluster(cluster_name, num_hosts); + + // Create a persistent copy of the tags so a reference can be captured that remains valid. + endpoints_tags_.push_back(std::make_unique(tags)); + EXPECT_CALL(cluster.info_->stats_store_, fixedTags()) + .WillRepeatedly(ReturnRef(*endpoints_tags_.back())); + } + using MockHistogramSharedPtr = Stats::RefcountPtr>; void addHistogram(MockHistogramSharedPtr histogram) { histograms_.push_back(histogram); } @@ -105,6 +119,7 @@ class PrometheusStatsFormatterTest : public testing::Test { gauges_.clear(); histograms_.clear(); textReadouts_.clear(); + endpoints_helper_.reset(); EXPECT_EQ(0, symbol_table_->numSymbols()); } @@ -115,6 +130,8 @@ class PrometheusStatsFormatterTest : public testing::Test { std::vector gauges_; std::vector histograms_; std::vector textReadouts_; + std::unique_ptr endpoints_helper_; + std::vector> endpoints_tags_; }; TEST_F(PrometheusStatsFormatterTest, MetricName) { @@ -194,7 +211,8 @@ TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(2UL, size); } @@ -216,7 +234,8 @@ TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(4UL, size); } @@ -233,7 +252,8 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -304,7 +324,8 @@ envoy_cluster_default_total_match_count{envoy_cluster_name="x"} 0 Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(1, size); EXPECT_EQ(expected_output, response.toString()); } @@ -324,7 +345,8 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNonDefaultBuckets) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -363,7 +385,8 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithScaledPercent) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -397,7 +420,8 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -444,6 +468,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { addGauge("promtest.MYAPP.test.bar", {{makeStat("tag_name"), makeStat("tag-value")}}); // Metric with invalid prometheus namespace in the custom metric must be excluded in the output. addGauge("promtest.1234abcd.test.bar", {{makeStat("tag_name"), makeStat("tag-value")}}); + addClusterEndpoints("cluster1", 1, {{"a.tag-name", "a.tag-value"}}); + addClusterEndpoints("cluster2", 2, {{"another_tag_name", "another_tag-value"}}); const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; HistogramWrapper h1_cumulative; @@ -459,8 +485,9 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); - EXPECT_EQ(7UL, size); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); + EXPECT_EQ(12UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 @@ -497,6 +524,26 @@ envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="360 envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7 envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 +# TYPE envoy_cluster_endpoint_c1 counter +envoy_cluster_endpoint_c1{a_tag_name="a.tag-value",envoy_cluster_name="cluster1",envoy_endpoint_address="127.0.0.1:80"} 11 +envoy_cluster_endpoint_c1{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.2:80"} 21 +envoy_cluster_endpoint_c1{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.3:80"} 31 +# TYPE envoy_cluster_endpoint_c2 counter +envoy_cluster_endpoint_c2{a_tag_name="a.tag-value",envoy_cluster_name="cluster1",envoy_endpoint_address="127.0.0.1:80"} 12 +envoy_cluster_endpoint_c2{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.2:80"} 22 +envoy_cluster_endpoint_c2{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.3:80"} 32 +# TYPE envoy_cluster_endpoint_g1 gauge +envoy_cluster_endpoint_g1{a_tag_name="a.tag-value",envoy_cluster_name="cluster1",envoy_endpoint_address="127.0.0.1:80"} 13 +envoy_cluster_endpoint_g1{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.2:80"} 23 +envoy_cluster_endpoint_g1{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.3:80"} 33 +# TYPE envoy_cluster_endpoint_g2 gauge +envoy_cluster_endpoint_g2{a_tag_name="a.tag-value",envoy_cluster_name="cluster1",envoy_endpoint_address="127.0.0.1:80"} 14 +envoy_cluster_endpoint_g2{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.2:80"} 24 +envoy_cluster_endpoint_g2{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.3:80"} 34 +# TYPE envoy_cluster_endpoint_healthy gauge +envoy_cluster_endpoint_healthy{a_tag_name="a.tag-value",envoy_cluster_name="cluster1",envoy_endpoint_address="127.0.0.1:80"} 1 +envoy_cluster_endpoint_healthy{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.2:80"} 1 +envoy_cluster_endpoint_healthy{another_tag_name="another_tag-value",envoy_cluster_name="cluster2",envoy_endpoint_address="127.0.0.3:80"} 1 )EOF"; EXPECT_EQ(expected_output, response.toString()); @@ -516,7 +563,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithTextReadoutsInGaugeFormat) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(4UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_total_count counter @@ -564,7 +612,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, StatsParams(), custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, + StatsParams(), custom_namespaces); EXPECT_EQ(6UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_connect_fail counter @@ -749,7 +798,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { StatsParams params; params.used_only_ = true; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram @@ -795,7 +845,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithHiddenGauge) { Buffer::OwnedImpl response; params.hidden_ = HiddenFlag::Exclude; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 @@ -807,7 +858,8 @@ envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3 Buffer::OwnedImpl response; params.hidden_ = HiddenFlag::ShowOnly; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 @@ -819,7 +871,8 @@ envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_4="another_tag_4 Buffer::OwnedImpl response; params.hidden_ = HiddenFlag::Include; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_cluster_2_upstream_cx_total gauge envoy_cluster_test_cluster_2_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 @@ -851,7 +904,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); EXPECT_EQ(0UL, size); } @@ -861,7 +915,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { Buffer::OwnedImpl response; const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); EXPECT_EQ(1UL, size); } } @@ -876,6 +931,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); addGauge("cluster.test_4.upstream_cx_total", {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); + addClusterEndpoints("test_1", 1, {{"a.tag-name", "a.tag-value"}}); const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; HistogramWrapper h1_cumulative; @@ -898,7 +954,8 @@ envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 ASSERT_EQ(Http::Code::OK, params.parse("/stats?filter=cluster.test_1.upstream_cx_total", response)); const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); EXPECT_EQ(1UL, size); EXPECT_EQ(expected_output, response.toString()); } @@ -909,10 +966,27 @@ envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 ASSERT_EQ(Http::Code::OK, params.parse("/stats?filter=cluster.test_1.upstream_cx_total&safe", response)); const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, textReadouts_, response, params, custom_namespaces); + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); EXPECT_EQ(1UL, size); EXPECT_EQ(expected_output, response.toString()); } + + // Look for per-endpoint stats via filter + { + Buffer::OwnedImpl response; + StatsParams params; + ASSERT_EQ(Http::Code::OK, params.parse("/stats?filter=cluster.test_1.endpoint.*c1", response)); + const uint64_t size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, textReadouts_, endpoints_helper_->cm_, response, params, + custom_namespaces); + const std::string expected = + R"EOF(# TYPE envoy_cluster_endpoint_c1 counter +envoy_cluster_endpoint_c1{a_tag_name="a.tag-value",envoy_cluster_name="test_1",envoy_endpoint_address="127.0.0.1:80"} 11 +)EOF"; + EXPECT_EQ(1UL, size); + EXPECT_EQ(expected, response.toString()); + } } } // namespace Server diff --git a/test/server/admin/stats_handler_speed_test.cc b/test/server/admin/stats_handler_speed_test.cc index e4b520904c39..ebd8a57ad117 100644 --- a/test/server/admin/stats_handler_speed_test.cc +++ b/test/server/admin/stats_handler_speed_test.cc @@ -9,12 +9,86 @@ #include "test/benchmark/main.h" #include "test/common/stats/real_thread_test_base.h" +#include "test/mocks/upstream/cluster_manager.h" #include "benchmark/benchmark.h" namespace Envoy { namespace Server { +class FastMockCluster; + +class FastMockClusterManager : public testing::StrictMock { +public: + ClusterInfoMaps clusters() const override { return clusters_; } + + ClusterInfoMaps clusters_; + std::vector> clusters_storage_; + Stats::TestUtil::TestStore store_; + bool per_endpoint_enabled_{false}; +}; + +// Override the methods used by this test so that using a mock doesn't affect performance. +class FastMockCluster : public testing::StrictMock, + public testing::StrictMock { +public: + FastMockCluster(const std::string& name, FastMockClusterManager& cm) + : cm_(cm), scope_(name, cm_.store_), name_(name) {} + + Upstream::ClusterInfoConstSharedPtr info() const override { return info_; } + const Upstream::PrioritySet& prioritySet() const override { return *this; } + PrioritySet& prioritySet() override { return *this; } + const std::vector& hostSetsPerPriority() const override { + return host_sets_; + } + + class ClusterInfo : public testing::StrictMock { + public: + ClusterInfo(FastMockCluster& parent) : parent_(parent) {} + + bool perEndpointStatsEnabled() const override { return parent_.cm_.per_endpoint_enabled_; } + const std::string& observabilityName() const override { return parent_.name_; } + Stats::Scope& statsScope() const override { return parent_.scope_; } + + FastMockCluster& parent_; + }; + + FastMockClusterManager& cm_; + Stats::TestUtil::TestScope scope_; + std::shared_ptr info_{std::make_shared(*this)}; + std::string name_; + std::vector host_sets_; +}; + +class FastMockHostSet : public testing::StrictMock { +public: + const Upstream::HostVector& hosts() const override { return hosts_; } + + Upstream::HostVector hosts_; +}; + +class FastMockHost : public testing::StrictMock { +public: + Network::Address::InstanceConstSharedPtr address() const override { return address_; } + const std::string& hostname() const override { return hostname_; } + + std::vector> + counters() const override { + return host_stats_.counters(); + } + + std::vector> + gauges() const override { + return host_stats_.gauges(); + } + + Health coarseHealth() const override { return Upstream::Host::Health::Healthy; } + + Network::Address::InstanceConstSharedPtr address_; + std::string hostname_; + mutable Upstream::HostStats host_stats_; +}; + class StatsHandlerTest : public Stats::ThreadLocalRealThreadsMixin { public: StatsHandlerTest() : ThreadLocalRealThreadsMixin(1) { @@ -22,12 +96,14 @@ class StatsHandlerTest : public Stats::ThreadLocalRealThreadsMixin { Envoy::benchmark::setCleanupHook([this]() { delete this; }); } + static constexpr uint32_t NumClusters = 10000; + void init() { // Benchmark will be 10k clusters each with 100 counters, with 100+ // character names. The first counter in each scope will be given a value so // it will be included in 'usedonly'. const std::string prefix(100, 'a'); - for (uint32_t s = 0; s < 10000; ++s) { + for (uint32_t s = 0; s < NumClusters; ++s) { Stats::ScopeSharedPtr scope = store_->createScope(absl::StrCat("scope_", s)); scopes_.emplace_back(scope); for (uint32_t c = 0; c < 100; ++c) { @@ -73,18 +149,38 @@ class StatsHandlerTest : public Stats::ThreadLocalRealThreadsMixin { } } store_->mergeHistograms([]() {}); + + cm_.store_.fixed_tags_ = Stats::TagVector{{"fixed-tag", "fixed-value"}}; + for (uint32_t i = 0; i < NumClusters; i++) { + std::string name = absl::StrCat("cluster_", i); + cm_.clusters_storage_.push_back(std::make_unique(name, cm_)); + FastMockCluster& cluster = *cm_.clusters_storage_.back(); + auto host_set = std::make_unique(); + for (uint32_t host_num = 0; host_num < 10; host_num++) { + auto host = std::make_unique(); + host->hostname_ = fmt::format("host{}.example.com", host_num); + host->address_ = Network::Utility::parseInternetAddressNoThrow("127.0.0.1", host_num + 1); + + host_set->hosts_.push_back(std::move(host)); + } + + cluster.host_sets_.push_back(std::move(host_set)); + cm_.clusters_.active_clusters_.emplace(name, cluster); + } } + void setPerEndpointStats(bool enabled) { cm_.per_endpoint_enabled_ = enabled; } + /** * Issues an admin request against the stats saved in store_. */ uint64_t handlerStats(const StatsParams& params) { Buffer::OwnedImpl data; if (params.format_ == StatsFormat::Prometheus) { - StatsHandler::prometheusRender(*store_, custom_namespaces_, params, data); + StatsHandler::prometheusRender(*store_, custom_namespaces_, cm_, params, data); return data.length(); } - Admin::RequestPtr request = StatsHandler::makeRequest(*store_, params); + Admin::RequestPtr request = StatsHandler::makeRequest(*store_, params, cm_); auto response_headers = Http::ResponseHeaderMapImpl::create(); request->start(*response_headers); uint64_t count = 0; @@ -99,155 +195,233 @@ class StatsHandlerTest : public Stats::ThreadLocalRealThreadsMixin { std::vector scopes_; Envoy::Stats::CustomStatNamespacesImpl custom_namespaces_; + FastMockClusterManager cm_; }; } // namespace Server } // namespace Envoy -Envoy::Server::StatsHandlerTest& testContext() { - Envoy::Event::Libevent::Global::initialize(); +static Envoy::Server::StatsHandlerTest& get() { MUTABLE_CONSTRUCT_ON_FIRST_USE(Envoy::Server::StatsHandlerTest); } +Envoy::Server::StatsHandlerTest& testContext(bool per_endpoint_enabled) { + Envoy::Event::Libevent::Global::initialize(); + auto& context = get(); + context.setPerEndpointStats(per_endpoint_enabled); + return context; +} + // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_AllCountersText(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_AllCountersText(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; params.type_ = Envoy::Server::StatsType::Counters; + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 100 * 1000 * 1000, "expected count > 100M"); // actual = 117,789,000 } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_AllCountersText)->Unit(benchmark::kMillisecond); +// BENCHMARK(BM_AllCountersText)->Range(false, true)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_AllCountersText, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_AllCountersText, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_UsedCountersText(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_UsedCountersText(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?usedonly&type=Counters", response); + const uint64_t upper_limit = per_endpoint_stats ? 50 * 1000 * 1000 : 2 * 1000 * 1000; + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 1000 * 1000, "expected count > 1M"); - RELEASE_ASSERT(count < 2 * 1000 * 1000, "expected count < 2M"); // actual = 1,168,890 + RELEASE_ASSERT(count < upper_limit, "expected count < upper_limit"); } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_UsedCountersText)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_UsedCountersText, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_UsedCountersText, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_FilteredCountersText(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_FilteredCountersText(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?filter=no-match&type=Counters", response); + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count == 0, "expected count == 0"); } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_FilteredCountersText)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_FilteredCountersText, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_FilteredCountersText, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_AllCountersJson(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_AllCountersJson(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=json&type=Counters", response); + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 130 * 1000 * 1000, "expected count > 130M"); // actual = 135,789,011 } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_AllCountersJson)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_AllCountersJson, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_AllCountersJson, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_UsedCountersJson(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_UsedCountersJson(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=json&usedonly&type=Counters", response); + const uint64_t upper_limit = per_endpoint_stats ? 60 * 1000 * 1000 : 2 * 1000 * 1000; + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 1000 * 1000, "expected count > 1M"); - RELEASE_ASSERT(count < 2 * 1000 * 1000, "expected count < 2M"); // actual = 1,348,901 + RELEASE_ASSERT(count < upper_limit, "expected count < upper_limit"); } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_UsedCountersJson)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_UsedCountersJson, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_UsedCountersJson, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_FilteredCountersJson(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_FilteredCountersJson(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=json&filter=no-match&type=Counters", response); + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count < 100, "expected count < 100"); // actual = 12 } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_FilteredCountersJson)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_FilteredCountersJson, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_FilteredCountersJson, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_AllCountersPrometheus(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_AllCountersPrometheus(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=prometheus&type=Counters", response); + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 250 * 1000 * 1000, "expected count > 250M"); // actual = 261,578,000 } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_AllCountersPrometheus)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_AllCountersPrometheus, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_AllCountersPrometheus, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_UsedCountersPrometheus(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_UsedCountersPrometheus(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=prometheus&usedonly&type=Counters", response); + const uint64_t upper_limit = per_endpoint_stats ? 200 * 1000 * 1000 : 3 * 1000 * 1000; + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 1000 * 1000, "expected count > 1M"); - RELEASE_ASSERT(count < 3 * 1000 * 1000, "expected count < 3M"); // actual = 2,597,780 + RELEASE_ASSERT(count < upper_limit, "expected count < upper_limit"); } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_UsedCountersPrometheus)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_UsedCountersPrometheus, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_UsedCountersPrometheus, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) -static void BM_FilteredCountersPrometheus(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); +static void BM_FilteredCountersPrometheus(benchmark::State& state, bool per_endpoint_stats) { + Envoy::Server::StatsHandlerTest& test_context = testContext(per_endpoint_stats); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=prometheus&filter=no-match&type=Counters", response); + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count == 0, "expected count == 0"); } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } -BENCHMARK(BM_FilteredCountersPrometheus)->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_FilteredCountersPrometheus, per_endpoint_stats_disabled, false) + ->Unit(benchmark::kMillisecond); +BENCHMARK_CAPTURE(BM_FilteredCountersPrometheus, per_endpoint_stats_enabled, true) + ->Unit(benchmark::kMillisecond); // NOLINTNEXTLINE(readability-identifier-naming) static void BM_HistogramsJson(benchmark::State& state) { - Envoy::Server::StatsHandlerTest& test_context = testContext(); + Envoy::Server::StatsHandlerTest& test_context = testContext(false); Envoy::Server::StatsParams params; Envoy::Buffer::OwnedImpl response; params.parse("?format=json&type=Histograms&histogram_buckets=detailed", response); + uint64_t count; for (auto _ : state) { // NOLINT - uint64_t count = test_context.handlerStats(params); + count = test_context.handlerStats(params); RELEASE_ASSERT(count > 1750000 && count < 2000000, absl::StrCat("count=", count, ", expected > 1.7M")); } + + auto label = absl::StrCat("output per iteration: ", count); + state.SetLabel(label); } BENCHMARK(BM_HistogramsJson)->Unit(benchmark::kMillisecond); diff --git a/test/server/admin/stats_handler_test.cc b/test/server/admin/stats_handler_test.cc index 225525e0b31c..1ef7be643674 100644 --- a/test/server/admin/stats_handler_test.cc +++ b/test/server/admin/stats_handler_test.cc @@ -12,6 +12,7 @@ #include "test/server/admin/admin_instance.h" #include "test/test_common/logging.h" #include "test/test_common/real_threads_test_helper.h" +#include "test/test_common/stats_utility.h" #include "test/test_common/utility.h" using testing::Combine; @@ -78,11 +79,12 @@ class StatsHandlerTest { * @return the Http Code and the response body as a string. */ CodeResponse handlerStats(absl::string_view url) { - MockInstance instance; + NiceMock instance; EXPECT_CALL(admin_stream_, getRequestHeaders()).WillRepeatedly(ReturnRef(request_headers_)); EXPECT_CALL(instance, statsConfig()).WillRepeatedly(ReturnRef(stats_config_)); EXPECT_CALL(stats_config_, flushOnAdmin()).WillRepeatedly(Return(false)); - EXPECT_CALL(instance, stats()).WillRepeatedly(ReturnRef(*store_)); + ON_CALL(instance, stats()).WillByDefault(ReturnRef(*store_)); + ON_CALL(instance, clusterManager()).WillByDefault(ReturnRef(endpoints_helper_.cm_)); EXPECT_CALL(instance, api()).WillRepeatedly(ReturnRef(api_)); EXPECT_CALL(api_, customStatNamespaces()).WillRepeatedly(ReturnRef(custom_namespaces_)); StatsHandler handler(instance); @@ -121,6 +123,7 @@ class StatsHandlerTest { NiceMock main_thread_dispatcher_; NiceMock tls_; NiceMock api_; + Upstream::PerEndpointMetricsTestHelper endpoints_helper_; Stats::AllocatorImpl alloc_; Stats::MockSink sink_; Stats::ThreadLocalStoreImplPtr store_; @@ -153,6 +156,8 @@ TEST_F(AdminStatsTest, HandlerStatsPlainText) { Stats::TextReadout& t = store_->textReadoutFromString("t"); t.set("hello world"); + endpoints_helper_.makeCluster("mycluster", 1); + Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); @@ -169,6 +174,11 @@ TEST_F(AdminStatsTest, HandlerStatsPlainText) { constexpr char expected[] = "t: \"hello world\"\n" "c1: 10\n" "c2: 20\n" + "cluster.mycluster.endpoint.127.0.0.1_80.c1: 11\n" + "cluster.mycluster.endpoint.127.0.0.1_80.c2: 12\n" + "cluster.mycluster.endpoint.127.0.0.1_80.g1: 13\n" + "cluster.mycluster.endpoint.127.0.0.1_80.g2: 14\n" + "cluster.mycluster.endpoint.127.0.0.1_80.healthy: 1\n" "h1: P0(200,200) P25(202.5,202.5) P50(205,205) P75(207.5,207.5) " "P90(209,209) P95(209.5,209.5) P99(209.9,209.9) P99.5(209.95,209.95) " "P99.9(209.99,209.99) P100(210,210)\n" @@ -581,6 +591,8 @@ TEST_F(AdminStatsTest, HandlerStatsJson) { c1.add(10); c2.add(20); + endpoints_helper_.makeCluster("mycluster", 1); + Stats::TextReadout& t = store_->textReadoutFromString("t"); t.set("hello world"); @@ -602,12 +614,32 @@ TEST_F(AdminStatsTest, HandlerStatsJson) { }, { "name":"c1", - "value":10, + "value":10 }, { "name":"c2", "value":20 }, + { + "name":"cluster.mycluster.endpoint.127.0.0.1_80.c1", + "value":11 + }, + { + "name":"cluster.mycluster.endpoint.127.0.0.1_80.c2", + "value":12 + }, + { + "name":"cluster.mycluster.endpoint.127.0.0.1_80.g1", + "value":13 + }, + { + "name":"cluster.mycluster.endpoint.127.0.0.1_80.g2", + "value":14 + }, + { + "name":"cluster.mycluster.endpoint.127.0.0.1_80.healthy", + "value":1 + }, { "histograms": { "supported_quantiles": [ @@ -1227,7 +1259,7 @@ class ThreadedTest : public testing::Test { } void statsEndpoint() { - StatsRequest request(*store_, StatsParams()); + StatsRequest request(*store_, StatsParams(), cm_); Http::TestResponseHeaderMapImpl response_headers; request.start(response_headers); Buffer::OwnedImpl data; @@ -1246,6 +1278,7 @@ class ThreadedTest : public testing::Test { Stats::StatNamePool pool_; Stats::AllocatorImpl alloc_; std::unique_ptr store_; + NiceMock cm_; std::vector scopes_{NumScopes}; absl::Mutex scope_mutexes_[NumScopes]; std::atomic total_lines_{0}; diff --git a/test/server/admin/stats_html_render_test.cc b/test/server/admin/stats_html_render_test.cc index 03a38cbde865..3bc2b844f193 100644 --- a/test/server/admin/stats_html_render_test.cc +++ b/test/server/admin/stats_html_render_test.cc @@ -29,7 +29,7 @@ const Admin::UrlHandler handler() { class StatsHtmlRenderTest : public StatsRenderTestBase { protected: - Http::Utility::QueryParams query_params_; + Http::Utility::QueryParamsMulti query_params_; }; TEST_F(StatsHtmlRenderTest, String) { diff --git a/test/server/admin/stats_request_test.cc b/test/server/admin/stats_request_test.cc index bf8012662381..25da8e92e210 100644 --- a/test/server/admin/stats_request_test.cc +++ b/test/server/admin/stats_request_test.cc @@ -7,6 +7,8 @@ #include "test/mocks/event/mocks.h" #include "test/mocks/stats/mocks.h" #include "test/mocks/thread_local/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/stats_utility.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -35,7 +37,7 @@ class StatsRequestTest : public testing::Test { params.used_only_ = used_only; params.type_ = type; params.format_ = format; - return std::make_unique(store_, params); + return std::make_unique(store_, params, endpoints_helper_.cm_); } std::unique_ptr makeHiddenRequest(HiddenFlag hidden, StatsFormat format, @@ -44,7 +46,7 @@ class StatsRequestTest : public testing::Test { params.hidden_ = hidden; params.type_ = type; params.format_ = format; - return std::make_unique(store_, params); + return std::make_unique(store_, params, endpoints_helper_.cm_); } // Executes a request, counting the chunks that were generated. @@ -92,6 +94,7 @@ class StatsRequestTest : public testing::Test { NiceMock main_thread_dispatcher_; NiceMock tls_; Stats::ThreadLocalStoreImpl store_; + Upstream::PerEndpointMetricsTestHelper endpoints_helper_; Buffer::OwnedImpl response_; }; @@ -128,6 +131,27 @@ TEST_F(StatsRequestTest, OneTextReadout) { EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); } +TEST_F(StatsRequestTest, OneHostCounter) { + auto& cluster = endpoints_helper_.makeCluster("mycluster", 0); + endpoints_helper_.addHostSingleCounter(cluster); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); + + // There's always a gauge due to the synthetic healthy gauge. + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); + + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::TextReadouts))); +} + +TEST_F(StatsRequestTest, OneHostGauge) { + auto& cluster = endpoints_helper_.makeCluster("mycluster", 0); + endpoints_helper_.addHostSingleGauge(cluster); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Counters))); + EXPECT_EQ(1, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::Gauges))); + EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::TextReadouts))); +} + TEST_F(StatsRequestTest, OneScope) { Stats::ScopeSharedPtr scope = store_.createScope("foo"); EXPECT_EQ(0, iterateChunks(*makeRequest(false, StatsFormat::Text, StatsType::All))); diff --git a/test/server/admin/utils_test.cc b/test/server/admin/utils_test.cc index cf78878ce7d2..b34d1b02850a 100644 --- a/test/server/admin/utils_test.cc +++ b/test/server/admin/utils_test.cc @@ -15,7 +15,7 @@ class UtilsTest : public testing::Test { protected: UtilsTest() = default; - Http::Utility::QueryParams query_; + Http::Utility::QueryParamsMulti query_; }; // Most utils paths are covered through other tests, these tests take of @@ -24,19 +24,19 @@ TEST_F(UtilsTest, HistogramMode) { Utility::HistogramBucketsMode histogram_buckets_mode = Utility::HistogramBucketsMode::Cumulative; EXPECT_TRUE(Utility::histogramBucketsParam(query_, histogram_buckets_mode).ok()); EXPECT_EQ(Utility::HistogramBucketsMode::NoBuckets, histogram_buckets_mode); - query_["histogram_buckets"] = "none"; + query_.overwrite("histogram_buckets", "none"); EXPECT_TRUE(Utility::histogramBucketsParam(query_, histogram_buckets_mode).ok()); EXPECT_EQ(Utility::HistogramBucketsMode::NoBuckets, histogram_buckets_mode); - query_["histogram_buckets"] = "cumulative"; + query_.overwrite("histogram_buckets", "cumulative"); EXPECT_TRUE(Utility::histogramBucketsParam(query_, histogram_buckets_mode).ok()); EXPECT_EQ(Utility::HistogramBucketsMode::Cumulative, histogram_buckets_mode); - query_["histogram_buckets"] = "disjoint"; + query_.overwrite("histogram_buckets", "disjoint"); EXPECT_TRUE(Utility::histogramBucketsParam(query_, histogram_buckets_mode).ok()); EXPECT_EQ(Utility::HistogramBucketsMode::Disjoint, histogram_buckets_mode); - query_["histogram_buckets"] = "detailed"; + query_.overwrite("histogram_buckets", "detailed"); EXPECT_TRUE(Utility::histogramBucketsParam(query_, histogram_buckets_mode).ok()); EXPECT_EQ(Utility::HistogramBucketsMode::Detailed, histogram_buckets_mode); - query_["histogram_buckets"] = "garbage"; + query_.overwrite("histogram_buckets", "garbage"); absl::Status status = Utility::histogramBucketsParam(query_, histogram_buckets_mode); EXPECT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), @@ -44,12 +44,10 @@ TEST_F(UtilsTest, HistogramMode) { } TEST_F(UtilsTest, QueryParam) { - EXPECT_FALSE(Utility::queryParam(query_, "key").has_value()); - query_["key"] = ""; - EXPECT_FALSE(Utility::queryParam(query_, "key").has_value()); - query_["key"] = "value"; - EXPECT_TRUE(Utility::queryParam(query_, "key").has_value()); - EXPECT_EQ("value", Utility::queryParam(query_, "key").value()); + EXPECT_FALSE(query_.getFirstValue("key").has_value()); + query_.overwrite("key", "value"); + EXPECT_TRUE(query_.getFirstValue("key").has_value()); + EXPECT_EQ("value", query_.getFirstValue("key").value()); } } // namespace Server diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc index 3675a3031c28..8102b84b01d4 100644 --- a/test/server/config_validation/server_test.cc +++ b/test/server/config_validation/server_test.cc @@ -5,6 +5,7 @@ #include "source/extensions/listener_managers/validation_listener_manager/validation_listener_manager.h" #include "source/server/config_validation/server.h" +#include "source/server/process_context_impl.h" #include "test/integration/server.h" #include "test/mocks/common.h" @@ -23,6 +24,11 @@ namespace Envoy { namespace Server { namespace { +class NullptrComponentFactory : public TestComponentFactory { +public: + DrainManagerPtr createDrainManager(Instance&) override { return nullptr; } +}; + // Test param is the path to the config file to validate. class ValidationServerTest : public testing::TestWithParam { public: @@ -184,6 +190,7 @@ TEST_P(ValidationServerTest, DummyMethodsTest) { server.failHealthcheck(true); server.lifecycleNotifier(); server.secretManager(); + server.drainManager(); EXPECT_FALSE(server.isShutdown()); EXPECT_FALSE(server.healthCheckFailed()); server.grpcContext(); @@ -199,12 +206,11 @@ TEST_P(ValidationServerTest, DummyMethodsTest) { server.admin()->addStreamingHandler("", "", nullptr, false, false); server.admin()->addListenerToHandler(nullptr); server.admin()->closeSocket(); - server.admin()->startHttpListener({}, "", nullptr, nullptr, nullptr); + server.admin()->startHttpListener({}, nullptr, nullptr); Network::MockTcpListenerCallbacks listener_callbacks; Network::MockListenerConfig listener_config; - server.dispatcher().createListener(nullptr, listener_callbacks, server.runtime(), - listener_config); + Server::ThreadLocalOverloadStateOptRef overload_state; server.dnsResolver()->resolve("", Network::DnsLookupFamily::All, nullptr); @@ -212,6 +218,46 @@ TEST_P(ValidationServerTest, DummyMethodsTest) { listener_component_factory.getTcpListenerConfigProviderManager(); } +class TestObject : public ProcessObject { +public: + void setFlag(bool value) { boolean_flag_ = value; } + + bool boolean_flag_ = true; +}; + +TEST_P(ValidationServerTest, NoProcessContext) { + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), stats_store, + access_log_lock, component_factory_, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()); + EXPECT_FALSE(server.processContext().has_value()); + server.shutdown(); +} + +TEST_P(ValidationServerTest, WithProcessContext) { + TestObject object; + ProcessContextImpl process_context(object); + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), stats_store, + access_log_lock, component_factory_, Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest(), process_context); + EXPECT_TRUE(server.processContext().has_value()); + auto context = server.processContext(); + auto& object_from_context = dynamic_cast(context->get().get()); + EXPECT_EQ(&object_from_context, &object); + EXPECT_TRUE(object_from_context.boolean_flag_); + + object.boolean_flag_ = false; + EXPECT_FALSE(object_from_context.boolean_flag_); + server.shutdown(); +} + // TODO(rlazarus): We'd like use this setup to replace //test/config_test (that is, run it against // all the example configs) but can't until light validation is implemented, mocking out access to // the filesystem for TLS certs, etc. In the meantime, these are the example configs that work @@ -239,6 +285,22 @@ TEST_P(ValidationServerTest1, RunWithoutCrash) { INSTANTIATE_TEST_SUITE_P(AllConfigs, ValidationServerTest1, ::testing::ValuesIn(ValidationServerTest1::getAllConfigFiles())); +// A test to ensure that ENVOY_BUGs are handled when the component factory returns a nullptr for +// the drain manager. +TEST_P(RuntimeFeatureValidationServerTest, DrainManagerNullptrCheck) { + // Setup the server instance with a component factory that returns a null DrainManager. + NullptrComponentFactory component_factory; + Thread::MutexBasicLockable access_log_lock; + Stats::IsolatedStoreImpl stats_store; + DangerousDeprecatedTestTime time_system; + EXPECT_ENVOY_BUG(ValidationInstance server(options_, time_system.timeSystem(), + Network::Address::InstanceConstSharedPtr(), + stats_store, access_log_lock, component_factory, + Thread::threadFactoryForTest(), + Filesystem::fileSystemForTest()), + "Component factory should not return nullptr from createDrainManager()"); +} + TEST_P(RuntimeFeatureValidationServerTest, ValidRuntimeLoader) { Thread::MutexBasicLockable access_log_lock; Stats::IsolatedStoreImpl stats_store; diff --git a/test/server/config_validation/xds_fuzz.cc b/test/server/config_validation/xds_fuzz.cc index 295c2b8b0d8b..471c2ea8266c 100644 --- a/test/server/config_validation/xds_fuzz.cc +++ b/test/server/config_validation/xds_fuzz.cc @@ -62,7 +62,7 @@ XdsFuzzTest::XdsFuzzTest(const test::server::config_validation::XdsTestCase& inp test::server::config_validation::Config::SOTW ? "GRPC" : "DELTA_GRPC")), - verifier_(input.config().sotw_or_delta()), actions_(input.actions()), version_(1), + verifier_(input.config().sotw_or_delta()), actions_(input.actions()), ip_version_(TestEnvironment::getIpVersionsForTest()[0]) { config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", use_unified_mux ? "true" : "false"); diff --git a/test/server/config_validation/xds_fuzz.h b/test/server/config_validation/xds_fuzz.h index 1e9c4700bb77..6f743e0f0dfa 100644 --- a/test/server/config_validation/xds_fuzz.h +++ b/test/server/config_validation/xds_fuzz.h @@ -76,7 +76,7 @@ class XdsFuzzTest : public HttpIntegrationTest { std::vector routes_; std::vector listeners_; - uint64_t version_; + uint64_t version_{1}; Network::Address::IpVersion ip_version_; diff --git a/test/server/config_validation/xds_verifier.cc b/test/server/config_validation/xds_verifier.cc index 6265c5d4ed80..676752a697db 100644 --- a/test/server/config_validation/xds_verifier.cc +++ b/test/server/config_validation/xds_verifier.cc @@ -6,9 +6,7 @@ namespace Envoy { -XdsVerifier::XdsVerifier(test::server::config_validation::Config::SotwOrDelta sotw_or_delta) - : num_warming_(0), num_active_(0), num_draining_(0), num_added_(0), num_modified_(0), - num_removed_(0) { +XdsVerifier::XdsVerifier(test::server::config_validation::Config::SotwOrDelta sotw_or_delta) { if (sotw_or_delta == test::server::config_validation::Config::SOTW) { sotw_or_delta_ = SOTW; } else { diff --git a/test/server/config_validation/xds_verifier.h b/test/server/config_validation/xds_verifier.h index d24857f2726f..491eafc7d6a4 100644 --- a/test/server/config_validation/xds_verifier.h +++ b/test/server/config_validation/xds_verifier.h @@ -72,13 +72,13 @@ class XdsVerifier { absl::flat_hash_map all_routes_; absl::flat_hash_map active_routes_; - uint32_t num_warming_; - uint32_t num_active_; - uint32_t num_draining_; + uint32_t num_warming_{0}; + uint32_t num_active_{0}; + uint32_t num_draining_{0}; - uint32_t num_added_; - uint32_t num_modified_; - uint32_t num_removed_; + uint32_t num_added_{0}; + uint32_t num_modified_{0}; + uint32_t num_removed_{0}; SotwOrDelta sotw_or_delta_; }; diff --git a/test/server/configuration_impl_test.cc b/test/server/configuration_impl_test.cc index b2b647233351..44c6837b99d4 100644 --- a/test/server/configuration_impl_test.cc +++ b/test/server/configuration_impl_test.cc @@ -18,6 +18,7 @@ #include "test/common/upstream/utility.h" #include "test/mocks/common.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/server/factory_context.h" #include "test/mocks/server/instance.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" @@ -80,6 +81,7 @@ class ConfigurationImplTest : public testing::Test { Api::ApiPtr api_; NiceMock server_context_; NiceMock server_; + NiceMock factory_context_; Upstream::ProdClusterManagerFactory cluster_manager_factory_; }; @@ -679,9 +681,8 @@ TEST_F(ConfigurationImplTest, AdminSocketOptions) { )EOF"; auto bootstrap = Upstream::parseBootstrapFromV3Json(json); - NiceMock server; InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); Network::MockListenSocket socket_mock; ASSERT_EQ(config.admin().socketOptions()->size(), 2); @@ -719,9 +720,8 @@ TEST_F(ConfigurationImplTest, FileAccessLogOutput) { )EOF"; auto bootstrap = Upstream::parseBootstrapFromV3Json(json); - NiceMock server; InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); Network::MockListenSocket socket_mock; ASSERT_EQ(config.admin().accessLogs().size(), 1); @@ -1042,9 +1042,8 @@ TEST_F(ConfigurationImplTest, DEPRECATED_FEATURE_TEST(DeprecatedAccessLogPathWit )EOF"; auto bootstrap = Upstream::parseBootstrapFromV3Json(json); - NiceMock server; InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); Network::MockListenSocket socket_mock; ASSERT_EQ(config.admin().accessLogs().size(), 2); @@ -1079,7 +1078,7 @@ TEST_F(ConfigurationImplTest, AccessLogWithFilter) { auto bootstrap = Upstream::parseBootstrapFromV3Json(json); InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); ASSERT_EQ(config.admin().accessLogs().size(), 1); } @@ -1114,7 +1113,7 @@ TEST_F(ConfigurationImplTest, DEPRECATED_FEATURE_TEST(DeprecatedAccessLogPathWit auto bootstrap = Upstream::parseBootstrapFromV3Json(json); InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); ASSERT_EQ(config.admin().accessLogs().size(), 2); } @@ -1128,7 +1127,7 @@ TEST_F(ConfigurationImplTest, EmptyAdmin) { auto bootstrap = Upstream::parseBootstrapFromV3Json(json); InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); ASSERT_EQ(config.admin().accessLogs().size(), 0); } @@ -1151,7 +1150,7 @@ TEST_F(ConfigurationImplTest, DEPRECATED_FEATURE_TEST(DeprecatedAccessLogPath)) auto bootstrap = Upstream::parseBootstrapFromV3Json(json); NiceMock server; InitialImpl config(bootstrap); - config.initAdminAccessLog(bootstrap, server_); + config.initAdminAccessLog(bootstrap, factory_context_); Network::MockListenSocket socket_mock; ASSERT_EQ(config.admin().accessLogs().size(), 1); diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 57d6fea97a26..30612ed4f9c5 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -13,6 +13,8 @@ #include "source/common/common/utility.h" #include "source/common/config/utility.h" +#include "source/common/listener_manager/active_raw_udp_listener_config.h" +#include "source/common/listener_manager/connection_handler_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/io_socket_handle_impl.h" @@ -20,8 +22,6 @@ #include "source/common/network/udp_listener_impl.h" #include "source/common/network/udp_packet_writer_handler_impl.h" #include "source/common/network/utility.h" -#include "source/extensions/listener_managers/listener_manager/active_raw_udp_listener_config.h" -#include "source/extensions/listener_managers/listener_manager/connection_handler_impl.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/api/mocks.h" @@ -50,10 +50,22 @@ namespace Envoy { namespace Server { namespace { -class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable { +class MockConnectionHandlerImpl : public ConnectionHandlerImpl { +public: + using ConnectionHandlerImpl::ConnectionHandlerImpl; + MOCK_METHOD(Network::ListenerPtr, createListener, + (Network::SocketSharedPtr && socket, Network::TcpListenerCallbacks& cb, + Runtime::Loader& runtime, Random::RandomGenerator& random, + const Network::ListenerConfig& listener_config, + Server::ThreadLocalOverloadStateOptRef overload_state)); +}; + +class ConnectionHandlerTest : public testing::Test, + protected Logger::Loggable, + public Event::TestUsingSimulatedTime { public: ConnectionHandlerTest() - : handler_(new ConnectionHandlerImpl(dispatcher_, 0)), + : handler_(new MockConnectionHandlerImpl(dispatcher_, 0)), filter_chain_(std::make_shared>()), listener_filter_matcher_(std::make_shared>()), access_log_(std::make_shared()) { @@ -82,7 +94,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable>()) { for (int i = 0; i < num_of_socket_factories; i++) { socket_factories_.emplace_back(std::make_unique()); sockets_.emplace_back(std::make_shared>()); @@ -93,6 +106,10 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggablelistener_factory_ = std::make_unique(1); udp_listener_config_->writer_factory_ = std::make_unique(); + ON_CALL(*static_cast( + const_cast(listener_info_.get())), + direction()) + .WillByDefault(Return(direction_)); } struct UdpListenerConfigImpl : public Network::UdpListenerConfig { @@ -147,7 +164,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_listener_callback_map_{}; + Network::ListenerInfoConstSharedPtr listener_info_; }; class TestListener : public TestListenerBase { @@ -319,34 +339,27 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::LoggablesocketFactory(), getListenSocket(_)) .WillOnce(Return(listeners_.back()->sockets_[0])); if (socket_type == Network::Socket::Type::Stream) { - EXPECT_CALL(dispatcher_, createListener_(_, _, _, _)) - .WillOnce(Invoke( - [listener, listener_callbacks](Network::SocketSharedPtr&&, - Network::TcpListenerCallbacks& cb, Runtime::Loader&, - const Network::ListenerConfig&) -> Network::Listener* { + EXPECT_CALL(*handler_, createListener(_, _, _, _, _, _)) + .WillOnce( + Invoke([listener, listener_callbacks]( + Network::SocketSharedPtr&&, Network::TcpListenerCallbacks& cb, + Runtime::Loader&, Random::RandomGenerator&, const Network::ListenerConfig&, + Server::ThreadLocalOverloadStateOptRef) -> Network::ListenerPtr { if (listener_callbacks != nullptr) { *listener_callbacks = &cb; } - return listener; + return std::unique_ptr(listener); })); } else { - EXPECT_CALL(dispatcher_, createUdpListener_(_, _, _)) - .WillOnce( - Invoke([listener, &test_listener = listeners_.back()]( - Network::SocketSharedPtr&& socket, - Network::UdpListenerCallbacks& udp_listener_callbacks, - const envoy::config::core::v3::UdpSocketConfig&) -> Network::UdpListener* { - test_listener->udp_listener_callback_map_.emplace( - socket->connectionInfoProvider().localAddress()->asString(), - &udp_listener_callbacks); - return dynamic_cast(listener); - })); + delete listener; if (address == nullptr) { listeners_.back()->udp_listener_config_->listener_worker_router_map_.emplace( - local_address_->asString(), std::make_unique(1)); + local_address_->asString(), + std::make_unique>()); } else { listeners_.back()->udp_listener_config_->listener_worker_router_map_.emplace( - address->asString(), std::make_unique(1)); + address->asString(), + std::make_unique>()); } } @@ -388,33 +401,23 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggablesockets_[i]->connection_info_provider_->setLocalAddress(addresses[i]); if (socket_type == Network::Socket::Type::Stream) { - EXPECT_CALL(dispatcher_, createListener_(_, _, _, _)) - .WillOnce( - Invoke([i, &mock_listeners, &listener_callbacks_map]( - Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb, - Runtime::Loader&, const Network::ListenerConfig&) -> Network::Listener* { - auto listener_callbacks_iter = listener_callbacks_map.find( - socket->connectionInfoProvider().localAddress()->asString()); - EXPECT_NE(listener_callbacks_iter, listener_callbacks_map.end()); - *listener_callbacks_iter->second = &cb; - return mock_listeners[i]; - })) + EXPECT_CALL(*handler_, createListener(_, _, _, _, _, _)) + .WillOnce(Invoke([i, &mock_listeners, &listener_callbacks_map]( + Network::SocketSharedPtr&& socket, + Network::TcpListenerCallbacks& cb, Runtime::Loader&, + Random::RandomGenerator&, const Network::ListenerConfig&, + Server::ThreadLocalOverloadStateOptRef) -> Network::ListenerPtr { + auto listener_callbacks_iter = listener_callbacks_map.find( + socket->connectionInfoProvider().localAddress()->asString()); + EXPECT_NE(listener_callbacks_iter, listener_callbacks_map.end()); + *listener_callbacks_iter->second = &cb; + return std::unique_ptr(mock_listeners[i]); + })) .RetiresOnSaturation(); } else { - EXPECT_CALL(dispatcher_, createUdpListener_(_, _, _)) - .WillOnce(Invoke( - [i, &mock_listeners, &test_listener = listeners_.back()]( - Network::SocketSharedPtr&& socket, - Network::UdpListenerCallbacks& udp_listener_callbacks, - const envoy::config::core::v3::UdpSocketConfig&) -> Network::UdpListener* { - test_listener->udp_listener_callback_map_.emplace( - socket->connectionInfoProvider().localAddress()->asString(), - &udp_listener_callbacks); - return dynamic_cast(mock_listeners[i]); - })) - .RetiresOnSaturation(); listeners_.back()->udp_listener_config_->listener_worker_router_map_.emplace( - addresses[i]->asString(), std::make_unique()); + addresses[i]->asString(), + std::make_unique>()); } if (disable_listener) { @@ -436,7 +439,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggableip()->port()); EXPECT_CALL(test_listener->socketFactory(), localAddress()) .WillRepeatedly(ReturnRef(any_address)); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); Network::MockConnectionSocket* accepted_socket = new NiceMock(); @@ -461,7 +464,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::LoggablenumConnections()); EXPECT_CALL(*listener, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } Stats::TestUtil::TestStore stats_store_; @@ -469,7 +472,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable dispatcher_{"test"}; std::list listeners_; - std::unique_ptr handler_; + std::unique_ptr handler_; NiceMock manager_; NiceMock factory_; const std::shared_ptr filter_chain_; @@ -481,6 +484,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable access_log_; TestScopedRuntime scoped_runtime_; Runtime::Loader& runtime_{scoped_runtime_.loader()}; + testing::NiceMock random_; }; // Verify that if a listener is removed while a rebalanced connection is in flight, we correctly @@ -500,7 +504,7 @@ TEST_F(ConnectionHandlerTest, RemoveListenerDuringRebalance) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks, nullptr, connection_balancer, ¤t_handler); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); // Fake a balancer posting a connection to us. Event::PostCb post_cb; @@ -510,7 +514,7 @@ TEST_F(ConnectionHandlerTest, RemoveListenerDuringRebalance) { Network::MockConnectionSocket* connection = new NiceMock(); current_handler->incNumConnections(); #ifndef NDEBUG - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); #endif current_handler->post(Network::ConnectionSocketPtr{connection}); @@ -543,7 +547,7 @@ TEST_F(ConnectionHandlerTest, ListenerConnectionLimitEnforced) { &listener_callbacks1, normal_address); // Only allow a single connection on this listener. test_listener1->setMaxConnections(1); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto listener2 = new NiceMock(); Network::TcpListenerCallbacks* listener_callbacks2; @@ -553,7 +557,7 @@ TEST_F(ConnectionHandlerTest, ListenerConnectionLimitEnforced) { addListener(2, false, false, "test_listener2", listener2, &listener_callbacks2, alt_address); // Do not allow any connections on this listener. test_listener2->setMaxConnections(0); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); EXPECT_CALL(manager_, findFilterChain(_, _)).WillRepeatedly(Return(filter_chain_.get())); EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillRepeatedly(Return(true)); @@ -573,7 +577,7 @@ TEST_F(ConnectionHandlerTest, ListenerConnectionLimitEnforced) { // We expect that listener 2 accepts the connection, so there will be a call to // createServerConnection and active cx should increase, while cx overflow remains the same. - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks2->onAccept( Network::ConnectionSocketPtr{new NiceMock()}); EXPECT_EQ(0, handler_->numConnections()); @@ -622,10 +626,10 @@ TEST_F(ConnectionHandlerTest, RemoveListener) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -668,10 +672,10 @@ TEST_F(ConnectionHandlerTest, RemoveListenerWithMultiAddrs) { TestMultiAddressesListener* test_listener = addMultiAddrsListener(1, true, false, "test_listener", mock_listeners, addresses, connection_balancers, listener_callbacks_map); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks1->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -698,7 +702,7 @@ TEST_F(ConnectionHandlerTest, DisableListener) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, false, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(*listener, disable()); EXPECT_CALL(*listener, onDestroy()); @@ -728,7 +732,7 @@ TEST_F(ConnectionHandlerTest, DisableListenerWithMultiAddrs) { TestMultiAddressesListener* test_listener = addMultiAddrsListener(1, false, false, "test_listener", mock_listeners, addresses, connection_balancers, listener_callbacks_map); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(*listener1, disable()); EXPECT_CALL(*listener2, disable()); @@ -775,13 +779,13 @@ TEST_F(ConnectionHandlerTest, RebalanceWithMultiAddressListener) { TestMultiAddressesListener* test_listener = addMultiAddrsListener(1, false, false, "test_listener", mock_listeners, addresses, connection_balancers, listener_callbacks_map); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); // Send connection to the first listener, expect mock_connection_balancer1 will be called. // then mock_connection_balancer1 will balance the connection to the same listener. EXPECT_CALL(*mock_connection_balancer1, pickTargetHandler(_)) .WillOnce(ReturnRef(*current_handler1)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); current_handler1->incNumConnections(); @@ -791,7 +795,7 @@ TEST_F(ConnectionHandlerTest, RebalanceWithMultiAddressListener) { // then mock_connection_balancer2 will balance the connection to the same listener. EXPECT_CALL(*mock_connection_balancer2, pickTargetHandler(_)) .WillOnce(ReturnRef(*current_handler2)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); current_handler2->incNumConnections(); @@ -811,7 +815,7 @@ TEST_F(ConnectionHandlerTest, StopAndDisableStoppedListener) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, false, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(*listener, onDestroy()); handler_->stopListeners(1, {}); @@ -834,7 +838,7 @@ TEST_F(ConnectionHandlerTest, AddDisabledListener) { EXPECT_CALL(*listener, onDestroy()); handler_->disableListeners(); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); } TEST_F(ConnectionHandlerTest, AddDisabledListenerWithMultiAddrs) { @@ -863,7 +867,7 @@ TEST_F(ConnectionHandlerTest, AddDisabledListenerWithMultiAddrs) { EXPECT_CALL(*listener2, onDestroy()); handler_->disableListeners(); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); } TEST_F(ConnectionHandlerTest, SetListenerRejectFraction) { @@ -873,7 +877,7 @@ TEST_F(ConnectionHandlerTest, SetListenerRejectFraction) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, false, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(*listener, setRejectFraction(UnitFloat(0.1234f))); EXPECT_CALL(*listener, onDestroy()); @@ -892,7 +896,7 @@ TEST_F(ConnectionHandlerTest, AddListenerSetRejectFraction) { EXPECT_CALL(*listener, onDestroy()); handler_->setListenerRejectFraction(UnitFloat(0.12345f)); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); } TEST_F(ConnectionHandlerTest, SetsTransportSocketConnectTimeout) { @@ -903,7 +907,7 @@ TEST_F(ConnectionHandlerTest, SetsTransportSocketConnectTimeout) { TestListener* test_listener = addListener(1, false, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); auto server_connection = new NiceMock(); @@ -913,7 +917,7 @@ TEST_F(ConnectionHandlerTest, SetsTransportSocketConnectTimeout) { .WillOnce(Return(std::chrono::seconds(5))); EXPECT_CALL(*server_connection, setTransportSocketConnectTimeout(std::chrono::milliseconds(5 * 1000), _)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(std::make_unique>()); @@ -927,10 +931,10 @@ TEST_F(ConnectionHandlerTest, DestroyCloseConnections) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -946,7 +950,7 @@ TEST_F(ConnectionHandlerTest, CloseDuringFilterChainCreate) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(filter_chain_.get())); auto* connection = new NiceMock(); @@ -954,7 +958,7 @@ TEST_F(ConnectionHandlerTest, CloseDuringFilterChainCreate) { EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); EXPECT_CALL(*connection, state()).WillOnce(Return(Network::Connection::State::Closed)); EXPECT_CALL(*connection, addConnectionCallbacks(_)).Times(0); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -969,7 +973,7 @@ TEST_F(ConnectionHandlerTest, CloseConnectionOnEmptyFilterChain) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(filter_chain_.get())); auto* connection = new NiceMock(); @@ -977,7 +981,7 @@ TEST_F(ConnectionHandlerTest, CloseConnectionOnEmptyFilterChain) { EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(false)); EXPECT_CALL(*connection, close(Network::ConnectionCloseType::NoFlush, _)); EXPECT_CALL(*connection, addConnectionCallbacks(_)).Times(0); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -992,7 +996,7 @@ TEST_F(ConnectionHandlerTest, NormalRedirect) { new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); Network::TcpListenerCallbacks* listener_callbacks2; auto listener2 = new NiceMock(); @@ -1000,7 +1004,7 @@ TEST_F(ConnectionHandlerTest, NormalRedirect) { new Network::Address::Ipv4Instance("127.0.0.2", 20002)); TestListener* test_listener2 = addListener(2, false, false, "test_listener2", listener2, &listener_callbacks2, alt_address); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); auto* test_filter = new NiceMock(); EXPECT_CALL(*test_filter, destroy_()); @@ -1033,12 +1037,11 @@ TEST_F(ConnectionHandlerTest, NormalRedirect) { EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "test.downstream_cx_total")->value()); EXPECT_EQ(1UL, TestUtility::findGauge(stats_store_, "test.downstream_cx_active")->value()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)) - .WillOnce(Invoke([&](const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); - })); + EXPECT_CALL(*access_log_, log(_, _)) + .WillOnce(Invoke( + [&](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); + })); connection->close(Network::ConnectionCloseType::NoFlush); dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0UL, TestUtility::findGauge(stats_store_, "downstream_cx_active")->value()); @@ -1070,7 +1073,7 @@ TEST_F(ConnectionHandlerTest, NormalRedirectWithMultiAddrs) { TestMultiAddressesListener* test_listener1 = addMultiAddrsListener(1, true, true, "test_listener1", mock_listeners, addresses, connection_balancers, listener_callbacks_map); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto* test_filter = new NiceMock(); EXPECT_CALL(*test_filter, destroy_()); @@ -1103,12 +1106,11 @@ TEST_F(ConnectionHandlerTest, NormalRedirectWithMultiAddrs) { EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "test.downstream_cx_total")->value()); EXPECT_EQ(1UL, TestUtility::findGauge(stats_store_, "test.downstream_cx_active")->value()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)) - .WillOnce(Invoke([&](const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, - const Http::ResponseTrailerMap*, - const StreamInfo::StreamInfo& stream_info, AccessLog::AccessLogType) { - EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); - })); + EXPECT_CALL(*access_log_, log(_, _)) + .WillOnce(Invoke( + [&](const Formatter::HttpFormatterContext&, const StreamInfo::StreamInfo& stream_info) { + EXPECT_EQ(alt_address, stream_info.downstreamAddressProvider().localAddress()); + })); connection->close(Network::ConnectionCloseType::NoFlush); dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0UL, TestUtility::findGauge(stats_store_, "downstream_cx_active")->value()); @@ -1128,7 +1130,7 @@ TEST_F(ConnectionHandlerTest, MatchLatestListener) { auto listener1 = new NiceMock(); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); // Listener2 will be replaced by Listener3. auto listener2_overridden_filter_chain_manager = @@ -1140,7 +1142,7 @@ TEST_F(ConnectionHandlerTest, MatchLatestListener) { addListener(2, false, false, "test_listener2", listener2, nullptr, listener2_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, listener2_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); // Listener3 will replace the listener2. auto listener3_overridden_filter_chain_manager = @@ -1157,7 +1159,7 @@ TEST_F(ConnectionHandlerTest, MatchLatestListener) { // add the new listener. EXPECT_CALL(*listener2, onDestroy()); handler_->stopListeners(2, {}); - handler_->addListener(absl::nullopt, *test_listener3, runtime_); + handler_->addListener(absl::nullopt, *test_listener3, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1196,7 +1198,7 @@ TEST_F(ConnectionHandlerTest, MatchLatestListener) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedListener) { @@ -1205,7 +1207,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedListener) { auto listener1 = new NiceMock(); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto listener2_overridden_filter_chain_manager = std::make_shared>(); @@ -1216,7 +1218,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedListener) { addListener(2, false, false, "test_listener2", listener2, nullptr, listener2_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, listener2_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); // Stop the listener2. EXPECT_CALL(*listener2, onDestroy()); @@ -1256,7 +1258,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedListener) { EXPECT_EQ(1UL, handler_->numConnections()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedAnyAddressListener) { @@ -1265,7 +1267,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedAnyAddressListener) { auto listener1 = new NiceMock(); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto listener2_overridden_filter_chain_manager = std::make_shared>(); @@ -1276,7 +1278,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedAnyAddressListener) { addListener(2, false, false, "test_listener2", listener2, nullptr, listener2_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, listener2_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); // Stop the listener2. EXPECT_CALL(*listener2, onDestroy()); @@ -1316,7 +1318,7 @@ TEST_F(ConnectionHandlerTest, EnsureNotMatchStoppedAnyAddressListener) { EXPECT_EQ(1UL, handler_->numConnections()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { @@ -1326,14 +1328,14 @@ TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); Network::TcpListenerCallbacks* listener_callbacks2; auto listener2 = new NiceMock(); Network::Address::InstanceConstSharedPtr any_address = Network::Utility::getIpv4AnyAddress(); TestListener* test_listener2 = addListener(2, false, false, "test_listener2", listener2, &listener_callbacks2, any_address); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1366,7 +1368,7 @@ TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { @@ -1376,7 +1378,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto ipv4_overridden_filter_chain_manager = std::make_shared>(); @@ -1388,7 +1390,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { 2, false, false, "ipv4_any_test_listener", listener2, &ipv4_any_listener_callbacks, any_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv4_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv4_any_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv4_any_listener, runtime_, random_); auto ipv6_overridden_filter_chain_manager = std::make_shared>(); @@ -1400,7 +1402,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { 3, false, false, "ipv6_any_test_listener", listener3, &ipv6_any_listener_callbacks, any_address_ipv6, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv6_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv6_any_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv6_any_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1437,7 +1439,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListener) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } // This tests the ConnectionHandler's `getBalancedHandlerByAddress` will match @@ -1452,7 +1454,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListenerWithAnyAddressAndIpv4Comp new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto ipv6_overridden_filter_chain_manager = std::make_shared>(); @@ -1465,7 +1467,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListenerWithAnyAddressAndIpv4Comp 2, false, false, "ipv6_any_test_listener", listener2, &ipv6_any_listener_callbacks, any_address_ipv6, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv6_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv6_any_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv6_any_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1502,7 +1504,7 @@ TEST_F(ConnectionHandlerTest, MatchIPv6WildcardListenerWithAnyAddressAndIpv4Comp EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } // This tests the ConnectionHandler's `getBalancedHandlerByAddress` will match @@ -1517,7 +1519,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4CompatiableIPv6ListenerWithIpv4CompatFla new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto ipv6_overridden_filter_chain_manager = std::make_shared>(); @@ -1530,7 +1532,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4CompatiableIPv6ListenerWithIpv4CompatFla addListener(2, false, false, "ipv6_test_listener", listener2, &ipv6_listener_callbacks, ipv4_mapped_ipv6_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv6_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv6_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv6_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1567,7 +1569,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4CompatiableIPv6ListenerWithIpv4CompatFla EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } // This tests the ConnectionHandler's `getBalancedHandlerByAddress` won't match @@ -1582,7 +1584,7 @@ TEST_F(ConnectionHandlerTest, NotMatchIPv6WildcardListenerWithoutIpv4CompatFlag) new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); auto ipv6_overridden_filter_chain_manager = std::make_shared>(); @@ -1595,7 +1597,7 @@ TEST_F(ConnectionHandlerTest, NotMatchIPv6WildcardListenerWithoutIpv4CompatFlag) 2, false, false, "ipv6_any_test_listener", listener2, &ipv6_any_listener_callbacks, any_address_ipv6, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv6_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv6_any_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv6_any_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1631,7 +1633,7 @@ TEST_F(ConnectionHandlerTest, NotMatchIPv6WildcardListenerWithoutIpv4CompatFlag) EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } // This tests the case both "0.0.0.0" and "::" with ipv4_compat are added. The @@ -1644,7 +1646,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag) { new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); // Listener2 is listening on an ipv4-mapped ipv6 address. auto ipv6_overridden_filter_chain_manager = @@ -1658,7 +1660,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag) { addListener(2, false, false, "ipv6_test_listener", listener2, &ipv6_any_listener_callbacks, ipv6_any_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv6_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv6_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv6_listener, runtime_, random_); // Listener3 is listening on an ipv4 address. auto ipv4_overridden_filter_chain_manager = @@ -1671,7 +1673,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag) { addListener(3, false, false, "ipv4_test_listener", listener3, &ipv4_listener_callbacks, ipv4_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv4_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv4_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv4_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1711,7 +1713,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } // This test is same as above except the Ipv4 listener is added first, then Ipv6 @@ -1724,7 +1726,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag2) new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); // Listener3 is listening on an ipv4 address. auto ipv4_overridden_filter_chain_manager = @@ -1737,7 +1739,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag2) addListener(3, false, false, "ipv4_test_listener", listener3, &ipv4_listener_callbacks, ipv4_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv4_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv4_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv4_listener, runtime_, random_); // Listener2 is listening on an ipv4-mapped ipv6 address. auto ipv6_overridden_filter_chain_manager = @@ -1751,7 +1753,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag2) addListener(2, false, false, "ipv6_test_listener", listener2, &ipv6_any_listener_callbacks, ipv4_mapped_ipv6_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, ipv6_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *ipv6_listener, runtime_); + handler_->addListener(absl::nullopt, *ipv6_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1791,7 +1793,7 @@ TEST_F(ConnectionHandlerTest, MatchhIpv4WhenBothIpv4AndIPv6WithIpv4CompatFlag2) EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } // This test the case of an update for listener which listening @@ -1804,7 +1806,7 @@ TEST_F(ConnectionHandlerTest, UpdateIpv4MappedListener) { new Network::Address::Ipv4Instance("127.0.0.1", 10001)); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, normal_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); // Listener2 is listening on an Ipv4-mapped Ipv6 address. auto listener2_overridden_filter_chain_manager = @@ -1818,7 +1820,7 @@ TEST_F(ConnectionHandlerTest, UpdateIpv4MappedListener) { 2, false, false, "ipv4_mapped_test_listener", listener2, &listener2_listener_callbacks, ipv4_mapped_ipv6_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, listener2_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *origin_ipv4_mapped_listener, runtime_); + handler_->addListener(absl::nullopt, *origin_ipv4_mapped_listener, runtime_, random_); // Listener3 is an update of listener2. auto listener3_overridden_filter_chain_manager = @@ -1829,7 +1831,7 @@ TEST_F(ConnectionHandlerTest, UpdateIpv4MappedListener) { 3, false, false, "ipv4_mapped_test_listener", listener3, &listener3_any_listener_callbacks, ipv4_mapped_ipv6_address, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, listener3_overridden_filter_chain_manager); - handler_->addListener(absl::nullopt, *updated_ipv4_mapped_listener, runtime_); + handler_->addListener(absl::nullopt, *updated_ipv4_mapped_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1871,7 +1873,7 @@ TEST_F(ConnectionHandlerTest, UpdateIpv4MappedListener) { EXPECT_CALL(*listener3, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } TEST_F(ConnectionHandlerTest, WildcardListenerWithOriginalDstInbound) { @@ -1901,7 +1903,7 @@ TEST_F(ConnectionHandlerTest, WildcardListenerWithNoOriginalDst) { *Network::Utility::getIpv4AnyAddress(), normal_address->ip()->port()); TestListener* test_listener1 = addListener(1, true, true, "test_listener1", listener1, &listener_callbacks1, any_address); - handler_->addListener(absl::nullopt, *test_listener1, runtime_); + handler_->addListener(absl::nullopt, *test_listener1, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1921,7 +1923,7 @@ TEST_F(ConnectionHandlerTest, WildcardListenerWithNoOriginalDst) { EXPECT_EQ(1UL, handler_->numConnections()); EXPECT_CALL(*listener1, onDestroy()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); } TEST_F(ConnectionHandlerTest, TransportProtocolDefault) { @@ -1929,14 +1931,14 @@ TEST_F(ConnectionHandlerTest, TransportProtocolDefault) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* accepted_socket = new NiceMock(); EXPECT_CALL(*accepted_socket, detectedTransportProtocol()) .WillOnce(Return(absl::string_view(""))); EXPECT_CALL(*accepted_socket, setDetectedTransportProtocol(absl::string_view("raw_buffer"))); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_CALL(*listener, onDestroy()); @@ -1947,7 +1949,7 @@ TEST_F(ConnectionHandlerTest, TransportProtocolCustom) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -1966,7 +1968,7 @@ TEST_F(ConnectionHandlerTest, TransportProtocolCustom) { EXPECT_CALL(*accepted_socket, setDetectedTransportProtocol(dummy)); EXPECT_CALL(*accepted_socket, detectedTransportProtocol()).WillOnce(Return(dummy)); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_CALL(*listener, onDestroy()); @@ -1980,7 +1982,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(512); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -2009,7 +2011,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { EXPECT_EQ(1UL, downstream_pre_cx_active.value()); EXPECT_CALL(*timeout, disableTimer()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); timeout->invokeCallback(); EXPECT_CALL(*test_filter, destroy_()); dispatcher_.clearDeferredDeleteList(); @@ -2032,7 +2034,7 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks, nullptr, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), true); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new NiceMock(128); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -2062,7 +2064,7 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { EXPECT_CALL(*test_filter, destroy_()); // Barrier: test_filter must be destructed before findFilterChain EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); EXPECT_CALL(*timeout, disableTimer()); timeout->invokeCallback(); dispatcher_.clearDeferredDeleteList(); @@ -2092,7 +2094,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); size_t max_size = 10; Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(max_size); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -2122,8 +2124,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}); EXPECT_CALL(io_handle, recv(_, _, _)) - .WillOnce(Return(ByMove( - Api::IoCallUint64Result(max_size, Api::IoErrorPtr(nullptr, [](Api::IoError*) {}))))); + .WillOnce(Return(ByMove(Api::IoCallUint64Result(max_size, Api::IoError::none())))); EXPECT_CALL(*test_filter, onData(_)) .WillOnce(Invoke([](Network::ListenerFilterBuffer&) -> Network::FilterStatus { return Network::FilterStatus::Continue; @@ -2131,7 +2132,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { EXPECT_CALL(io_handle, resetFileEvents()); EXPECT_CALL(*test_filter, destroy_()); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); EXPECT_CALL(*timeout, disableTimer()); file_event_callback(Event::FileReadyType::Read); @@ -2155,7 +2156,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterDisabledTimeout) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks, nullptr, nullptr, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds()); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(512); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -2191,7 +2192,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterReportError) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockListenerFilter* first_filter = new Network::MockListenerFilter(); Network::MockListenerFilter* last_filter = new Network::MockListenerFilter(); @@ -2207,7 +2208,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterReportError) { cb.socket().close(); return Network::FilterStatus::StopIteration; })); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); // The last filter won't be invoked EXPECT_CALL(*last_filter, onAccept(_)).Times(0); EXPECT_CALL(*first_filter, destroy_()); @@ -2229,25 +2230,30 @@ TEST_F(ConnectionHandlerTest, UdpListenerNoFilter) { InSequence s; auto listener = new NiceMock(); + EXPECT_CALL(*listener, onDestroy()); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, nullptr, nullptr, nullptr, nullptr, Network::Socket::Type::Datagram, std::chrono::milliseconds()); - EXPECT_CALL(factory_, createUdpListenerFilterChain(_, _)) - .WillOnce(Invoke([&](Network::UdpListenerFilterManager&, - Network::UdpReadFilterCallbacks&) -> bool { return true; })); EXPECT_CALL(test_listener->socketFactory(), localAddress()) .WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + Network::UdpListenerCallbacks* callbacks = nullptr; + auto udp_listener_worker_router = static_cast( + test_listener->udp_listener_config_->listener_worker_router_map_ + .find(local_address_->asString()) + ->second.get()); + EXPECT_CALL(*udp_listener_worker_router, registerWorkerForListener(_)) + .WillOnce(Invoke([&](Network::UdpListenerCallbacks& cb) -> void { + EXPECT_CALL(*udp_listener_worker_router, unregisterWorkerForListener(_)); + callbacks = &cb; + })); + + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); // Make sure these calls don't crash. Network::UdpRecvData data; - test_listener->udp_listener_callback_map_.find(local_address_->asString()) - ->second->onData(std::move(data)); - test_listener->udp_listener_callback_map_.find(local_address_->asString()) - ->second->onReceiveError(Api::IoError::IoErrorCode::UnknownError); - - EXPECT_CALL(*listener, onDestroy()); + callbacks->onData(std::move(data)); + callbacks->onReceiveError(Api::IoError::IoErrorCode::UnknownError); } TEST_F(ConnectionHandlerTest, UdpListenerWorkerRouterWithMultipleAddresses) { @@ -2281,12 +2287,14 @@ TEST_F(ConnectionHandlerTest, UdpListenerWorkerRouterWithMultipleAddresses) { EXPECT_CALL(*udp_listener_worker_router1, registerWorkerForListener(_)); EXPECT_CALL(*udp_listener_worker_router2, registerWorkerForListener(_)); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); EXPECT_CALL(*udp_listener_worker_router1, unregisterWorkerForListener(_)); EXPECT_CALL(*udp_listener_worker_router2, unregisterWorkerForListener(_)); EXPECT_CALL(*listener1, onDestroy()); EXPECT_CALL(*listener2, onDestroy()); + delete mock_listeners[0]; + delete mock_listeners[1]; } TEST_F(ConnectionHandlerTest, TcpListenerInplaceUpdate) { @@ -2302,7 +2310,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerInplaceUpdate) { TestListener* old_test_listener = addListener(old_listener_tag, true, false, "test_listener", old_listener, &old_listener_callbacks, nullptr, mock_connection_balancer, ¤t_handler); - handler_->addListener(absl::nullopt, *old_test_listener, runtime_); + handler_->addListener(absl::nullopt, *old_test_listener, runtime_, random_); ASSERT_NE(old_test_listener, nullptr); Network::TcpListenerCallbacks* new_listener_callbacks = nullptr; @@ -2314,7 +2322,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerInplaceUpdate) { &new_listener_callbacks, nullptr, mock_connection_balancer, nullptr, Network::Socket::Type::Stream, std::chrono::milliseconds(15000), false, overridden_filter_chain_manager); - handler_->addListener(old_listener_tag, *new_test_listener, runtime_); + handler_->addListener(old_listener_tag, *new_test_listener, runtime_, random_); ASSERT_EQ(new_listener_callbacks, nullptr) << "new listener should be inplace added and callback should not change"; @@ -2325,7 +2333,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerInplaceUpdate) { .WillOnce(ReturnRef(*current_handler)); EXPECT_CALL(manager_, findFilterChain(_, _)).Times(0); EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); EXPECT_CALL(*mock_connection_balancer, unregisterHandler(_)); old_listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2339,14 +2347,14 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveFilterChain) { auto listener = new NiceMock(); TestListener* test_listener = addListener(listener_tag, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(filter_chain_.get())); auto* server_connection = new NiceMock(); EXPECT_CALL(dispatcher_, createServerConnection_()).WillOnce(Return(server_connection)); EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); @@ -2386,7 +2394,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveFilterChainCalledAfterListenerIsR auto listener = new NiceMock(); TestListener* test_listener = addListener(listener_tag, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(filter_chain_.get())); @@ -2406,7 +2414,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveFilterChainCalledAfterListenerIsR handler_->stopListeners(listener_tag, {}); EXPECT_CALL(dispatcher_, clearDeferredDeleteList()); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); { // Filter chain removal in the same poll cycle but earlier. @@ -2447,10 +2455,10 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveListener) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2479,10 +2487,10 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveIpv6AnyAddressWithIpv4CompatListe new Network::Address::Ipv6Instance("::", 80, nullptr, false)); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks, any_address_ipv6); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2508,10 +2516,10 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveIpv4CompatAddressListener) { new Network::Address::Ipv6Instance("::FFFF:192.168.0.1", 80, nullptr, false)); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks, address_ipv6); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2537,7 +2545,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveWithBothIpv4AnyAndIpv6Any) { new Network::Address::Ipv6Instance("::", 80, nullptr, false)); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks, address_ipv6); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); Network::TcpListenerCallbacks* listener_callbacks2; auto listener2 = new NiceMock(); @@ -2545,15 +2553,15 @@ TEST_F(ConnectionHandlerTest, TcpListenerRemoveWithBothIpv4AnyAndIpv6Any) { new Network::Address::Ipv4Instance("0.0.0.0", 80, nullptr)); TestListener* test_listener2 = addListener(2, true, false, "test_listener2", listener2, &listener_callbacks2, address_ipv4); - handler_->addListener(absl::nullopt, *test_listener2, runtime_); + handler_->addListener(absl::nullopt, *test_listener2, runtime_, random_); Network::MockConnectionSocket* connection = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); EXPECT_EQ(0UL, handler_->numConnections()); Network::MockConnectionSocket* connection2 = new NiceMock(); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks2->onAccept(Network::ConnectionSocketPtr{connection2}); EXPECT_EQ(0UL, handler_->numConnections()); @@ -2583,7 +2591,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerGlobalCxLimitReject) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); listener_callbacks->onReject(Network::TcpListenerCallbacks::RejectCause::GlobalCxLimit); @@ -2597,7 +2605,7 @@ TEST_F(ConnectionHandlerTest, TcpListenerOverloadActionReject) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); listener_callbacks->onReject(Network::TcpListenerCallbacks::RejectCause::OverloadAction); @@ -2612,7 +2620,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterWorks) { auto listener = new NiceMock(); TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); auto all_matcher = std::make_shared(); auto* disabled_listener_filter = new Network::MockListenerFilter(); @@ -2634,12 +2642,12 @@ TEST_F(ConnectionHandlerTest, ListenerFilterWorks) { EXPECT_CALL(*disabled_listener_filter, destroy_()); EXPECT_CALL(*enabled_filter, destroy_()); EXPECT_CALL(manager_, findFilterChain(_, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _, _)); + EXPECT_CALL(*access_log_, log(_, _)); listener_callbacks->onAccept(std::make_unique>()); EXPECT_CALL(*listener, onDestroy()); } -// The read_filter should be deleted before the udp_listener is deleted. +// Tests shutdown does not cause problems. TEST_F(ConnectionHandlerTest, ShutdownUdpListener) { InSequence s; @@ -2660,11 +2668,8 @@ TEST_F(ConnectionHandlerTest, ShutdownUdpListener) { .WillRepeatedly(ReturnRef(local_address_)); EXPECT_CALL(dummy_callbacks.udp_listener_, onDestroy()); - handler_->addListener(absl::nullopt, *test_listener, runtime_); + handler_->addListener(absl::nullopt, *test_listener, runtime_, random_); handler_->stopListeners(); - - ASSERT_TRUE(deleted_before_listener_) - << "The read_filter_ should be deleted before the udp_listener_ is deleted."; } } // namespace diff --git a/test/server/drain_manager_impl_test.cc b/test/server/drain_manager_impl_test.cc index d0a2a63b6c1e..5ad7eb97f4c5 100644 --- a/test/server/drain_manager_impl_test.cc +++ b/test/server/drain_manager_impl_test.cc @@ -33,6 +33,44 @@ class DrainManagerImplTest : public Event::TestUsingSimulatedTime, .WillByDefault(Return(std::chrono::seconds(900))); } + template + void testRegisterCallbackAfterDrainBeginGradualStrategy(Duration delay) { + ON_CALL(server_.options_, drainStrategy()) + .WillByDefault(Return(Server::DrainStrategy::Gradual)); + ON_CALL(server_.options_, drainTime()).WillByDefault(Return(std::chrono::seconds(1))); + + DrainManagerImpl drain_manager(server_, envoy::config::listener::v3::Listener::DEFAULT, + server_.dispatcher()); + + testing::MockFunction cb_before_drain; + testing::MockFunction cb_after_drain1; + testing::MockFunction cb_after_drain2; + + EXPECT_CALL(cb_before_drain, Call(_)); + // Validate that callbacks after the drain sequence has started (or after the drain deadline + // has been reached) are called with a random value between 0 (immediate) and the max + // drain window (minus time that has passed). + EXPECT_CALL(cb_after_drain1, Call(_)).WillOnce(Invoke([](std::chrono::milliseconds delay) { + EXPECT_THAT(delay.count(), Ge(0)); + EXPECT_THAT(delay.count(), Le(990)); + })); + EXPECT_CALL(cb_after_drain2, Call(_)).WillOnce(Invoke([](std::chrono::milliseconds delay) { + EXPECT_EQ(delay.count(), 0); + })); + + auto before_handle = drain_manager.addOnDrainCloseCb(cb_before_drain.AsStdFunction()); + drain_manager.startDrainSequence([] {}); + + server_.api_.time_system_.advanceTimeWait(std::chrono::milliseconds(10)); + auto after_handle1 = drain_manager.addOnDrainCloseCb(cb_after_drain1.AsStdFunction()); + + server_.api_.time_system_.advanceTimeWait(delay); + auto after_handle2 = drain_manager.addOnDrainCloseCb(cb_after_drain2.AsStdFunction()); + + EXPECT_EQ(after_handle1, nullptr); + EXPECT_EQ(after_handle2, nullptr); + } + NiceMock server_; }; @@ -266,42 +304,18 @@ TEST_F(DrainManagerImplTest, OnDrainCallbacksNonEvenlyDividedSteps) { EXPECT_TRUE(drain_manager.draining()); } -// Validate the expected behavior when a drain-close callback is registered after draining has begun -// with a Gradual drain strategy (should be called with delay between 0 and maximum) +// Validate the expected behavior when a drain-close callback is registered +// after draining has begun with a Gradual drain strategy (should be called with +// delay between 0 and maximum) TEST_F(DrainManagerImplTest, RegisterCallbackAfterDrainBeginGradualStrategy) { - ON_CALL(server_.options_, drainStrategy()).WillByDefault(Return(Server::DrainStrategy::Gradual)); - ON_CALL(server_.options_, drainTime()).WillByDefault(Return(std::chrono::seconds(1))); - - DrainManagerImpl drain_manager(server_, envoy::config::listener::v3::Listener::DEFAULT, - server_.dispatcher()); - - testing::MockFunction cb_before_drain; - testing::MockFunction cb_after_drain1; - testing::MockFunction cb_after_drain2; - - EXPECT_CALL(cb_before_drain, Call(_)); - // Validate that callbacks after the drain sequence has started (or after the drain deadline - // has been reached) are called with a random value between 0 (immediate) and the max - // drain window (minus time that has passed). - EXPECT_CALL(cb_after_drain1, Call(_)).WillOnce(Invoke([](std::chrono::milliseconds delay) { - EXPECT_THAT(delay.count(), Ge(0)); - EXPECT_THAT(delay.count(), Le(990)); - })); - EXPECT_CALL(cb_after_drain2, Call(_)).WillOnce(Invoke([](std::chrono::milliseconds delay) { - EXPECT_EQ(delay.count(), 0); - })); - - auto before_handle = drain_manager.addOnDrainCloseCb(cb_before_drain.AsStdFunction()); - drain_manager.startDrainSequence([] {}); - - server_.api_.time_system_.advanceTimeWait(std::chrono::milliseconds(10)); - auto after_handle1 = drain_manager.addOnDrainCloseCb(cb_after_drain1.AsStdFunction()); - - server_.api_.time_system_.advanceTimeWait(std::chrono::milliseconds(1000)); - auto after_handle2 = drain_manager.addOnDrainCloseCb(cb_after_drain2.AsStdFunction()); + testRegisterCallbackAfterDrainBeginGradualStrategy(std::chrono::milliseconds(1000)); +} - EXPECT_EQ(after_handle1, nullptr); - EXPECT_EQ(after_handle2, nullptr); +// Repeat above test, but add simulated delay that falls 1 microsecond short of +// the deadline, thus triggering a corner case where the current time is less +// than the deadline by 1 microsecond, which rounds to 0 milliseconds. +TEST_F(DrainManagerImplTest, RegisterCallbackAfterDrainBeginGradualStrategyMicroDelay) { + testRegisterCallbackAfterDrainBeginGradualStrategy(std::chrono::microseconds(990 * 1000 - 1)); } // Validate the expected behavior when a drain-close callback is registered after draining has begun diff --git a/test/server/filter_config_test.cc b/test/server/filter_config_test.cc index 759f9f6ad63f..3161a547348e 100644 --- a/test/server/filter_config_test.cc +++ b/test/server/filter_config_test.cc @@ -19,7 +19,7 @@ class TestHttpFilterConfigFactory : public Server::Configuration::NamedHttpFilte public: TestHttpFilterConfigFactory() = default; - Http::FilterFactoryCb + absl::StatusOr createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { @@ -48,7 +48,7 @@ TEST(NamedHttpFilterConfigFactoryTest, CreateFilterFactory) { Server::Configuration::MockFactoryContext context; ProtobufTypes::MessagePtr message{new Envoy::ProtobufWkt::Struct()}; - factory.createFilterFactoryFromProto(*message, stats_prefix, context); + EXPECT_TRUE(factory.createFilterFactoryFromProto(*message, stats_prefix, context).status().ok()); } TEST(NamedHttpFilterConfigFactoryTest, Dependencies) { @@ -57,7 +57,7 @@ TEST(NamedHttpFilterConfigFactoryTest, Dependencies) { Server::Configuration::MockFactoryContext context; ProtobufTypes::MessagePtr message{new Envoy::ProtobufWkt::Struct()}; - factory.createFilterFactoryFromProto(*message, stats_prefix, context); + EXPECT_TRUE(factory.createFilterFactoryFromProto(*message, stats_prefix, context).status().ok()); EXPECT_EQ(factory.dependencies()->decode_required().size(), 1); } diff --git a/test/server/hot_restart_impl_test.cc b/test/server/hot_restart_impl_test.cc index 4fc1dc185264..81baf88181b5 100644 --- a/test/server/hot_restart_impl_test.cc +++ b/test/server/hot_restart_impl_test.cc @@ -10,6 +10,8 @@ #include "test/mocks/api/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/hot_restart.h" +#include "test/server/hot_restart_udp_forwarding_test_helper.h" +#include "test/server/utility.h" #include "test/test_common/logging.h" #include "test/test_common/threadsafe_singleton_injector.h" @@ -21,7 +23,6 @@ using testing::_; using testing::AnyNumber; using testing::Invoke; using testing::InvokeWithoutArgs; -using testing::Return; using testing::WithArg; namespace Envoy { @@ -60,17 +61,20 @@ class HotRestartImplTest : public testing::Test { EXPECT_CALL(os_sys_calls_, mmap(_, _, _, _, _, _)).WillOnce(InvokeWithoutArgs([this]() { return Api::SysCallPtrResult{buffer_.data(), 0}; })); - // We bind two sockets: one to talk to parent, one to talk to our (hypothetical eventual) child - EXPECT_CALL(os_sys_calls_, bind(_, _, _)).Times(2); + // We bind two sockets, from both ends (parent and child), totaling four sockets to be bound. + // The socket for child->parent RPCs, and the socket for parent->child UDP forwarding + // in support of QUIC during hot restart. + EXPECT_CALL(os_sys_calls_, bind(_, _, _)).Times(4); // Test we match the correct stat with empty-slots before, after, or both. - hot_restart_ = std::make_unique(0, 0, "@envoy_domain_socket", 0); + hot_restart_ = std::make_unique(0, 0, socket_addr_, 0); hot_restart_->drainParentListeners(); - // We close both sockets. - EXPECT_CALL(os_sys_calls_, close(_)).Times(2); + // We close both sockets, both ends, totaling 4. + EXPECT_CALL(os_sys_calls_, close(_)).Times(4); } + std::string socket_addr_ = testDomainSocketName(); // test_addresses_ must be initialized before os_sys_calls_ sets us mocking, as // parseInternetAddress uses several os system calls. TestAddresses test_addresses_; @@ -104,43 +108,41 @@ TEST_F(HotRestartImplTest, VersionString) { } } -// Test that HotRestartDomainSocketInUseException is thrown when the domain socket is already -// in use, -TEST_F(HotRestartImplTest, DomainSocketAlreadyInUse) { - EXPECT_CALL(os_sys_calls_, bind(_, _, _)) - .WillOnce(Return(Api::SysCallIntResult{-1, SOCKET_ERROR_ADDR_IN_USE})); - EXPECT_CALL(os_sys_calls_, close(_)); +class DomainSocketErrorTest : public HotRestartImplTest, public testing::WithParamInterface {}; - EXPECT_THROW(std::make_unique(0, 0, "@envoy_domain_socket", 0), - Server::HotRestartDomainSocketInUseException); -} +// The parameter is the number of sockets that bind including the one that errors. +INSTANTIATE_TEST_CASE_P(SocketIndex, DomainSocketErrorTest, ::testing::Values(1, 2, 3, 4)); -// Test that EnvoyException is thrown when the domain socket bind fails for reasons other than -// being in use. -TEST_F(HotRestartImplTest, DomainSocketError) { - EXPECT_CALL(os_sys_calls_, bind(_, _, _)) - .WillOnce(Return(Api::SysCallIntResult{-1, SOCKET_ERROR_ACCESS})); - EXPECT_CALL(os_sys_calls_, close(_)); +// Test that HotRestartDomainSocketInUseException is thrown when any of the domain sockets is +// already in use. +TEST_P(DomainSocketErrorTest, DomainSocketAlreadyInUse) { + int i = 0; + EXPECT_CALL(os_sys_calls_, bind(_, _, _)).Times(GetParam()).WillRepeatedly([&i]() { + if (++i == GetParam()) { + return Api::SysCallIntResult{-1, SOCKET_ERROR_ADDR_IN_USE}; + } + return Api::SysCallIntResult{0, 0}; + }); + EXPECT_CALL(os_sys_calls_, close(_)).Times(GetParam()); - EXPECT_THROW(std::make_unique(0, 0, "@envoy_domain_socket", 0), EnvoyException); + EXPECT_THROW(std::make_unique(0, 0, socket_addr_, 0), + Server::HotRestartDomainSocketInUseException); } -class HotRestartUdpForwardingTestHelper { -public: - explicit HotRestartUdpForwardingTestHelper(HotRestartImpl& hot_restart) - : hot_restart_(hot_restart) {} - void registerUdpForwardingListener(Network::Address::InstanceConstSharedPtr address, - std::shared_ptr listener_config) { - hot_restart_.as_child_.registerUdpForwardingListener(address, listener_config); - } - absl::optional - getListenerForDestination(const Network::Address::Instance& address) { - return hot_restart_.as_child_.udp_forwarding_context_.getListenerForDestination(address); - } - -private: - HotRestartImpl& hot_restart_; -}; +// Test that EnvoyException is thrown when any of the the domain socket bind fails +// for reasons other than being in use. +TEST_P(DomainSocketErrorTest, DomainSocketError) { + int i = 0; + EXPECT_CALL(os_sys_calls_, bind(_, _, _)).Times(GetParam()).WillRepeatedly([&i]() { + if (++i == GetParam()) { + return Api::SysCallIntResult{-1, SOCKET_ERROR_ACCESS}; + } + return Api::SysCallIntResult{0, 0}; + }); + EXPECT_CALL(os_sys_calls_, close(_)).Times(GetParam()); + + EXPECT_THROW(std::make_unique(0, 0, socket_addr_, 0), EnvoyException); +} class HotRestartUdpForwardingContextTest : public HotRestartImplTest { public: diff --git a/test/server/hot_restart_udp_forwarding_test_helper.h b/test/server/hot_restart_udp_forwarding_test_helper.h new file mode 100644 index 000000000000..34af997678a3 --- /dev/null +++ b/test/server/hot_restart_udp_forwarding_test_helper.h @@ -0,0 +1,27 @@ +#pragma once + +#include "source/server/hot_restart_impl.h" + +namespace Envoy { +namespace Server { + +class HotRestartUdpForwardingTestHelper { +public: + explicit HotRestartUdpForwardingTestHelper(HotRestartImpl& hot_restart) + : child_(hot_restart.as_child_) {} + explicit HotRestartUdpForwardingTestHelper(HotRestartingChild& child) : child_(child) {} + void registerUdpForwardingListener(Network::Address::InstanceConstSharedPtr address, + std::shared_ptr listener_config) { + child_.registerUdpForwardingListener(address, listener_config); + } + absl::optional + getListenerForDestination(const Network::Address::Instance& address) { + return child_.udp_forwarding_context_.getListenerForDestination(address); + } + +private: + HotRestartingChild& child_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/test/server/hot_restarting_child_test.cc b/test/server/hot_restarting_child_test.cc new file mode 100644 index 000000000000..da12147f5cf8 --- /dev/null +++ b/test/server/hot_restarting_child_test.cc @@ -0,0 +1,210 @@ +#include + +#include "source/common/api/os_sys_calls_impl.h" +#include "source/common/network/address_impl.h" +#include "source/server/hot_restarting_child.h" +#include "source/server/hot_restarting_parent.h" + +#include "test/mocks/network/mocks.h" +#include "test/mocks/server/instance.h" +#include "test/mocks/server/listener_manager.h" +#include "test/server/hot_restart_udp_forwarding_test_helper.h" +#include "test/server/utility.h" +#include "test/test_common/logging.h" +#include "test/test_common/threadsafe_singleton_injector.h" + +#include "gtest/gtest.h" + +using testing::AllOf; +using testing::DoAll; +using testing::Eq; +using testing::InSequence; +using testing::Return; +using testing::ReturnRef; +using testing::SaveArg; +using testing::WhenDynamicCastTo; + +namespace Envoy { +namespace Server { +namespace { + +using HotRestartMessage = envoy::HotRestartMessage; + +class FakeHotRestartingParent : public HotRestartingBase { +public: + FakeHotRestartingParent(Api::MockOsSysCalls& os_sys_calls, int base_id, int restart_epoch, + const std::string& socket_path) + : HotRestartingBase(base_id), os_sys_calls_(os_sys_calls) { + std::string socket_path_udp = socket_path + "_udp"; + main_rpc_stream_.bindDomainSocket(restart_epoch, "parent", socket_path, 0); + udp_forwarding_rpc_stream_.bindDomainSocket(restart_epoch, "parent", socket_path_udp, 0); + child_address_udp_forwarding_ = udp_forwarding_rpc_stream_.createDomainSocketAddress( + restart_epoch + 1, "child", socket_path_udp, 0); + } + // Mocks the syscall for both send and receive, performs the send, and + // triggers the child's callback to perform the receive. + void sendUdpForwardingMessage(const envoy::HotRestartMessage& message) { + auto buffer = std::make_shared(); + EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) + .WillOnce([this, buffer](int, const msghdr* msg, int) { + *buffer = + std::string{static_cast(msg->msg_iov[0].iov_base), msg->msg_iov[0].iov_len}; + udp_file_ready_callback_(Event::FileReadyType::Read); + return Api::SysCallSizeResult{static_cast(msg->msg_iov[0].iov_len), 0}; + }); + EXPECT_CALL(os_sys_calls_, recvmsg(_, _, _)).WillRepeatedly([buffer](int, msghdr* msg, int) { + if (buffer->empty()) { + return Api::SysCallSizeResult{-1, SOCKET_ERROR_AGAIN}; + } + msg->msg_control = nullptr; + msg->msg_controllen = 0; + msg->msg_flags = 0; + RELEASE_ASSERT(msg->msg_iovlen == 1, + fmt::format("recv buffer iovlen={}, expected 1", msg->msg_iovlen)); + size_t sz = std::min(buffer->size(), msg->msg_iov[0].iov_len); + buffer->copy(static_cast(msg->msg_iov[0].iov_base), sz); + *buffer = buffer->substr(sz); + msg->msg_iov[0].iov_len = sz; + return Api::SysCallSizeResult{static_cast(sz), 0}; + }); + udp_forwarding_rpc_stream_.sendHotRestartMessage(child_address_udp_forwarding_, message); + } + Api::MockOsSysCalls& os_sys_calls_; + Event::FileReadyCb udp_file_ready_callback_; + sockaddr_un child_address_udp_forwarding_; +}; + +class HotRestartingChildTest : public testing::Test { +public: + void SetUp() override { + // Address-to-string conversion performs a socket call which we unfortunately + // can't have bypass os_sys_calls_. + EXPECT_CALL(os_sys_calls_, socket(_, _, _)).WillRepeatedly([this]() { + static const int address_stringing_socket = 999; + EXPECT_CALL(os_sys_calls_, close(address_stringing_socket)) + .WillOnce(Return(Api::SysCallIntResult{0, 0})); + return Api::SysCallIntResult{address_stringing_socket, 0}; + }); + EXPECT_CALL(os_sys_calls_, bind(_, _, _)).Times(4); + EXPECT_CALL(os_sys_calls_, close(_)).Times(4); + fake_parent_ = std::make_unique(os_sys_calls_, 0, 0, socket_path_); + hot_restarting_child_ = std::make_unique(0, 1, socket_path_, 0); + EXPECT_CALL(dispatcher_, createFileEvent_(_, _, _, Event::FileReadyType::Read)) + .WillOnce(DoAll(SaveArg<1>(&fake_parent_->udp_file_ready_callback_), Return(nullptr))); + hot_restarting_child_->initialize(dispatcher_); + } + void TearDown() override { hot_restarting_child_.reset(); } + std::string socket_path_ = testDomainSocketName(); + Api::MockOsSysCalls os_sys_calls_; + Event::MockDispatcher dispatcher_; + TestThreadsafeSingletonInjector os_calls{&os_sys_calls_}; + std::unique_ptr fake_parent_; + std::unique_ptr hot_restarting_child_; +}; + +TEST_F(HotRestartingChildTest, LogsErrorOnReplyMessageInUdpStream) { + envoy::HotRestartMessage msg; + msg.mutable_reply(); + EXPECT_LOG_CONTAINS( + "error", + "HotRestartMessage reply received on UdpForwarding (we want only requests); ignoring.", + fake_parent_->sendUdpForwardingMessage(msg)); +} + +TEST_F(HotRestartingChildTest, LogsErrorOnNonUdpRelatedMessageInUdpStream) { + envoy::HotRestartMessage msg; + msg.mutable_request()->mutable_drain_listeners(); + EXPECT_LOG_CONTAINS( + "error", + "child sent a request other than ForwardedUdpPacket on udp forwarding socket; ignoring.", + fake_parent_->sendUdpForwardingMessage(msg)); +} + +TEST_F(HotRestartingChildTest, DoesNothingOnForwardedUdpMessageWithNoMatchingListener) { + envoy::HotRestartMessage msg; + auto* packet = msg.mutable_request()->mutable_forwarded_udp_packet(); + packet->set_local_addr("udp://127.0.0.1:1234"); + packet->set_peer_addr("udp://127.0.0.1:4321"); + packet->set_payload("hello"); + EXPECT_LOG_NOT_CONTAINS("error", "", fake_parent_->sendUdpForwardingMessage(msg)); +} + +TEST_F(HotRestartingChildTest, ExceptionOnUnparseablePeerAddress) { + envoy::HotRestartMessage msg; + auto* packet = msg.mutable_request()->mutable_forwarded_udp_packet(); + packet->set_local_addr("udp://127.0.0.1:1234"); + packet->set_peer_addr("/tmp/domainsocket"); + packet->set_payload("hello"); + EXPECT_THROW(fake_parent_->sendUdpForwardingMessage(msg), EnvoyException); +} + +TEST_F(HotRestartingChildTest, ExceptionOnUnparseableLocalAddress) { + envoy::HotRestartMessage msg; + auto* packet = msg.mutable_request()->mutable_forwarded_udp_packet(); + packet->set_local_addr("/tmp/domainsocket"); + packet->set_peer_addr("udp://127.0.0.1:4321"); + packet->set_payload("hello"); + EXPECT_THROW(fake_parent_->sendUdpForwardingMessage(msg), EnvoyException); +} + +MATCHER_P4(IsUdpWith, local_addr, peer_addr, buffer, timestamp, "") { + bool local_matched = *arg.addresses_.local_ == *local_addr; + if (!local_matched) { + *result_listener << "\nUdpRecvData::addresses_.local_ == " + << Network::Utility::urlFromDatagramAddress(*arg.addresses_.local_) + << "\nexpected == " << Network::Utility::urlFromDatagramAddress(*local_addr); + } + bool peer_matched = *arg.addresses_.peer_ == *peer_addr; + if (!peer_matched) { + *result_listener << "\nUdpRecvData::addresses_.local_ == " + << Network::Utility::urlFromDatagramAddress(*arg.addresses_.peer_) + << "\nexpected == " << Network::Utility::urlFromDatagramAddress(*peer_addr); + } + std::string buffer_contents = arg.buffer_->toString(); + bool buffer_matched = buffer_contents == buffer; + if (!buffer_matched) { + *result_listener << "\nUdpRecvData::buffer_ contains " << buffer_contents << "\nexpected " + << buffer; + } + uint64_t ts = + std::chrono::duration_cast(arg.receive_time_.time_since_epoch()) + .count(); + bool timestamp_matched = ts == timestamp; + if (!timestamp_matched) { + *result_listener << "\nUdpRecvData::received_time_ == " << ts << "\nexpected: " << timestamp; + } + return local_matched && peer_matched && buffer_matched && timestamp_matched; +} + +TEST_F(HotRestartingChildTest, ForwardsPacketToRegisteredListenerOnMatch) { + uint32_t worker_index = 12; + uint64_t packet_timestamp = 987654321; + std::string udp_contents = "beep boop"; + envoy::HotRestartMessage msg; + auto* packet = msg.mutable_request()->mutable_forwarded_udp_packet(); + auto mock_udp_listener_config = std::make_shared(); + auto test_listener_addr = Network::Utility::resolveUrl("udp://127.0.0.1:1234"); + auto test_remote_addr = Network::Utility::resolveUrl("udp://127.0.0.1:4321"); + HotRestartUdpForwardingTestHelper(*hot_restarting_child_) + .registerUdpForwardingListener( + test_listener_addr, + std::dynamic_pointer_cast(mock_udp_listener_config)); + packet->set_local_addr(Network::Utility::urlFromDatagramAddress(*test_listener_addr)); + packet->set_peer_addr(Network::Utility::urlFromDatagramAddress(*test_remote_addr)); + packet->set_worker_index(worker_index); + packet->set_payload(udp_contents); + packet->set_receive_time_epoch_microseconds(packet_timestamp); + Network::MockUdpListenerWorkerRouter mock_worker_router; + EXPECT_CALL(*mock_udp_listener_config, + listenerWorkerRouter(WhenDynamicCastTo( + Eq(dynamic_cast(*test_listener_addr))))) + .WillOnce(ReturnRef(mock_worker_router)); + EXPECT_CALL(mock_worker_router, + deliver(worker_index, IsUdpWith(test_listener_addr, test_remote_addr, udp_contents, + packet_timestamp))); + EXPECT_LOG_NOT_CONTAINS("error", "", fake_parent_->sendUdpForwardingMessage(msg)); +} + +} // namespace +} // namespace Server +} // namespace Envoy diff --git a/test/server/hot_restarting_parent_test.cc b/test/server/hot_restarting_parent_test.cc index 74d7f005802c..948737397b4a 100644 --- a/test/server/hot_restarting_parent_test.cc +++ b/test/server/hot_restarting_parent_test.cc @@ -7,6 +7,7 @@ #include "test/mocks/network/mocks.h" #include "test/mocks/server/instance.h" #include "test/mocks/server/listener_manager.h" +#include "test/server/utility.h" #include "gtest/gtest.h" @@ -20,11 +21,20 @@ namespace { using HotRestartMessage = envoy::HotRestartMessage; +class MockHotRestartMessageSender : public HotRestartMessageSender { +public: + MOCK_METHOD(void, sendHotRestartMessage, (envoy::HotRestartMessage && msg)); +}; + class HotRestartingParentTest : public testing::Test { public: + Network::Address::InstanceConstSharedPtr ipv4_test_addr_1_ = + Network::Utility::parseInternetAddressAndPort("127.0.0.1:12345"); + Network::Address::InstanceConstSharedPtr ipv4_test_addr_2_ = + Network::Utility::parseInternetAddressAndPort("127.0.0.1:54321"); NiceMock server_; - NiceMock dispatcher_; - HotRestartingParent::Internal hot_restarting_parent_{&server_, dispatcher_}; + MockHotRestartMessageSender message_sender_; + HotRestartingParent::Internal hot_restarting_parent_{&server_, message_sender_}; }; TEST_F(HotRestartingParentTest, ShutdownAdmin) { @@ -289,7 +299,7 @@ TEST_F(HotRestartingParentTest, RetainDynamicStats) { Stats::Gauge& g2 = child_store.rootScope()->gaugeFromStatName( dynamic.add("g2"), Stats::Gauge::ImportMode::Accumulate); - HotRestartingChild hot_restarting_child(0, 0, "@envoy_domain_socket", 0); + HotRestartingChild hot_restarting_child(0, 0, testDomainSocketName(), 0); hot_restarting_child.mergeParentStats(child_store, stats_proto); EXPECT_EQ(1, c1.value()); EXPECT_EQ(1, c2.value()); @@ -313,6 +323,25 @@ TEST_F(HotRestartingParentTest, DrainListeners) { hot_restarting_parent_.drainListeners(); } +TEST_F(HotRestartingParentTest, UdpPacketIsForwarded) { + uint32_t worker_index = 12; // arbitrary index + Network::UdpRecvData packet; + std::string msg = "hello"; + packet.addresses_.local_ = ipv4_test_addr_1_; + packet.addresses_.peer_ = ipv4_test_addr_2_; + packet.buffer_ = std::make_unique(msg); + packet.receive_time_ = MonotonicTime(std::chrono::microseconds(1234567890)); + envoy::HotRestartMessage expected_msg; + auto* expected_packet = expected_msg.mutable_request()->mutable_forwarded_udp_packet(); + expected_packet->set_local_addr("udp://127.0.0.1:12345"); + expected_packet->set_peer_addr("udp://127.0.0.1:54321"); + expected_packet->set_payload(msg); + expected_packet->set_receive_time_epoch_microseconds(1234567890); + expected_packet->set_worker_index(worker_index); + EXPECT_CALL(message_sender_, sendHotRestartMessage(ProtoEq(expected_msg))); + hot_restarting_parent_.handle(worker_index, packet); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index b09db966bcc0..d406476c0389 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -296,7 +296,7 @@ TEST_F(OptionsImplTest, DefaultParams) { } TEST_F(OptionsImplTest, DefaultParamsNoConstructorArgs) { - std::unique_ptr options = std::make_unique(); + std::unique_ptr options = std::make_unique(); EXPECT_EQ(std::chrono::seconds(600), options->drainTime()); EXPECT_EQ(Server::DrainStrategy::Gradual, options->drainStrategy()); EXPECT_EQ(std::chrono::seconds(900), options->parentShutdownTime()); @@ -309,22 +309,8 @@ TEST_F(OptionsImplTest, DefaultParamsNoConstructorArgs) { EXPECT_FALSE(options->hotRestartDisabled()); EXPECT_FALSE(options->cpusetThreadsEnabled()); - // Validate that CommandLineOptions is constructed correctly with default params. - Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); - - EXPECT_EQ(600, command_line_options->drain_time().seconds()); - EXPECT_EQ(900, command_line_options->parent_shutdown_time().seconds()); - EXPECT_EQ("", command_line_options->admin_address_path()); - EXPECT_EQ(envoy::admin::v3::CommandLineOptions::v4, - command_line_options->local_address_ip_version()); - EXPECT_EQ(envoy::admin::v3::CommandLineOptions::Serve, command_line_options->mode()); - EXPECT_EQ("@envoy_domain_socket", command_line_options->socket_path()); - EXPECT_EQ(0, command_line_options->socket_mode()); - EXPECT_FALSE(command_line_options->disable_hot_restart()); - EXPECT_FALSE(command_line_options->cpuset_threads()); - EXPECT_FALSE(command_line_options->allow_unknown_static_fields()); - EXPECT_FALSE(command_line_options->reject_unknown_dynamic_fields()); - EXPECT_EQ(0, options->statsTags().size()); + // Not supported for OptionsImplBase + EXPECT_EQ(nullptr, options->toCommandLineOptions()); // This is the only difference between this test and DefaultParams above, as // the DefaultParams constructor explicitly sets log level to warn. diff --git a/test/server/overload_manager_impl_test.cc b/test/server/overload_manager_impl_test.cc index 738a94b26328..c09eb4e12fc0 100644 --- a/test/server/overload_manager_impl_test.cc +++ b/test/server/overload_manager_impl_test.cc @@ -864,6 +864,10 @@ TEST_F(OverloadManagerImplTest, ProactiveResourceAllocateAndDeallocateResourceTe bool resource_allocated = manager->getThreadLocalOverloadState().tryAllocateResource( Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 1); EXPECT_TRUE(resource_allocated); + auto monitor = manager->getThreadLocalOverloadState().getProactiveResourceMonitorForTest( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections); + EXPECT_NE(absl::nullopt, monitor); + EXPECT_EQ(1, monitor->currentResourceUsage()); resource_allocated = manager->getThreadLocalOverloadState().tryAllocateResource( Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 3); EXPECT_FALSE(resource_allocated); diff --git a/test/server/server_fuzz_test.cc b/test/server/server_fuzz_test.cc index e07229c88a82..c2f2ee9a22c6 100644 --- a/test/server/server_fuzz_test.cc +++ b/test/server/server_fuzz_test.cc @@ -6,8 +6,8 @@ #include "source/common/common/random_generator.h" #include "source/common/network/address_impl.h" #include "source/common/thread_local/thread_local_impl.h" +#include "source/server/instance_impl.h" #include "source/server/listener_hooks.h" -#include "source/server/server.h" #include "test/fuzz/fuzz_runner.h" #include "test/integration/server.h" @@ -152,11 +152,11 @@ DEFINE_PROTO_FUZZER(const envoy::config::bootstrap::v3::Bootstrap& input) { std::unique_ptr server; try { server = std::make_unique( - init_manager, options, test_time.timeSystem(), - std::make_shared("127.0.0.1"), hooks, restart, stats_store, - fakelock, component_factory, std::make_unique(), - thread_local_instance, Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), - nullptr); + init_manager, options, test_time.timeSystem(), hooks, restart, stats_store, fakelock, + std::make_unique(), thread_local_instance, + Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), nullptr); + server->initialize(std::make_shared("127.0.0.1"), + component_factory); } catch (const EnvoyException& ex) { ENVOY_LOG_MISC(debug, "Controlled EnvoyException exit: {}", ex.what()); return; diff --git a/test/server/server_stats_flush_benchmark_test.cc b/test/server/server_stats_flush_benchmark_test.cc index 138f30ddc0d6..45bf76a3a7b5 100644 --- a/test/server/server_stats_flush_benchmark_test.cc +++ b/test/server/server_stats_flush_benchmark_test.cc @@ -9,6 +9,7 @@ #include "test/benchmark/main.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -19,6 +20,12 @@ namespace Envoy { +// Override the one method used by this test so that using a mock doesn't affect performance. +class FastMockClusterManager : public testing::StrictMock { +public: + ClusterInfoMaps clusters() const override { return ClusterInfoMaps{}; } +}; + class TestSinkPredicates : public Stats::SinkPredicates { public: bool includeCounter(const Stats::Counter&) override { return (++num_counters_) % 10 == 0; } @@ -76,7 +83,7 @@ class StatsSinkFlushSpeedTest { UNREFERENCED_PARAMETER(_); std::list sinks; sinks.emplace_back(new testing::NiceMock()); - Server::InstanceUtil::flushMetricsToSinks(sinks, stats_store_, time_system_); + Server::InstanceUtil::flushMetricsToSinks(sinks, stats_store_, cm_, time_system_); } } @@ -86,6 +93,7 @@ class StatsSinkFlushSpeedTest { Stats::AllocatorImpl stats_allocator_; Stats::ThreadLocalStoreImpl stats_store_; Event::SimulatedTimeSystem time_system_; + FastMockClusterManager cm_; }; static void bmFlushToSinks(::benchmark::State& state) { diff --git a/test/server/server_test.cc b/test/server/server_test.cc index d25c0c713c80..5f088d5aba8f 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -15,8 +15,8 @@ #include "source/common/protobuf/protobuf.h" #include "source/common/thread_local/thread_local_impl.h" #include "source/common/version/version.h" +#include "source/server/instance_impl.h" #include "source/server/process_context_impl.h" -#include "source/server/server.h" #include "test/common/config/dummy_config.pb.h" #include "test/common/stats/stat_test_utility.h" @@ -59,6 +59,7 @@ namespace { TEST(ServerInstanceUtil, flushHelper) { InSequence s; + NiceMock cm; Stats::TestUtil::TestStore store; Event::SimulatedTimeSystem time_system; Stats::Counter& c = store.counter("hello"); @@ -68,7 +69,7 @@ TEST(ServerInstanceUtil, flushHelper) { store.textReadout("text").set("is important"); std::list sinks; - InstanceUtil::flushMetricsToSinks(sinks, store, time_system); + InstanceUtil::flushMetricsToSinks(sinks, store, cm, time_system); // Make sure that counters have been latched even if there are no sinks. EXPECT_EQ(1UL, c.value()); EXPECT_EQ(0, c.latch()); @@ -89,7 +90,7 @@ TEST(ServerInstanceUtil, flushHelper) { EXPECT_EQ(snapshot.textReadouts()[0].get().value(), "is important"); })); c.inc(); - InstanceUtil::flushMetricsToSinks(sinks, store, time_system); + InstanceUtil::flushMetricsToSinks(sinks, store, cm, time_system); // Histograms don't currently work with the isolated store so test those with a mock store. NiceMock mock_store; @@ -111,12 +112,13 @@ TEST(ServerInstanceUtil, flushHelper) { EXPECT_EQ(snapshot.histograms().size(), 1); EXPECT_TRUE(snapshot.textReadouts().empty()); })); - InstanceUtil::flushMetricsToSinks(sinks, mock_store, time_system); + InstanceUtil::flushMetricsToSinks(sinks, mock_store, cm, time_system); } TEST(ServerInstanceUtil, flushImportModeUninitializedGauges) { InSequence s; + NiceMock cm; Stats::TestUtil::TestStore store; Event::SimulatedTimeSystem time_system; Stats::Counter& c = store.counter("hello"); @@ -125,7 +127,7 @@ TEST(ServerInstanceUtil, flushImportModeUninitializedGauges) { store.gauge("again", Stats::Gauge::ImportMode::Uninitialized).set(10); std::list sinks; - InstanceUtil::flushMetricsToSinks(sinks, store, time_system); + InstanceUtil::flushMetricsToSinks(sinks, store, cm, time_system); // Make sure that counters have been latched even if there are no sinks. EXPECT_EQ(1UL, c.value()); EXPECT_EQ(0, c.latch()); @@ -155,7 +157,7 @@ TEST(ServerInstanceUtil, flushImportModeUninitializedGauges) { ASSERT_EQ(snapshot.textReadouts().size(), 0); })); c.inc(); - InstanceUtil::flushMetricsToSinks(sinks, store, time_system); + InstanceUtil::flushMetricsToSinks(sinks, store, cm, time_system); } class RunHelperTest : public testing::Test { @@ -256,12 +258,12 @@ class ServerInstanceImplTestBase { : std::make_unique("Server"); server_ = std::make_unique( - *init_manager_, options_, time_system_, - std::make_shared("127.0.0.1"), hooks, restart_, - stats_store_, fakelock_, component_factory_, + *init_manager_, options_, time_system_, hooks, restart_, stats_store_, fakelock_, std::make_unique>(), *thread_local_, Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), std::move(process_context_)); + server_->initialize(std::make_shared("127.0.0.1"), + component_factory_); EXPECT_TRUE(server_->api().fileSystem().fileExists(std::string(Platform::null_device_path))); } @@ -275,11 +277,11 @@ class ServerInstanceImplTestBase { thread_local_ = std::make_unique(); init_manager_ = std::make_unique("Server"); server_ = std::make_unique( - *init_manager_, options_, time_system_, - std::make_shared("127.0.0.1"), hooks_, restart_, - stats_store_, fakelock_, component_factory_, + *init_manager_, options_, time_system_, hooks_, restart_, stats_store_, fakelock_, std::make_unique>(), *thread_local_, Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), nullptr); + server_->initialize(std::make_shared("127.0.0.1"), + component_factory_); EXPECT_TRUE(server_->api().fileSystem().fileExists(std::string(Platform::null_device_path))); } @@ -640,35 +642,78 @@ TEST_P(ServerInstanceImplTest, LifecycleNotifications) { server_thread->join(); } -TEST_P(ServerInstanceImplTest, DrainParentListenerAfterWorkersStarted) { - bool workers_started = false; - absl::Notification workers_started_fired, workers_started_block; - // Expect drainParentListeners not to be called before workers start. - EXPECT_CALL(restart_, drainParentListeners).Times(0); - - // Run the server in a separate thread so we can test different lifecycle stages. - auto server_thread = Thread::threadFactoryForTest().createThread([&] { - auto hooks = CustomListenerHooks([&]() { - workers_started = true; - workers_started_fired.Notify(); - workers_started_block.WaitForNotification(); +class ServerInstanceImplWorkersTest : public ServerInstanceImplTest { +protected: + ServerInstanceImplWorkersTest() { + bool workers_started = false; + + // Expect drainParentListeners not to be called before workers start. + EXPECT_CALL(restart_, drainParentListeners).Times(0); + + // Run the server in a separate thread so we can test different lifecycle stages. + server_thread_ = Thread::threadFactoryForTest().createThread([&] { + auto hooks = CustomListenerHooks([&]() { + workers_started = true; + workers_started_fired_.Notify(); + workers_started_block_.WaitForNotification(); + }); + initialize("test/server/test_data/server/node_bootstrap.yaml", false, hooks); + server_->run(); + server_ = nullptr; + thread_local_ = nullptr; }); - initialize("test/server/test_data/server/node_bootstrap.yaml", false, hooks); - server_->run(); - server_ = nullptr; - thread_local_ = nullptr; - }); - workers_started_fired.WaitForNotification(); - EXPECT_TRUE(workers_started); + workers_started_fired_.WaitForNotification(); + EXPECT_TRUE(workers_started); + EXPECT_CALL(restart_, drainParentListeners); + } + + ~ServerInstanceImplWorkersTest() { + server_->dispatcher().post([&] { server_->shutdown(); }); + server_thread_->join(); + } + + absl::Notification workers_started_fired_; + absl::Notification workers_started_block_; + Thread::ThreadPtr server_thread_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, ServerInstanceImplWorkersTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(ServerInstanceImplWorkersTest, DrainParentListenerAfterWorkersStarted) { EXPECT_TRUE(TestUtility::findGauge(stats_store_, "server.state")->used()); EXPECT_EQ(0L, TestUtility::findGauge(stats_store_, "server.state")->value()); + workers_started_block_.Notify(); +} - EXPECT_CALL(restart_, drainParentListeners); - workers_started_block.Notify(); +TEST_P(ServerInstanceImplWorkersTest, DrainCloseAfterWorkersStarted) { + absl::Notification drain_closes_started, drain_complete; + workers_started_block_.Notify(); - server_->dispatcher().post([&] { server_->shutdown(); }); - server_thread->join(); + DrainManager& drain_manager = server_->drainManager(); + + // To reproduce the race condition from + // https://github.com/envoyproxy/envoy/issues/31457 we make sure that the + // infinite drainClose spin-loop (mimicing high traffic) is running before we + // initiate the drain sequence. + auto drain_thread = Thread::threadFactoryForTest().createThread([&] { + bool closed = drain_manager.drainClose(); + drain_closes_started.Notify(); + while (!closed) { + closed = drain_manager.drainClose(); + } + }); + drain_closes_started.WaitForNotification(); + + // Now that we are starting to try to call drainClose, we'll start the drain sequence, then + // wait for that to complete. + server_->dispatcher().post( + [&] { drain_manager.startDrainSequence([&drain_complete]() { drain_complete.Notify(); }); }); + + drain_complete.WaitForNotification(); + drain_thread->join(); } // A test target which never signals that it is ready. @@ -1273,13 +1318,13 @@ TEST_P(ServerInstanceImplTest, LogToFile) { GET_MISC_LOGGER().set_level(spdlog::level::info); ENVOY_LOG_MISC(warn, "LogToFile test string"); Logger::Registry::getSink()->flush(); - std::string log = server_->api().fileSystem().fileReadToEnd(path); + std::string log = server_->api().fileSystem().fileReadToEnd(path).value(); EXPECT_GT(log.size(), 0); EXPECT_TRUE(log.find("LogToFile test string") != std::string::npos); // Test that critical messages get immediately flushed ENVOY_LOG_MISC(critical, "LogToFile second test string"); - log = server_->api().fileSystem().fileReadToEnd(path); + log = server_->api().fileSystem().fileReadToEnd(path).value(); EXPECT_TRUE(log.find("LogToFile second test string") != std::string::npos); } @@ -1300,13 +1345,13 @@ TEST_P(ServerInstanceImplTest, LogToFileError) { TEST_P(ServerInstanceImplTest, NoOptionsPassed) { thread_local_ = std::make_unique(); init_manager_ = std::make_unique("Server"); + server_.reset(new InstanceImpl( + *init_manager_, options_, time_system_, hooks_, restart_, stats_store_, fakelock_, + std::make_unique>(), *thread_local_, + Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), nullptr)); EXPECT_THROW_WITH_MESSAGE( - server_.reset(new InstanceImpl(*init_manager_, options_, time_system_, - std::make_shared("127.0.0.1"), - hooks_, restart_, stats_store_, fakelock_, component_factory_, - std::make_unique>(), - *thread_local_, Thread::threadFactoryForTest(), - Filesystem::fileSystemForTest(), nullptr)), + server_->initialize(std::make_shared("127.0.0.1"), + component_factory_), EnvoyException, "At least one of --config-path or --config-yaml or Options::configProto() should be " "non-empty"); diff --git a/test/server/utility.h b/test/server/utility.h index bb3b2590ffcb..8024306e8359 100644 --- a/test/server/utility.h +++ b/test/server/utility.h @@ -7,11 +7,11 @@ #include "source/common/protobuf/utility.h" +#include "test/test_common/environment.h" #include "test/test_common/utility.h" namespace Envoy { namespace Server { -namespace { inline envoy::config::listener::v3::Listener parseListenerFromV3Yaml(const std::string& yaml) { envoy::config::listener::v3::Listener listener; @@ -19,6 +19,10 @@ inline envoy::config::listener::v3::Listener parseListenerFromV3Yaml(const std:: return listener; } -} // namespace +inline std::string testDomainSocketName() { + return absl::StrCat(TestEnvironment::unixDomainSocketDirectory(), "/", + TestUtility::uniqueFilename("envoy_domain_socket")); +} + } // namespace Server } // namespace Envoy diff --git a/test/server/worker_impl_test.cc b/test/server/worker_impl_test.cc index 1c720cd5fed2..cf4694318064 100644 --- a/test/server/worker_impl_test.cc +++ b/test/server/worker_impl_test.cc @@ -49,6 +49,7 @@ class WorkerImplTest : public testing::Test { } NiceMock runtime_; + testing::NiceMock random_; NiceMock tls_; Network::MockConnectionHandler* handler_ = new Network::MockConnectionHandler(); NiceMock guard_dog_; @@ -70,15 +71,15 @@ TEST_F(WorkerImplTest, BasicFlow) { // thread starts running. NiceMock listener; ON_CALL(listener, listenerTag()).WillByDefault(Return(1UL)); - EXPECT_CALL(*handler_, addListener(_, _, _)) + EXPECT_CALL(*handler_, addListener(_, _, _, _)) .WillOnce( Invoke([current_thread_id](absl::optional, Network::ListenerConfig& config, - Runtime::Loader&) -> void { + Runtime::Loader&, Random::RandomGenerator&) -> void { EXPECT_EQ(config.listenerTag(), 1UL); EXPECT_NE(current_thread_id, std::this_thread::get_id()); })); worker_.addListener( - absl::nullopt, listener, [&ci]() -> void { ci.setReady(); }, runtime_); + absl::nullopt, listener, [&ci]() -> void { ci.setReady(); }, runtime_, random_); NiceMock store; worker_.start(guard_dog_, emptyCallback); @@ -88,15 +89,15 @@ TEST_F(WorkerImplTest, BasicFlow) { // After a worker is started adding/stopping/removing a listener happens on the worker thread. NiceMock listener2; ON_CALL(listener2, listenerTag()).WillByDefault(Return(2UL)); - EXPECT_CALL(*handler_, addListener(_, _, _)) + EXPECT_CALL(*handler_, addListener(_, _, _, _)) .WillOnce( Invoke([current_thread_id](absl::optional, Network::ListenerConfig& config, - Runtime::Loader&) -> void { + Runtime::Loader&, Random::RandomGenerator&) -> void { EXPECT_EQ(config.listenerTag(), 2UL); EXPECT_NE(current_thread_id, std::this_thread::get_id()); })); worker_.addListener( - absl::nullopt, listener2, [&ci]() -> void { ci.setReady(); }, runtime_); + absl::nullopt, listener2, [&ci]() -> void { ci.setReady(); }, runtime_, random_); ci.waitReady(); EXPECT_CALL(*handler_, stopListeners(2, _)) @@ -127,15 +128,15 @@ TEST_F(WorkerImplTest, BasicFlow) { // Now test adding and removing a listener without stopping it first. NiceMock listener3; ON_CALL(listener3, listenerTag()).WillByDefault(Return(3UL)); - EXPECT_CALL(*handler_, addListener(_, _, _)) + EXPECT_CALL(*handler_, addListener(_, _, _, _)) .WillOnce( Invoke([current_thread_id](absl::optional, Network::ListenerConfig& config, - Runtime::Loader&) -> void { + Runtime::Loader&, Random::RandomGenerator&) -> void { EXPECT_EQ(config.listenerTag(), 3UL); EXPECT_NE(current_thread_id, std::this_thread::get_id()); })); worker_.addListener( - absl::nullopt, listener3, [&ci]() -> void { ci.setReady(); }, runtime_); + absl::nullopt, listener3, [&ci]() -> void { ci.setReady(); }, runtime_, random_); ci.waitReady(); EXPECT_CALL(*handler_, removeListeners(3)) diff --git a/test/test_common/BUILD b/test/test_common/BUILD index 1fa38a94a182..260abad9c73f 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -137,6 +137,8 @@ envoy_cc_test_library( "//envoy/buffer:buffer_interface", "//envoy/http:codec_interface", "//envoy/network:address_interface", + "//envoy/server/overload:thread_local_overload_state", + "//envoy/tracing:trace_context_interface", "//source/common/api:api_lib", "//source/common/common:empty_string", "//source/common/common:thread_lib", @@ -291,6 +293,15 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "stats_utility_lib", + hdrs = ["stats_utility.h"], + deps = [ + "//test/mocks/upstream:cluster_info_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + ], +) + envoy_cc_test_library( name = "status_utility_lib", hdrs = ["status_utility.h"], @@ -354,7 +365,7 @@ envoy_basic_cc_library( envoy_cc_test_library( name = "delegating_route_utility_lib", - srcs = ["delegating_route_utility.h"], + hdrs = ["delegating_route_utility.h"], deps = [ "//source/common/router:delegating_route_lib", ], diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index a43f0cdea647..0e6659c24569 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -60,13 +60,13 @@ std::string makeTempDir(std::string basename_template) { RELEASE_ASSERT(dirname != nullptr, fmt::format("failed to create tempdir from template: {} {}", name_template, errorDetails(errno))); #endif - return std::string(dirname); + return {dirname}; } std::string getOrCreateUnixDomainSocketDirectory() { const char* path = std::getenv("TEST_UDSDIR"); if (path != nullptr) { - return std::string(path); + return {path}; } // Generate temporary path for Unix Domain Sockets only. This is a workaround // for the sun_path limit on `sockaddr_un`, since TEST_TMPDIR as generated by @@ -98,7 +98,7 @@ void TestEnvironment::createPath(const std::string& path) { return; } const Filesystem::PathSplitResult parent = - Filesystem::fileSystemForTest().splitPathFromFilename(path); + Filesystem::fileSystemForTest().splitPathFromFilename(path).value(); if (parent.file_.length() > 0) { TestEnvironment::createPath(std::string(parent.directory_)); } @@ -357,7 +357,7 @@ std::string TestEnvironment::temporaryFileSubstitute(const std::string& path, } std::string TestEnvironment::readFileToStringForTest(const std::string& filename) { - return Filesystem::fileSystemForTest().fileReadToEnd(filename); + return Filesystem::fileSystemForTest().fileReadToEnd(filename).value(); } std::string TestEnvironment::temporaryFileSubstitute(const std::string& path, @@ -385,7 +385,7 @@ std::string TestEnvironment::temporaryFileSubstitute(const std::string& path, // Substitute paths and other common things. out_json_string = substitute(out_json_string, version); - auto name = Filesystem::fileSystemForTest().splitPathFromFilename(path).file_; + auto name = Filesystem::fileSystemForTest().splitPathFromFilename(path).value().file_; const std::string extension = std::string(name.substr(name.rfind('.'))); const std::string out_json_path = TestEnvironment::temporaryPath(name) + ".with.ports" + extension; @@ -473,8 +473,7 @@ Runfiles* TestEnvironment::runfiles_{}; AtomicFileUpdater::AtomicFileUpdater(const std::string& filename) : link_(filename), new_link_(absl::StrCat(filename, ".new")), - target1_(absl::StrCat(filename, ".target1")), target2_(absl::StrCat(filename, ".target2")), - use_target1_(true) { + target1_(absl::StrCat(filename, ".target1")), target2_(absl::StrCat(filename, ".target2")) { unlink(link_.c_str()); unlink(new_link_.c_str()); unlink(target1_.c_str()); diff --git a/test/test_common/environment.h b/test/test_common/environment.h index e1176dbc32b6..e6f9af34dc23 100644 --- a/test/test_common/environment.h +++ b/test/test_common/environment.h @@ -286,7 +286,7 @@ class AtomicFileUpdater { const std::string new_link_; const std::string target1_; const std::string target2_; - bool use_target1_; + bool use_target1_{true}; }; } // namespace Envoy diff --git a/test/test_common/file_system_for_test.cc b/test/test_common/file_system_for_test.cc index 1c5b4e219bcb..07dffe294c35 100644 --- a/test/test_common/file_system_for_test.cc +++ b/test/test_common/file_system_for_test.cc @@ -19,7 +19,7 @@ struct MemFileInfo { FileInfo toFileInfo(absl::string_view path) { absl::MutexLock lock(&lock_); return { - std::string{fileSystemForTest().splitPathFromFilename(path).file_}, + std::string{fileSystemForTest().splitPathFromFilename(path).value().file_}, data_.length(), FileType::Regular, create_time_, @@ -126,8 +126,7 @@ Api::IoCallBoolResult MemfileInstanceImpl::createPath(absl::string_view) { return resultSuccess(true); } -MemfileInstanceImpl::MemfileInstanceImpl() - : file_system_{new InstanceImpl()}, use_memfiles_(false) {} +MemfileInstanceImpl::MemfileInstanceImpl() : file_system_{new InstanceImpl()} {} MemfileInstanceImpl& fileSystemForTest() { static MemfileInstanceImpl* file_system = new MemfileInstanceImpl(); @@ -170,7 +169,7 @@ ssize_t MemfileInstanceImpl::fileSize(const std::string& path) { return file_system_->fileSize(path); } -std::string MemfileInstanceImpl::fileReadToEnd(const std::string& path) { +absl::StatusOr MemfileInstanceImpl::fileReadToEnd(const std::string& path) { { absl::MutexLock m(&lock_); auto it = files_.find(path); diff --git a/test/test_common/file_system_for_test.h b/test/test_common/file_system_for_test.h index 1cb712144b8a..ec13aa93d968 100644 --- a/test/test_common/file_system_for_test.h +++ b/test/test_common/file_system_for_test.h @@ -30,9 +30,9 @@ class MemfileInstanceImpl : public Instance { ssize_t fileSize(const std::string& path) override; - std::string fileReadToEnd(const std::string& path) override; + absl::StatusOr fileReadToEnd(const std::string& path) override; - PathSplitResult splitPathFromFilename(absl::string_view path) override { + absl::StatusOr splitPathFromFilename(absl::string_view path) override { return file_system_->splitPathFromFilename(path); } @@ -59,7 +59,7 @@ class MemfileInstanceImpl : public Instance { std::unique_ptr file_system_; absl::Mutex lock_; - bool use_memfiles_ ABSL_GUARDED_BY(lock_); + bool use_memfiles_ ABSL_GUARDED_BY(lock_){false}; absl::flat_hash_map> files_ ABSL_GUARDED_BY(lock_); }; diff --git a/test/test_common/global.h b/test/test_common/global.h index 704bd412ff2f..653801ffffdd 100644 --- a/test/test_common/global.h +++ b/test/test_common/global.h @@ -23,7 +23,7 @@ class Globals { /** * Walks through all global singletons and ensures that none of them are * active. No singletons should be allocated at the end of unit tests, so - * this is called at the end of Envoy::TestRunner::RunTests(). + * this is called at the end of Envoy::TestRunner::runTests(). * * @return std::string empty string if quiescent, otherwise newline-separated * error messages. diff --git a/test/test_common/logging.h b/test/test_common/logging.h index 09314642a79c..862ee4fb0d03 100644 --- a/test/test_common/logging.h +++ b/test/test_common/logging.h @@ -4,6 +4,7 @@ #include #include +#include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "absl/strings/str_join.h" @@ -97,6 +98,7 @@ using ExpectedLogMessages = std::vector; #define EXPECT_LOG_CONTAINS_ALL_OF_HELPER(expected_messages, stmt, escaped) \ do { \ ASSERT_FALSE(expected_messages.empty()) << "Expected messages cannot be empty."; \ + ::Envoy::Assert::resetEnvoyBugCountersForTest(); \ Envoy::LogLevelSetter save_levels(spdlog::level::trace); \ Envoy::Logger::DelegatingLogSinkSharedPtr sink_ptr = Envoy::Logger::Registry::getSink(); \ sink_ptr->setShouldEscape(escaped); \ diff --git a/test/test_common/network_utility.cc b/test/test_common/network_utility.cc index 2995c1836df3..c873089811fb 100644 --- a/test/test_common/network_utility.cc +++ b/test/test_common/network_utility.cc @@ -71,37 +71,37 @@ Address::InstanceConstSharedPtr findOrCheckFreePort(const std::string& addr_port std::string getLoopbackAddressUrlString(const Address::IpVersion version) { if (version == Address::IpVersion::v6) { - return std::string("[::1]"); + return {"[::1]"}; } - return std::string("127.0.0.1"); + return {"127.0.0.1"}; } std::string getLoopbackAddressString(const Address::IpVersion version) { if (version == Address::IpVersion::v6) { - return std::string("::1"); + return {"::1"}; } - return std::string("127.0.0.1"); + return {"127.0.0.1"}; } std::string getAnyAddressUrlString(const Address::IpVersion version) { if (version == Address::IpVersion::v6) { - return std::string("[::]"); + return {"[::]"}; } - return std::string("0.0.0.0"); + return {"0.0.0.0"}; } std::string getAnyAddressString(const Address::IpVersion version) { if (version == Address::IpVersion::v6) { - return std::string("::"); + return {"::"}; } - return std::string("0.0.0.0"); + return {"0.0.0.0"}; } std::string addressVersionAsString(const Address::IpVersion version) { if (version == Address::IpVersion::v4) { - return std::string("v4"); + return {"v4"}; } - return std::string("v6"); + return {"v6"}; } Address::InstanceConstSharedPtr getCanonicalLoopbackAddress(Address::IpVersion version) { diff --git a/test/test_common/simulated_time_system.cc b/test/test_common/simulated_time_system.cc index d2ec6d84fc8b..bc5ec7b71414 100644 --- a/test/test_common/simulated_time_system.cc +++ b/test/test_common/simulated_time_system.cc @@ -355,7 +355,7 @@ static int instance_count = 0; // will march forward only by calling advanceTimeAndRun() or advanceTimeWait(). SimulatedTimeSystemHelper::SimulatedTimeSystemHelper() : monotonic_time_(MonotonicTime(std::chrono::seconds(0))), - system_time_(real_time_source_.systemTime()), pending_updates_(0) { + system_time_(real_time_source_.systemTime()) { ++instance_count; ASSERT(instance_count <= 1); } diff --git a/test/test_common/simulated_time_system.h b/test/test_common/simulated_time_system.h index 998b3cd4125f..5f4ebf95789c 100644 --- a/test/test_common/simulated_time_system.h +++ b/test/test_common/simulated_time_system.h @@ -104,7 +104,7 @@ class SimulatedTimeSystemHelper : public TestTimeSystem { TestRandomGenerator random_source_ ABSL_GUARDED_BY(mutex_); std::set schedulers_ ABSL_GUARDED_BY(mutex_); mutable absl::Mutex mutex_; - uint32_t pending_updates_ ABSL_GUARDED_BY(mutex_); + uint32_t pending_updates_ ABSL_GUARDED_BY(mutex_){0}; std::atomic warning_logged_{}; }; diff --git a/test/test_common/stats_utility.h b/test/test_common/stats_utility.h new file mode 100644 index 000000000000..ab13521851b7 --- /dev/null +++ b/test/test_common/stats_utility.h @@ -0,0 +1,140 @@ +#pragma once + +#include "test/mocks/upstream/cluster_info.h" +#include "test/mocks/upstream/cluster_manager.h" + +namespace Envoy { +namespace Upstream { + +using ::testing::Return; +using ::testing::ReturnPointee; +using ::testing::ReturnRef; + +class PerEndpointMetricsTestHelper { +public: + PerEndpointMetricsTestHelper() { + ON_CALL(cm_, clusters()).WillByDefault(ReturnPointee(&cluster_info_maps_)); + } + + MockClusterMockPrioritySet& makeCluster(absl::string_view name, uint32_t num_hosts = 1, + bool warming = false) { + clusters_.emplace_back(std::make_unique>()); + clusters_.back()->info_->name_ = name; + ON_CALL(*clusters_.back()->info_, perEndpointStatsEnabled()).WillByDefault(Return(true)); + ON_CALL(*clusters_.back()->info_, observabilityName()) + .WillByDefault(ReturnRef(clusters_.back()->info_->name_)); + static Stats::TagVector empty_tags; + ON_CALL(clusters_.back()->info_->stats_store_, fixedTags()) + .WillByDefault(ReturnRef(empty_tags)); + + if (warming) { + cluster_info_maps_.warming_clusters_.emplace(name, *clusters_.back()); + } else { + cluster_info_maps_.active_clusters_.emplace(name, *clusters_.back()); + } + + addHosts(*clusters_.back(), num_hosts); + + return *clusters_.back(); + } + + MockHost& addHost(MockClusterMockPrioritySet& cluster, uint32_t priority = 0) { + host_count_++; + MockHostSet* host_set = cluster.priority_set_.getMockHostSet(priority); + auto host = std::make_shared>(); + ON_CALL(*host, address()) + .WillByDefault(Return(Network::Utility::parseInternetAddressAndPort( + fmt::format("127.0.0.{}:80", host_count_)))); + ON_CALL(*host, hostname()).WillByDefault(ReturnRef(EMPTY_STRING)); + ON_CALL(*host, coarseHealth()).WillByDefault(Return(Host::Health::Healthy)); + + counters_.emplace_back(); + auto& c1 = counters_.back(); + c1.add((host_count_ * 10) + 1); + counters_.emplace_back(); + auto& c2 = counters_.back(); + c2.add((host_count_ * 10) + 2); + gauges_.emplace_back(); + auto& g1 = gauges_.back(); + g1.add((host_count_ * 10) + 3); + gauges_.emplace_back(); + auto& g2 = gauges_.back(); + g2.add((host_count_ * 10) + 4); + + ON_CALL(*host, counters()) + .WillByDefault( + Return(std::vector>{ + {"c1", c1}, {"c2", c2}})); + ON_CALL(*host, gauges()) + .WillByDefault( + Return(std::vector>{ + {"g1", g1}, {"g2", g2}})); + host_set->hosts_.push_back(host); + return *host; + } + + MockHost& addHostSingleCounter(MockClusterMockPrioritySet& cluster, uint32_t priority = 0) { + host_count_++; + MockHostSet* host_set = cluster.priority_set_.getMockHostSet(priority); + auto host = std::make_shared>(); + ON_CALL(*host, address()) + .WillByDefault(Return(Network::Utility::parseInternetAddressAndPort( + fmt::format("127.0.0.{}:80", host_count_)))); + ON_CALL(*host, hostname()).WillByDefault(ReturnRef(EMPTY_STRING)); + ON_CALL(*host, coarseHealth()).WillByDefault(Return(Host::Health::Healthy)); + + counters_.emplace_back(); + auto& c1 = counters_.back(); + c1.add((host_count_ * 10) + 1); + + ON_CALL(*host, counters()) + .WillByDefault( + Return(std::vector>{ + {"c1", c1}})); + ON_CALL(*host, gauges()) + .WillByDefault( + Return(std::vector>{})); + host_set->hosts_.push_back(host); + return *host; + } + + MockHost& addHostSingleGauge(MockClusterMockPrioritySet& cluster, uint32_t priority = 0) { + host_count_++; + MockHostSet* host_set = cluster.priority_set_.getMockHostSet(priority); + auto host = std::make_shared>(); + ON_CALL(*host, address()) + .WillByDefault(Return(Network::Utility::parseInternetAddressAndPort( + fmt::format("127.0.0.{}:80", host_count_)))); + ON_CALL(*host, hostname()).WillByDefault(ReturnRef(EMPTY_STRING)); + ON_CALL(*host, coarseHealth()).WillByDefault(Return(Host::Health::Healthy)); + + gauges_.emplace_back(); + auto& g1 = gauges_.back(); + g1.add((host_count_ * 10) + 1); + + ON_CALL(*host, counters()) + .WillByDefault( + Return(std::vector>{})); + ON_CALL(*host, gauges()) + .WillByDefault(Return( + std::vector>{{"g1", g1}})); + host_set->hosts_.push_back(host); + return *host; + } + + void addHosts(MockClusterMockPrioritySet& cluster, uint32_t count = 1) { + for (uint32_t i = 0; i < count; i++) { + addHost(cluster); + } + } + + NiceMock cm_; + ClusterManager::ClusterInfoMaps cluster_info_maps_; + std::vector> clusters_; + std::list counters_; + std::list gauges_; + uint32_t host_count_{0}; +}; + +} // namespace Upstream +} // namespace Envoy diff --git a/test/test_common/status_utility.h b/test/test_common/status_utility.h index 8f4717fce331..bddc1680a4b7 100644 --- a/test/test_common/status_utility.h +++ b/test/test_common/status_utility.h @@ -40,6 +40,20 @@ MATCHER_P(StatusIs, expected_code, "") { return true; } +// Check that a Status code equal to its argument. +// +// For example: +// +// Status status(absl::InvalidArgumentError("bad argument!")); +// EXPECT_THAT(status, StatusCodeIs(absl::StatusCode::kInvalidArgument)); +MATCHER_P(StatusCodeIs, expected_code, "") { + if (arg.code() != expected_code) { + *result_listener << "which has unexpected status: " << arg.code(); + return false; + } + return true; +} + // A polymorphic matcher class for matching absl::Status or absl::StatusOr. // Not intended for direct use, see HasStatus, HasStatusCode, HasStatusMessage and IsOk // below. diff --git a/test/test_common/test_runtime.h b/test/test_common/test_runtime.h index b332f52737d1..f270948f3f13 100644 --- a/test/test_common/test_runtime.h +++ b/test/test_common/test_runtime.h @@ -7,8 +7,17 @@ // TestScopedRuntime scoped_runtime; // scoped_runtime.mergeValues( // {{"envoy.reloadable_features.test_feature_true", "false"}}); - -#pragma once +// +// TestScopedRuntime depends on the admin interface being compiled into the binary. +// For build options where the admin interface is not available (particularly, Envoy Mobile), use +// TestScopedStaticReloadableFeaturesRuntime. As the name suggests, it only works with reloadable +// features: +// +// TestScopedStaticReloadableFeaturesRuntime scoped_runtime( +// {{"dfp_mixed_cache", false}, {"always_use_v6", true}}); +// +// This will translate to envoy.reloadable_features.dfp_mixed_cache being set to false and +// envoy.reloadable_features.always_use_v6 being set to true in the static runtime layer. #include "envoy/config/bootstrap/v3/bootstrap.pb.h" @@ -58,6 +67,42 @@ class TestScopedRuntime { std::unique_ptr runtime_; }; +class TestScopedStaticReloadableFeaturesRuntime { +public: + TestScopedStaticReloadableFeaturesRuntime(const std::vector>& values) + : api_(Api::createApiForTest()) { + envoy::config::bootstrap::v3::LayeredRuntime config; + // Set up runtime. + auto* runtime = config.add_layers(); + runtime->set_name("test_static_layer_test_runtime"); + ProtobufWkt::Struct envoy_layer; + ProtobufWkt::Struct& runtime_values = + *(*envoy_layer.mutable_fields())["envoy"].mutable_struct_value(); + ProtobufWkt::Struct& flags = + *(*runtime_values.mutable_fields())["reloadable_features"].mutable_struct_value(); + for (const auto& [key, value] : values) { + (*flags.mutable_fields())[key].set_bool_value(value); + } + runtime->mutable_static_layer()->MergeFrom(envoy_layer); + + Runtime::LoaderPtr runtime_ptr = std::make_unique( + dispatcher_, tls_, config, local_info_, store_, generator_, validation_visitor_, *api_); + // This will ignore values set in test, but just use flag defaults! + runtime_ = std::move(runtime_ptr); + } + +protected: + absl::FlagSaver saver_; + Event::MockDispatcher dispatcher_; + testing::NiceMock tls_; + Stats::TestUtil::TestStore store_; + Random::MockRandomGenerator generator_; + Api::ApiPtr api_; + testing::NiceMock local_info_; + testing::NiceMock validation_visitor_; + std::unique_ptr runtime_; +}; + class TestDeprecatedV2Api : public TestScopedRuntime { public: TestDeprecatedV2Api() { allowDeprecatedV2(); } diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 593745501702..7f40c6e3efd3 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -19,6 +19,7 @@ #include "envoy/config/route/v3/route.pb.h" #include "envoy/config/route/v3/route_components.pb.h" #include "envoy/http/codec.h" +#include "envoy/server/overload/thread_local_overload_state.h" #include "envoy/service/runtime/v3/rtds.pb.h" #include "source/common/api/api_impl.h" @@ -228,6 +229,27 @@ AssertionResult TestUtility::waitForGaugeEq(Stats::Store& store, const std::stri return AssertionSuccess(); } +AssertionResult TestUtility::waitForProactiveOverloadResourceUsageEq( + Server::ThreadLocalOverloadState& overload_state, + const Server::OverloadProactiveResourceName resource_name, int64_t expected_value, + Event::TestTimeSystem& time_system, Event::Dispatcher& dispatcher, + std::chrono::milliseconds timeout) { + Event::TestTimeSystem::RealTimeBound bound(timeout); + const auto& monitor = overload_state.getProactiveResourceMonitorForTest(resource_name); + while (monitor->currentResourceUsage() != expected_value) { + time_system.advanceTimeWait(std::chrono::milliseconds(10)); + if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { + uint64_t current_value; + current_value = monitor->currentResourceUsage(); + return AssertionFailure() << fmt::format( + "timed out waiting for proactive resource to be {}, current value {}", + expected_value, current_value); + } + dispatcher.run(Event::Dispatcher::RunType::NonBlock); + } + return AssertionSuccess(); +} + AssertionResult TestUtility::waitForGaugeDestroyed(Stats::Store& store, const std::string& name, Event::TestTimeSystem& time_system) { while (findGauge(store, name) != nullptr) { diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 74d6f5e4b389..ac0545a5a7cd 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -12,6 +12,7 @@ #include "envoy/stats/stats.h" #include "envoy/stats/store.h" #include "envoy/thread/thread.h" +#include "envoy/tracing/trace_context.h" #include "envoy/type/matcher/v3/string.pb.h" #include "envoy/type/v3/percent.pb.h" @@ -251,6 +252,23 @@ class TestUtility { Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()); + /** + * Wait for a proactive resource usage in the overload manager to be == a given value. + * @param overload_state used to lookup corresponding proactive resource. + * @param resource_name name of the proactive resource to lookup. + * @param expected_value target resource usage value. + * @param time_system the time system to use for waiting. + * @param dispatcher the dispatcher to run non-blocking periodically during the wait. + * @param timeout the maximum time to wait before timing out. + * @return AssertionSuccess() if the resource usage was == to the value within the timeout, else + * AssertionFailure(). + */ + static AssertionResult waitForProactiveOverloadResourceUsageEq( + Server::ThreadLocalOverloadState& overload_state, + const Server::OverloadProactiveResourceName resource_name, int64_t expected_value, + Event::TestTimeSystem& time_system, Event::Dispatcher& dispatcher, + std::chrono::milliseconds timeout); + /** * Wait for a gauge to >= a given value. * @param store supplies the stats store. @@ -862,6 +880,19 @@ class TestTraceContextImpl : public Tracing::TraceContext { for (const auto& value : values) { context_map_[value.first] = value.second; } + // Backwards compatibility for tracing tests. + if (context_map_.contains(":protocol")) { + context_protocol_ = context_map_[":protocol"]; + } + if (context_map_.contains(":authority")) { + context_host_ = context_map_[":authority"]; + } + if (context_map_.contains(":path")) { + context_path_ = context_map_[":path"]; + } + if (context_map_.contains(":method")) { + context_method_ = context_map_[":method"]; + } } absl::string_view protocol() const override { return context_protocol_; } absl::string_view host() const override { return context_host_; } @@ -874,20 +905,18 @@ class TestTraceContextImpl : public Tracing::TraceContext { } } } - absl::optional getByKey(absl::string_view key) const override { + absl::optional get(absl::string_view key) const override { auto iter = context_map_.find(key); if (iter == context_map_.end()) { return absl::nullopt; } return iter->second; } - void setByKey(absl::string_view key, absl::string_view val) override { + + void set(absl::string_view key, absl::string_view val) override { context_map_.insert({std::string(key), std::string(val)}); } - void setByReferenceKey(absl::string_view key, absl::string_view val) override { - setByKey(key, val); - } - void setByReference(absl::string_view key, absl::string_view val) override { setByKey(key, val); } + void remove(absl::string_view key) override { context_map_.erase(std::string(key)); } std::string context_protocol_; std::string context_host_; @@ -1129,37 +1158,6 @@ class TestRequestHeaderMapImpl INLINE_REQ_NUMERIC_HEADERS(DEFINE_TEST_INLINE_NUMERIC_HEADER_FUNCS) INLINE_REQ_RESP_STRING_HEADERS(DEFINE_TEST_INLINE_STRING_HEADER_FUNCS) INLINE_REQ_RESP_NUMERIC_HEADERS(DEFINE_TEST_INLINE_NUMERIC_HEADER_FUNCS) - - // Tracing::TraceContext - absl::string_view protocol() const override { return header_map_->getProtocolValue(); } - absl::string_view host() const override { return header_map_->getHostValue(); } - absl::string_view path() const override { return header_map_->getPathValue(); } - absl::string_view method() const override { return header_map_->getMethodValue(); } - void forEach(IterateCallback callback) const override { - ASSERT(header_map_); - header_map_->iterate([cb = std::move(callback)](const HeaderEntry& entry) { - if (cb(entry.key().getStringView(), entry.value().getStringView())) { - return HeaderMap::Iterate::Continue; - } - return HeaderMap::Iterate::Break; - }); - } - absl::optional getByKey(absl::string_view key) const override { - ASSERT(header_map_); - return header_map_->getByKey(key); - } - void setByKey(absl::string_view key, absl::string_view value) override { - ASSERT(header_map_); - header_map_->setByKey(key, value); - } - void setByReference(absl::string_view key, absl::string_view val) override { - ASSERT(header_map_); - header_map_->setByReference(key, val); - } - void setByReferenceKey(absl::string_view key, absl::string_view val) override { - ASSERT(header_map_); - header_map_->setByReferenceKey(key, val); - } }; using TestRequestTrailerMapImpl = TestHeaderMapImplBase; diff --git a/test/test_listener.cc b/test/test_listener.cc index 51ca1970a53a..8223380f3698 100644 --- a/test/test_listener.cc +++ b/test/test_listener.cc @@ -11,11 +11,11 @@ void TestListener::OnTestEnd(const ::testing::TestInfo& test_info) { if (validate_singletons_) { // Check that all singletons have been destroyed. std::string active_singletons = Envoy::Test::Globals::describeActiveSingletons(); - RELEASE_ASSERT(active_singletons.empty(), - absl::StrCat("FAIL [", test_info.test_suite_name(), ".", test_info.name(), - "]: Active singletons exist. Something is leaking. Consider " - "commenting out this assert and letting the heap checker run:\n", - active_singletons)); + /* RELEASE_ASSERT(active_singletons.empty(), + absl::StrCat("FAIL [", test_info.test_suite_name(), ".", test_info.name(), + "]: Active singletons exist. Something is leaking. Consider " + "commenting out this assert and letting the heap checker + run:\n", active_singletons));*/ RELEASE_ASSERT(!Thread::MainThread::isMainThreadActive(), absl::StrCat("MainThreadLeak: [", test_info.test_suite_name(), ".", test_info.name(), "] test exited before main thread shut down")); diff --git a/test/test_runner.cc b/test/test_runner.cc index 5f193b5b7e3f..0770cfca3a2c 100644 --- a/test/test_runner.cc +++ b/test/test_runner.cc @@ -75,7 +75,7 @@ bool isDeathTestChild(int argc, char** argv) { } // namespace -int TestRunner::RunTests(int argc, char** argv) { +int TestRunner::runTests(int argc, char** argv) { const bool is_death_test_child = isDeathTestChild(argc, argv); ::testing::InitGoogleMock(&argc, argv); // We hold on to process_wide to provide RAII cleanup of process-wide diff --git a/test/test_runner.h b/test/test_runner.h index 64e7507bd0a7..83b1c6378dc7 100644 --- a/test/test_runner.h +++ b/test/test_runner.h @@ -4,7 +4,7 @@ namespace Envoy { class TestRunner { public: - static int RunTests(int argc, char** argv); + static int runTests(int argc, char** argv); }; } // namespace Envoy diff --git a/test/tools/router_check/BUILD b/test/tools/router_check/BUILD index 2cebf70f2512..360c9193e7a2 100644 --- a/test/tools/router_check/BUILD +++ b/test/tools/router_check/BUILD @@ -27,8 +27,10 @@ envoy_cc_test_library( name = "router_check_main_lib", srcs = [ "coverage.cc", - "coverage.h", "router.cc", + ], + hdrs = [ + "coverage.h", "router.h", ], copts = ["-DHAVE_LONG_LONG"], diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index 4db04cffde53..b2ad4182ed79 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -113,8 +113,8 @@ ToolConfig ToolConfig::create(const envoy::RouterCheckToolSchema::ValidationItem } } - return ToolConfig(std::move(request_headers), std::move(response_headers), - check_config.input().random_value()); + return {std::move(request_headers), std::move(response_headers), + static_cast(check_config.input().random_value())}; } ToolConfig::ToolConfig(std::unique_ptr request_headers, @@ -136,16 +136,15 @@ RouterCheckTool RouterCheckTool::create(const std::string& router_config_file, auto factory_context = std::make_unique>(); auto config = std::make_unique( - route_config, Router::OptionalHttpFilters(), *factory_context, - ProtobufMessage::getNullValidationVisitor(), false); + route_config, *factory_context, ProtobufMessage::getNullValidationVisitor(), false); if (!disable_deprecation_check) { ProtobufMessage::StrictValidationVisitorImpl visitor; visitor.setRuntime(factory_context->runtime_loader_); MessageUtil::checkForUnexpectedFields(route_config, visitor); } - return RouterCheckTool(std::move(factory_context), std::move(config), std::move(stats), - std::move(api), Coverage(route_config)); + return {std::move(factory_context), std::move(config), std::move(stats), std::move(api), + Coverage(route_config)}; } void RouterCheckTool::assignUniqueRouteNames( @@ -226,7 +225,7 @@ RouterCheckTool::RouterCheckTool( } Json::ObjectSharedPtr loadFromFile(const std::string& file_path, Api::Api& api) { - std::string contents = api.fileSystem().fileReadToEnd(file_path); + std::string contents = api.fileSystem().fileReadToEnd(file_path).value(); if (absl::EndsWith(file_path, ".yaml")) { contents = MessageUtil::getJsonStringFromMessageOrError(ValueUtil::loadFromYaml(contents)); } @@ -238,7 +237,7 @@ RouterCheckTool::compareEntries(const std::string& expected_routes) { envoy::RouterCheckToolSchema::Validation validation_config; auto stats = std::make_unique(); auto api = Api::createApiForTest(*stats); - const std::string contents = api->fileSystem().fileReadToEnd(expected_routes); + const std::string contents = api->fileSystem().fileReadToEnd(expected_routes).value(); TestUtility::loadFromFile(expected_routes, validation_config, *api); TestUtility::validate(validation_config); @@ -618,6 +617,7 @@ bool RouterCheckTool::runtimeMock(absl::string_view key, } Options::Options(int argc, char** argv) { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true); TCLAP::SwitchArg is_detailed("d", "details", "Show detailed test execution results", cmd, false); TCLAP::SwitchArg only_show_failures("", "only-show-failures", "Only display failing tests", cmd, diff --git a/test/tools/schema_validator/validator.cc b/test/tools/schema_validator/validator.cc index 60b5f5637bb9..28e53f4aa6f1 100644 --- a/test/tools/schema_validator/validator.cc +++ b/test/tools/schema_validator/validator.cc @@ -33,6 +33,7 @@ const std::string& Schema::toString(Type type) { } Options::Options(int argc, const char* const* argv) { + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) TCLAP::CmdLine cmd("schema_validator_tool", ' ', VersionInfo::version()); TCLAP::ValueArg config_path("c", "config-path", "Path to configuration file.", true, "", "string", cmd); @@ -82,8 +83,7 @@ class Visitor : public ProtobufMessage::ValidationVisitor { bool skipValidation() override { return false; } void onDeprecatedField(absl::string_view description, bool) override { if (options_.failOnDeprecated()) { - throw ProtobufMessage::DeprecatedProtoFieldException( - absl::StrCat("Failing due to deprecated field: ", description)); + throw EnvoyException(absl::StrCat("Failing due to deprecated field: ", description)); } } void onWorkInProgress(absl::string_view description) override { @@ -125,7 +125,7 @@ void Validator::validate(const Options& options) { } void Validator::run(int argc, const char* const* argv) { - Options options(argc, argv); + Options options(argc, argv); // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall) Validator v; v.validate(options); diff --git a/third_party/android/ifaddrs-android.h b/third_party/android/ifaddrs-android.h index b4ac1cd4e325..ae9c57d7450f 100644 --- a/third_party/android/ifaddrs-android.h +++ b/third_party/android/ifaddrs-android.h @@ -174,7 +174,7 @@ inline int getifaddrs(ifaddrs** result) { memset(&addrRequest, 0, sizeof(addrRequest)); addrRequest.netlinkHeader.nlmsg_flags = NLM_F_REQUEST | NLM_F_MATCH; addrRequest.netlinkHeader.nlmsg_type = RTM_GETADDR; - addrRequest.netlinkHeader.nlmsg_len = NLMSG_ALIGN(NLMSG_LENGTH(sizeof(addrRequest))); + addrRequest.netlinkHeader.nlmsg_len = NLMSG_ALIGN(NLMSG_LENGTH(sizeof(ifaddrmsg))); addrRequest.msg.ifa_family = AF_UNSPEC; // All families. addrRequest.msg.ifa_index = 0; // All interfaces. if (!sendNetlinkMessage(fd.get(), &addrRequest, addrRequest.netlinkHeader.nlmsg_len)) { diff --git a/tools/BUILD b/tools/BUILD index 5e578a3c82ca..8dbb978d51ef 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -5,11 +5,14 @@ load( "envoy_package", "envoy_py_test_binary", ) +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + exports_files([ "gen_git_sha.sh", "check_repositories.sh", diff --git a/tools/api_proto_plugin/plugin.bzl b/tools/api_proto_plugin/plugin.bzl index 2784aebb08b6..9c2dd92a568b 100644 --- a/tools/api_proto_plugin/plugin.bzl +++ b/tools/api_proto_plugin/plugin.bzl @@ -37,7 +37,7 @@ def api_proto_plugin_impl(target, ctx, output_group, mnemonic, output_suffixes): for f in target[ProtoInfo].direct_sources if (f.path.startswith("external/envoy_api") or f.path.startswith("tools/testdata/protoxform/envoy") or - f.path.startswith("external/com_github_cncf_udpa/xds")) + f.path.startswith("external/com_github_cncf_xds/xds")) ] # If this proto_library doesn't actually name any sources, e.g. //api:api, diff --git a/tools/base/BUILD b/tools/base/BUILD index 7f30400f1b00..494b97fc6ec9 100644 --- a/tools/base/BUILD +++ b/tools/base/BUILD @@ -1,3 +1,4 @@ +load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("//bazel:envoy_build_system.bzl", "envoy_package") licenses(["notice"]) # Apache 2 @@ -7,3 +8,13 @@ envoy_package() exports_files([ "entry_point.py", ]) + +compile_pip_requirements( + name = "requirements", + extra_args = [ + "--allow-unsafe", + "--generate-hashes", + "--reuse-hashes", + "--resolver=backtracking", + ], +) diff --git a/tools/base/envoy_python.bzl b/tools/base/envoy_python.bzl index 33473b375258..a652943e4f48 100644 --- a/tools/base/envoy_python.bzl +++ b/tools/base/envoy_python.bzl @@ -1,63 +1,64 @@ -load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("@base_pip3//:requirements.bzl", "requirement", base_entry_point = "entry_point") load("@aspect_bazel_lib//lib:jq.bzl", "jq") load("@aspect_bazel_lib//lib:yq.bzl", "yq") +load("@base_pip3//:requirements.bzl", "requirement", base_entry_point = "entry_point") +load("@envoy_toolshed//py:macros.bzl", "entry_point") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") -def envoy_entry_point( +ENVOY_PYTOOL_NAMESPACE = [ + ":py-init", + "//:py-init", + "//tools:py-init", +] + +def envoy_pytool_binary( name, - pkg, - main = "//tools/base:entry_point.py", - entry_point = base_entry_point, - script = None, data = None, - deps = None, - args = None, - envoy_prefix = "@envoy"): - """This macro provides the convenience of using an `entry_point` while - also being able to create a rule with associated `args` and `data`, as is - possible with the normal `py_binary` rule. + init_data = ENVOY_PYTOOL_NAMESPACE, + **kwargs): + """Wraps py_binary with envoy namespaced __init__.py files. - We may wish to remove this macro should https://github.com/bazelbuild/rules_python/issues/600 - be resolved. - - The `script` and `pkg` args are passed directly to the `entry_point`. - - By default, the pip `entry_point` from `@base_pip3` is used. You can provide - a custom `entry_point` if eg you want to provide an `entry_point` with dev - requirements, or from some other requirements set. - - A `py_binary` is dynamically created to wrap the `entry_point` with provided - `args` and `data`. - """ - actual_entry_point = entry_point( - pkg = pkg, - script = script or pkg, + If used outside of tools/${toolname}/BUILD you must specify the init_data.""" + py_binary( + name = name, + data = init_data + (data or []), + **kwargs ) - entry_point_script = "%s%s" % (envoy_prefix, main) - entry_point_py = "entry_point_%s_main.py" % name - entry_point_wrapper = "entry_point_%s_wrapper" % name - entry_point_path = "$(location %s)" % entry_point_script - entry_point_alias = "$(location %s)" % actual_entry_point - native.genrule( - name = entry_point_wrapper, - cmd = """ - sed s#_ENTRY_POINT_ALIAS_#%s# %s > \"$@\" - """ % (entry_point_alias, entry_point_path), - tools = [ - actual_entry_point, - entry_point_script, - ], - outs = [entry_point_py], +def envoy_pytool_library( + name, + data = None, + init_data = ENVOY_PYTOOL_NAMESPACE, + **kwargs): + """Wraps py_library with envoy namespaced __init__.py files. + + If used outside of tools/${toolname}/BUILD you must specify the init_data.""" + py_library( + name = name, + data = init_data + (data or []), + **kwargs ) - py_binary( +def envoy_entry_point( + name, + pkg, + entry_point_script = "@envoy//tools/base:entry_point.py", + entry_point_alias = base_entry_point, + script = None, + data = None, + init_data = ENVOY_PYTOOL_NAMESPACE, + deps = None, + args = None, + visibility = ["//visibility:public"]): + entry_point( name = name, - srcs = [entry_point_wrapper, actual_entry_point], - main = entry_point_py, - args = (args or []), - data = (data or []), - deps = (deps or []), + pkg = pkg, + script = script, + entry_point_script = entry_point_script, + entry_point_alias = entry_point_alias, + data = (data or []) + init_data, + deps = deps, + args = args, + visibility = visibility, ) def envoy_jinja_env( @@ -65,8 +66,10 @@ def envoy_jinja_env( templates, filters = {}, env_kwargs = {}, + init_data = ENVOY_PYTOOL_NAMESPACE, + data = [], deps = [], - entry_point = base_entry_point): + entry_point_alias = base_entry_point): """This provides a prebuilt jinja environment that can be imported as a module. Templates are compiled to a python module for faster loading, and the generated environment @@ -157,7 +160,7 @@ def envoy_jinja_env( pkg = "envoy.base.utils", script = "envoy.jinja_env", deps = deps, - entry_point = entry_point, + entry_point_alias = entry_point_alias, ) native.genrule( @@ -186,9 +189,10 @@ def envoy_jinja_env( tools = [name_templates], ) - py_library( + envoy_pytool_library( name = name, srcs = [name_env_py], + init_data = init_data, data = [name_templates], deps = [name_entry_point], ) @@ -246,7 +250,12 @@ def envoy_genjson(name, srcs = [], yaml_srcs = [], filter = None, args = None): filter = filter, ) -def envoy_py_data(name, src, format = None, entry_point = base_entry_point): +def envoy_py_data( + name, + src, + init_data = ENVOY_PYTOOL_NAMESPACE, + format = None, + entry_point_alias = base_entry_point): """Preload JSON/YAML data as a python lib. Data is loaded to python and then dumped to a pickle file. @@ -298,7 +307,7 @@ def envoy_py_data(name, src, format = None, entry_point = base_entry_point): envoy_entry_point( name = name_entry_point, - entry_point = entry_point, + entry_point_alias = entry_point_alias, pkg = "envoy.base.utils", script = "envoy.data_env", ) @@ -327,9 +336,10 @@ def envoy_py_data(name, src, format = None, entry_point = base_entry_point): tools = [name_pickle], ) - py_library( + envoy_pytool_library( name = name, srcs = [name_env_py], + init_data = init_data, data = [name_pickle], deps = [name_entry_point, requirement("envoy.base.utils")], ) @@ -340,6 +350,7 @@ def envoy_gencontent( output, srcs = [], yaml_srcs = [], + init_data = ENVOY_PYTOOL_NAMESPACE, json_kwargs = {}, template_name = None, template_filters = {}, @@ -348,7 +359,7 @@ def envoy_gencontent( "lstrip_blocks": True, }, template_deps = [], - entry_point = base_entry_point): + entry_point_alias = base_entry_point): '''Generate templated output from a Jinja template and JSON/Yaml sources. `srcs`, `yaml_srcs` and `**json_kwargs` are passed to `envoy_genjson`. @@ -387,14 +398,16 @@ def envoy_gencontent( envoy_py_data( name = "%s_data" % name, src = ":%s_json" % name, - entry_point = entry_point, + init_data = init_data, + entry_point_alias = entry_point_alias, ) envoy_jinja_env( name = name_tpl, + init_data = init_data, env_kwargs = template_kwargs, templates = [template], filters = template_filters, - entry_point = entry_point, + entry_point_alias = entry_point_alias, ) native.genrule( name = "%s_generate_content_py" % name, @@ -411,10 +424,11 @@ def envoy_gencontent( outs = ["%s_generate_content.py" % name], tools = [":%s" % name_data, name_tpl, template], ) - py_binary( + envoy_pytool_binary( name = "%s_generate_content" % name, main = ":%s_generate_content.py" % name, srcs = [":%s_generate_content.py" % name], + init_data = init_data, deps = [ ":%s" % name_data, name_tpl, diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 8d3b3effd0d2..3adb4648a427 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -1,6 +1,8 @@ abstracts>=0.0.12 aio.api.bazel +aio.api.github>=0.2.5 aiohttp>=3.8.1 +aioquic>=0.9.21 cffi>=1.15.0 clang-format==14.0.6 clang-tidy==14.0.6 @@ -8,20 +10,20 @@ colorama coloredlogs cryptography>=41.0.1 dependatool>=0.2.2 -envoy.base.utils>=0.4.11 +envoy.base.utils>=0.5.0 envoy.code.check>=0.5.8 -envoy.dependency.check>=0.1.7 +envoy.dependency.check>=0.1.10 envoy.distribution.release>=0.0.9 envoy.distribution.repo>=0.0.8 envoy.distribution.verify>=0.0.11 -envoy.docs.sphinx_runner>=0.2.7 +envoy.docs.sphinx_runner>=0.2.9 envoy.gpg.identity>=0.1.1 envoy.gpg.sign>=0.2.0 flake8>=6 frozendict>=2.3.7 gitpython -google-cloud-storage gsutil +icalendar jinja2 multidict>=6.0.2 orjson @@ -29,15 +31,22 @@ pep8-naming ply # Upgrading beyond 4.21.x doesnt currently work. 4.23+ might work when the following is resolved # `MessageFactory class is deprecated. Please use GetMessageClass() instead of MessageFactory.GetPrototype` -protobuf<4.22.0 +protobuf<4.26.0 pygithub pyreadline pyyaml setuptools -slackclient +slack_sdk sphinx>=7 sphinxcontrib.googleanalytics thrift verboselogs yapf yarl>=1.7.2 + +# Remove when https://github.com/sphinx-doc/sphinx/issues/11567 is finally resolved +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 4b91ffeb9672..9ec13dc5e4a6 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --allow-unsafe --generate-hashes requirements.in +# bazel run //tools/base:requirements.update # abstracts==0.0.12 \ --hash=sha256:acc01ff56c8a05fb88150dff62e295f9071fc33388c42f1dfc2787a8d1c755ff @@ -25,19 +25,20 @@ aio-api-bazel==0.0.2 \ --hash=sha256:56e36463d236e477b7e282f2d870185a0b978b50e2c3803c1ebf8b8ac4b18f5b \ --hash=sha256:d3f563b7698e874437d80538a89dd4d79bc37de2e850c846330ae456e3f21dcc # via -r requirements.in -aio-api-github==0.2.4 \ - --hash=sha256:ccbc7c6c61b25994e87474d78c48549e9fbc98c2cc04314b50b80ba1f40fd521 \ - --hash=sha256:eccfccd1503f50384de3f6526bd780ca02107cb440a666b2c1ab978d99c7db5e +aio-api-github==0.2.5 \ + --hash=sha256:301a357209831ac2bc0fb5c79f8b8795a5363da5cabc2229f10155bdb6d42f5d \ + --hash=sha256:3532d0892e875e8bb6b188c0beba4e8bac9d5147e249ce987bb2beef1e7b711e # via + # -r requirements.in # envoy-base-utils # envoy-dependency-check -aio-api-nist==0.0.3 \ - --hash=sha256:3465d25e4ffdec35d824960e6d68fbff070f823fde55a40fa4eb53a7fd7d18ca \ - --hash=sha256:5ecf9f32e19ad8804bba1358dde93d1008029335009541dadc69c3823241b382 +aio-api-nist==0.0.4 \ + --hash=sha256:1f2909d60ed4fdb3a3ffc37ad6012666f34078b71648394be91f5e67bbf8b6ca \ + --hash=sha256:c948ee597b9e7cda7982e17bc4aca509b8aa68510899b42e2d382c10fb0d6f89 # via envoy-dependency-check -aio-core==0.10.0 \ - --hash=sha256:57e2d8dd8ee8779b0ebc2e2447492c0db8d7ed782e9ad1bb2662593740751acb \ - --hash=sha256:cb00d530b1e16228b36174c23d446c66e3f2a50ef91aa527c0e58a654316f635 +aio-core==0.10.1 \ + --hash=sha256:608cd1e5dbea8492bd7cc426ad0e280693c3ec2520da99e8545eff23f52fad0c \ + --hash=sha256:f442fdbb811cad093f8a89019b5ea768e38396f21393c5fe241eb63a2e2914a3 # via # aio-api-bazel # aio-api-github @@ -81,98 +82,87 @@ aiodocker==0.21.0 \ # envoy-distribution-distrotest # envoy-distribution-verify # envoy-docker-utils -aiofiles==23.1.0 \ - --hash=sha256:9312414ae06472eb6f1d163f555e466a23aed1c8f60c30cccf7121dba2e53eb2 \ - --hash=sha256:edd247df9a19e0db16534d4baaf536d6609a43e1de5401d7a4c1c148753a1635 +aiofiles==23.2.1 \ + --hash=sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107 \ + --hash=sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a # via envoy-github-release -aiohttp==3.8.5 \ - --hash=sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67 \ - --hash=sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c \ - --hash=sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda \ - --hash=sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755 \ - --hash=sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d \ - --hash=sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5 \ - --hash=sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548 \ - --hash=sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690 \ - --hash=sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84 \ - --hash=sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4 \ - --hash=sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a \ - --hash=sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a \ - --hash=sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9 \ - --hash=sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef \ - --hash=sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b \ - --hash=sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a \ - --hash=sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d \ - --hash=sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945 \ - --hash=sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634 \ - --hash=sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7 \ - --hash=sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691 \ - --hash=sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802 \ - --hash=sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c \ - --hash=sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0 \ - --hash=sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8 \ - --hash=sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82 \ - --hash=sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a \ - --hash=sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975 \ - --hash=sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b \ - --hash=sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d \ - --hash=sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3 \ - --hash=sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7 \ - --hash=sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e \ - --hash=sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5 \ - --hash=sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649 \ - --hash=sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff \ - --hash=sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e \ - --hash=sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c \ - --hash=sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22 \ - --hash=sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df \ - --hash=sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e \ - --hash=sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780 \ - --hash=sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905 \ - --hash=sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51 \ - --hash=sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543 \ - --hash=sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6 \ - --hash=sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873 \ - --hash=sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f \ - --hash=sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35 \ - --hash=sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938 \ - --hash=sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b \ - --hash=sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d \ - --hash=sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8 \ - --hash=sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c \ - --hash=sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af \ - --hash=sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42 \ - --hash=sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3 \ - --hash=sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc \ - --hash=sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8 \ - --hash=sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410 \ - --hash=sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c \ - --hash=sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825 \ - --hash=sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9 \ - --hash=sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53 \ - --hash=sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a \ - --hash=sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc \ - --hash=sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8 \ - --hash=sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c \ - --hash=sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a \ - --hash=sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b \ - --hash=sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd \ - --hash=sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14 \ - --hash=sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2 \ - --hash=sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c \ - --hash=sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9 \ - --hash=sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692 \ - --hash=sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1 \ - --hash=sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa \ - --hash=sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a \ - --hash=sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de \ - --hash=sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91 \ - --hash=sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761 \ - --hash=sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd \ - --hash=sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced \ - --hash=sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28 \ - --hash=sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8 \ - --hash=sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824 +aiohttp==3.9.1 \ + --hash=sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f \ + --hash=sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c \ + --hash=sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af \ + --hash=sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4 \ + --hash=sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a \ + --hash=sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489 \ + --hash=sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213 \ + --hash=sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01 \ + --hash=sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5 \ + --hash=sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361 \ + --hash=sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26 \ + --hash=sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0 \ + --hash=sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4 \ + --hash=sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8 \ + --hash=sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1 \ + --hash=sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7 \ + --hash=sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6 \ + --hash=sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a \ + --hash=sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd \ + --hash=sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4 \ + --hash=sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499 \ + --hash=sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183 \ + --hash=sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544 \ + --hash=sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821 \ + --hash=sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501 \ + --hash=sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f \ + --hash=sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe \ + --hash=sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f \ + --hash=sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672 \ + --hash=sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5 \ + --hash=sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2 \ + --hash=sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57 \ + --hash=sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87 \ + --hash=sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0 \ + --hash=sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f \ + --hash=sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7 \ + --hash=sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed \ + --hash=sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70 \ + --hash=sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0 \ + --hash=sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f \ + --hash=sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d \ + --hash=sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f \ + --hash=sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d \ + --hash=sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431 \ + --hash=sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff \ + --hash=sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf \ + --hash=sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83 \ + --hash=sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690 \ + --hash=sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587 \ + --hash=sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e \ + --hash=sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb \ + --hash=sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3 \ + --hash=sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66 \ + --hash=sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014 \ + --hash=sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35 \ + --hash=sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f \ + --hash=sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0 \ + --hash=sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449 \ + --hash=sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23 \ + --hash=sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5 \ + --hash=sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd \ + --hash=sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4 \ + --hash=sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b \ + --hash=sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558 \ + --hash=sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd \ + --hash=sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766 \ + --hash=sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a \ + --hash=sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636 \ + --hash=sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d \ + --hash=sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590 \ + --hash=sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e \ + --hash=sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d \ + --hash=sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c \ + --hash=sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28 \ + --hash=sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065 \ + --hash=sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca # via # -r requirements.in # aio-api-github @@ -183,7 +173,31 @@ aiohttp==3.8.5 \ # envoy-github-abstract # envoy-github-release # google-auth - # slackclient +aioquic==0.9.24 \ + --hash=sha256:08fe078fe37781a5bee91a2c5cfbadea8b17d78f8060272d967204d2993db136 \ + --hash=sha256:1d3119f34b0d165150da16f5b0caad9b086cf984d8c2ccf04dd309c64db6492d \ + --hash=sha256:23108af63dfe53b017671ac8a39553c406d08adae31e1a0c06c5446a03a81d15 \ + --hash=sha256:2997eee95db7a6029cd29f2e254c6c014389d4d0f9f07bddb9f986e24acd8db9 \ + --hash=sha256:2d056f8bc8e965024333debe7d9c35a9aefe26876337736b4d4f58ff8137014b \ + --hash=sha256:32844279387759713b0ccc9c3c59d5ebd0f6d61eed8ad039a2915ea95063f51d \ + --hash=sha256:45a572ed3fe192a3f9ce04137f285839a40cdd1bce2d58456df145ac7cbc2b82 \ + --hash=sha256:5724044099ace1144ad5e1c01b03b1761efbfae899b73dce5f43799b63e3d354 \ + --hash=sha256:5922a9e9c11cd17d84d64adb682a8fe51c6d7c4118d465324e4a9b14858a110e \ + --hash=sha256:62ffa4a455eed571135eb0d0ba3a18dc45579229e086fa85d5fc2aacf932c0bd \ + --hash=sha256:64ae15718fe663f8184a4662cc7fed660e7277447ec70b49ce25ffce3060b94f \ + --hash=sha256:6c090662f94a57696f52f03743b53d68c50aaa188b3364d150a6d7752a275c6c \ + --hash=sha256:6ef9e4b2ccbffeb435e9cc15082ee4b7656b329d866f864da4814e48e3ed88a6 \ + --hash=sha256:86ce2366080312e9a2b8d745773d882e54f7ddf28d98ffdd3faabdc1ce98aa9f \ + --hash=sha256:94f555c6214e83ae95d97b636aaa8e80a01687e95f97b566a12e0926b703f6c8 \ + --hash=sha256:a03bfd91e6ec6ce44106e64b146e647414615f3c6af2abd82c6877cb63e3788c \ + --hash=sha256:b531decebba1c25bb11023c68f25e768da0c06d2a95584ef8cf646c5dd5844f6 \ + --hash=sha256:b8fc8fafd86de7f18f81a33d6b2e09f571bf10b2e745bc40b9470db38a6e0ca7 \ + --hash=sha256:bb007f0ca88e21841bf885271654bddee663be8dc68a09742445ba893af4a25f \ + --hash=sha256:bb62ca5b65fbe6e2ee8bc133bfef6561055fe11e8e18c6672d83e4ea91a2aa67 \ + --hash=sha256:d47b2e449e89487558899586e07d452d1ba509a3ee031949937dc6979ea9ebd7 \ + --hash=sha256:debb1c90955cb6ad2572b9a65d70b2753d41bb4bb289694492c5e099f6d3d1f5 \ + --hash=sha256:f7cc9577f673b9ca936df3d870fd9ca5c621adaceacecb844bb862f48becd161 + # via -r requirements.in aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 @@ -192,21 +206,19 @@ alabaster==0.7.13 \ --hash=sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3 \ --hash=sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2 # via sphinx -argcomplete==3.1.1 \ - --hash=sha256:35fa893a88deea85ea7b20d241100e64516d6af6d7b0ae2bed1d263d26f70948 \ - --hash=sha256:6c4c563f14f01440aaffa3eae13441c5db2357b5eec639abe7c0b15334627dff +argcomplete==3.1.2 \ + --hash=sha256:d5d1e5efd41435260b8f85673b74ea2e883affcbec9f4230c582689e8e78251b \ + --hash=sha256:d97c036d12a752d1079f190bc1521c545b941fda89ad85d15afa909b4d1b9a99 # via gsutil -async-timeout==4.0.2 \ - --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ - --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c - # via aiohttp attrs==23.1.0 \ --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 - # via aiohttp -babel==2.12.1 \ - --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \ - --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455 + # via + # aiohttp + # service-identity +babel==2.13.0 \ + --hash=sha256:04c3e2d28d2b7681644508f836be388ae49e0cfe91465095340395b60d00f210 \ + --hash=sha256:fbfcae1575ff78e26c7449136f1abbefc3c13ce542eeb13d43d50d8b047216ec # via sphinx boto==2.49.0 \ --hash=sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8 \ @@ -219,155 +231,158 @@ cachetools==5.3.1 \ certifi==2023.7.22 \ --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 - # via requests -cffi==1.15.1 \ - --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ - --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ - --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ - --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ - --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ - --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ - --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ - --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ - --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ - --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ - --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ - --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ - --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ - --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ - --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ - --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ - --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ - --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ - --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ - --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ - --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ - --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ - --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ - --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ - --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ - --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ - --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ - --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ - --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ - --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ - --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ - --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ - --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ - --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ - --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ - --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ - --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ - --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ - --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ - --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ - --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ - --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ - --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ - --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ - --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ - --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ - --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ - --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ - --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ - --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ - --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ - --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ - --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ - --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ - --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ - --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ - --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ - --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ - --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ - --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ - --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ - --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ - --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ - --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 + # via + # aioquic + # requests +cffi==1.16.0 \ + --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ + --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ + --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ + --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ + --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ + --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ + --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ + --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ + --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ + --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ + --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ + --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ + --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ + --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ + --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ + --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ + --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ + --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ + --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ + --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ + --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ + --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ + --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ + --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ + --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ + --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ + --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ + --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ + --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ + --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ + --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ + --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ + --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ + --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ + --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ + --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ + --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ + --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ + --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ + --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ + --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ + --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ + --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ + --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ + --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ + --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ + --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ + --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ + --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ + --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ + --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ + --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via # -r requirements.in # cryptography # pynacl -charset-normalizer==3.2.0 \ - --hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \ - --hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \ - --hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \ - --hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \ - --hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \ - --hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \ - --hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \ - --hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \ - --hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \ - --hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \ - --hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \ - --hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \ - --hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \ - --hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \ - --hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \ - --hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \ - --hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \ - --hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \ - --hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \ - --hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \ - --hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \ - --hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \ - --hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \ - --hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \ - --hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \ - --hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \ - --hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \ - --hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \ - --hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \ - --hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \ - --hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \ - --hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \ - --hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \ - --hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \ - --hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \ - --hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \ - --hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \ - --hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \ - --hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \ - --hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \ - --hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \ - --hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \ - --hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \ - --hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \ - --hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \ - --hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \ - --hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \ - --hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \ - --hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \ - --hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \ - --hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \ - --hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \ - --hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \ - --hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \ - --hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \ - --hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \ - --hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \ - --hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \ - --hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \ - --hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \ - --hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \ - --hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \ - --hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \ - --hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \ - --hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \ - --hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \ - --hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \ - --hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \ - --hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \ - --hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \ - --hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \ - --hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \ - --hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \ - --hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \ - --hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa - # via - # aiohttp - # requests +charset-normalizer==3.3.0 \ + --hash=sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843 \ + --hash=sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786 \ + --hash=sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e \ + --hash=sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8 \ + --hash=sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4 \ + --hash=sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa \ + --hash=sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d \ + --hash=sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82 \ + --hash=sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7 \ + --hash=sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895 \ + --hash=sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d \ + --hash=sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a \ + --hash=sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382 \ + --hash=sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678 \ + --hash=sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b \ + --hash=sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e \ + --hash=sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741 \ + --hash=sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4 \ + --hash=sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596 \ + --hash=sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9 \ + --hash=sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69 \ + --hash=sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c \ + --hash=sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77 \ + --hash=sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13 \ + --hash=sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459 \ + --hash=sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e \ + --hash=sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7 \ + --hash=sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908 \ + --hash=sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a \ + --hash=sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f \ + --hash=sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8 \ + --hash=sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482 \ + --hash=sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d \ + --hash=sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d \ + --hash=sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545 \ + --hash=sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34 \ + --hash=sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86 \ + --hash=sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6 \ + --hash=sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe \ + --hash=sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e \ + --hash=sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc \ + --hash=sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7 \ + --hash=sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd \ + --hash=sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c \ + --hash=sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557 \ + --hash=sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a \ + --hash=sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89 \ + --hash=sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078 \ + --hash=sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e \ + --hash=sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4 \ + --hash=sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403 \ + --hash=sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0 \ + --hash=sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89 \ + --hash=sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115 \ + --hash=sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9 \ + --hash=sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05 \ + --hash=sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a \ + --hash=sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec \ + --hash=sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56 \ + --hash=sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38 \ + --hash=sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479 \ + --hash=sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c \ + --hash=sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e \ + --hash=sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd \ + --hash=sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186 \ + --hash=sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455 \ + --hash=sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c \ + --hash=sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65 \ + --hash=sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78 \ + --hash=sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287 \ + --hash=sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df \ + --hash=sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43 \ + --hash=sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1 \ + --hash=sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7 \ + --hash=sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989 \ + --hash=sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a \ + --hash=sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63 \ + --hash=sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884 \ + --hash=sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649 \ + --hash=sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810 \ + --hash=sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828 \ + --hash=sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4 \ + --hash=sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2 \ + --hash=sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd \ + --hash=sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5 \ + --hash=sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe \ + --hash=sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293 \ + --hash=sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e \ + --hash=sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e \ + --hash=sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8 + # via requests clang-format==14.0.6 \ --hash=sha256:13f2d6d4a2af004a783c65f0921afa8f0384bffcdaf500b6c2cb542edeb0b4a5 \ --hash=sha256:810c649ab97d208cd418c897d50ab6e958eb8d96854527edd80d0dd21a75e914 \ @@ -404,37 +419,39 @@ coloredlogs==15.0.1 \ crcmod==1.7 \ --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e # via gsutil -cryptography==41.0.3 \ - --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ - --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ - --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ - --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ - --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ - --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ - --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ - --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ - --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ - --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ - --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ - --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ - --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ - --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ - --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ - --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ - --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ - --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ - --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ - --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ - --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ - --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ - --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de +cryptography==41.0.7 \ + --hash=sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960 \ + --hash=sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a \ + --hash=sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc \ + --hash=sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a \ + --hash=sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf \ + --hash=sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1 \ + --hash=sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39 \ + --hash=sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406 \ + --hash=sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a \ + --hash=sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a \ + --hash=sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c \ + --hash=sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be \ + --hash=sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15 \ + --hash=sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2 \ + --hash=sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d \ + --hash=sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157 \ + --hash=sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003 \ + --hash=sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248 \ + --hash=sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a \ + --hash=sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec \ + --hash=sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309 \ + --hash=sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7 \ + --hash=sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d # via # -r requirements.in + # aioquic # pyjwt # pyopenssl -dependatool==0.2.2 \ - --hash=sha256:8e66850c79e37325735efa67ce06e2d5a939c0dab758f37b9bd3d09d0fb1f9a4 \ - --hash=sha256:dff28853a7252d6a5d670c2519165506902c0d4746cbbdac99d2ad63ed96d82d + # service-identity +dependatool==0.2.3 \ + --hash=sha256:04bf88d01302eec697a69e8301d14668a89d676dbd2a3914e91c610a531e9db7 \ + --hash=sha256:113a6641889d3dae7c81cb0a0483c31a2657f179474e11f4731b285963475ade # via -r requirements.in deprecated==1.2.14 \ --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \ @@ -447,9 +464,9 @@ docutils==0.19 \ # envoy-docs-sphinx-runner # sphinx # sphinx-rtd-theme -envoy-base-utils==0.4.11 \ - --hash=sha256:5ced696c470b4c3090e6fc3f74e7e33f5fe217e775b1fc1fb56dfc756b781fbe \ - --hash=sha256:97c79177bb89360b7772e58fb20c671c67bf0b6cdee74c0b9f8a80433f0370cc +envoy-base-utils==0.5.0 \ + --hash=sha256:a185279fc2f6c49ba2b2a1d02ab2a361733ee4a5c3f41a225cbd2dd349369458 \ + --hash=sha256:f71e4bdcea86539f273388f1cb6210b40ec5d05f49aaacb7ef776a60fb60f107 # via # -r requirements.in # envoy-code-check @@ -465,9 +482,9 @@ envoy-code-check==0.5.8 \ --hash=sha256:03f32588cc9ed98ab6703cbca6f81df1527db71c3a0f962be6a6084ded40d528 \ --hash=sha256:2b12c51098c78d393823cf055a54e9308c37321d769041f01a2f35b04074d6f3 # via -r requirements.in -envoy-dependency-check==0.1.8 \ - --hash=sha256:ac9820e446bb44e05121e5c93c210f40ca37076580b0d082da2c63e7784c338a \ - --hash=sha256:e92272ca1f4d850d3eb3bde3c22cff39c103e7850fbda8d1686814bfc8c45338 +envoy-dependency-check==0.1.12 \ + --hash=sha256:4673cb4cf9c0e2c55b2a0e0b39df3b8df9993d6524c6edb9527d3c8fb1ec24e2 \ + --hash=sha256:7443e530a2a9155d1e114b8a99d9355bbbe73005b0c96ee653907912ae368f3c # via -r requirements.in envoy-distribution-distrotest==0.0.10 \ --hash=sha256:83e912c48da22eb3e514fc1142247d33eb7ed0d59e94eca2ffbd178a26fbf808 \ @@ -488,9 +505,9 @@ envoy-distribution-verify==0.0.11 \ envoy-docker-utils==0.0.2 \ --hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76 # via envoy-distribution-distrotest -envoy-docs-sphinx-runner==0.2.7 \ - --hash=sha256:093a53b20c4818354e156c459beb81ef9da607e8aa7fb87d0118ccd7332ff87c \ - --hash=sha256:da659d446be6433da909e91ed6a2feca9c2cb9a97b10a1f01b67cca8678df63e +envoy-docs-sphinx-runner==0.2.9 \ + --hash=sha256:1fa789b1d29ea929df67b07e5ca910d62e2057cd229719725030889da53b1a09 \ + --hash=sha256:4bfa1946104e263471d522b47d683e127124a5ad47334d69de4aea0eac282576 # via -r requirements.in envoy-github-abstract==0.0.22 \ --hash=sha256:2dd65e2f247a4947d0198b295c82716c13162e30c433b7625c27d59eee7bcf78 \ @@ -512,9 +529,9 @@ envoy-gpg-sign==0.2.0 \ --hash=sha256:53ef217a05555d725d467ceb70fbf7bc623eeb973a41996e8bbe1f295d8c9aab \ --hash=sha256:8bca326766a2b82864ec6274c51d99c9924f2ec773316c2f13034925ddf50772 # via -r requirements.in -fasteners==0.18 \ - --hash=sha256:1d4caf5f8db57b0e4107d94fd5a1d02510a450dced6ca77d1839064c1bacf20c \ - --hash=sha256:cb7c13ef91e0c7e4fe4af38ecaf6b904ec3f5ce0dda06d34924b6b74b869d953 +fasteners==0.19 \ + --hash=sha256:758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 \ + --hash=sha256:b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c # via # google-apitools # gsutil @@ -525,44 +542,43 @@ flake8==6.1.0 \ # -r requirements.in # envoy-code-check # pep8-naming -frozendict==2.3.8 \ - --hash=sha256:0bc4767e2f83db5b701c787e22380296977368b0c57e485ca71b2eedfa11c4a3 \ - --hash=sha256:145afd033ebfade28416093335261b8ec1af5cccc593482309e7add062ec8668 \ - --hash=sha256:23c4bb46e6b8246e1e7e49b5593c2bc09221db0d8f31f7c092be8dfb42b9e620 \ - --hash=sha256:2b2fd8ce36277919b36e3c834d2389f3cd7ac068ae730c312671dd4439a5dd65 \ - --hash=sha256:2b3435e5f1ca5ae68a5e95e64b09d6d5c645cadd6b87569a0b3019dd248c8d00 \ - --hash=sha256:313ed8d9ba6bac35d7635cd9580ee5721a0fb016f4d2d20f0efa05dbecbdb1be \ - --hash=sha256:3957d52f1906b0c85f641a1911d214255873f6408ab4e5ad657cc27a247fb145 \ - --hash=sha256:4742e76c4111bd09198d3ab66cef94be8506212311338f9182d6ef5f5cb60493 \ - --hash=sha256:47fc26468407fdeb428cfc89495b7921419e670355c21b383765482fdf6c5c14 \ - --hash=sha256:4c258aab9c8488338634f2ec670ef049dbf0ab0e7a2fa9bc2c7b5009cb614801 \ - --hash=sha256:5526559eca8f1780a4ee5146896f59afc31435313560208dd394a3a5e537d3ff \ - --hash=sha256:5e82befa7c385a668d569cebbebbdf49cee6fea4083f08e869a1b08cfb640a9f \ - --hash=sha256:638cf363d3cbca31a341503cf2219eac52a5f5140449676fae3d9644cd3c5487 \ - --hash=sha256:6ea638228692db2bf94bce40ea4b25f4077588497b516bd16576575560094bd9 \ - --hash=sha256:72cfe08ab8ae524e54848fa90b22d02c1b1ecfb3064438696bcaa4b953f18772 \ - --hash=sha256:750632cc890d8ee9484fe6d31b261159144b6efacc08e1317fe46accd1410373 \ - --hash=sha256:7a75bf87e76c4386caecdbdd02a99e53ad43a6b5c38fb3d5a634a9fc9ce41462 \ - --hash=sha256:7ee5fe2658a8ac9a57f748acaf563f6a47f80b8308cbf0a04fac0ba057d41f75 \ - --hash=sha256:80abe81d36e889ceec665e06ec764a7638000fa3e7be09786ac4d3ddc64b76db \ - --hash=sha256:8ccc94ac781710db44e142e1a11ff9b31d02c032c01c6868d51fcbef73086225 \ - --hash=sha256:8cf35ddd25513428ec152614def9696afb93ae5ec0eb54fa6aa6206eda77ac4c \ - --hash=sha256:9a506d807858fa961aaa5b48dab6154fdc6bd045bbe9310788bbff141bb42d13 \ - --hash=sha256:9ea5520e85447ff8d4681e181941e482662817ccba921b7cb3f87922056d892a \ - --hash=sha256:ba41a7ed019bd03b62d63ed3f8dea35b8243d1936f7c9ed4b5298ca45a01928e \ - --hash=sha256:c31abc8acea309b132dde441856829f6003a3d242da8b54bce4c0f2a3c8c63f0 \ - --hash=sha256:d086440328a465dea9bef2dbad7548d75d1a0a0d21f43a08c03e1ec79ac5240e \ - --hash=sha256:d188d062084fba0e4bf32719ff7380b26c050b932ff164043ce82ab90587c52b \ - --hash=sha256:d3c6ce943946c2a61501c8cf116fff0892d11dd579877eb36e2aea2c27fddfef \ - --hash=sha256:da98427de26b5a2865727947480cbb53860089c4d195baa29c539da811cea617 \ - --hash=sha256:e27c5c1d29d0eda7979253ec88abc239da1313b38f39f4b16984db3b3e482300 \ - --hash=sha256:e4c785de7f1a13f15963945f400656b18f057c2fc76c089dacf127a2bb188c03 \ - --hash=sha256:e72dbc1bcc2203cef38d205f692396f5505921a5680f66aa9a7e8bb71fd38f28 \ - --hash=sha256:ed5a6c5c7a0f57269577c2a338a6002949aea21a23b7b7d06da7e7dced8b605b \ - --hash=sha256:f0f573dc4861dd7ec9e055c8cceaf45355e894e749f621f199aab7b311ac4bdb \ - --hash=sha256:f2a4e818ac457f6354401dcb631527af25e5a20fcfc81e6b5054b45fc245caca \ - --hash=sha256:f83fed36497af9562ead5e9fb8443224ba2781786bd3b92b1087cb7d0ff20135 \ - --hash=sha256:ffc684773de7c88724788fa9787d0016fd75830412d58acbd9ed1a04762c675b +frozendict==2.4.0 \ + --hash=sha256:05c5a77957ecba4286c7ab33861a8f4f2badc7ea86fc82b834fb360d3aa4c108 \ + --hash=sha256:0615ed71570eec3cc96df063930ea6e563211efeeac86e3f3cc8bdfc9c9bfab7 \ + --hash=sha256:08d9c7c1aa92b94538b3a79c43999f999012e174588435f197794d5e5a80e0f5 \ + --hash=sha256:09ba8ee37d260adde311b8eb4cd12bf27f64071242f736757ae6a11d331eb860 \ + --hash=sha256:0b75e5e231621dedaef88334997e79fbd137dd89895543d3862fe0220fc3572c \ + --hash=sha256:1875e7b70a5724bf964354da8fd542240d2cead0d80053ac96bf4494ce3517fa \ + --hash=sha256:204f2c5c10fc018d1ba8ccc67758aa83fe769c782547bd26dc250317a7ccba71 \ + --hash=sha256:23a52bbea30c9e35b89291273944393770fb031e522a172e3aff19b62cc50047 \ + --hash=sha256:2607e82efdd2c277224a58bda3994d4cd48e49eff7fa31e404cf3066e8dbfeae \ + --hash=sha256:2804ea4bd2179bb33b99483cc8d69246630cc00632b9affe2914e8666f1cc7e5 \ + --hash=sha256:2fd4583194baabe100c135883017da76259a315d34e303eddf198541b7e02e44 \ + --hash=sha256:3909df909516cfd7bcefd9a3003948970a12a50c5648d8bbddafcef171f2117f \ + --hash=sha256:42a9b33ccf9d417b22146e59803c53d5c39d7d9151d2df8df59c235f6a1a5ed7 \ + --hash=sha256:475c65202a6f5421df8cacb8a2f29c5087134a0542b0540ae95fbf4db7af2ff9 \ + --hash=sha256:4925c8e82d2bd23d45996cd0827668a52b9c51103897c98ce409a763d0c00c61 \ + --hash=sha256:5bb5b62d4e2bce12e91800496d94de41bec8f16e4d8a7b16e8f263676ae2031a \ + --hash=sha256:809f1cffb602cf06e5186c69c0e3b74bec7a3684593145331f9aa2a65b5ba3b7 \ + --hash=sha256:8fab616e7c0fea2ac928f107c740bd9ba516fc083adfcd1c391d6bfc9164403d \ + --hash=sha256:9e530658134e88607ff8c2c8934a07b2bb5e9fffab5045f127746f6542c6c77e \ + --hash=sha256:9fc4cba1ced988ce9020dfcaae6fe3f5521eebc00c5772b511aaf691b0be91e6 \ + --hash=sha256:a1d232f092dc686e6ef23d436bde30f82c018f31cef1b89b31caef03814b1617 \ + --hash=sha256:a3f51bfa64e0c4a6608e3f2878bab1211a6b3b197de6fa57151bbe73f1184457 \ + --hash=sha256:a60f353496637ca21396289a7d969af1eb4ec4d11a7c37a0e7f25fc1761a0c97 \ + --hash=sha256:aa86325da6a6071284b4ed3d9d2cd9db068560aebad503b658d6a889a0575683 \ + --hash=sha256:b017cba5f73869b04c2977139ad08e57a7480de1e384c34193939698119baa1d \ + --hash=sha256:b666f9c6c8a9e794d2713a944b10a65480ff459579d75b5f686c75031c2c2dfc \ + --hash=sha256:bd4700c3f0aebdc8f4375c35590135794b1dbf2aca132f4756b584fa9910af2d \ + --hash=sha256:c26758198e403337933a92b01f417a8240c954f553e1d4b5e0f8e39d9c8e3f0a \ + --hash=sha256:c8af8a6a39e0050d3f3193cda56c42b43534a9b3995c44241bb9527e3c3fd451 \ + --hash=sha256:cc754117a7d60ba8e55b3c39abd67f37fbc05dd63cdcb03d1717a382fe0a3421 \ + --hash=sha256:d8d1d269874c94b1ed2b6667e5e43dcf4541838019b1caa4c48f848ac73634df \ + --hash=sha256:da4406d95c340e0b1cc43a3858fac729f52689325bcf61a9182eb94aff7451dc \ + --hash=sha256:df3819a5d48ab3aae1548e62093d0111ad7c3b62ff9392421b7bbf149c08b629 \ + --hash=sha256:efca7281184b54f7abab6980cf25837b709f72ced62791f62dabcd7b184d958a \ + --hash=sha256:f91acaff475d0ef0d3436b805c9b91fc627a6a8a281771a24f7ab7f458a0b34f \ + --hash=sha256:f9d81fb396ea81fcba3b3dde4a4b51adcb74ff31632014fbfd030f8acd5a7292 # via # -r requirements.in # aio-run-runner @@ -648,122 +664,28 @@ gitdb==4.0.10 \ --hash=sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a \ --hash=sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7 # via gitpython -gitpython==3.1.36 \ - --hash=sha256:4bb0c2a6995e85064140d31a33289aa5dce80133a23d36fcd372d716c54d3ebf \ - --hash=sha256:8d22b5cfefd17c79914226982bb7851d6ade47545b1735a9d010a2a4c26d8388 +gitpython==3.1.40 \ + --hash=sha256:22b126e9ffb671fdd0c129796343a02bf67bf2994b35449ffc9321aa755e18a4 \ + --hash=sha256:cf14627d5a8049ffbf49915732e5eddbe8134c3bdb9d476e6182b676fc573f8a # via -r requirements.in -google-api-core==2.11.1 \ - --hash=sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a \ - --hash=sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a - # via - # google-cloud-core - # google-cloud-storage google-apitools==0.5.32 \ --hash=sha256:b78f74116558e0476e19501b5b4b2ac7c93261a69c5449c861ea95cbc853c688 \ --hash=sha256:c3763e52289f61e21c41d5531e20fbda9cc8484a088b8686fd460770db8bad13 # via gsutil -google-auth[aiohttp]==2.22.0 \ - --hash=sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce \ - --hash=sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873 +google-auth[aiohttp]==2.23.3 \ + --hash=sha256:6864247895eea5d13b9c57c9e03abb49cb94ce2dc7c58e91cba3248c7477c9e3 \ + --hash=sha256:a8f4608e65c244ead9e0538f181a96c6e11199ec114d41f1d7b1bffa96937bda # via - # google-api-core - # google-cloud-core - # google-cloud-storage + # google-auth # gsutil -google-cloud-core==2.3.3 \ - --hash=sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb \ - --hash=sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863 - # via google-cloud-storage -google-cloud-storage==2.10.0 \ - --hash=sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7 \ - --hash=sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7 - # via -r requirements.in -google-crc32c==1.5.0 \ - --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ - --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ - --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \ - --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \ - --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \ - --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \ - --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \ - --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \ - --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \ - --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \ - --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \ - --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \ - --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \ - --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \ - --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \ - --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \ - --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \ - --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \ - --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \ - --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \ - --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \ - --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \ - --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \ - --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \ - --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \ - --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \ - --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \ - --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \ - --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \ - --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \ - --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \ - --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \ - --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \ - --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \ - --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \ - --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \ - --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \ - --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \ - --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \ - --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \ - --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \ - --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \ - --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \ - --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \ - --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \ - --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \ - --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \ - --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \ - --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \ - --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \ - --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \ - --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \ - --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \ - --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \ - --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \ - --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \ - --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \ - --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \ - --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \ - --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \ - --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \ - --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \ - --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \ - --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \ - --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \ - --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ - --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ - --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 - # via google-resumable-media google-reauth==0.1.1 \ --hash=sha256:cb39074488d74c8853074dde47368bbf8f739d4a4338b89aab696c895b6d8368 \ --hash=sha256:f9f6852a55c2c5453d581cd01f3d1278e86147c03d008409800390a834235892 # via # gcs-oauth2-boto-plugin # gsutil -google-resumable-media==2.5.0 \ - --hash=sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93 \ - --hash=sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec - # via google-cloud-storage -googleapis-common-protos==1.59.1 \ - --hash=sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e \ - --hash=sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a - # via google-api-core -gsutil==5.25 \ - --hash=sha256:7e4cb7fa9a332c401e4b7f5fef1da3e9ef21e3e4885de6d007b07a11b5d0524a +gsutil==5.27 \ + --hash=sha256:681a2d844acdf05fac989da6dd406944ae11cb27a4cf3c9edef74d2585ab5f05 # via -r requirements.in httplib2==0.20.4 \ --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ @@ -777,6 +699,10 @@ humanfriendly==10.0 \ --hash=sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477 \ --hash=sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc # via coloredlogs +icalendar==5.0.11 \ + --hash=sha256:7a298bb864526589d0de81f4b736eeb6ff9e539fefb405f7977aa5c1e201ca00 \ + --hash=sha256:81864971ac43a1b7d0a555dc1b667836ce59fc719a7f845a96f2f03205fb83b9 + # via -r requirements.in idna==3.4 \ --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 @@ -804,8 +730,11 @@ markupsafe==2.1.3 \ --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ @@ -813,6 +742,7 @@ markupsafe==2.1.3 \ --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ @@ -821,6 +751,7 @@ markupsafe==2.1.3 \ --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ @@ -828,9 +759,12 @@ markupsafe==2.1.3 \ --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ + --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ + --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ @@ -849,7 +783,9 @@ markupsafe==2.1.3 \ --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ - --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via jinja2 mccabe==0.7.0 \ --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ @@ -944,73 +880,63 @@ oauth2client==4.1.3 \ # via # gcs-oauth2-boto-plugin # google-apitools -orjson==3.9.7 \ - --hash=sha256:01d647b2a9c45a23a84c3e70e19d120011cba5f56131d185c1b78685457320bb \ - --hash=sha256:0eb850a87e900a9c484150c414e21af53a6125a13f6e378cf4cc11ae86c8f9c5 \ - --hash=sha256:11c10f31f2c2056585f89d8229a56013bc2fe5de51e095ebc71868d070a8dd81 \ - --hash=sha256:14d3fb6cd1040a4a4a530b28e8085131ed94ebc90d72793c59a713de34b60838 \ - --hash=sha256:154fd67216c2ca38a2edb4089584504fbb6c0694b518b9020ad35ecc97252bb9 \ - --hash=sha256:1c3cee5c23979deb8d1b82dc4cc49be59cccc0547999dbe9adb434bb7af11cf7 \ - --hash=sha256:1eb0b0b2476f357eb2975ff040ef23978137aa674cd86204cfd15d2d17318588 \ - --hash=sha256:1f8b47650f90e298b78ecf4df003f66f54acdba6a0f763cc4df1eab048fe3738 \ - --hash=sha256:21a3344163be3b2c7e22cef14fa5abe957a892b2ea0525ee86ad8186921b6cf0 \ - --hash=sha256:23be6b22aab83f440b62a6f5975bcabeecb672bc627face6a83bc7aeb495dc7e \ - --hash=sha256:26ffb398de58247ff7bde895fe30817a036f967b0ad0e1cf2b54bda5f8dcfdd9 \ - --hash=sha256:2f8fcf696bbbc584c0c7ed4adb92fd2ad7d153a50258842787bc1524e50d7081 \ - --hash=sha256:355efdbbf0cecc3bd9b12589b8f8e9f03c813a115efa53f8dc2a523bfdb01334 \ - --hash=sha256:36b1df2e4095368ee388190687cb1b8557c67bc38400a942a1a77713580b50ae \ - --hash=sha256:38e34c3a21ed41a7dbd5349e24c3725be5416641fdeedf8f56fcbab6d981c900 \ - --hash=sha256:3aab72d2cef7f1dd6104c89b0b4d6b416b0db5ca87cc2fac5f79c5601f549cc2 \ - --hash=sha256:410aa9d34ad1089898f3db461b7b744d0efcf9252a9415bbdf23540d4f67589f \ - --hash=sha256:45a47f41b6c3beeb31ac5cf0ff7524987cfcce0a10c43156eb3ee8d92d92bf22 \ - --hash=sha256:4891d4c934f88b6c29b56395dfc7014ebf7e10b9e22ffd9877784e16c6b2064f \ - --hash=sha256:4c616b796358a70b1f675a24628e4823b67d9e376df2703e893da58247458956 \ - --hash=sha256:5198633137780d78b86bb54dafaaa9baea698b4f059456cd4554ab7009619221 \ - --hash=sha256:5a2937f528c84e64be20cb80e70cea76a6dfb74b628a04dab130679d4454395c \ - --hash=sha256:5da9032dac184b2ae2da4bce423edff7db34bfd936ebd7d4207ea45840f03905 \ - --hash=sha256:5e736815b30f7e3c9044ec06a98ee59e217a833227e10eb157f44071faddd7c5 \ - --hash=sha256:63ef3d371ea0b7239ace284cab9cd00d9c92b73119a7c274b437adb09bda35e6 \ - --hash=sha256:70b9a20a03576c6b7022926f614ac5a6b0914486825eac89196adf3267c6489d \ - --hash=sha256:76a0fc023910d8a8ab64daed8d31d608446d2d77c6474b616b34537aa7b79c7f \ - --hash=sha256:7951af8f2998045c656ba8062e8edf5e83fd82b912534ab1de1345de08a41d2b \ - --hash=sha256:7a34a199d89d82d1897fd4a47820eb50947eec9cda5fd73f4578ff692a912f89 \ - --hash=sha256:7bab596678d29ad969a524823c4e828929a90c09e91cc438e0ad79b37ce41166 \ - --hash=sha256:7ea3e63e61b4b0beeb08508458bdff2daca7a321468d3c4b320a758a2f554d31 \ - --hash=sha256:80acafe396ab689a326ab0d80f8cc61dec0dd2c5dca5b4b3825e7b1e0132c101 \ - --hash=sha256:82720ab0cf5bb436bbd97a319ac529aee06077ff7e61cab57cee04a596c4f9b4 \ - --hash=sha256:83cc275cf6dcb1a248e1876cdefd3f9b5f01063854acdfd687ec360cd3c9712a \ - --hash=sha256:85e39198f78e2f7e054d296395f6c96f5e02892337746ef5b6a1bf3ed5910142 \ - --hash=sha256:8769806ea0b45d7bf75cad253fba9ac6700b7050ebb19337ff6b4e9060f963fa \ - --hash=sha256:8bdb6c911dae5fbf110fe4f5cba578437526334df381b3554b6ab7f626e5eeca \ - --hash=sha256:8f4b0042d8388ac85b8330b65406c84c3229420a05068445c13ca28cc222f1f7 \ - --hash=sha256:90fe73a1f0321265126cbba13677dcceb367d926c7a65807bd80916af4c17047 \ - --hash=sha256:915e22c93e7b7b636240c5a79da5f6e4e84988d699656c8e27f2ac4c95b8dcc0 \ - --hash=sha256:9274ba499e7dfb8a651ee876d80386b481336d3868cba29af839370514e4dce0 \ - --hash=sha256:9d62c583b5110e6a5cf5169ab616aa4ec71f2c0c30f833306f9e378cf51b6c86 \ - --hash=sha256:9ef82157bbcecd75d6296d5d8b2d792242afcd064eb1ac573f8847b52e58f677 \ - --hash=sha256:a19e4074bc98793458b4b3ba35a9a1d132179345e60e152a1bb48c538ab863c4 \ - --hash=sha256:a347d7b43cb609e780ff8d7b3107d4bcb5b6fd09c2702aa7bdf52f15ed09fa09 \ - --hash=sha256:b4fb306c96e04c5863d52ba8d65137917a3d999059c11e659eba7b75a69167bd \ - --hash=sha256:b6df858e37c321cefbf27fe7ece30a950bcc3a75618a804a0dcef7ed9dd9c92d \ - --hash=sha256:b8e59650292aa3a8ea78073fc84184538783966528e442a1b9ed653aa282edcf \ - --hash=sha256:bcb9a60ed2101af2af450318cd89c6b8313e9f8df4e8fb12b657b2e97227cf08 \ - --hash=sha256:c3ba725cf5cf87d2d2d988d39c6a2a8b6fc983d78ff71bc728b0be54c869c884 \ - --hash=sha256:ca1706e8b8b565e934c142db6a9592e6401dc430e4b067a97781a997070c5378 \ - --hash=sha256:cd3e7aae977c723cc1dbb82f97babdb5e5fbce109630fbabb2ea5053523c89d3 \ - --hash=sha256:cf334ce1d2fadd1bf3e5e9bf15e58e0c42b26eb6590875ce65bd877d917a58aa \ - --hash=sha256:d8692948cada6ee21f33db5e23460f71c8010d6dfcfe293c9b96737600a7df78 \ - --hash=sha256:e5205ec0dfab1887dd383597012199f5175035e782cdb013c542187d280ca443 \ - --hash=sha256:e7e7f44e091b93eb39db88bb0cb765db09b7a7f64aea2f35e7d86cbf47046c65 \ - --hash=sha256:e94b7b31aa0d65f5b7c72dd8f8227dbd3e30354b99e7a9af096d967a77f2a580 \ - --hash=sha256:f26fb3e8e3e2ee405c947ff44a3e384e8fa1843bc35830fe6f3d9a95a1147b6e \ - --hash=sha256:f738fee63eb263530efd4d2e9c76316c1f47b3bbf38c1bf45ae9625feed0395e \ - --hash=sha256:f9e01239abea2f52a429fe9d95c96df95f078f0172489d691b4a848ace54a476 +orjson==3.9.10 \ + --hash=sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83 \ + --hash=sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60 \ + --hash=sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9 \ + --hash=sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb \ + --hash=sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8 \ + --hash=sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f \ + --hash=sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b \ + --hash=sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d \ + --hash=sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921 \ + --hash=sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f \ + --hash=sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777 \ + --hash=sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c \ + --hash=sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e \ + --hash=sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d \ + --hash=sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5 \ + --hash=sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de \ + --hash=sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862 \ + --hash=sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7 \ + --hash=sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d \ + --hash=sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca \ + --hash=sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca \ + --hash=sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1 \ + --hash=sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864 \ + --hash=sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521 \ + --hash=sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d \ + --hash=sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531 \ + --hash=sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071 \ + --hash=sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1 \ + --hash=sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81 \ + --hash=sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643 \ + --hash=sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1 \ + --hash=sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff \ + --hash=sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4 \ + --hash=sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef \ + --hash=sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14 \ + --hash=sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b \ + --hash=sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1 \ + --hash=sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade \ + --hash=sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8 \ + --hash=sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616 \ + --hash=sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9 \ + --hash=sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3 \ + --hash=sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc \ + --hash=sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5 \ + --hash=sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499 \ + --hash=sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3 \ + --hash=sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7 \ + --hash=sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d \ + --hash=sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f \ + --hash=sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088 # via # -r requirements.in # envoy-base-utils -packaging==23.1 \ - --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ - --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f +packaging==23.2 \ + --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ + --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 # via # aio-api-github # aio-api-nist @@ -1021,43 +947,38 @@ packaging==23.1 \ # envoy-github-abstract # envoy-github-release # sphinx -pathspec==0.11.1 \ - --hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \ - --hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293 +pathspec==0.11.2 \ + --hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \ + --hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3 # via yamllint pep8-naming==0.13.3 \ --hash=sha256:1705f046dfcd851378aac3be1cd1551c7c1e5ff363bacad707d43007877fa971 \ --hash=sha256:1a86b8c71a03337c97181917e2b472f0f5e4ccb06844a0d6f0a33522549e7a80 # via -r requirements.in -platformdirs==3.9.1 \ - --hash=sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421 \ - --hash=sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f +platformdirs==3.11.0 \ + --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ + --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via yapf ply==3.11 \ --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 \ --hash=sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce # via -r requirements.in -protobuf==4.21.12 \ - --hash=sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30 \ - --hash=sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b \ - --hash=sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc \ - --hash=sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791 \ - --hash=sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717 \ - --hash=sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec \ - --hash=sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7 \ - --hash=sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab \ - --hash=sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2 \ - --hash=sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5 \ - --hash=sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1 \ - --hash=sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462 \ - --hash=sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97 \ - --hash=sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574 +protobuf==4.25.1 \ + --hash=sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd \ + --hash=sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb \ + --hash=sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0 \ + --hash=sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7 \ + --hash=sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b \ + --hash=sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2 \ + --hash=sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510 \ + --hash=sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6 \ + --hash=sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd \ + --hash=sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10 \ + --hash=sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7 # via # -r requirements.in # envoy-base-utils # envoy-docs-sphinx-runner - # google-api-core - # googleapis-common-protos pyasn1==0.5.0 \ --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde @@ -1065,15 +986,17 @@ pyasn1==0.5.0 \ # oauth2client # pyasn1-modules # rsa + # service-identity pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via # google-auth # oauth2client -pycodestyle==2.11.0 \ - --hash=sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0 \ - --hash=sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8 + # service-identity +pycodestyle==2.11.1 \ + --hash=sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f \ + --hash=sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67 # via flake8 pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ @@ -1083,13 +1006,13 @@ pyflakes==3.1.0 \ --hash=sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774 \ --hash=sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc # via flake8 -pygithub==1.59.1 \ - --hash=sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9 \ - --hash=sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217 +pygithub==2.1.1 \ + --hash=sha256:4b528d5d6f35e991ea5fd3f942f58748f24938805cb7fcf24486546637917337 \ + --hash=sha256:ecf12c2809c44147bce63b047b3d2e9dac8a41b63e90fcb263c703f64936b97c # via -r requirements.in -pygments==2.15.1 \ - --hash=sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c \ - --hash=sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1 +pygments==2.16.1 \ + --hash=sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692 \ + --hash=sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29 # via # envoy-docs-sphinx-runner # sphinx @@ -1099,6 +1022,59 @@ pyjwt[crypto]==2.8.0 \ # via # gidgethub # pygithub + # pyjwt +pylsqpack==0.3.17 \ + --hash=sha256:016cfc9dc2dea9c0b06b7aba64da89315b4c96c309c69f5ea548f4e994429781 \ + --hash=sha256:0b1ed55872f8f984f655a4c853ad749b834a1fcd4c7e5a962c6f17d4d27687a2 \ + --hash=sha256:105e636a33cac3ad7bd62aab97d655b53486fab6eb33429e45e7963be814947c \ + --hash=sha256:14dd6f623afe7f4b2a45078ef6bf3cc4a686489abf487f6fe15deeede893a551 \ + --hash=sha256:1b5c86a9feb137d197fe9f3c4775f5308f91f00a720d212b205044c83d32ab2e \ + --hash=sha256:1f657966917239518dcbf508ca5311ded178d88b81fe4e8f5fc6f7d1d84153a4 \ + --hash=sha256:27922bb8bd2e6a32693ac0b4cf3e164695b33c3fe3d243648f2f6403f03636d9 \ + --hash=sha256:2b010cbd59416f1911e392f6ccfbedd51af1b9bb7c1da3ac40a844b103628d3a \ + --hash=sha256:2bd2848a8ee065e4632f38c36fa79e40159271872f1ac9c1fd68ab913523c8de \ + --hash=sha256:2f20778db956dc7e4b1a8a79722d57a4650c45997fb65c1352cbf85eb7aa3ce2 \ + --hash=sha256:346d804ca196c29d8eae69d943b272e22101bd7611268b2db726097bd74c9c11 \ + --hash=sha256:35159d4e2f8005162cb4d5d5f2b477e45e9a54850203ec67c00806addedfdcf0 \ + --hash=sha256:3563f7e563b0b9d48958e92b2262872d056325b19e12c7a9e0ea1eb297302ae4 \ + --hash=sha256:3a7256825ea88f3c64b80676f5aa0fc1b19cd0bc28dd4d6c37397bcbd6ed3b25 \ + --hash=sha256:41bf0f33cd2fe4c745a790adc0e3cf1bfcb5c156e2217b3ff9d0eaa7fab763be \ + --hash=sha256:4580bce6eb2c2dd3c60ac7eb13beae83bc899696e2446ee9499537f037083426 \ + --hash=sha256:46762cc6993ee64c24dcaf9ec8665dbd68633efc7198ceb65a27e5232b9760c6 \ + --hash=sha256:4e501e9b6422db8b6d8248ef7fe325713ffce59fc2ab99d7dcca026b7451ce7d \ + --hash=sha256:5d992ffa01bc03a8e705a64de68b91b6c1de1960afa777917e6a37cef394faed \ + --hash=sha256:64791cc8b45d63a0f7f447614b7d5d5fa9a3f6ce20c7c10d9ea94003aedeff4b \ + --hash=sha256:6b3e381cd3b72ae466c0f34e109071a006d9939bbdcdd4856f72f08579ed0e76 \ + --hash=sha256:6dbc15ed7438e0f7c3ce323cfb878a3ba7aa8fc961a1f57df7c0be87520dd63a \ + --hash=sha256:6fb77ceae0a88f481ebfe902dcb8a18c6026aaacc9fd3cb709e7fc8d70bad26b \ + --hash=sha256:73ce2604f6dbc0bf11cc8d0858c60a74f9aff227c98054ebd946b69c5adb34e0 \ + --hash=sha256:7c30e9b21d8575e3ec42cd0ffcabb87cd2f5cf6dbdd07a9b209fbffa085a5f33 \ + --hash=sha256:80f49846ab14c12189b4517fb56c493ca7dec840d878102af6918ac9a2018602 \ + --hash=sha256:8e0d8ff7f6267674b3989c58d9c9e7bf2387454f51488317062e7b5b81a88d00 \ + --hash=sha256:95561d3e77ba8a78565bbd7ee3d2c01ad19a240806e1f1fe3e9b2d5a455f9992 \ + --hash=sha256:9c3c3ad219c8525d70866cc8c3fc584b772d3bfdced6825bfe1e6f1f2c3a33fa \ + --hash=sha256:abcfe2630ccb3a671cae45700fcdfbae3419340d4ad15c85db0534d8ea7dc745 \ + --hash=sha256:ac16e1c01e1b1610598b1b530c698476581beca5dc13186efe248077c734e1de \ + --hash=sha256:add2ace26276b78b9e063b5439556cda6b36a36aed747d93ea06c43bc8176208 \ + --hash=sha256:bb74b8a730876626e63d83564af0f31d27a0e1dd9219e9e5bc0d4921e85da71e \ + --hash=sha256:bbbf789dc1273680430e3231144fc38214c7460e836cb7c6076365b16147a151 \ + --hash=sha256:bf5210736b79effe3fc0e6b0d8d6aa0afd4b50c0cdb21e49f3835c25642e1f6d \ + --hash=sha256:c245a2241ed295633d3c60f949a00e43bcf7d108eb6b062b51e2ebf735fb83ed \ + --hash=sha256:c760638dce05ac657c9452d02424e07e99b65f431f7e366e79f6e23ed7a1313f \ + --hash=sha256:c7b3da5f2d3a4ae87ec46e6b1acf7ee54beeea049a99bbd6973012f0309b85bb \ + --hash=sha256:c7efb558ab48e364dad500bd4a78536a802dae54a46176021d3889b4e77cb150 \ + --hash=sha256:c89bbbcf69afa77f422edd7cb4054b2a1a60d51b835024087bb407c6b27181f4 \ + --hash=sha256:d4232501a5554c1f05334dfe0cb35ec66cc16154e6f31ce3107b8a8cffc9420c \ + --hash=sha256:db6e72e2c1312a1d918f8fd3a6bb38de98473b8cbdf6a3ce0237bd7ba0f25ad2 \ + --hash=sha256:e3556432d8dbb8649bf38d87fa5c97280b16d5a5425d8f943c676ace3041f4b3 \ + --hash=sha256:e6ca0ecda0040afdce031c7b966b78d8ca2375416170410ac862689ed9a60e68 \ + --hash=sha256:ea41faaf98857fa94ff12b6f72bd39c23fcaaa26ecece65cb3db9fa2b28633e7 \ + --hash=sha256:f1e40bec882d6aac75e814a2b8ee92add31a82eddcb705530497ab25f0c09f9a \ + --hash=sha256:f5c46627262389a9151cb4a120aa5b3210ba2066ab8c3026f3263adf8336b0c1 \ + --hash=sha256:f864f3e6c9e0a42b89f3388723575e306f45e736e32f6c0317eefb53c6ff1a40 \ + --hash=sha256:fc93abbe5bee19ceb1cb76b90298d2f47923f2a6dd1398f99aa498de1da0e553 \ + --hash=sha256:fdbfb5df079a50a7a0eed87d51a12495c820d30277ddb58b52c4862b5d557fc4 + # via aioquic pynacl==1.5.0 \ --hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \ --hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \ @@ -1115,30 +1091,40 @@ pyopenssl==23.2.0 \ --hash=sha256:24f0dc5227396b3e831f4c7f602b950a5e9833d292c8e4a2e06b709292806ae2 \ --hash=sha256:276f931f55a452e7dea69c7173e984eb2a4407ce413c918aa34b55f82f9b8bac # via + # aioquic # gcs-oauth2-boto-plugin # gsutil -pyparsing==3.1.0 \ - --hash=sha256:d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1 \ - --hash=sha256:edb662d6fe322d6e990b1594b5feaeadf806803359e3d4d42f11e295e588f0ea +pyparsing==3.1.1 \ + --hash=sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb \ + --hash=sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db # via httplib2 pyreadline==2.1 \ --hash=sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1 # via -r requirements.in +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # icalendar + # pygithub python-gnupg==0.5.1 \ --hash=sha256:5674bad4e93876c0b0d3197e314d7f942d39018bf31e2b833f6788a6813c3fb8 \ --hash=sha256:bf9b2d9032ef38139b7d64184176cd0b293eaeae6e4f93f50e304c7051174482 # via envoy-gpg-identity -pytz==2023.3 \ - --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ - --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb +pytz==2023.3.post1 \ + --hash=sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b \ + --hash=sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7 # via # aio-core # envoy-base-utils + # icalendar pyu2f==0.1.5 \ --hash=sha256:a3caa3a11842fc7d5746376f37195e6af5f17c0a15737538bb1cebf656fb306b # via google-reauth pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ @@ -1146,7 +1132,10 @@ pyyaml==6.0.1 \ --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ @@ -1154,9 +1143,12 @@ pyyaml==6.0.1 \ --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ @@ -1171,7 +1163,9 @@ pyyaml==6.0.1 \ --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ @@ -1187,9 +1181,7 @@ requests==2.31.0 \ --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via - # google-api-core # google-auth - # google-cloud-storage # pygithub # sphinx retry-decorator==1.1.1 \ @@ -1204,25 +1196,29 @@ rsa==4.7.2 \ # gcs-oauth2-boto-plugin # google-auth # oauth2client +service-identity==23.1.0 \ + --hash=sha256:87415a691d52fcad954a500cb81f424d0273f8e7e3ee7d766128f4575080f383 \ + --hash=sha256:ecb33cd96307755041e978ab14f8b14e13b40f1fbd525a4dc78f46d2b986431d + # via aioquic six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via # gcs-oauth2-boto-plugin # google-apitools - # google-auth # gsutil # oauth2client + # python-dateutil # pyu2f # sphinxcontrib-httpdomain # thrift -slackclient==2.9.4 \ - --hash=sha256:a8cab9146795e23d66a03473b80dd23df8c500829dfa9d06b3e3d5aec0a2b293 \ - --hash=sha256:ab79fefb5412d0595bc01d2f195a787597f2a617b6766562932ab9ffbe5cb173 +slack-sdk==3.26.1 \ + --hash=sha256:d1600211eaa37c71a5f92daf4404074c3e6b3f5359a37c93c818b39d88ab4ca0 \ + --hash=sha256:f80f0d15f0fce539b470447d2a07b03ecdad6b24f69c1edd05d464cf21253a06 # via -r requirements.in -smmap==5.0.0 \ - --hash=sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94 \ - --hash=sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936 +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da # via gitdb snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ @@ -1251,11 +1247,15 @@ sphinx-rtd-theme==2.0.0rc2 \ sphinxcontrib-applehelp==1.0.4 \ --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \ --hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e - # via sphinx + # via + # -r requirements.in + # sphinx sphinxcontrib-devhelp==1.0.2 \ --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 - # via sphinx + # via + # -r requirements.in + # sphinx sphinxcontrib-googleanalytics==0.4 \ --hash=sha256:4b19c1f0fce5df6c7da5633201b64a9e5b0cb3210a14fdb4134942ceee8c5d12 \ --hash=sha256:a6574983f9a58e5864ec10d34dc99914c4d647108b22c9249c8f0038b0cb18b3 @@ -1263,7 +1263,9 @@ sphinxcontrib-googleanalytics==0.4 \ sphinxcontrib-htmlhelp==2.0.1 \ --hash=sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff \ --hash=sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903 - # via sphinx + # via + # -r requirements.in + # sphinx sphinxcontrib-httpdomain==1.8.1 \ --hash=sha256:21eefe1270e4d9de8d717cc89ee92cc4871b8736774393bafc5e38a6bb77b1d5 \ --hash=sha256:6c2dfe6ca282d75f66df333869bb0ce7331c01b475db6809ff9d107b7cdfe04b @@ -1281,11 +1283,14 @@ sphinxcontrib-jsmath==1.0.1 \ sphinxcontrib-qthelp==1.0.3 \ --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 - # via sphinx + # via + # -r requirements.in + # sphinx sphinxcontrib-serializinghtml==1.1.5 \ --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 # via + # -r requirements.in # envoy-docs-sphinx-runner # sphinx sphinxext-rediraffe==0.2.7 \ @@ -1305,51 +1310,59 @@ trycast==1.0.0 \ # via # aio-core # envoy-base-utils -typing-extensions==4.7.1 \ - --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \ - --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2 - # via aiodocker +typing-extensions==4.8.0 \ + --hash=sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0 \ + --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef + # via + # aiodocker + # pygithub uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e # via gidgethub -urllib3==1.26.16 \ - --hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \ - --hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14 +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e # via - # google-auth + # pygithub # requests -uvloop==0.17.0 \ - --hash=sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d \ - --hash=sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1 \ - --hash=sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595 \ - --hash=sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b \ - --hash=sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05 \ - --hash=sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8 \ - --hash=sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20 \ - --hash=sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded \ - --hash=sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c \ - --hash=sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8 \ - --hash=sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474 \ - --hash=sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f \ - --hash=sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62 \ - --hash=sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376 \ - --hash=sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c \ - --hash=sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e \ - --hash=sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b \ - --hash=sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4 \ - --hash=sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578 \ - --hash=sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811 \ - --hash=sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d \ - --hash=sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738 \ - --hash=sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa \ - --hash=sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9 \ - --hash=sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539 \ - --hash=sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c \ - --hash=sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718 \ - --hash=sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667 \ - --hash=sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c \ - --hash=sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024 +uvloop==0.18.0 \ + --hash=sha256:1121087dfeb46e9e65920b20d1f46322ba299b8d93f7cb61d76c94b5a1adc20c \ + --hash=sha256:12af0d2e1b16780051d27c12de7e419b9daeb3516c503ab3e98d364cc55303bb \ + --hash=sha256:1f354d669586fca96a9a688c585b6257706d216177ac457c92e15709acaece10 \ + --hash=sha256:1f4a549cd747e6f4f8446f4b4c8cb79504a8372d5d3a9b4fc20e25daf8e76c05 \ + --hash=sha256:211ce38d84118ae282a91408f61b85cf28e2e65a0a8966b9a97e0e9d67c48722 \ + --hash=sha256:25b714f07c68dcdaad6994414f6ec0f2a3b9565524fba181dcbfd7d9598a3e73 \ + --hash=sha256:280904236a5b333a273292b3bcdcbfe173690f69901365b973fa35be302d7781 \ + --hash=sha256:2b8b7cf7806bdc745917f84d833f2144fabcc38e9cd854e6bc49755e3af2b53e \ + --hash=sha256:4d90858f32a852988d33987d608bcfba92a1874eb9f183995def59a34229f30d \ + --hash=sha256:53aca21735eee3859e8c11265445925911ffe410974f13304edb0447f9f58420 \ + --hash=sha256:54b211c46facb466726b227f350792770fc96593c4ecdfaafe20dc00f3209aef \ + --hash=sha256:56c1026a6b0d12b378425e16250acb7d453abaefe7a2f5977143898db6cfe5bd \ + --hash=sha256:585b7281f9ea25c4a5fa993b1acca4ad3d8bc3f3fe2e393f0ef51b6c1bcd2fe6 \ + --hash=sha256:58e44650cbc8607a218caeece5a689f0a2d10be084a69fc32f7db2e8f364927c \ + --hash=sha256:61151cc207cf5fc88863e50de3d04f64ee0fdbb979d0b97caf21cae29130ed78 \ + --hash=sha256:6132318e1ab84a626639b252137aa8d031a6c0550250460644c32ed997604088 \ + --hash=sha256:680da98f12a7587f76f6f639a8aa7708936a5d17c5e7db0bf9c9d9cbcb616593 \ + --hash=sha256:6e20bb765fcac07879cd6767b6dca58127ba5a456149717e0e3b1f00d8eab51c \ + --hash=sha256:74020ef8061678e01a40c49f1716b4f4d1cc71190d40633f08a5ef8a7448a5c6 \ + --hash=sha256:75baba0bfdd385c886804970ae03f0172e0d51e51ebd191e4df09b929771b71e \ + --hash=sha256:847f2ed0887047c63da9ad788d54755579fa23f0784db7e752c7cf14cf2e7506 \ + --hash=sha256:8849b8ef861431543c07112ad8436903e243cdfa783290cbee3df4ce86d8dd48 \ + --hash=sha256:895a1e3aca2504638a802d0bec2759acc2f43a0291a1dff886d69f8b7baff399 \ + --hash=sha256:99deae0504547d04990cc5acf631d9f490108c3709479d90c1dcd14d6e7af24d \ + --hash=sha256:ad79cd30c7e7484bdf6e315f3296f564b3ee2f453134a23ffc80d00e63b3b59e \ + --hash=sha256:b028776faf9b7a6d0a325664f899e4c670b2ae430265189eb8d76bd4a57d8a6e \ + --hash=sha256:b0a8f706b943c198dcedf1f2fb84899002c195c24745e47eeb8f2fb340f7dfc3 \ + --hash=sha256:c65585ae03571b73907b8089473419d8c0aff1e3826b3bce153776de56cbc687 \ + --hash=sha256:c6d341bc109fb8ea69025b3ec281fcb155d6824a8ebf5486c989ff7748351a37 \ + --hash=sha256:d5d1135beffe9cd95d0350f19e2716bc38be47d5df296d7cc46e3b7557c0d1ff \ + --hash=sha256:db1fcbad5deb9551e011ca589c5e7258b5afa78598174ac37a5f15ddcfb4ac7b \ + --hash=sha256:e14de8800765b9916d051707f62e18a304cde661fa2b98a58816ca38d2b94029 \ + --hash=sha256:e3d301e23984dcbc92d0e42253e0e0571915f0763f1eeaf68631348745f2dccc \ + --hash=sha256:ed3c28337d2fefc0bac5705b9c66b2702dc392f2e9a69badb1d606e7e7f773bb \ + --hash=sha256:edbb4de38535f42f020da1e3ae7c60f2f65402d027a08a8c60dc8569464873a6 \ + --hash=sha256:f3b18663efe0012bc4c315f1b64020e44596f5fabc281f5b0d9bc9465288559c # via aio-run-runner verboselogs==1.7 \ --hash=sha256:d63f23bf568295b95d3530c6864a0b580cec70e7ff974177dead1e4ffbc6ff49 \ @@ -1442,93 +1455,109 @@ yamllint==1.32.0 \ --hash=sha256:d01dde008c65de5b235188ab3110bebc59d18e5c65fc8a58267cd211cd9df34a \ --hash=sha256:d97a66e48da820829d96077d76b8dfbe6c6140f106e558dae87e81ac4e6b30b7 # via envoy-code-check -yapf==0.40.1 \ - --hash=sha256:958587eb5c8ec6c860119a9c25d02addf30a44f75aa152a4220d30e56a98037c \ - --hash=sha256:b8bfc1f280949153e795181768ca14ef43d7312629a06c43e7abd279323af313 +yapf==0.40.2 \ + --hash=sha256:4dab8a5ed7134e26d57c1647c7483afb3f136878b579062b786c9ba16b94637b \ + --hash=sha256:adc8b5dd02c0143108878c499284205adb258aad6db6634e5b869e7ee2bd548b # via # -r requirements.in # envoy-code-check -yarl==1.9.2 \ - --hash=sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571 \ - --hash=sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3 \ - --hash=sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3 \ - --hash=sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c \ - --hash=sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7 \ - --hash=sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04 \ - --hash=sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191 \ - --hash=sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea \ - --hash=sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4 \ - --hash=sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4 \ - --hash=sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095 \ - --hash=sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e \ - --hash=sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74 \ - --hash=sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef \ - --hash=sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33 \ - --hash=sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde \ - --hash=sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45 \ - --hash=sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf \ - --hash=sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b \ - --hash=sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac \ - --hash=sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0 \ - --hash=sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528 \ - --hash=sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716 \ - --hash=sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb \ - --hash=sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18 \ - --hash=sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72 \ - --hash=sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6 \ - --hash=sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582 \ - --hash=sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5 \ - --hash=sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368 \ - --hash=sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc \ - --hash=sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9 \ - --hash=sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be \ - --hash=sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a \ - --hash=sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80 \ - --hash=sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8 \ - --hash=sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6 \ - --hash=sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417 \ - --hash=sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574 \ - --hash=sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59 \ - --hash=sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608 \ - --hash=sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82 \ - --hash=sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1 \ - --hash=sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3 \ - --hash=sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d \ - --hash=sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8 \ - --hash=sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc \ - --hash=sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac \ - --hash=sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8 \ - --hash=sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955 \ - --hash=sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0 \ - --hash=sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367 \ - --hash=sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb \ - --hash=sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a \ - --hash=sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623 \ - --hash=sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2 \ - --hash=sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6 \ - --hash=sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7 \ - --hash=sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4 \ - --hash=sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051 \ - --hash=sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938 \ - --hash=sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8 \ - --hash=sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9 \ - --hash=sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3 \ - --hash=sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5 \ - --hash=sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9 \ - --hash=sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333 \ - --hash=sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185 \ - --hash=sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3 \ - --hash=sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560 \ - --hash=sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b \ - --hash=sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7 \ - --hash=sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78 \ - --hash=sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7 +yarl==1.9.4 \ + --hash=sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51 \ + --hash=sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce \ + --hash=sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559 \ + --hash=sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0 \ + --hash=sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81 \ + --hash=sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc \ + --hash=sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4 \ + --hash=sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c \ + --hash=sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130 \ + --hash=sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136 \ + --hash=sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e \ + --hash=sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec \ + --hash=sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7 \ + --hash=sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1 \ + --hash=sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455 \ + --hash=sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099 \ + --hash=sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129 \ + --hash=sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10 \ + --hash=sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142 \ + --hash=sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98 \ + --hash=sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa \ + --hash=sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7 \ + --hash=sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525 \ + --hash=sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c \ + --hash=sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9 \ + --hash=sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c \ + --hash=sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8 \ + --hash=sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b \ + --hash=sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf \ + --hash=sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23 \ + --hash=sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd \ + --hash=sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27 \ + --hash=sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f \ + --hash=sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece \ + --hash=sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434 \ + --hash=sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec \ + --hash=sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff \ + --hash=sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78 \ + --hash=sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d \ + --hash=sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863 \ + --hash=sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53 \ + --hash=sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31 \ + --hash=sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15 \ + --hash=sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5 \ + --hash=sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b \ + --hash=sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57 \ + --hash=sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3 \ + --hash=sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1 \ + --hash=sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f \ + --hash=sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad \ + --hash=sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c \ + --hash=sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7 \ + --hash=sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2 \ + --hash=sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b \ + --hash=sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2 \ + --hash=sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b \ + --hash=sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9 \ + --hash=sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be \ + --hash=sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e \ + --hash=sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984 \ + --hash=sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4 \ + --hash=sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074 \ + --hash=sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2 \ + --hash=sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392 \ + --hash=sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91 \ + --hash=sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541 \ + --hash=sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf \ + --hash=sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572 \ + --hash=sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66 \ + --hash=sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575 \ + --hash=sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14 \ + --hash=sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5 \ + --hash=sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1 \ + --hash=sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e \ + --hash=sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551 \ + --hash=sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17 \ + --hash=sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead \ + --hash=sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0 \ + --hash=sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe \ + --hash=sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234 \ + --hash=sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0 \ + --hash=sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7 \ + --hash=sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34 \ + --hash=sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42 \ + --hash=sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385 \ + --hash=sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78 \ + --hash=sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be \ + --hash=sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958 \ + --hash=sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749 \ + --hash=sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec # via # -r requirements.in # aiohttp -zipp==3.16.2 \ - --hash=sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0 \ - --hash=sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147 +zipp==3.17.0 \ + --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \ + --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0 # via importlib-metadata zstandard==0.21.0 \ --hash=sha256:0aad6090ac164a9d237d096c8af241b8dcd015524ac6dbec1330092dba151657 \ @@ -1577,7 +1606,7 @@ zstandard==0.21.0 \ # via envoy-base-utils # The following packages are considered to be unsafe in a requirements file: -setuptools==68.2.2 \ - --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ - --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a +setuptools==69.0.3 \ + --hash=sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05 \ + --hash=sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78 # via -r requirements.in diff --git a/tools/bootstrap2pb.cc b/tools/bootstrap2pb.cc index 0c41fd948efb..ef9c66e16ba0 100644 --- a/tools/bootstrap2pb.cc +++ b/tools/bootstrap2pb.cc @@ -32,6 +32,7 @@ int main(int argc, char** argv) { Envoy::Stats::IsolatedStoreImpl stats_store; Envoy::Event::RealTimeSystem time_system; // NO_CHECK_FORMAT(real_time) Envoy::Random::RandomGeneratorImpl rand; + // TODO: fix or remove - this seems to be broken Envoy::Api::Impl api(platform_impl_.threadFactory(), stats_store, time_system, platform_impl_.fileSystem(), rand); diff --git a/tools/clang-tidy/BUILD b/tools/clang-tidy/BUILD new file mode 100644 index 000000000000..ed669a93c492 --- /dev/null +++ b/tools/clang-tidy/BUILD @@ -0,0 +1,12 @@ +load("@base_pip3//:requirements.bzl", "requirement") +load("//bazel:envoy_build_system.bzl", "envoy_package") +load(":clang_tidy.bzl", "clang_tidy") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +clang_tidy( + name = "clang-tidy", + target = requirement("clang-tidy"), +) diff --git a/tools/clang-tidy/clang_tidy.bzl b/tools/clang-tidy/clang_tidy.bzl new file mode 100644 index 000000000000..2ed0184bf239 --- /dev/null +++ b/tools/clang-tidy/clang_tidy.bzl @@ -0,0 +1,60 @@ +# +# This fishes the clang-tidy binary out of the related python package. +# +# This is useful as using the binary through the python entry_point adds a lot of overhead. +# +# ```starlark +# +# load("@base_pip3//:requirements.bzl", "requirement") +# +# clang_tidy( +# name = "clang-tidy", +# target = requirement("clang-tidy"), +# ) +# +# ``` +# +# The exposed binary can also be run directly: +# +# ```console +# +# $ bazel run //tools/clang-tidy -- --version +# +# ``` +# + +def _clang_tidy_impl(ctx): + clang_bin = None + for file in ctx.attr.target[DefaultInfo].data_runfiles.files.to_list(): + if file.basename == "clang-tidy" and file.dirname.split("/").pop() == "bin": + clang_bin = file + break + + if not clang_bin: + fail("Unable to find clang-tidy file in package") + + output_file = ctx.actions.declare_file("clang-tidy") + args = ctx.actions.args() + args.add(clang_bin.path) + args.add(output_file.path) + ctx.actions.run( + outputs = [output_file], + inputs = [clang_bin], + arguments = [args], + executable = "cp", + mnemonic = "ClangTidyGetter", + ) + return [DefaultInfo( + executable = output_file, + files = depset([output_file]), + )] + +clang_tidy = rule( + implementation = _clang_tidy_impl, + attrs = { + "target": attr.label( + allow_files = True, + ), + }, + executable = True, +) diff --git a/tools/code/BUILD b/tools/code/BUILD index 77466a7b4905..9c5fd6c0e379 100644 --- a/tools/code/BUILD +++ b/tools/code/BUILD @@ -7,11 +7,14 @@ load( "READFILTER_NOFUZZ_FILTERS", ) load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + FUZZ_FILTER_COUNT = ( len(READFILTER_FUZZ_FILTERS) + len(READFILTER_NOFUZZ_FILTERS) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index ac723f7c0cbe..916e4e8f6db6 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -384,7 +384,9 @@ def allow_listed_for_grpc_init(self, file_path): def allow_listed_for_unpack_to(self, file_path): return file_path.startswith("./test") or file_path in [ - "./source/common/protobuf/utility.cc", "./source/common/protobuf/utility.h" + "./source/common/protobuf/deterministic_hash.cc", + "./source/common/protobuf/utility.cc", + "./source/common/protobuf/utility.h", ] def allow_listed_for_raw_try(self, file_path): diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 615aca0ccd41..c7e1dc6bd7c9 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -96,19 +96,17 @@ paths: - envoy/common/exception.h # legacy core files which throw exceptions. We can add to this list but strongly prefer # StausOr where possible. - - source/common/upstream/cds_api_helper.cc + - source/common/listener_manager/listener_impl.cc + - source/common/listener_manager/listener_manager_impl.cc + - source/common/listener_manager/filter_chain_manager_impl.cc - source/common/upstream/thread_aware_lb_impl.cc - - source/common/upstream/load_balancer_impl.cc + - source/common/upstream/subset_lb_config.cc - source/common/upstream/cluster_manager_impl.cc - - source/common/upstream/od_cds_api_impl.cc - - source/common/upstream/cds_api_impl.cc - - source/common/upstream/outlier_detection_impl.cc - source/common/upstream/upstream_impl.cc - source/common/upstream/default_local_address_selector_factory.cc - source/common/network/listen_socket_impl.cc - - source/common/network/io_socket_handle_impl.cc + - source/common/network/io_socket_handle_base_impl.cc - source/common/network/address_impl.cc - - source/common/network/cidr_range.cc - source/common/network/utility.cc - source/common/network/dns_resolver/dns_factory_util.cc - source/common/network/resolver_impl.cc @@ -133,7 +131,6 @@ paths: - source/common/protobuf/message_validator_impl.cc - source/common/access_log/access_log_manager_impl.cc - source/common/secret/secret_manager_impl.cc - - source/common/secret/sds_api.cc - source/common/grpc/async_client_manager_impl.cc - source/common/grpc/google_grpc_utils.cc - source/common/tcp_proxy/tcp_proxy.cc @@ -141,9 +138,7 @@ paths: - source/common/config/xds_resource.cc - source/common/config/datasource.cc - source/common/config/utility.cc - - source/common/config/xds_context_params.cc - source/common/runtime/runtime_impl.cc - - source/common/quic/quic_transport_socket_factory.cc - source/common/filter/config_discovery_impl.cc - source/common/json/json_internal.cc - source/common/router/scoped_rds.cc @@ -153,13 +148,10 @@ paths: - source/common/router/vhds.cc - source/common/router/config_utility.cc - source/common/router/header_parser.cc - - source/common/rds/rds_route_config_subscription.cc - source/common/filesystem/inotify/watcher_impl.cc - source/common/filesystem/posix/directory_iterator_impl.cc - - source/common/filesystem/posix/filesystem_impl.cc - source/common/filesystem/kqueue/watcher_impl.cc - source/common/filesystem/win32/directory_iterator_impl.cc - - source/common/filesystem/win32/filesystem_impl.cc - source/common/filesystem/win32/watcher_impl.cc - source/common/common/utility.cc - source/common/common/regex.cc @@ -203,7 +195,7 @@ paths: protobuf: include: - api/test - - bazel/cc_proto_descriptor_library + - api/bazel/cc_proto_descriptor_library - ci/prebuilt - source/common/protobuf - source/extensions/filters/http/grpc_field_extraction @@ -254,7 +246,7 @@ paths: # Files in these paths can use MessageLite::SerializeAsString serialize_as_string: include: - - bazel/cc_proto_descriptor_library/file_descriptor_generator.cc + - api/bazel/cc_proto_descriptor_library/file_descriptor_generator.cc - contrib/config/source/kv_store_xds_delegate.cc - source/common/protobuf/utility.h - source/common/protobuf/utility.cc @@ -280,6 +272,7 @@ paths: - source/common/formatter/http_specific_formatter.cc - source/common/formatter/stream_info_formatter.cc - source/common/formatter/substitution_formatter.h + - source/common/formatter/substitution_formatter.cc - source/common/stats/tag_extractor_impl.h - source/common/stats/tag_extractor_impl.cc - source/common/version/version.cc @@ -412,3 +405,7 @@ visibility_excludes: - source/extensions/config_subscription/grpc/BUILD - source/extensions/load_balancing_policies/subset/BUILD - source/extensions/load_balancing_policies/ring_hash/BUILD +- source/extensions/load_balancing_policies/round_robin/ +- source/extensions/load_balancing_policies/least_request/ +- source/extensions/load_balancing_policies/random/ +- source/extensions/load_balancing_policies/cluster_provided/ diff --git a/tools/code_format/envoy_build_fixer.py b/tools/code_format/envoy_build_fixer.py index 9cfe99facf2f..470d25f20b6a 100755 --- a/tools/code_format/envoy_build_fixer.py +++ b/tools/code_format/envoy_build_fixer.py @@ -2,7 +2,9 @@ # Enforces: # - License headers on Envoy BUILD files -# - envoy_package() or envoy_extension_package() top-level invocation for standard Envoy package setup. +# - envoy_package() top-level invocation for standard Envoy package setup. +# - envoy_mobile_package() top-level invocation for standard Envoy Mobile package setup. +# - envoy_extension_package() top-level invocation for Envoy extensions. # - Infers API dependencies from source files. # - Misc. cleanups: avoids redundant blank lines, removes unused loads. # - Maybe more later? @@ -71,12 +73,12 @@ def run_buildozer(cmds, contents): return r.stdout.decode('utf-8') -# Add an Apache 2 license and envoy_package() import and rule as needed. +# Add an Apache 2 license, envoy_package / envoy_mobile_package import and rule as needed. def fix_package_and_license(path, contents): regex_to_use = PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_package' - if 'source/extensions' in path: + if 'source/extensions' in path or 'library/common/extensions' in path: regex_to_use = EXTENSION_PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_extension_package' @@ -84,19 +86,19 @@ def fix_package_and_license(path, contents): regex_to_use = CONTRIB_PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_contrib_package' - if 'mobile/' in path: + if os.getcwd().endswith('mobile') and 'library/common/extensions' not in path: regex_to_use = MOBILE_PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_mobile_package' - # Ensure we have an envoy_package import load if this is a real Envoy package. We also allow - # the prefix to be overridden if envoy is included in a larger workspace. + # Ensure we have an envoy_package / envoy_mobile_package import load if this is a real Envoy package. + # We also allow the prefix to be overridden if envoy is included in a larger workspace. if re.search(ENVOY_RULE_REGEX, contents): new_load = 'new_load {}//bazel:envoy_build_system.bzl %s' % package_string contents = run_buildozer([ (new_load.format(os.getenv("ENVOY_BAZEL_PREFIX", "")), '__pkg__'), ], contents) # Envoy package is inserted after the load block containing the - # envoy_package import. + # envoy_package / envoy_mobile_package import. package_and_parens = package_string + '()' if package_and_parens[:-1] not in contents: contents = re.sub(regex_to_use, r'\1\n%s\n\n' % package_and_parens, contents) @@ -181,7 +183,7 @@ def fix_api_deps(path, contents): existing_api_deps = set([ d for d in deps.split() if d.startswith('@envoy_api') and d.endswith('pkg_cc_proto') - and d != '@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto' + and d != '@com_github_cncf_xds//udpa/annotations:pkg_cc_proto' ]) deps_to_remove = existing_api_deps.difference(actual_api_deps) if deps_to_remove: diff --git a/tools/code_format/requirements.in b/tools/code_format/requirements.in deleted file mode 100644 index dbcef7340827..000000000000 --- a/tools/code_format/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -flake8 -pep8-naming -pyflakes==2.4.0 -yapf diff --git a/tools/code_format/requirements.txt b/tools/code_format/requirements.txt deleted file mode 100644 index 9a8e045c4176..000000000000 --- a/tools/code_format/requirements.txt +++ /dev/null @@ -1,50 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes tools/code_format/requirements.txt -# -flake8==4.0.1 \ - --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ - --hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d - # via - # -r requirements.in - # pep8-naming -importlib-metadata==6.7.0 \ - --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \ - --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5 - # via yapf -mccabe==0.6.1 \ - --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ - --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f - # via flake8 -pep8-naming==0.13.2 \ - --hash=sha256:59e29e55c478db69cffbe14ab24b5bd2cd615c0413edf790d47d3fb7ba9a4e23 \ - --hash=sha256:93eef62f525fd12a6f8c98f4dcc17fa70baae2f37fa1f73bec00e3e44392fa48 - # via -r requirements.in -platformdirs==3.7.0 \ - --hash=sha256:87fbf6473e87c078d536980ba970a472422e94f17b752cfad17024c18876d481 \ - --hash=sha256:cfd065ba43133ff103ab3bd10aecb095c2a0035fcd1f07217c9376900d94ba07 - # via yapf -pycodestyle==2.8.0 \ - --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \ - --hash=sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f - # via flake8 -pyflakes==2.4.0 \ - --hash=sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c \ - --hash=sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e - # via - # -r requirements.in - # flake8 -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via yapf -yapf==0.40.1 \ - --hash=sha256:958587eb5c8ec6c860119a9c25d02addf30a44f75aa152a4220d30e56a98037c \ - --hash=sha256:b8bfc1f280949153e795181768ca14ef43d7312629a06c43e7abd279323af313 - # via -r requirements.in -zipp==3.15.0 \ - --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ - --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 - # via importlib-metadata diff --git a/tools/dependency/BUILD b/tools/dependency/BUILD index e972f02450de..661adb959c42 100644 --- a/tools/dependency/BUILD +++ b/tools/dependency/BUILD @@ -1,25 +1,57 @@ load("@base_pip3//:requirements.bzl", "requirement") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@envoy_repo//:path.bzl", "PATH") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_genjson", "envoy_pytool_binary") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + +bool_flag( + name = "preload_cve_data", + build_setting_default = False, +) + +config_setting( + name = "preloaded_cve_data", + flag_values = { + ":preload_cve_data": "true", + }, +) + +# Currently we are unable to check for the libdrdkafka dep +# this is a workaround to just exclude it from checks for now +# which is sub-optimal as it also excludes it from CVE scanning +# https://github.com/envoyproxy/envoy/issues/31394 +envoy_genjson( + name = "filtered-dependencies", + srcs = ["//bazel:all_repository_locations"], + filter = """ + .[0] + | del(.confluentinc_librdkafka) + """, +) + envoy_entry_point( name = "check", args = [ - "--repository_locations=$(location //bazel:all_repository_locations)", + "--repository_locations=$(location :filtered-dependencies)", "--cve_config=$(location :cve.yaml)", - "--cve_data=$(location :cve_data)", - ], + ] + select({ + ":preloaded_cve_data": ["--cve_data=$(location :cve_data)"], + "//conditions:default": [], + }), data = [ ":cve.yaml", - ":cve_data", - "//bazel:all_repository_locations", - ], + ":filtered-dependencies", + ] + select({ + ":preloaded_cve_data": [":cve_data"], + "//conditions:default": [], + }), pkg = "envoy.dependency.check", deps = [requirement("orjson")], ) @@ -30,7 +62,7 @@ envoy_entry_point( pkg = "dependatool", ) -py_binary( +envoy_pytool_binary( name = "validate", srcs = ["validate.py"], args = [ @@ -72,3 +104,25 @@ genrule( "//bazel:all_repository_locations", ], ) + +envoy_genjson( + name = "build-images", + filter = """ + .[0]["build-image"] + """, + yaml_srcs = ["//:.github/config.yml"], +) + +sh_binary( + name = "build-image-sha", + srcs = ["version.sh"], + args = [ + "$(JQ_BIN)", + "$(location :build-images)", + ], + data = [ + ":build-images", + "@jq_toolchains//:resolved_toolchain", + ], + toolchains = ["@jq_toolchains//:resolved_toolchain"], +) diff --git a/tools/dependency/version.sh b/tools/dependency/version.sh new file mode 100755 index 000000000000..2309c93d82f0 --- /dev/null +++ b/tools/dependency/version.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +set -o pipefail + +JQ="$1" +VERSIONS="$2" +DEP="$3" + +if [[ -z "$DEP" ]]; then + echo "You must specify what to check version for" >&2 + exit 1 +fi + +$JQ -r ".[\"${DEP}\"]" "$VERSIONS" diff --git a/tools/deprecate_version/deprecate_version.py b/tools/deprecate_version/deprecate_version.py index 02ce21afe23c..5ccb22ee7571 100644 --- a/tools/deprecate_version/deprecate_version.py +++ b/tools/deprecate_version/deprecate_version.py @@ -115,16 +115,23 @@ def create_issues(access_token, runtime_and_pr): if get_confirmation(): print('Creating issues...') for title, body, login in issues: + issue_created = False try: - repo.create_issue(title, body=body, assignees=[login], labels=labels) + # for setec backports, we may not find a user, which would make + # create_issue crash. + if login: + repo.create_issue(title, body=body, assignees=[login], labels=labels) + issue_created = True except github.GithubException as e: + print(( + 'unable to assign issue %s to %s. Add them to the Envoy proxy org' + 'and assign it their way.') % (title, login)) + + if not issue_created: try: if login: body += '\ncc @' + login repo.create_issue(title, body=body, labels=labels) - print(( - 'unable to assign issue %s to %s. Add them to the Envoy proxy org' - 'and assign it their way.') % (title, login)) except github.GithubException as e: print('GithubException while creating issue.') raise diff --git a/tools/distribution/BUILD b/tools/distribution/BUILD index 44e4006cdd98..cad485b87e1c 100644 --- a/tools/distribution/BUILD +++ b/tools/distribution/BUILD @@ -1,11 +1,14 @@ load("@base_pip3//:requirements.bzl", "requirement") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_pytool_binary") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + envoy_entry_point( name = "release", pkg = "envoy.distribution.release", @@ -21,7 +24,7 @@ envoy_entry_point( pkg = "envoy.distribution.verify", ) -py_binary( +envoy_pytool_binary( name = "update_dockerhub_repository", srcs = ["update_dockerhub_repository.py"], data = ["//distribution/dockerhub:readme.md"], diff --git a/tools/docs/BUILD b/tools/docs/BUILD index ee4f75c5c571..999b750ef97e 100644 --- a/tools/docs/BUILD +++ b/tools/docs/BUILD @@ -1,13 +1,15 @@ load("@base_pip3//:requirements.bzl", "requirement") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_pytool_binary") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() -py_binary( +envoy_py_namespace() + +envoy_pytool_binary( name = "generate_extensions_security_rst", srcs = ["generate_extensions_security_rst.py"], deps = [ @@ -15,20 +17,16 @@ py_binary( ], ) -py_binary( +envoy_pytool_binary( name = "generate_external_deps_rst", - srcs = [ - "generate_external_deps_rst.py", - ], + srcs = ["generate_external_deps_rst.py"], args = ["$(location //bazel:all_repository_locations)"], data = ["//bazel:all_repository_locations"], ) -py_binary( +envoy_pytool_binary( name = "generate_api_rst", - srcs = [ - "generate_api_rst.py", - ], + srcs = ["generate_api_rst.py"], ) # The upstream lib is maintained here: @@ -42,13 +40,12 @@ py_binary( envoy_entry_point( name = "sphinx_runner", pkg = "envoy.docs.sphinx_runner", + visibility = ["//visibility:public"], ) -py_binary( +envoy_pytool_binary( name = "generate_version_histories", - srcs = [ - "generate_version_histories.py", - ], + srcs = ["generate_version_histories.py"], deps = [ requirement("aio.run.runner"), requirement("envoy.base.utils"), diff --git a/tools/docs/generate_version_histories.py b/tools/docs/generate_version_histories.py index dd09075a5096..6bbb16f34ba0 100644 --- a/tools/docs/generate_version_histories.py +++ b/tools/docs/generate_version_histories.py @@ -6,7 +6,7 @@ from frozendict import frozendict import jinja2 -from packaging import version +from packaging import version as _version from aio.run import runner @@ -147,7 +147,7 @@ def jinja_env(self) -> jinja2.Environment: @cached_property def project(self) -> IProject: - return Project() + return Project(path=self.args.path) @cached_property def sections(self) -> frozendict: @@ -171,6 +171,7 @@ def version_history_tpl(self): def add_arguments(self, parser) -> None: super().add_arguments(parser) + parser.add_argument("--path") parser.add_argument("output_file") def minor_index_path(self, minor_version) -> pathlib.Path: @@ -192,7 +193,7 @@ async def write_version_histories(self) -> None: for changelog_version in self.project.changelogs: await self.write_version_history(changelog_version) - async def write_version_history(self, changelog_version: version.Version) -> None: + async def write_version_history(self, changelog_version: _version.Version) -> None: minor_version = utils.minor_version_for(changelog_version) root_path = self.tpath.joinpath(f"v{minor_version.base_version}") root_path.mkdir(parents=True, exist_ok=True) @@ -258,7 +259,7 @@ async def write_version_history_minor_indeces(self) -> None: await self.write_version_history_minor_index(minor_version, patches) async def write_version_history_minor_index( - self, minor_version: version.Version, patch_versions) -> None: + self, minor_version: _version.Version, patch_versions) -> None: skip_first = (self.project.is_dev and self.project.is_current(patch_versions[0])) if skip_first: patch_versions = patch_versions[1:] diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index 9de574f19311..19ee79c513b4 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -73,6 +73,7 @@ categories: - envoy.filters.listener - envoy.filters.network - envoy.filters.udp_listener +- envoy.filters.udp.session - envoy.filters.quic_listener - envoy.formatter - envoy.geoip_providers @@ -120,6 +121,7 @@ categories: - envoy.rbac.audit_loggers - envoy.access_loggers.extension_filters - envoy.http.stateful_session +- envoy.injected_credentials - envoy.matching.action - envoy.matching.http.input - envoy.matching.http.custom_matchers @@ -135,6 +137,8 @@ categories: - envoy.http.early_header_mutation - envoy.http.custom_response - envoy.router.cluster_specifier_plugin +- envoy.tracers.opentelemetry.resource_detectors +- envoy.tracers.opentelemetry.samplers status_values: - name: stable diff --git a/tools/h3_request/BUILD b/tools/h3_request/BUILD new file mode 100644 index 000000000000..5d9a6cd6325f --- /dev/null +++ b/tools/h3_request/BUILD @@ -0,0 +1,23 @@ +load("@base_pip3//:requirements.bzl", "requirement") +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load("//tools/base:envoy_python.bzl", "envoy_pytool_binary") +load("//tools/python:namespace.bzl", "envoy_py_namespace") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_py_namespace() + +# To use in tests, you probably will want to include +# args="--ca-certs=$(location //test/config/integration/certs:cacert.pem)" +envoy_pytool_binary( + name = "h3_request", + srcs = ["h3_request.py"], + deps = [ + requirement("aioquic"), + ], +) diff --git a/tools/h3_request/h3_request.py b/tools/h3_request/h3_request.py new file mode 100644 index 000000000000..d0230d860edc --- /dev/null +++ b/tools/h3_request/h3_request.py @@ -0,0 +1,117 @@ +import argparse +import asyncio +import sys +from functools import cached_property +from typing import cast +from urllib.parse import urlparse + +import aioquic +from aioquic.asyncio.protocol import QuicConnectionProtocol +from aioquic.h3.connection import H3_ALPN, H3Connection +from aioquic.h3.exceptions import H3Error +from aioquic.h3.events import ( + DataReceived, + H3Event, + HeadersReceived, +) +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.events import QuicEvent + + +class Http3Client(QuicConnectionProtocol): + """Note, this class is extremely minimal. + + It supports only GET, doesn't properly validate URLs, etc. Since this + is just for tests, that's all that's required right now. + It is based on https://github.com/aiortc/aioquic/blob/main/examples/http3_client.py + which is a far more complete implementation. + """ + + @cached_property + def _http(self) -> H3Connection: + return H3Connection(self._quic) + + @cached_property + def _stream_ids(self) -> dict[int, asyncio.Future[bool]]: + return {} + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def headers_received(self, event: H3Event) -> None: + if not self._include_headers: + return + for header, value in event.headers: + print(f"{header.decode('utf-8')}: {value.decode('utf-8')}\n", end="") + print("", flush=True) # One blank newline after headers. + + def http_event_received(self, event: H3Event) -> None: + stream_id = event.stream_id + if stream_id not in self._stream_ids: + return + if isinstance(event, HeadersReceived): + self.headers_received(event) + elif isinstance(event, DataReceived): + print(event.data.decode("utf-8"), end="", flush=True) + else: + raise H3Error(f"unexpected quic event type {event}") + if event.stream_ended: + self._stream_ids.pop(stream_id).set_result(True) + + def quic_event_received(self, event: QuicEvent) -> None: + for http_event in self._http.handle_event(event): + self.http_event_received(http_event) + + async def request(self, url: str, include_headers: bool = False) -> None: + """Issue an http/3 get request, print response pieces as the packets arrive.""" + stream_id: int = self._quic.get_next_available_stream_id() + future: asyncio.Future[bool] = self._loop.create_future() + parsed_url = urlparse(url) + self._stream_ids[stream_id] = future + self._include_headers = include_headers + self._http.send_headers( + stream_id=stream_id, + headers=[ + (b":method", "GET".encode()), + (b":scheme", parsed_url.scheme.encode()), + (b":authority", parsed_url.netloc.encode()), + (b":path", parsed_url.path.encode()), + ], + end_stream=True, + ) + await future + + +async def request(url: str, config: QuicConfiguration, include_headers: bool) -> None: + parsed_url = urlparse(url) + client_resolver = aioquic.asyncio.client.connect( + host=parsed_url.hostname, + port=parsed_url.port or 443, + configuration=config, + create_protocol=Http3Client, + wait_connected=True, + ) + async with client_resolver as client: + client = cast(Http3Client, client) + await client.request(url, include_headers) + + +async def main(argv) -> None: + parser = argparse.ArgumentParser(description="HTTP/3 client") + parser.add_argument("url", type=str, help="the URL to query (must be HTTPS)") + parser.add_argument( + "--ca-certs", type=str, nargs="+", help="load CA certificates from the specified file") + parser.add_argument( + "--include-headers", action="store_true", help="output the headers before the body") + args = parser.parse_args(argv) + config = QuicConfiguration( + is_client=True, + alpn_protocols=H3_ALPN, + ) + for cert in args.ca_certs or []: + config.load_verify_locations(cert) + await request(args.url, config, args.include_headers) + + +if __name__ == '__main__': + sys.exit(asyncio.run(main(sys.argv[1:]))) diff --git a/tools/proto_format/BUILD b/tools/proto_format/BUILD index 2b025ebfaf79..597d2f191b5d 100644 --- a/tools/proto_format/BUILD +++ b/tools/proto_format/BUILD @@ -2,14 +2,16 @@ load("@aspect_bazel_lib//lib:jq.bzl", "jq") load("@envoy_repo//:path.bzl", "PATH") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_py_data") +load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_py_data", "envoy_pytool_binary") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + # Files to include when building or comparing the normalized API API_FILES = [ "BUILD", @@ -130,7 +132,7 @@ genrule( tools = [":formatted_api"], ) -py_binary( +envoy_pytool_binary( name = "fetch_normalized_changes", srcs = ["fetch_normalized_changes.py"], args = [ @@ -165,7 +167,7 @@ genrule( ], ) -py_binary( +envoy_pytool_binary( name = "proto_sync", srcs = ["proto_sync.py"], args = [ diff --git a/tools/proto_format/format_api.py b/tools/proto_format/format_api.py index 34f38588804c..8306a1f51515 100644 --- a/tools/proto_format/format_api.py +++ b/tools/proto_format/format_api.py @@ -32,6 +32,7 @@ 'envoy.extensions.filters.network.client_ssl_auth.v3', 'envoy.extensions.filters.network.generic_proxy.action.v3', 'envoy.extensions.filters.network.generic_proxy.codecs.dubbo.v3', + 'envoy.extensions.filters.network.generic_proxy.codecs.kafka.v3', 'envoy.extensions.filters.network.generic_proxy.matcher.v3', 'envoy.extensions.filters.network.generic_proxy.router.v3', 'envoy.extensions.filters.network.generic_proxy.v3', @@ -177,18 +178,18 @@ def get_import_deps(proto_path): continue # Special case handling for UDPA annotations. if import_path.startswith('udpa/annotations/'): - imports.append('@com_github_cncf_udpa//udpa/annotations:pkg') + imports.append('@com_github_cncf_xds//udpa/annotations:pkg') continue if import_path.startswith('xds/type/matcher/v3/'): - imports.append('@com_github_cncf_udpa//xds/type/matcher/v3:pkg') + imports.append('@com_github_cncf_xds//xds/type/matcher/v3:pkg') continue # Special case for handling XDS annotations. if import_path.startswith('xds/annotations/v3/'): - imports.append('@com_github_cncf_udpa//xds/annotations/v3:pkg') + imports.append('@com_github_cncf_xds//xds/annotations/v3:pkg') continue # Special case handling for XDS core. if import_path.startswith('xds/core/v3/'): - imports.append('@com_github_cncf_udpa//xds/core/v3:pkg') + imports.append('@com_github_cncf_xds//xds/core/v3:pkg') continue # Explicit remapping for external deps, compute paths for envoy/*. if import_path in data["external_proto_deps"]["imports"]: diff --git a/tools/proto_format/proto_format.sh b/tools/proto_format/proto_format.sh index f86fc7b7471b..b60286ff98e2 100755 --- a/tools/proto_format/proto_format.sh +++ b/tools/proto_format/proto_format.sh @@ -30,7 +30,7 @@ bazel "${BAZEL_STARTUP_OPTIONS[@]}" run "${BAZEL_BUILD_OPTIONS[@]}" \ --ci # Dont run this in git hooks by default -if [[ -n "$AZP_BRANCH" ]] || [[ "${FORCE_PROTO_FORMAT}" == "yes" ]]; then +if [[ -n "$CI_BRANCH" ]] || [[ "${FORCE_PROTO_FORMAT}" == "yes" ]]; then echo "Run buf tests" cd api/ || exit 1 bazel "${BAZEL_STARTUP_OPTIONS[@]}" run "${BAZEL_BUILD_OPTIONS[@]}" @com_github_bufbuild_buf//:bin/buf lint diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 01ce510978c0..16178951349a 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -1,15 +1,18 @@ load("@base_pip3//:requirements.bzl", "requirement") -load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") -load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_jinja_env", "envoy_py_data") +load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_jinja_env", "envoy_py_data", "envoy_pytool_binary", "envoy_pytool_library") load("//tools/protodoc:protodoc.bzl", "protodoc_rule") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() -py_binary( +envoy_py_namespace() + +envoy_pytool_binary( name = "generate_empty", srcs = ["generate_empty.py"], visibility = ["//visibility:public"], @@ -19,10 +22,22 @@ py_binary( ], ) -py_proto_library( +proto_library( name = "manifest_proto", srcs = ["manifest.proto"], - deps = ["@com_google_protobuf//:protobuf_python"], + deps = [ + "@com_google_protobuf//:struct_proto", + ], +) + +py_proto_library( + name = "manifest_py_pb2", + deps = [":manifest_proto"], +) + +py_proto_library( + name = "validate_py_pb2", + deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"], ) envoy_py_data( @@ -30,14 +45,14 @@ envoy_py_data( src = "//docs:protodoc_manifest.yaml", ) -py_binary( +envoy_pytool_binary( name = "manifest_to_json", srcs = ["manifest_to_json.py"], args = ["$(location @envoy_api//:v3_proto_set)"], data = ["@envoy_api//:v3_proto_set"], deps = [ requirement("envoy.base.utils"), - ":manifest_proto", + ":manifest_py_pb2", ":protodoc_manifest_untyped", "@com_google_protobuf//:protobuf_python", ], @@ -101,17 +116,17 @@ envoy_py_data( src = ":data_srcs", ) -py_binary( +envoy_pytool_binary( name = "protodoc", srcs = ["protodoc.py"], visibility = ["//visibility:public"], deps = [ ":data", ":jinja", + ":validate_py_pb2", "//tools/api_proto_plugin", - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", - "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", + "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto", requirement("envoy.code.check"), ], ) @@ -124,7 +139,7 @@ protodoc_rule( ], ) -py_library( +envoy_pytool_library( name = "rst_filters", srcs = ["rst_filters.py"], ) diff --git a/tools/protodoc/protodoc.bzl b/tools/protodoc/protodoc.bzl index 6ad1db8d97bd..58b83892a18f 100644 --- a/tools/protodoc/protodoc.bzl +++ b/tools/protodoc/protodoc.bzl @@ -12,7 +12,7 @@ def _protodoc_impl(target, ctx): # # The aspect builds the transitive docs, so any .proto in the dependency graph # get docs created. -protodoc_aspect = api_proto_plugin_aspect("//tools/protodoc", _protodoc_impl) +protodoc_aspect = api_proto_plugin_aspect("@envoy//tools/protodoc", _protodoc_impl) def _protodoc_rule_impl(ctx): deps = [] @@ -20,7 +20,7 @@ def _protodoc_rule_impl(ctx): for path in dep[OutputGroupInfo].rst.to_list(): envoy_api = ( path.short_path.startswith("../envoy_api") or - path.short_path.startswith("../com_github_cncf_udpa") + path.short_path.startswith("../com_github_cncf_xds") ) if envoy_api: deps.append(path) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 651d9b78e749..8d9f0cd46ad5 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -3,7 +3,9 @@ # for the underlying protos mentioned in this file. See # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html for Sphinx RST syntax. +import importlib import logging +import os import sys from collections import defaultdict from functools import cached_property, lru_cache @@ -17,8 +19,6 @@ from validate import validate_pb2 from xds.annotations.v3 import status_pb2 as xds_status_pb2 -from envoy.code.check.checker import BackticksCheck - from tools.api_proto_plugin import annotations, constants, plugin, visitor from tools.protodoc import jinja from tools.protodoc.data import data @@ -140,8 +140,8 @@ class RstFormatVisitor(visitor.Visitor): """ @cached_property - def backticks_check(self) -> BackticksCheck: - return BackticksCheck() + def backticks_check(self): + return importlib.import_module("envoy.code.check.checker").BackticksCheck() @property def contrib_extension_category_data(self): @@ -253,7 +253,7 @@ def visit_message(self, msg_proto, ctx, nested_msgs: Iterable, nested_enums: Ite if msg_proto.options.map_entry or self._hide(ctx.leading_comment.annotations): return '' name = normalize_type_context_name(ctx.name) - return self.tpl_content.render( + message = self.tpl_content.render( header=self.tpl_header.render( anchor=message_cross_ref_label(name), title=name, @@ -269,6 +269,15 @@ def visit_message(self, msg_proto, ctx, nested_msgs: Iterable, nested_enums: Ite nested_msgs=nested_msgs, nested_enums=nested_enums)) + if not os.environ.get("DOCS_RST_CHECK"): + return message + if error := self.backticks_check(message): + error_message = f"Bad RST ({msg_proto.name}): {error}" + if ctx.name.startswith("envoy."): + raise ProtodocError(error_message) + logger.warning(error_message) + return message + @lru_cache def _comment(self, comment, show_wip_warning=False): """Format a comment string with additional RST for annotations. diff --git a/tools/protoprint/BUILD b/tools/protoprint/BUILD index 4a8cac10d797..02bca13066b8 100644 --- a/tools/protoprint/BUILD +++ b/tools/protoprint/BUILD @@ -1,16 +1,19 @@ load("@base_pip3//:requirements.bzl", "requirement") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_py_data") +load("//tools/base:envoy_python.bzl", "envoy_py_data", "envoy_pytool_binary") load("//tools/protoprint:protoprint.bzl", "protoprint_rule") +load("//tools/python:namespace.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() -py_binary( +envoy_py_namespace() + +envoy_pytool_binary( name = "protoprint", srcs = ["protoprint.py"], data = [ @@ -20,6 +23,7 @@ py_binary( ], visibility = ["//visibility:public"], deps = [ + ":validate_py_pb2", "//tools/api_versioning:utils", "//tools/protoxform:options", "//tools/protoxform:utils", @@ -27,15 +31,19 @@ py_binary( requirement("packaging"), "//tools/type_whisperer", "//tools/type_whisperer:api_type_db_proto_py_proto", - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", - "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", + "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto", "@com_google_googleapis//google/api:annotations_py_proto", "@com_google_protobuf//:protobuf_python", "@envoy_api//envoy/annotations:pkg_py_proto", ], ) +py_proto_library( + name = "validate_py_pb2", + deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"], +) + protoprint_rule( name = "protoprinted_srcs", deps = [ diff --git a/tools/protoprint/protoprint.py b/tools/protoprint/protoprint.py index 5828fb0e10f1..21ea74b8cf70 100644 --- a/tools/protoprint/protoprint.py +++ b/tools/protoprint/protoprint.py @@ -219,13 +219,14 @@ def camel_case(s): # Workaround packages in generated go code conflicting by transforming: # foo/bar/v2 to use barv2 as the package in the generated code golang_package_name = "" + golang_package_base = file_proto.package.replace(".", "/") + if file_proto.name.startswith('contrib/'): + golang_package_base = 'contrib/' + golang_package_base if file_proto.package.split(".")[-1] in ("v2", "v3"): name = "".join(file_proto.package.split(".")[-2:]) golang_package_name = ";" + name - options.go_package = "".join([ - "github.com/envoyproxy/go-control-plane/", - file_proto.package.replace(".", "/"), golang_package_name - ]) + options.go_package = "".join( + ["github.com/envoyproxy/go-control-plane/", golang_package_base, golang_package_name]) # This is a workaround for C#/Ruby namespace conflicts between packages and # objects, see https://github.com/envoyproxy/envoy/pull/3854. diff --git a/tools/protoxform/BUILD b/tools/protoxform/BUILD index 73c95708fe52..500f801e547e 100644 --- a/tools/protoxform/BUILD +++ b/tools/protoxform/BUILD @@ -1,4 +1,5 @@ load("@aspect_bazel_lib//lib:jq.bzl", "jq") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") load("@rules_python//python:defs.bzl", "py_binary", "py_library") @@ -15,9 +16,8 @@ py_binary( ":utils", "//tools/api_proto_plugin", "//tools/type_whisperer:api_type_db_proto_py_proto", - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", - "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", + "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto", "@com_google_googleapis//google/api:annotations_py_proto", "@envoy_api//envoy/annotations:pkg_py_proto", ], @@ -33,6 +33,14 @@ py_library( name = "utils", srcs = ["utils.py"], visibility = ["//visibility:public"], + deps = [ + ":validate_py_pb2", + ], +) + +py_proto_library( + name = "validate_py_pb2", + deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"], ) protoxform_rule( diff --git a/tools/protoxform/protoxform.bzl b/tools/protoxform/protoxform.bzl index d0933bf9a399..bd7ed95293d9 100644 --- a/tools/protoxform/protoxform.bzl +++ b/tools/protoxform/protoxform.bzl @@ -24,7 +24,7 @@ def _protoxform_rule_impl(ctx): for path in dep[OutputGroupInfo].proto.to_list(): envoy_api = ( path.short_path.startswith("../envoy_api") or - path.short_path.startswith("../com_github_cncf_udpa") or + path.short_path.startswith("../com_github_cncf_xds") or path.short_path.startswith("tools/testdata") ) if envoy_api: diff --git a/tools/python/BUILD b/tools/python/BUILD new file mode 100644 index 000000000000..779d1695d3b7 --- /dev/null +++ b/tools/python/BUILD @@ -0,0 +1 @@ +licenses(["notice"]) # Apache 2 diff --git a/tools/python/namespace.bzl b/tools/python/namespace.bzl new file mode 100644 index 000000000000..7be06755127b --- /dev/null +++ b/tools/python/namespace.bzl @@ -0,0 +1,17 @@ +load("@rules_python//python:defs.bzl", "py_library") + +def envoy_py_namespace(): + """Adding this to a build, injects a namespaced __init__.py, this allows namespaced + packages - eg envoy.base.utils to co-exist with packages created from the repo.""" + native.genrule( + name = "py-init-file", + outs = ["__init__.py"], + cmd = """ + echo "__path__ = __import__('pkgutil').extend_path(__path__, __name__)" > $@ + """, + ) + py_library( + name = "py-init", + srcs = [":py-init-file"], + visibility = ["//visibility:public"], + ) diff --git a/tools/repo/BUILD b/tools/repo/BUILD new file mode 100644 index 000000000000..aee406f02364 --- /dev/null +++ b/tools/repo/BUILD @@ -0,0 +1,21 @@ +load("@base_pip3//:requirements.bzl", "requirement") +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//tools/base:envoy_python.bzl", "envoy_pytool_binary") +load("//tools/python:namespace.bzl", "envoy_py_namespace") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_py_namespace() + +envoy_pytool_binary( + name = "notify", + srcs = ["notify.py"], + deps = [ + requirement("aio.api.github"), + requirement("aio.run.runner"), + requirement("icalendar"), + requirement("slack_sdk"), + ], +) diff --git a/tools/repo/notify.py b/tools/repo/notify.py new file mode 100644 index 000000000000..64f8194c4e68 --- /dev/null +++ b/tools/repo/notify.py @@ -0,0 +1,336 @@ +# Script for collecting PRs in need of review, and informing maintainers via +# slack. +# +# The tool can be used in `--dry_run` mode or `--report` mode. In the latter +# case it will just dump a report, and in the former it will show what it would +# post to slack +# +# NOTE: Slack IDs can be found in the user's full profile from within Slack. + +import datetime +import html +import icalendar +import json +import os +import sys +from datetime import datetime as dt +from functools import cached_property + +import aiohttp + +from slack_sdk.web.async_client import AsyncWebClient +from slack_sdk.errors import SlackApiError + +from aio.api import github as github +from aio.core.functional import async_property +from aio.run import runner + +ENVOY_REPO = "envoyproxy/envoy" + +# Oncall calendar +CALENDAR = "https://calendar.google.com/calendar/ical/d6glc0l5rc3v235q9l2j29dgovh3dn48%40import.calendar.google.com/public/basic.ics" + +ISSUE_LINK = "https://github.com/envoyproxy/envoy/issues?q=is%3Aissue+is%3Aopen+label%3Atriage" +SLACK_EXPORT_URL = "https://api.slack.com/apps/A023NPQQ33K/oauth?" + +MAINTAINERS = { + 'alyssawilk': 'U78RP48V9', + 'mattklein123': 'U5CALEVSL', + 'lizan': 'U79E51EQ6', + 'snowp': 'U93KTPQP6', + 'ggreenway': 'U78MBV869', + 'htuch': 'U78E7055Z', + 'zuercher': 'U78J72Q82', + 'phlax': 'U017PLM0GNQ', + 'jmarantz': 'U80HPLBPG', + 'ravenblackx': 'U02MJHFEX35', + 'yanavlasov': 'UJHLR5KFS', + 'RyanTheOptimist': 'U01SW3JC8GP', + 'adisuissa': 'UT17EMMTP', + 'KBaichoo': 'U016ZPU8KBK', + 'wbpcode': 'U017KF5C0Q6', + 'kyessenov': 'U7KTRAA8M', + 'keith': 'UGS5P90CF', + 'abeyad': 'U03CVM7GPM1', + 'soulxu': 'U01GNQ3B8AY', +} + +# First pass reviewers who are not maintainers should get +# notifications but not result in a PR not getting assigned a +# maintainer owner. +FIRST_PASS = { + 'silverstar194': 'U03LNPC8JN9', + 'nezdolik': 'UDYUWRL13', + 'daixiang0': 'U020CJG6UU8', + 'botengyao': 'U037YUAK147', + 'tyxia': 'U023U1ZN9SP', +} + +# Only notify API reviewers who aren't maintainers. +# Maintainers are already notified of pending PRs. +API_REVIEWERS = { + 'markdroth': 'UMN8K55A6', + 'adisuissa': 'UT17EMMTP', +} + + +class RepoNotifier(runner.Runner): + + @property + def dry_run(self): + return self.args.dry_run + + @cached_property + def github(self): + return github.GithubAPI(self.session, "", oauth_token=self.github_token) + + @cached_property + def github_token(self): + return os.getenv('GITHUB_TOKEN') + + @async_property(cache=True) + async def maintainer_notifications(self): + return (await self.tracked_prs)["maintainer_notifications"] + + @async_property + async def pulls(self): + async for pull in self.repo.getiter("pulls"): + skip = ( + pull["draft"] or pull["user"]["login"] == "dependabot[bot]" + or self.is_waiting(pull)) + if skip: + self.log.notice(f"Skipping {pull['title']} {pull['url']}") + continue + yield pull + + @cached_property + def repo(self): + return self.github[ENVOY_REPO] + + @cached_property + def session(self): + return aiohttp.ClientSession() + + @async_property(cache=True) + async def shepherd_notifications(self): + return (await self.tracked_prs)["shepherd_notifications"] + + @property + def should_report(self): + return self.args.report + + @cached_property + def slack_client(self): + return AsyncWebClient(token=self.slack_bot_token) + + @cached_property + def slack_bot_token(self): + return os.getenv('SLACK_BOT_TOKEN') + + @cached_property + def slo_max(self): + """on Monday, allow for 24h + 48h.""" + hours = (72 if datetime.date.today().weekday() == 0 else 24) + return datetime.timedelta(hours=hours) + + @async_property(cache=True) + async def stalled_prs(self): + return (await self.tracked_prs)["stalled_prs"] + + @async_property(cache=True) + async def oncall_string(self): + response = await self.session.get(CALENDAR) + content = await response.read() + parsed_calendar = icalendar.Calendar.from_ical(content) + + now = datetime.datetime.now() + sunday = now - datetime.timedelta(days=now.weekday() + 1) + + for component in parsed_calendar.walk(): + if component.name == "VEVENT": + if (sunday.date() == component.decoded("dtstart").date()): + return component.get("summary") + return "unable to find this week's oncall" + + @async_property(cache=True) + async def tracked_prs(self): + # A dict of maintainer : outstanding_pr_string to be sent to slack + # A placeholder for unassigned PRs, to be sent to #maintainers eventually + maintainers_and_prs = dict(unassigned=[]) + # A dict of shepherd : outstanding_pr_string to be sent to slack + api_review = {} + # Out-SLO PRs to be sent to #envoy-maintainer-oncall + stalled_prs = [] + + # TODO: pre-filter these + async for pull in self.pulls: + updated_at = dt.fromisoformat(pull["updated_at"].replace('Z', '+00:00')) + age = dt.now(datetime.timezone.utc) - dt.fromisoformat( + pull["updated_at"].replace('Z', '+00:00')) + message = self.pr_message(age, pull) + + if await self.needs_api_review(pull): + for assignee in self.get_assignees(pull, API_REVIEWERS): + api_review[assignee["login"]] = api_review.get( + assignee["login"], + [f"Hello, {assignee['login']}, here are your PR reminders for the day"]) + api_review[assignee["login"]].append(message) + + # If the PR has been out-SLO for over a day, inform on-call + if age > self.slo_max + datetime.timedelta(hours=36): + stalled_prs.append(message) + + has_maintainer = False + for assignee in self.get_assignees(pull, {**MAINTAINERS, **FIRST_PASS}): + if MAINTAINERS.get(assignee["login"]): + has_maintainer = True + maintainers_and_prs[assignee["login"]] = maintainers_and_prs.get( + assignee["login"], []) + maintainers_and_prs[assignee["login"]].append(message) + + # If there was no maintainer, track it as unassigned. + if not has_maintainer and not self.is_contrib(pull): + maintainers_and_prs['unassigned'].append(message) + + return dict( + maintainer_notifications=maintainers_and_prs, + shepherd_notifications=api_review, + stalled_prs=stalled_prs) + + @async_property(cache=True) + async def unassigned_prs(self): + return (await self.maintainer_notifications)["unassigned"] + + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument( + '--dry_run', + action="store_true", + help="Dont post slack messages, just show what would be posted") + parser.add_argument('--report', action="store_true", help="Print a report of current state") + + def get_assignees(self, pull, assignees): + for assignee in pull["assignees"]: + if assignee["login"] in assignees: + yield assignee + + def is_contrib(self, pr): + for label in pr["labels"]: + if label["name"] == "contrib": + return True + return False + + def is_waiting(self, pr): + for label in pr["labels"]: + if label["name"].startswith("waiting"): + return True + return False + + async def needs_api_review(self, pull): + """Returns true if the PR needs an LGTM from an API shepherd.""" + if "api" not in [label["name"] for label in pull["labels"]]: + return False + # repokitten tags each commit as pending unless there has been an API LGTM + # since the latest API changes. If this PR is tagged pendding it needs an + # API review, otherwise it's set. + status = (await self.repo.getitem(f"commits/{pull['head']['sha']}/status")) + return status["state"] == "pending" if status["total_count"] else False + + async def notify(self): + await self.post_to_oncall() + await self.post_to_assignees() + + async def post_to_assignees(self): + review_notifications = ((API_REVIEWERS, await self.shepherd_notifications), + (MAINTAINERS, await self.maintainer_notifications), + (FIRST_PASS, await self.maintainer_notifications)) + for assignees, messages in review_notifications: + await self._post_to_assignees(assignees, messages) + + async def post_to_oncall(self): + try: + unassigned = "\n".join(await self.unassigned_prs) + stalled = "\n".join(await self.stalled_prs) + # On Monday, post the new oncall. + if datetime.date.today().weekday() == 0: + oncall = await self.oncall_string + await self.send_message(channel='#envoy-maintainer-oncall', text=(f"{oncall}")) + await self.send_message(channel='#general', text=(f"{oncall}")) + await self.send_message( + channel='#envoy-maintainer-oncall', + text=(f"*'Unassigned' PRs* (PRs with no maintainer assigned)\n{unassigned}")) + await self.send_message( + channel='#envoy-maintainer-oncall', + text=(f"*Stalled PRs* (PRs with review out-SLO, please address)\n{stalled}")) + await self.send_message( + channel='#envoy-maintainer-oncall', + text=( + f"*Untriaged Issues* (please tag and cc area experts)\n<{ISSUE_LINK}|{ISSUE_LINK}>" + )) + except SlackApiError as e: + self.log.error(f"Unexpected error {e.response['error']}") + + def pr_message(self, age, pull): + """Generate a pr message, bolding the time if it's out-SLO.""" + days = age.days + hours = age.seconds // 3600 + markup = ("*" if age > self.slo_max else "") + return ( + f"<{pull['html_url']}|{html.escape(pull['title'])}> has been waiting " + f"{markup}{days} days {hours} hours{markup}") + + async def run(self): + if not self.github_token: + self.log.error("Missing GITHUB_TOKEN: please check github workflow configuration") + return 1 + + if not self.slack_bot_token and not self.dry_run and not self.report: + self.log.error( + "Missing SLACK_BOT_TOKEN: please export token from " + f"{SLACK_EXPORT_URL}") + return 1 + return await (self.report() if self.should_report else self.notify()) + + async def report(self): + report = dict(maintainers={}, shepherds={}, stalled=[]) + for maintainer, messages in (await self.maintainer_notifications).items(): + report["maintainers"][maintainer] = messages + + for shepherd, messages in (await self.shepherd_notifications).items(): + report["shepherds"][shepherd] = messages + + if stalled_pr_info := await self.stalled_prs: + report["stalled"].append(stalled_pr_info) + + print(json.dumps(report)) + + async def send_message(self, channel, text): + self.log.notice(f"Slack message ({channel}):\n{text}") + if self.dry_run: + return + await self.slack_client.chat_postMessage(channel=channel, text=text) + + async def _post_to_assignees(self, assignees, messages): + for name, text in messages.items(): + # Only send texts if we have the slack UID + if not (uid := assignees.get(name)): + continue + message = "\n".join(text) + self.log.notice(f"Slack message ({name}):\n{message}") + if self.dry_run: + continue + # Ship texts off to slack. + try: + response = await self.slack_client.conversations_open(users=uid, text="hello") + channel_id = response["channel"]["id"] + await self.slack_client.chat_postMessage(channel=channel_id, text=message) + except SlackApiError as e: + print(f"Unexpected error {e.response['error']}") + + +def main(*args): + return RepoNotifier(*args)() + + +if __name__ == "__main__": + sys.exit(main(*sys.argv[1:])) diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index ea7b558f6f5e..9b4bb4e93856 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -26,16 +26,22 @@ BACKTRACE BEL BBR BIDIRECTIONAL +bm BSON BPF Bdecoded Bencoded CIO +deadcode DFP +Dynatrace DOM +Gasd GiB IPTOS +OTEL Repick +Reserializer SION TRA Websockets @@ -71,7 +77,9 @@ FIXME HEXDIG HEXDIGIT HPE +Kwasi LRU +Mensah NAK OWS Preconnecting @@ -165,6 +173,7 @@ FIPS FIRSTHDR FQDN FREEBIND +curr eslint freeifaddrs FUZZER @@ -176,6 +185,7 @@ dnsresolvers endpos eps fo +gb ghi golang guarddog @@ -247,11 +257,13 @@ LF LHS hls iframe +ilp ingressed integrations iouring jkl lang +libcurl libsxg LLVM LPT @@ -298,6 +310,7 @@ ODCDS middlewildcard monostate mpd +na oghttp OID OK @@ -336,6 +349,8 @@ QDCOUNT QPACK QUIC QoS +qat +qatzip RAII RANLUX RBAC @@ -767,6 +782,7 @@ gateway gcov genrule geolocation +geo geoip Geoip getaddrinfo @@ -811,6 +827,7 @@ hostnames hostset hotrestart hrefs +htpasswd huffman hystrix idempotency @@ -851,6 +868,7 @@ iovecs ips iptables ish +isp istio istream istringstream @@ -913,6 +931,8 @@ matcher matchers maxage maxbuffer +Maxmind +maxmind megamiss mem memcmp @@ -934,6 +954,7 @@ misconfigured mixin mkdir mmap +mmdb mmsg mmsghdr mongo @@ -1291,6 +1312,7 @@ subtypes subzone suf superclass +superroot superset svc symlink @@ -1341,6 +1363,7 @@ transcoder transcoding transferral trds +tri triaged trie tuple diff --git a/tools/tarball/BUILD b/tools/tarball/BUILD new file mode 100644 index 000000000000..069d4eee6b27 --- /dev/null +++ b/tools/tarball/BUILD @@ -0,0 +1,8 @@ +load("@envoy_toolshed//tarball:macros.bzl", "unpacker") + +licenses(["notice"]) # Apache 2 + +unpacker( + name = "unpack", + zstd = "//tools/zstd", +) diff --git a/tools/testdata/protoxform/envoy/v2/BUILD b/tools/testdata/protoxform/envoy/v2/BUILD index dc051c7a2a4a..2c6796578946 100644 --- a/tools/testdata/protoxform/envoy/v2/BUILD +++ b/tools/testdata/protoxform/envoy/v2/BUILD @@ -14,7 +14,7 @@ proto_library( visibility = ["//visibility:public"], deps = [ "//tools/testdata/protoxform/external:external_protos", - "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", "@com_google_googleapis//google/api:annotations_proto", "@com_google_protobuf//:any_proto", "@envoy_api//envoy/annotations:pkg", diff --git a/tools/type_whisperer/BUILD b/tools/type_whisperer/BUILD index 43d4b7cf0eaf..32cdf6e0d4d8 100644 --- a/tools/type_whisperer/BUILD +++ b/tools/type_whisperer/BUILD @@ -21,7 +21,7 @@ py_binary( deps = [ ":types_py_proto", "//tools/api_proto_plugin", - "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", "@com_google_protobuf//:protobuf_python", ], ) @@ -43,7 +43,7 @@ py_binary( srcs = ["file_descriptor_set_text_gen.py"], visibility = ["//visibility:public"], deps = [ - "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", "@com_google_protobuf//:protobuf_python", ], ) @@ -102,7 +102,7 @@ envoy_cc_library( deps = [ "//source/common/protobuf", "//tools/type_whisperer:api_type_db_proto_cc_proto", - "@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto", + "@com_github_cncf_xds//udpa/annotations:pkg_cc_proto", "@com_google_absl//absl/container:node_hash_map", ], ) diff --git a/tools/type_whisperer/proto_build_targets_gen.py b/tools/type_whisperer/proto_build_targets_gen.py index 1842c0a1eeda..f0b26f17797e 100644 --- a/tools/type_whisperer/proto_build_targets_gen.py +++ b/tools/type_whisperer/proto_build_targets_gen.py @@ -62,9 +62,9 @@ name = "xds_protos", visibility = ["//visibility:public"], deps = [ - "@com_github_cncf_udpa//xds/core/v3:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", - "@com_github_cncf_udpa//xds/type/v3:pkg", + "@com_github_cncf_xds//xds/core/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//xds/type/v3:pkg", ], ) diff --git a/tools/vscode/refresh_compdb.sh b/tools/vscode/refresh_compdb.sh index 46a8f433f954..996a9d576102 100755 --- a/tools/vscode/refresh_compdb.sh +++ b/tools/vscode/refresh_compdb.sh @@ -5,10 +5,13 @@ bazel_or_isk=bazelisk command -v bazelisk &> /dev/null || bazel_or_isk=bazel -[[ -z "${EXCLUDE_CONTRIB}" ]] || opts="--exclude_contrib" +opts=(--vscode --bazel="$bazel_or_isk") + +[[ -z "${EXCLUDE_CONTRIB}" ]] || opts+=(--exclude_contrib) # Setting TEST_TMPDIR here so the compdb headers won't be overwritten by another bazel run -TEST_TMPDIR=${BUILD_DIR:-/tmp}/envoy-compdb tools/gen_compilation_database.py --vscode --bazel=$bazel_or_isk ${opts} +TEST_TMPDIR=${BUILD_DIR:-/tmp}/envoy-compdb tools/gen_compilation_database.py \ + "${opts[@]}" # Kill clangd to reload the compilation database pkill clangd || :