refactor(clustering/rpc): tunings and fixes for rubustness #37038

Workflow file for this run

.github/workflows/build_and_test.yml at 63c17e2

	name: Build & Test
	on:
	pull_request:
	paths-ignore:
	# ignore markdown files (CHANGELOG.md, README.md, etc.)
	- '*/.md'
	- '.github/workflows/release.yml'
	- 'changelog/**'
	- 'kong.conf.default'
	push:
	paths-ignore:
	# ignore markdown files (CHANGELOG.md, README.md, etc.)
	- '*/.md'
	# ignore PRs for the generated COPYRIGHT file
	- 'COPYRIGHT'
	branches:
	- master
	- release/*
	- test-please/*
	workflow_dispatch:
	inputs:
	coverage:
	description: 'Coverage enabled'
	required: false
	type: boolean
	default: false

	# cancel previous runs if new commits are pushed to the PR, but run for each commit on master
	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	env:
	BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build
	KONG_TEST_COVERAGE: ${{ inputs.coverage == true \|\| github.event_name == 'schedule' }}
	RUNNER_COUNT: 7

	jobs:
	metadata:
	name: Metadata
	runs-on: ubuntu-22.04
	outputs:
	old-kong-version: ${{ steps.old-kong-version.outputs.ref }}

	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # `git merge-base` requires the history

	- name: Get Old Kong Version
	id: old-kong-version
	run: \|
	KONG_VERSION=$(bash scripts/grep-kong-version.sh)
	major=$(echo "$KONG_VERSION" \| cut -d. -f1)
	minor=$(echo "$KONG_VERSION" \| cut -d. -f2)
	# if the minor version isn't 0, use the first release or starting point of the previous minor branch;
	# otherwise just leave it empty, so later the default branch or commit will be used.
	if [ "$minor" -ne 0 ]; then
	minor=$((minor - 1))
	git fetch origin master -t
	if [ $(git tag -l "$major.$minor.0") ]; then
	echo "ref=$major.$minor.0" >> $GITHUB_OUTPUT
	else
	git fetch origin release/$major.$minor.x
	COMMIT_HASH=$(git merge-base origin/master origin/release/$major.$minor.x)
	echo "ref=$COMMIT_HASH" >> $GITHUB_OUTPUT
	fi
	else
	echo "ref=" >> $GITHUB_OUTPUT
	fi

	build:
	uses: ./.github/workflows/build.yml
	with:
	relative-build-root: bazel-bin/build

	lint-and-doc-tests:
	name: Lint and Doc tests
	runs-on: ubuntu-22.04
	needs: build

	steps:
	- name: Bump max open files
	run: \|
	sudo echo 'kong soft nofile 65536' \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo 'kong hard nofile 65536' \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo "$(whoami) soft nofile 65536" \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo "$(whoami) hard nofile 65536" \| sudo tee -a /etc/security/limits.d/kong-ci.conf

	- name: Checkout Kong source code
	uses: actions/checkout@v4

	- name: Lookup build cache
	id: cache-deps
	uses: actions/cache@v4
	with:
	path: ${{ env.BUILD_ROOT }}
	key: ${{ needs.build.outputs.cache-key }}

	- name: Check test-helpers doc generation
	run: \|
	source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
	pushd ./spec && ldoc .

	- name: Check autodoc generation
	run: \|
	source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
	scripts/autodoc

	- name: Lint Lua code
	run: \|
	make lint

	- name: Validate rockspec file
	run: \|
	source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
	scripts/validate-rockspec

	- name: Check spec file misspelling
	run: \|
	scripts/check_spec_files_spelling.sh

	- name: Check labeler configuration
	run: scripts/check-labeler.pl .github/labeler.yml

	schedule:
	name: Schedule busted tests to run
	runs-on: ubuntu-22.04
	needs: build

	env:
	WORKFLOW_ID: ${{ github.run_id }}

	outputs:
	runners: ${{ steps.generate-runner-array.outputs.RUNNERS }}

	steps:
	- name: Checkout source code
	uses: actions/checkout@v4

	- name: Download runtimes file
	uses: Kong/gh-storage/download@v1
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	with:
	repo-path: Kong/gateway-action-storage/main/.ci/runtimes.json

	- name: Schedule tests
	uses: Kong/gateway-test-scheduler/schedule@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3
	with:
	test-suites-file: .ci/test_suites.json
	test-file-runtime-file: .ci/runtimes.json
	output-prefix: test-chunk.
	runner-count: ${{ env.RUNNER_COUNT }}
	static-mode: ${{ github.run_attempt > 1 }}

	- name: Upload schedule files
	uses: actions/upload-artifact@v4
	continue-on-error: true
	with:
	name: schedule-test-files
	path: test-chunk.*
	retention-days: 7

	- name: Generate runner array
	id: generate-runner-array
	run: \|
	echo "RUNNERS=[$(seq -s "," 1 $(( "$RUNNER_COUNT" )))]" >> "$GITHUB_OUTPUT"

	busted-tests:
	name: Busted test runner ${{ matrix.runner }}
	runs-on: ubuntu-22.04
	needs: [metadata,build,schedule]

	strategy:
	fail-fast: false
	matrix:
	runner: ${{ fromJSON(needs.schedule.outputs.runners) }}

	services:
	postgres:
	image: postgres:13
	env:
	POSTGRES_USER: kong
	POSTGRES_DB: kong
	POSTGRES_HOST_AUTH_METHOD: trust
	ports:
	- 5432:5432
	options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8

	grpcbin:
	image: kong/grpcbin
	ports:
	- 15002:9000
	- 15003:9001

	redis:
	image: redis
	ports:
	- 6379:6379
	- 6380:6380
	options: >-
	--name kong_redis

	zipkin:
	image: openzipkin/zipkin:2
	ports:
	- 9411:9411

	redis-auth:
	image: redis/redis-stack-server
	# Set health checks to wait until redis has started
	options: >-
	--health-cmd "redis-cli ping"
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 6381:6379
	env:
	REDIS_ARGS: "--requirepass passdefault"

	steps:
	- name: Bump max open files
	run: \|
	sudo echo 'kong soft nofile 65536' \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo 'kong hard nofile 65536' \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo "$(whoami) soft nofile 65536" \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo "$(whoami) hard nofile 65536" \| sudo tee -a /etc/security/limits.d/kong-ci.conf

	- name: Checkout Kong source code
	uses: actions/checkout@v4

	# used for plugin compatibility test
	- name: Checkout old version Kong source code
	uses: actions/checkout@v4
	with:
	path: kong-old
	# if the minor version is 0, `ref` will default to ''
	# which is same as in the previous step
	ref: ${{ needs.metadata.outputs.old-kong-version }}

	- name: Lookup build cache
	id: cache-deps
	uses: actions/cache@v4
	with:
	path: ${{ env.BUILD_ROOT }}
	key: ${{ needs.build.outputs.cache-key }}

	- name: Build WASM Test Filters
	uses: ./.github/actions/build-wasm-test-filters

	- name: Add gRPC test host names
	run: \|
	echo "127.0.0.1 grpcs_1.test" \| sudo tee -a /etc/hosts
	echo "127.0.0.1 grpcs_2.test" \| sudo tee -a /etc/hosts

	- name: Enable SSL for Redis
	run: \|
	docker cp ${{ github.workspace }} kong_redis:/workspace
	docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh
	docker restart kong_redis
	docker logs kong_redis

	- name: Run OpenTelemetry Collector
	run: \|
	mkdir -p ${{ github.workspace }}/tmp/otel
	touch ${{ github.workspace }}/tmp/otel/file_exporter.json
	sudo chmod 777 -R ${{ github.workspace }}/tmp/otel
	docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \
	-v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \
	-v ${{ github.workspace }}/tmp/otel:/etc/otel \
	--name opentelemetry-collector -d \
	otel/opentelemetry-collector-contrib:0.52.0 \
	--config=/etc/otel-collector-config.yaml
	sleep 2
	docker logs opentelemetry-collector

	- name: Install AWS SAM cli tool
	run: \|
	curl -L -s -o /tmp/aws-sam-cli.zip https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
	unzip -o /tmp/aws-sam-cli.zip -d /tmp/aws-sam-cli
	sudo /tmp/aws-sam-cli/install --update

	- name: Update PATH
	run: \|
	echo "$BUILD_ROOT/kong-dev/bin" >> $GITHUB_PATH
	echo "$BUILD_ROOT/kong-dev/openresty/nginx/sbin" >> $GITHUB_PATH
	echo "$BUILD_ROOT/kong-dev/openresty/bin" >> $GITHUB_PATH

	- name: Debug (nginx)
	run: \|
	echo nginx: $(which nginx)
	nginx -V 2>&1 \| sed -re 's/ --/\n--/g'
	ldd $(which nginx)

	- name: Debug (luarocks)
	run: \|
	echo luarocks: $(which luarocks)
	luarocks --version
	luarocks config

	- name: Tune up postgres max_connections
	run: \|
	# arm64 runners may use more connections due to more worker cores
	psql -hlocalhost -Ukong kong -tAc 'alter system set max_connections = 5000;'

	- name: Download test schedule file
	uses: actions/download-artifact@v4
	with:
	name: schedule-test-files

	- name: Generate helper environment variables
	run: \|
	echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV
	echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV
	echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV

	- name: Build & install dependencies
	run: \|
	make dev

	- name: Download test rerun information
	uses: actions/download-artifact@v4
	continue-on-error: true
	with:
	name: test-rerun-info-${{ matrix.runner }}

	- name: Download test runtime statistics from previous runs
	uses: actions/download-artifact@v4
	continue-on-error: true
	with:
	name: test-runtime-statistics-${{ matrix.runner }}

	- name: Run Tests
	env:
	KONG_TEST_PG_DATABASE: kong
	KONG_TEST_PG_USER: kong
	KONG_TEST_DATABASE: postgres
	KONG_SPEC_TEST_GRPCBIN_PORT: "15002"
	KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003"
	KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json
	KONG_SPEC_TEST_OLD_VERSION_KONG_PATH: ${{ github.workspace }}/kong-old
	DD_ENV: ci
	DD_SERVICE: kong-ce-ci
	DD_CIVISIBILITY_MANUAL_API_ENABLED: 1
	DD_CIVISIBILITY_AGENTLESS_ENABLED: true
	DD_TRACE_GIT_METADATA_ENABLED: true
	DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
	SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
	uses: Kong/gateway-test-scheduler/runner@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3
	with:
	tests-to-run-file: test-chunk.${{ matrix.runner }}.json
	failed-test-files-file: ${{ env.FAILED_TEST_FILES_FILE }}
	test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }}
	setup-venv-path: ${{ env.BUILD_ROOT }}

	- name: Upload error logs
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: busted-test-errlogs-${{ matrix.runner }}
	path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
	retention-days: 1

	- name: Upload test rerun information
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: test-rerun-info-${{ matrix.runner }}
	path: ${{ env.FAILED_TEST_FILES_FILE }}
	retention-days: 2

	- name: Upload test runtime statistics for offline scheduling
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: test-runtime-statistics-${{ matrix.runner }}
	path: ${{ env.TEST_FILE_RUNTIME_FILE }}
	retention-days: 7

	- name: Archive coverage stats file
	uses: actions/upload-artifact@v4
	if: ${{ always() && (inputs.coverage == true \|\| github.event_name == 'schedule') }}
	with:
	name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}-${{ matrix.runner }}
	retention-days: 1
	path: \|
	luacov.stats.out

	- name: Get kernel message
	if: failure()
	run: \|
	sudo dmesg -T

	pdk-tests:
	name: PDK tests
	runs-on: ubuntu-22.04
	needs: build

	steps:
	- name: Bump max open files
	run: \|
	sudo echo 'kong soft nofile 65536' \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo 'kong hard nofile 65536' \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo "$(whoami) soft nofile 65536" \| sudo tee -a /etc/security/limits.d/kong-ci.conf
	sudo echo "$(whoami) hard nofile 65536" \| sudo tee -a /etc/security/limits.d/kong-ci.conf

	- name: Checkout Kong source code
	uses: actions/checkout@v4

	- name: Lookup build cache
	id: cache-deps
	uses: actions/cache@v4
	with:
	path: ${{ env.BUILD_ROOT }}
	key: ${{ needs.build.outputs.cache-key }}

	- name: Install Test::Nginx
	run: \|
	CPAN_DOWNLOAD=./cpanm
	mkdir -p $CPAN_DOWNLOAD
	curl -o $CPAN_DOWNLOAD/cpanm https://cpanmin.us
	chmod +x $CPAN_DOWNLOAD/cpanm

	echo "Installing CPAN dependencies..."
	$CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
	$CPAN_DOWNLOAD/cpanm --notest Test::Nginx

	- name: Generate environment variables
	run: \|
	echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV

	- name: Tests
	env:
	TEST_SUITE: pdk
	run: \|
	source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
	if [[ $KONG_TEST_COVERAGE = true ]]; then
	export PDK_LUACOV=1
	fi
	eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
	prove -I. -r t

	- name: Upload error logs
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: PDK-test-errlogs
	path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
	retention-days: 1

	- name: Archive coverage stats file
	uses: actions/upload-artifact@v4
	if: ${{ always() && (inputs.coverage == true \|\| github.event_name == 'schedule') }}
	with:
	name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}
	retention-days: 1
	path: \|
	luacov.stats.out

	- name: Get kernel message
	if: failure()
	run: \|
	sudo dmesg -T

	cleanup-and-aggregate-stats:
	needs: [lint-and-doc-tests,pdk-tests,busted-tests]
	name: Cleanup and Luacov stats aggregator
	if: ${{ always() && (inputs.coverage == true \|\| github.event_name == 'schedule') }}
	runs-on: ubuntu-22.04

	steps:
	- name: Checkout source code
	uses: actions/checkout@v4

	- name: Install requirements
	run: \|
	sudo apt-get update && sudo apt-get install -y luarocks
	sudo luarocks install luacov
	sudo luarocks install luafilesystem

	# Download all archived coverage stats files
	- uses: actions/download-artifact@v4

	- name: Stats aggregation
	shell: bash
	run: \|
	lua .ci/luacov-stats-aggregator.lua "luacov-stats-out-" "luacov.stats.out" ${{ github.workspace }}/
	# The following prints a report with each file sorted by coverage percentage, and the total coverage
	printf "\n\nCoverage File\n\n"
	awk -v RS='Coverage\n-+\n' 'NR>1{print $0}' luacov.report.out \| grep -vE "^-\|^$" > summary.out
	cat summary.out \| grep -v "^Total" \| awk '{printf "%7d%% %s\n", $4, $1}' \| sort -n
	cat summary.out \| grep "^Total" \| awk '{printf "%7d%% %s\n", $4, $1}'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

refactor(clustering/rpc): tunings and fixes for rubustness #37038

Workflow file

refactor(clustering/rpc): tunings and fixes for rubustness #37038

Jobs

Run details

Workflow file for this run