ref(workflows): consolidate workflows based on their purpose #4

Workflow file for this run

.github/workflows/ci-unit-tests-os.yml at 9f251b7

	name: Multi-OS Unit Tests

	# Ensures that only one workflow task will run at a time. Previous builds, if
	# already in process, will get cancelled. Only the latest commit will be allowed
	# to run, cancelling any workflows in between
	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	on:
	workflow_dispatch:
	# we build Rust and Zcash parameter caches on main,
	# so they can be shared by all branches:
	# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache

	pull_request:
	paths:
	# code and tests
	- '*/.rs'
	# hard-coded checkpoints and proptest regressions
	- '*/.txt'
	# test data snapshots
	- '*/.snap'
	# dependencies
	- '**/Cargo.toml'
	- '**/Cargo.lock'
	- '**/deny.toml'
	# configuration files
	- '.cargo/config.toml'
	- '**/clippy.toml'
	# workflow definitions
	- '.github/workflows/ci-unit-tests-os.yml'
	- '.github/workflows/sub-build-docker-image.yml'

	push:
	branches:
	- main
	paths:
	# production code and test code
	- '*/.rs'
	# hard-coded checkpoints
	# TODO: skip proptest regressions?
	- '*/.txt'
	# test data snapshots
	- '*/.snap'
	# dependencies
	- '**/Cargo.toml'
	- '**/Cargo.lock'
	- '**/deny.toml'
	# configuration files
	- '.cargo/config.toml'
	- '**/clippy.toml'
	# workflow definitions
	- '.github/workflows/ci-unit-tests-os.yml'
	- '.github/workflows/sub-build-docker-image.yml'

	env:
	CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}
	RUST_LOG: ${{ vars.RUST_LOG }}
	RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }}
	RUST_LIB_BACKTRACE: ${{ vars.RUST_LIB_BACKTRACE }}
	COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }}

	jobs:
	########################################
	### Build and test Zebra on all OSes ###
	########################################
	test:
	name: Test ${{ matrix.rust }} on ${{ matrix.os }}${{ matrix.features }}
	# The large timeout is to accommodate:
	# - macOS and Windows builds (90 minutes, typically 30-70 minutes), and
	# - parameter downloads (an extra 100 minutes, but only when the cache expires)
	timeout-minutes: 190
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	# TODO: Windows was removed for now, see https://github.com/ZcashFoundation/zebra/issues/3801
	# TODO: macOS tests were removed for now, see https://github.com/ZcashFoundation/zebra/issues/6824
	os: [ubuntu-latest]
	rust: [stable, beta]
	features: ["", " --features getblocktemplate-rpcs"]
	exclude:
	# We're excluding macOS for the following reasons:
	# - the concurrent macOS runner limit is much lower than the Linux limit
	# - macOS is slower than Linux, and shouldn't have a build or test difference with Linux
	# - macOS is a second-tier Zebra support platform
	- os: macos-latest
	rust: beta
	- os: macos-latest
	features: " --features getblocktemplate-rpcs"
	# getblocktemplate-rpcs is an experimental feature, so we just need to test it on stable Rust
	# beta is unlikely to fail just for this feature, and if it does, we can fix it when it reaches stable.
	- os: ubuntu-latest
	rust: beta
	features: " --features getblocktemplate-rpcs"

	steps:
	- uses: actions/[email protected]
	with:
	persist-credentials: false
	- uses: r7kamura/[email protected]

	- name: Install last version of Protoc
	uses: arduino/[email protected]
	with:
	# TODO: increase to latest version after https://github.com/arduino/setup-protoc/issues/33 is fixed
	version: '23.x'
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	# Setup Rust with ${{ matrix.rust }} toolchain and minimal profile
	- name: Setup Rust
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --default-toolchain=${{ matrix.rust }} --profile=minimal


	- uses: Swatinem/[email protected]
	# TODO: change Rust cache target directory on Windows,
	# or remove this workaround once the build is more efficient (#3005).
	#with:
	# workspaces: ". -> C:\\zebra-target"
	with:
	# Split the getblocktemplate-rpcs cache from the regular cache, to avoid linker errors.
	# (These might be "disk full" errors, or they might be dependency resolution issues.)
	key: ${{ matrix.features }}

	- name: Change target output directory on Windows
	# Windows doesn't have enough space on the D: drive, so we redirect the build output to the
	# larger C: drive.
	# TODO: Remove this workaround once the build is more efficient (#3005).
	if: matrix.os == 'windows-latest'
	run: \|
	mkdir "C:\\zebra-target"
	echo "CARGO_TARGET_DIR=C:\\zebra-target" \| Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append

	- name: cargo fetch
	run: \|
	cargo fetch

	- name: Install LLVM on Windows
	if: matrix.os == 'windows-latest'
	run: \|
	choco install llvm -y
	echo "C:\Program Files\LLVM\bin" \| Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
	echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" \| Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

	- name: Skip network tests on Ubuntu and Windows
	# Ubuntu runners don't have reliable network or DNS during test steps.
	# Windows runners have an unreliable network.
	shell: bash
	if: matrix.os != 'macos-latest'
	run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV

	- name: Minimise proptest cases on macOS and Windows
	# We set cases to 1, because some tests already run 1 case by default.
	# We keep maximum shrink iterations at the default value, because it only happens on failure.
	#
	# Windows compilation and tests are slower than other platforms.
	# macOS runners do extra network tests, so they take longer.
	shell: bash
	if: matrix.os != 'ubuntu-latest'
	run: \|
	echo "PROPTEST_CASES=1" >> $GITHUB_ENV
	echo "PROPTEST_MAX_SHRINK_ITERS=1024" >> $GITHUB_ENV

	# Modified from:
	# https://github.com/zcash/librustzcash/blob/c48bb4def2e122289843ddb3cb2984c325c03ca0/.github/workflows/ci.yml#L20-L33
	#
	# TODO:
	# - split Fetch/Cache Zcash parameters into their own job,
	# and use `concurrency:` to limit it to one job per OS
	# - split get-params-path and download-params examples into their own crate,
	# to speed up compilation
	- name: Fetch path to Zcash parameters
	working-directory: ./zebra-consensus
	shell: bash
	run: echo "ZCASH_PARAMS=$(cargo run --release --example get-params-path)" >> $GITHUB_ENV
	- name: Cache Zcash parameters
	id: cache-params
	uses: actions/cache@v3
	with:
	path: ${{ env.ZCASH_PARAMS }}
	key: ${{ runner.os }}-sprout-and-sapling-params
	- name: Fetch Zcash parameters
	if: steps.cache-params.outputs.cache-hit != 'true'
	working-directory: ./zebra-consensus
	run: cargo run --release --example download-params

	# Run unit and basic acceptance tests, only showing command output if the test fails.
	#
	# If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
	- name: Run tests${{ matrix.features }}
	run: \|
	cargo test ${{ matrix.features }} --release --verbose --workspace

	# Explicitly run any tests that are usually #[ignored]

	- name: Run zebrad large sync tests${{ matrix.features }}
	# Skip the entire step on Ubuntu and Windows, because the test would be skipped anyway due to ZEBRA_SKIP_NETWORK_TESTS
	# Currently, this also skips large sync with `getblocktemplate-rpcs`,
	# but that is already covered by the Docker tests.
	if: matrix.os == 'macos-latest'
	run: \|
	cargo test ${{ matrix.features }} --release --verbose --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_

	# Install Zebra with lockfile dependencies, with no caching and default features
	install-from-lockfile-no-cache:
	name: Install zebrad from lockfile without cache on ubuntu-latest
	timeout-minutes: 60
	runs-on: ubuntu-latest

	steps:
	- uses: actions/[email protected]
	with:
	persist-credentials: false
	- uses: r7kamura/[email protected]

	# Setup Rust with stable toolchain and minimal profile
	- name: Setup Rust
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --default-toolchain=stable --profile=minimal

	- name: Install zebrad
	run: \|
	cargo install --locked --path ./zebrad/ zebrad

	# Check that Cargo.lock includes any Cargo.toml changes.
	# This check makes sure the `cargo-deny` crate dependency checks are accurate.
	check-cargo-lock:
	name: Check Cargo.lock is up to date
	timeout-minutes: 60
	runs-on: ubuntu-latest

	steps:
	- uses: actions/[email protected]
	with:
	persist-credentials: false
	- uses: r7kamura/[email protected]

	- name: Install last version of Protoc
	uses: arduino/[email protected]
	with:
	version: '23.x'
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	# Setup Rust with stable toolchain and minimal profile
	- name: Setup Rust
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --default-toolchain=stable --profile=minimal

	- uses: Swatinem/[email protected]
	with:
	shared-key: "clippy-cargo-lock"

	- name: Check Cargo.lock is up to date
	run: \|
	cargo check --locked --all-features --all-targets

	cargo-deny:
	name: Check deny.toml ${{ matrix.checks }} ${{ matrix.features }}
	runs-on: ubuntu-latest
	strategy:
	matrix:
	checks:
	- bans
	- sources
	# We don't need to check `--no-default-features` here, because (except in very rare cases):
	# - disabling features isn't going to add duplicate dependencies
	# - disabling features isn't going to add more crate sources
	features: ['', '--features default-release-binaries', '--all-features']
	# Always run the --all-features job, to get accurate "skip tree root was not found" warnings
	fail-fast: false

	# Prevent sudden announcement of a new advisory from failing ci:
	continue-on-error: ${{ matrix.checks == 'advisories' }}

	steps:
	- uses: actions/[email protected]
	with:
	persist-credentials: false
	- uses: r7kamura/[email protected]

	- name: Check ${{ matrix.checks }} with features ${{ matrix.features }}
	uses: EmbarkStudios/cargo-deny-action@v1
	with:
	# --all-features spuriously activates openssl, but we want to ban that dependency in
	# all of zebrad's production features for security reasons. But the --all-features job is
	# the only job that gives accurate "skip tree root was not found" warnings.
	# In other jobs, we expect some of these warnings, due to disabled features.
	command: check ${{ matrix.checks }} ${{ matrix.features == '--all-features' && '--allow banned' \|\| '--allow unmatched-skip-root' }}
	arguments: --workspace ${{ matrix.features }}

	unused-deps:
	name: Check for unused dependencies
	runs-on: ubuntu-latest

	steps:
	- name: Checkout git repository
	uses: actions/[email protected]
	with:
	persist-credentials: false
	- uses: r7kamura/[email protected]

	# Setup Rust with stable toolchain and minimal profile
	- name: Setup Rust
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --default-toolchain=stable --profile=minimal

	- name: Install cargo-machete
	uses: baptiste0928/[email protected]
	with:
	crate: cargo-machete

	- name: Check unused dependencies
	# Exclude macro and transitive dependencies by filtering them out of the output,
	# then if there are any more unused dependencies, fail the job.
	run: \|
	echo "-- full cargo machete output, including ignored dependencies --"
	cargo machete --skip-target-dir \|\| true
	echo "-- unused dependencies are below this line, full output is above --"
	if (cargo machete --skip-target-dir 2>/dev/null \|\| true) \| \
	grep -v -e gumdrop -e humantime-serde -e tinyvec -e zebra-utils -e "found the following" -e Cargo.toml -e Done; then
	echo "New unused dependencies were found, please remove them!"
	exit 1
	else
	echo "No unused dependencies found."
	fi

	########################################
	### Build and test Zebra with Docker ###
	########################################
	# Build the docker image used by the tests.
	#
	# The default network in the Zebra config in the image is mainnet, unless a manually triggered
	# workflow or repository variable is configured differently. Testnet jobs change that config to
	# testnet when running the image.
	build:
	name: Build CI Docker
	uses: ./.github/workflows/sub-build-docker-image.yml
	with:
	dockerfile_path: ./docker/Dockerfile
	dockerfile_target: tests
	image_name: ${{ vars.CI_IMAGE_NAME }}
	no_cache: ${{ inputs.no_cache \|\| false }}
	rust_backtrace: full
	rust_lib_backtrace: full
	rust_log: info

	# zebrad tests without cached state

	# TODO: make the non-cached-state tests use:
	# network: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# Run all the zebra tests, including tests that are ignored by default.
	# Skips tests that need a cached state disk or a lightwalletd binary.
	#
	# - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step.
	# - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.
	#
	# TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings
	test-all:
	name: Test all
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	# Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails.
	#
	# If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
	- name: Run zebrad tests
	run: \|
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK --name zebrad-tests --tty ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored
	env:
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# zebrad tests without cached state with `getblocktemplate-rpcs` feature
	#
	# Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature.
	test-all-getblocktemplate-rpcs:
	name: Test all with getblocktemplate-rpcs feature
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	- name: Run zebrad tests
	run: \|
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK --name zebrad-tests --tty -e ${{ inputs.network \|\| vars.ZCASH_NETWORK }} ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored
	env:
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# Run state tests with fake activation heights.
	#
	# This test changes zebra-chain's activation heights,
	# which can recompile all the Zebra crates,
	# so we want its build products to be cached separately.
	#
	# Also, we don't want to accidentally use the fake heights in other tests.
	#
	# (The gRPC feature is a zebrad feature, so it isn't needed here.)
	test-fake-activation-heights:
	name: Test with fake activation heights
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	- name: Run tests with fake activation heights
	run: \|
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
	env:
	TEST_FAKE_ACTIVATION_HEIGHTS: '1'
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# Test that Zebra syncs and checkpoints a few thousand blocks from an empty state.
	#
	# (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
	test-empty-sync:
	name: Test checkpoint sync from empty state
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	- name: Run zebrad large sync tests
	run: \|
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK --name zebrad-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_
	env:
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# Test launching lightwalletd with an empty lightwalletd and Zebra state.
	#
	# (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
	test-lightwalletd-integration:
	name: Test integration with lightwalletd
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	- name: Run tests with empty lightwalletd launch
	run: \|
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
	env:
	ZEBRA_TEST_LIGHTWALLETD: '1'
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# Test that Zebra works using the default config with the latest Zebra version
	test-configuration-file:
	name: Test Zebra default Docker config file
	timeout-minutes: 15
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	- name: Run tests using the default config
	run: \|
	set -ex
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start
	EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 \| grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; )
	docker stop default-conf-tests
	docker logs default-conf-tests
	exit "$EXIT_STATUS"
	env:
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

	# Test that Zebra works using the $ZEBRA_CONF_PATH config
	test-zebra-conf-path:
	name: Test Zebra custom Docker config file
	timeout-minutes: 15
	runs-on: ubuntu-latest
	needs: build
	steps:
	- uses: r7kamura/[email protected]

	- name: Inject slug/short variables
	uses: rlespinasse/github-slug-action@v4
	with:
	short-length: 7

	- name: Run tests using the $ZEBRA_CONF_PATH
	run: \|
	set -ex
	docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
	docker run -e NETWORK --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start
	EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 \| grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; )
	docker stop variable-conf-tests
	docker logs variable-conf-tests
	exit "$EXIT_STATUS"
	env:
	ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml'
	NETWORK: ${{ inputs.network \|\| vars.ZCASH_NETWORK }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ref(workflows): consolidate workflows based on their purpose #4

Workflow file

ref(workflows): consolidate workflows based on their purpose #4

Jobs

Run details

Workflow file for this run