From 9bcaeb1a44be7a4f5b59c8cebd3b6779bde07159 Mon Sep 17 00:00:00 2001 From: DaughterOfMars Date: Wed, 14 Aug 2024 04:39:06 -0400 Subject: [PATCH] chore(CI): Refactor rust CI workflows into a hierarchy (#1616) * chore(CI): Refactor rust CI workflows into a hierarchy * update name * update old paths-filter action * tweaks * remove rust cache from license check * update external too * add hierarchy top level workflow and gate other big workflows behind dprint * Add more to the hierarchy * parallelize more * more reworking * remove excess diff job * cleanup * more * move clippy back * move license check up * simplify call * Update .github/workflows/_e2e.yml Co-authored-by: Thoralf-M <46689931+Thoralf-M@users.noreply.github.com> * use diff for license check * add doc filter and update filters for workflows --------- Co-authored-by: Mario Sarcevic Co-authored-by: Thibault Martinez Co-authored-by: Thoralf-M <46689931+Thoralf-M@users.noreply.github.com> --- .github/actions/diffs/action.yml | 16 +- .github/workflows/_docs_lint.yml | 15 ++ .../{docusaurus.yml => _docusaurus.yml} | 10 +- .github/workflows/{e2e.yml => _e2e.yml} | 46 ++-- .github/workflows/_execution_cut.yml | 20 ++ .github/workflows/_external_rust_lints.yml | 35 +++ .github/workflows/_external_rust_tests.yml | 44 ++++ .github/workflows/_move_tests.yml | 17 ++ .../{mysticeti.yml => _mysticeti_tests.yml} | 26 +-- .github/workflows/_rust.yml | 58 +++++ .github/workflows/_rust_lints.yml | 43 ++++ .../workflows/{rust.yml => _rust_tests.yml} | 202 +----------------- .../{turborepo.yml => _turborepo.yml} | 11 +- .github/workflows/{typos.yml => _typos.yml} | 2 +- .github/workflows/docs-lint.yml | 36 ---- .github/workflows/dprint.yml | 24 --- .github/workflows/external.yml | 121 ----------- .github/workflows/hierarchy.yml | 130 +++++++++++ .github/workflows/labeler.yml | 2 +- 19 files changed, 408 insertions(+), 450 deletions(-) create mode 100644 .github/workflows/_docs_lint.yml rename .github/workflows/{docusaurus.yml => _docusaurus.yml} (70%) rename .github/workflows/{e2e.yml => _e2e.yml} (60%) create mode 100644 .github/workflows/_execution_cut.yml create mode 100644 .github/workflows/_external_rust_lints.yml create mode 100644 .github/workflows/_external_rust_tests.yml create mode 100644 .github/workflows/_move_tests.yml rename .github/workflows/{mysticeti.yml => _mysticeti_tests.yml} (87%) create mode 100644 .github/workflows/_rust.yml create mode 100644 .github/workflows/_rust_lints.yml rename .github/workflows/{rust.yml => _rust_tests.yml} (50%) rename .github/workflows/{turborepo.yml => _turborepo.yml} (89%) rename .github/workflows/{typos.yml => _typos.yml} (95%) delete mode 100644 .github/workflows/docs-lint.yml delete mode 100644 .github/workflows/dprint.yml delete mode 100644 .github/workflows/external.yml create mode 100644 .github/workflows/hierarchy.yml diff --git a/.github/actions/diffs/action.yml b/.github/actions/diffs/action.yml index ed03daeb86e..77b04956d22 100644 --- a/.github/actions/diffs/action.yml +++ b/.github/actions/diffs/action.yml @@ -22,7 +22,7 @@ runs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - name: Detect Changes - uses: dorny/paths-filter@v2.10.2 + uses: dorny/paths-filter@v3 id: diff with: filters: | @@ -32,13 +32,20 @@ runs: - "external-crates/**" - "narwhal/**" - "iota-execution/**" + - ".github/workflows/hierarchy.yml" - ".github/workflows/codecov.yml" - - ".github/workflows/rust.yml" - - ".github/workflows/external.yml" + - ".github/workflows/_rust.yml" + - ".github/workflows/_rust_tests.yml" + - ".github/workflows/_rust_lints.yml" + - ".github/workflows/_external_rust_tests.yml" + - ".github/workflows/_external_rust_lints.yml" + - ".github/workflows/_mysticeti_tests.yml" isDoc: - "docs/content/**" + - "docs/site/**" - "*.mdx" - - ".github/workflows/docs.yml" + - ".github/workflows/_docs_lint.yml" + - ".github/workflows/_docusaurus.yml" isOldDoc: - "doc/**" - "*.md" @@ -50,6 +57,7 @@ runs: - "Cargo.toml" - "examples/**" - "iota_programmability/**" + - ".github/workflows/_move_tests.yml" isReleaseNotesEligible: - "consensus/**" - "crates/**" diff --git a/.github/workflows/_docs_lint.yml b/.github/workflows/_docs_lint.yml new file mode 100644 index 00000000000..8fda79142af --- /dev/null +++ b/.github/workflows/_docs_lint.yml @@ -0,0 +1,15 @@ +name: Docs spellcheck and linting + +on: workflow_call + +jobs: + spelling: + name: Lint documentation + runs-on: [self-hosted] + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Spell Check Docs + uses: crate-ci/typos@v1.16.11 + with: + files: ./docs/content diff --git a/.github/workflows/docusaurus.yml b/.github/workflows/_docusaurus.yml similarity index 70% rename from .github/workflows/docusaurus.yml rename to .github/workflows/_docusaurus.yml index 6f4bb27562e..482214b0a3a 100644 --- a/.github/workflows/docusaurus.yml +++ b/.github/workflows/_docusaurus.yml @@ -1,14 +1,6 @@ name: Docusaurus Build -on: - push: - branches: [develop] - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: workflow_call jobs: build: diff --git a/.github/workflows/e2e.yml b/.github/workflows/_e2e.yml similarity index 60% rename from .github/workflows/e2e.yml rename to .github/workflows/_e2e.yml index 3a2f0c4e5c4..597d72657aa 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/_e2e.yml @@ -1,35 +1,33 @@ name: TS End-to-end Tests on: - pull_request: + workflow_call: + inputs: + isRust: + type: boolean + required: true + isExplorer: + type: boolean + required: true + isTypescriptSDK: + type: boolean + required: true + isWallet: + type: boolean + required: true + isGraphQlTransport: + type: boolean + required: true concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - diff: - runs-on: self-hosted - outputs: - isWallet: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), 'iota-wallet')) }} - isExplorer: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), 'iota-explorer')) }} - isTypescriptSDK: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), '@iota/iota-sdk')) }} - isGraphQlTransport: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), '@iota/graphql-transport')) }} - isRust: ${{ steps.diff.outputs.isRust }} - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Detect Changes (turbo) - uses: "./.github/actions/turbo-diffs" - id: turbo - - name: Detect Changes (diff) - uses: "./.github/actions/diffs" - id: diff - # Run e2e test against localnet built on the develop branch localnet: name: Localnet - needs: diff - if: needs.diff.outputs.isExplorer == 'true' || needs.diff.outputs.isTypescriptSDK == 'true' || needs.diff.outputs.isWallet == 'true' || needs.diff.outputs.isRust == 'true' + if: inputs.isExplorer || inputs.isTypescriptSDK || inputs.isWallet || inputs.isRust runs-on: self-hosted services: postgres: @@ -66,20 +64,20 @@ jobs: echo "E2E_RUN_LOCAL_NET_CMD=(RUST_LOG=info RUST_BACKTRACE=1 $(echo $PWD/target/debug/iota-test-validator) --with-indexer --pg-port 5432 --pg-db-name iota_indexer_v2 --graphql-host 127.0.0.1 --graphql-port 9125)" >> $GITHUB_ENV - name: Run TS SDK e2e tests - if: ${{ needs.diff.outputs.isTypescriptSDK == 'true' || needs.diff.outputs.isRust == 'true'}} + if: inputs.isTypescriptSDK || inputs.isRust run: pnpm dlx concurrently --kill-others --success command-1 "$E2E_RUN_LOCAL_NET_CMD" 'pnpm --filter @iota/iota-sdk test:e2e' - name: Run RPC/GraphQL compatibility e2e tests - if: ${{ needs.diff.outputs.isGraphQlTransport == 'true' || needs.diff.outputs.isRust == 'true'}} + if: inputs.isGraphQlTransport || inputs.isRust run: pnpm dlx concurrently --kill-others --success command-1 "$E2E_RUN_LOCAL_NET_CMD" 'pnpm --filter @iota/graphql-transport test:e2e' - name: Build explorer - if: ${{ needs.diff.outputs.isTypescriptSDK == 'true' || needs.diff.outputs.isExplorer == 'true' || needs.diff.outputs.isRust == 'true'}} + if: ${{ inputs.isTypescriptSDK || inputs.isExplorer || inputs.isRust}} run: pnpm turbo --filter=iota-explorer build - name: Run Explorer e2e tests # need to run Explorer e2e when its upstream(TS SDK and Rust) or itself is changed - if: ${{ needs.diff.outputs.isTypescriptSDK == 'true' || needs.diff.outputs.isExplorer == 'true' || needs.diff.outputs.isRust == 'true'}} + if: ${{ inputs.isTypescriptSDK || inputs.isExplorer || inputs.isRust}} run: pnpm --filter iota-explorer playwright test - uses: actions/upload-artifact@v3 if: always() diff --git a/.github/workflows/_execution_cut.yml b/.github/workflows/_execution_cut.yml new file mode 100644 index 00000000000..d22e1cada46 --- /dev/null +++ b/.github/workflows/_execution_cut.yml @@ -0,0 +1,20 @@ +name: Execution cut + +on: workflow_call + +jobs: + execution-cut: + name: cutting a new execution layer + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Install cargo-hakari, and cache the binary + uses: baptiste0928/cargo-install@1cd874a5478fdca35d868ccc74640c5aabbb8f1b # pin@v3.0.0 + with: + crate: cargo-hakari + locked: true + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Make cut + run: ./scripts/execution_layer.py cut for_ci_test + - name: Check execution builds + run: cargo build -p iota-execution diff --git a/.github/workflows/_external_rust_lints.yml b/.github/workflows/_external_rust_lints.yml new file mode 100644 index 00000000000..b84ef3a1b69 --- /dev/null +++ b/.github/workflows/_external_rust_lints.yml @@ -0,0 +1,35 @@ +name: Rust lints + +on: workflow_call + +jobs: + rustfmt: + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Install latest nightly + run: rustup toolchain install nightly --component rustfmt --allow-downgrade + - name: Check Rust formatting + run: | + cargo +nightly ci-fmt-external + + cargo-deny: + name: cargo-deny (advisories, licenses, bans, ...) + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - uses: EmbarkStudios/cargo-deny-action@v1 + + clippy: + needs: + - rustfmt + - cargo-deny + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - run: rustup component add clippy + + # See '.cargo/config' for list of enabled/disabled clippy lints + - name: cargo move-clippy + run: | + cargo ci-clippy-external diff --git a/.github/workflows/_external_rust_tests.yml b/.github/workflows/_external_rust_tests.yml new file mode 100644 index 00000000000..9171cc3bfdf --- /dev/null +++ b/.github/workflows/_external_rust_tests.yml @@ -0,0 +1,44 @@ +name: Rust tests + +on: workflow_call + +jobs: + changes: + runs-on: [self-hosted] + outputs: + components: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: dorny/paths-filter@v3 + id: filter + with: + list-files: "json" + filters: .github/external-crates-filters.yml + + test: + name: Test ${{ matrix.components }} + needs: changes + if: needs.changes.outputs.components != '[]' + env: + # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var + # causes #[sim_test] to only run under the deterministic `simtest` job, and not the + # non-deterministic `test` job. + IOTA_SKIP_SIMTESTS: 1 + strategy: + matrix: + components: ${{ fromJson(needs.changes.outputs.components) }} + fail-fast: false + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: taiki-e/install-action@nextest + - name: Install python dependencies + run: pip install pyopenssl --upgrade + - name: cargo test + run: > + cargo nextest run + --config-file .config/nextest.toml + --manifest-path external-crates/move/Cargo.toml + -E '!test(prove) and !test(run_all::simple_build_with_docs/args.txt) and !test(run_test::nested_deps_bad_parent/Move.toml) and rdeps(${{matrix.components}})' + -p ${{matrix.components}} + --profile ci diff --git a/.github/workflows/_move_tests.yml b/.github/workflows/_move_tests.yml new file mode 100644 index 00000000000..4c79d869f02 --- /dev/null +++ b/.github/workflows/_move_tests.yml @@ -0,0 +1,17 @@ +name: Move tests + +on: workflow_call + +jobs: + # This job ensures that Move unit tests are run if there are changes + # to Move code but not Rust code (If there are Rust changes, they + # will be run as part of a larger test suite). + move-test: + timeout-minutes: 10 + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - uses: taiki-e/install-action@nextest + - name: Run move tests + run: | + cargo nextest run -p iota-framework-tests -- unit_tests:: diff --git a/.github/workflows/mysticeti.yml b/.github/workflows/_mysticeti_tests.yml similarity index 87% rename from .github/workflows/mysticeti.yml rename to .github/workflows/_mysticeti_tests.yml index 4db44f57e80..3f034cadee2 100644 --- a/.github/workflows/mysticeti.yml +++ b/.github/workflows/_mysticeti_tests.yml @@ -1,15 +1,7 @@ name: Mysticeti tests on: - push: - branches: - - "develop" - - "devnet" - - "testnet" - - "mainnet" - - "releases/iota-*-release" - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: workflow_dispatch: inputs: iota_repo_ref: @@ -88,23 +80,7 @@ jobs: - name: cargo test run: cargo nextest run --profile ci -E 'rdeps(${{matrix.components}})' -p ${{matrix.components}} - diff: - runs-on: [self-hosted] - outputs: - isRust: ${{ steps.diff.outputs.isRust }} - isMove: ${{ steps.diff.outputs.isMove }} - isReleaseNotesEligible: ${{ steps.diff.outputs.isReleaseNotesEligible }} - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - with: - ref: ${{ github.event.inputs.iota_repo_ref || github.ref }} - - name: Detect Changes - uses: "./.github/actions/diffs" - id: diff - test-extra: - needs: diff - if: needs.diff.outputs.isRust == 'true' timeout-minutes: 45 env: # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml new file mode 100644 index 00000000000..c8a4d3267e3 --- /dev/null +++ b/.github/workflows/_rust.yml @@ -0,0 +1,58 @@ +name: Rust crates + +on: workflow_call + +env: + CARGO_TERM_COLOR: always + # Disable incremental compilation. + # + # Incremental compilation is useful as part of an edit-build-test-edit cycle, + # as it lets the compiler avoid recompiling code that hasn't changed. However, + # on CI, we're not making small edits; we're almost always building the entire + # project from scratch. Thus, incremental compilation on CI actually + # introduces *additional* overhead to support making future builds + # faster...but no future builds will ever occur in any given CI environment. + # + # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow + # for details. + CARGO_INCREMENTAL: 0 + # Allow more retries for network requests in cargo (downloading crates) and + # rustup (installing toolchains). This should help to reduce flaky CI failures + # from transient network timeouts or other issues. + CARGO_NET_RETRY: 10 + RUSTUP_MAX_RETRIES: 10 + # Don't emit giant backtraces in the CI logs. + RUST_BACKTRACE: short + RUSTDOCFLAGS: -D warnings + +jobs: + rust-lints: + uses: ./.github/workflows/_rust_lints.yml + + external-lints: + uses: ./.github/workflows/_external_rust_lints.yml + + rust-tests: + needs: + - rust-lints + - external-lints + uses: ./.github/workflows/_rust_tests.yml + + external-tests: + needs: + - rust-lints + - external-lints + uses: ./.github/workflows/_external_rust_tests.yml + + mysticeti-tests: + needs: + - rust-lints + - external-lints + uses: ./.github/workflows/_mysticeti_tests.yml + + execution-cut: + needs: + - rust-tests + - external-tests + - mysticeti-tests + uses: ./.github/workflows/_execution_cut.yml diff --git a/.github/workflows/_rust_lints.yml b/.github/workflows/_rust_lints.yml new file mode 100644 index 00000000000..ad86bee0f5a --- /dev/null +++ b/.github/workflows/_rust_lints.yml @@ -0,0 +1,43 @@ +name: Rust lints + +on: workflow_call + +jobs: + rustfmt: + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Install latest nightly + run: rustup toolchain install nightly --component rustfmt --allow-downgrade + - name: Check Rust formatting + run: cargo +nightly ci-fmt + + cargo-deny: + name: cargo-deny (advisories, licenses, bans, ...) + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - uses: EmbarkStudios/cargo-deny-action@v1 + + clippy: + needs: + - rustfmt + - cargo-deny + runs-on: [self-hosted] + steps: + - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # pin@v3.0.0 + # this avoids rate-limiting + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - run: rustup component add clippy + # TODO(bradh): debug and re-enable this; the caching is breaking the clippy build + # Enable caching of the 'librocksdb-sys' crate by additionally caching the + # 'librocksdb-sys' src directory which is managed by cargo + # - uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths + # with: + # path: ~/.cargo/registry/src/**/librocksdb-sys-* + + # See '.cargo/config' for list of enabled/disabled clippy lints + - name: Check Clippy Lints + run: cargo ci-clippy diff --git a/.github/workflows/rust.yml b/.github/workflows/_rust_tests.yml similarity index 50% rename from .github/workflows/rust.yml rename to .github/workflows/_rust_tests.yml index 25f847615f5..3d535545ac5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/_rust_tests.yml @@ -1,43 +1,6 @@ name: Rust tests -on: - push: - branches: - - "develop" - - "devnet" - - "testnet" - - "mainnet" - - "releases/iota-*-release" - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} - -env: - CARGO_TERM_COLOR: always - # Disable incremental compilation. - # - # Incremental compilation is useful as part of an edit-build-test-edit cycle, - # as it lets the compiler avoid recompiling code that hasn't changed. However, - # on CI, we're not making small edits; we're almost always building the entire - # project from scratch. Thus, incremental compilation on CI actually - # introduces *additional* overhead to support making future builds - # faster...but no future builds will ever occur in any given CI environment. - # - # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow - # for details. - CARGO_INCREMENTAL: 0 - # Allow more retries for network requests in cargo (downloading crates) and - # rustup (installing toolchains). This should help to reduce flaky CI failures - # from transient network timeouts or other issues. - CARGO_NET_RETRY: 10 - RUSTUP_MAX_RETRIES: 10 - # Don't emit giant backtraces in the CI logs. - RUST_BACKTRACE: short - # RUSTFLAGS: -D warnings - RUSTDOCFLAGS: -D warnings +on: workflow_call jobs: changes: @@ -54,6 +17,8 @@ jobs: test: name: Test ${{ matrix.components }} + needs: changes + if: needs.changes.outputs.components != '[]' env: # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var # causes #[sim_test] to only run under the deterministic `simtest` job, and not the @@ -64,7 +29,6 @@ jobs: components: ${{ fromJson(needs.changes.outputs.components) }} fail-fast: false runs-on: [self-hosted] - needs: changes steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - uses: taiki-e/install-action@nextest @@ -73,6 +37,8 @@ jobs: iota-sdk-changes: runs-on: [self-hosted] + outputs: + changed: ${{ steps.filter.outputs.iota-sdk }} steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - uses: dorny/paths-filter@v3 @@ -84,7 +50,7 @@ jobs: test-iota-sdk: name: Test iota-sdk - if: needs.iota-sdk-changes.outputs.iota-sdk == 'true' + if: needs.iota-sdk-changes.outputs.changed == 'true' env: # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var # causes #[sim_test] to only run under the deterministic `simtest` job, and not the @@ -101,56 +67,7 @@ jobs: # (we have the stardust iota-sdk package as a dependency) run: cargo nextest run --profile ci -E 'rdeps(iota-sdk)' -p iota-sdk@0.1.4 - diff: - runs-on: [self-hosted] - outputs: - isRust: ${{ steps.diff.outputs.isRust }} - isMove: ${{ steps.diff.outputs.isMove }} - isReleaseNotesEligible: ${{ steps.diff.outputs.isReleaseNotesEligible }} - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Detect Changes - uses: "./.github/actions/diffs" - id: diff - - license-check: - name: license-check - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths - - - name: Run cargo-license-template - run: cargo ci-license - - release-notes-description-check: - name: release-notes-check - needs: diff - if: needs.diff.outputs.isReleaseNotesEligible == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Check Pull Request Description - shell: bash - run: | - export PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") - export DESCRIPTION=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/iotaledger/iota/pulls/${PR_NUMBER}" \ - | jq --raw-output .body) - - if [[ "${DESCRIPTION}" == *"[x]"* ]]; then - if [[ "${DESCRIPTION}" =~ Release[[:space:]]notes[[:space:]]+.* ]]; then - exit 0 - else - echo "At least one '[x]' was checked under 'Type of Change (Check all that apply)', you need to add a blob under the 'Release Notes' section." - exit 1 - fi - fi - test-extra: - needs: diff - if: needs.diff.outputs.isRust == 'true' env: # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var # causes #[sim_test] to only run under the deterministic `simtest` job, and not the @@ -209,25 +126,8 @@ jobs: # run: | # scripts/simtest/stress-new-tests.sh -E 'rdeps(${{matrix.components}})' -p ${{matrix.components}} - # This job ensures that Move unit tests are run if there are changes - # to Move code but not Rust code (If there are Rust changes, they - # will be run as part of a larger test suite). - move-test: - needs: diff - if: needs.diff.outputs.isRust == 'false' && needs.diff.outputs.isMove == 'true' - timeout-minutes: 10 - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - uses: taiki-e/install-action@nextest - - name: Run move tests - run: | - cargo nextest run -p iota-framework-tests -- unit_tests:: - # # Disabled # rosetta-validation: - # needs: diff - # if: needs.diff.outputs.isRust == 'true' # timeout-minutes: 45 # runs-on: [self-hosted] # steps: @@ -260,98 +160,8 @@ jobs: # ./bin/rosetta-cli --configuration-file rosetta_cli.json check:data # shell: bash - clippy: - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # pin@v3.0.0 - # this avoids rate-limiting - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - run: rustup component add clippy - # TODO(bradh): debug and re-enable this; the caching is breaking the clippy build - # Enable caching of the 'librocksdb-sys' crate by additionally caching the - # 'librocksdb-sys' src directory which is managed by cargo - # - uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths - # with: - # path: ~/.cargo/registry/src/**/librocksdb-sys-* - - # See '.cargo/config' for list of enabled/disabled clippy lints - - name: Check Clippy Lints - run: cargo ci-clippy - - rustfmt: - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Install latest nightly - run: rustup toolchain install nightly --component rustfmt --allow-downgrade - - name: Check Rust formatting - run: cargo +nightly ci-fmt - - cargo-deny: - name: cargo-deny (advisories, licenses, bans, ...) - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - uses: EmbarkStudios/cargo-deny-action@v1 - - iota-excution-cut: - name: cutting a new execution layer - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Install cargo-hakari, and cache the binary - uses: baptiste0928/cargo-install@1cd874a5478fdca35d868ccc74640c5aabbb8f1b # pin@v3.0.0 - with: - crate: cargo-hakari - locked: true - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Make cut - run: ./scripts/execution_layer.py cut for_ci_test - - name: Check execution builds - run: cargo build -p iota-execution - - #indexer: - # name: indexer - # needs: diff - # if: needs.diff.outputs.isRust == 'true' - # timeout-minutes: 45 - # runs-on: [self-hosted] - # services: - # postgres: - # image: postgres - # env: - # POSTGRES_PASSWORD: postgrespw - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # ports: - # - 5432:5432 - # steps: - # - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - # - name: integration-test-with-postgres - # #We only have 1 instance of postgres DB, can only run test in single thread - # run: | - # cargo test --package iota-indexer --test integration_tests --features pg_integration -- --test-threads=1 - # env: - # POSTGRES_HOST: localhost - # POSTGRES_PORT: 5432 - graphql-rpc: name: graphql-rpc - needs: diff - if: needs.diff.outputs.isRust == 'true' timeout-minutes: 45 runs-on: [self-hosted] env: diff --git a/.github/workflows/turborepo.yml b/.github/workflows/_turborepo.yml similarity index 89% rename from .github/workflows/turborepo.yml rename to .github/workflows/_turborepo.yml index fe200a19a84..1272ed8b14f 100644 --- a/.github/workflows/turborepo.yml +++ b/.github/workflows/_turborepo.yml @@ -1,14 +1,6 @@ name: Turborepo CI -on: - push: - branches: [develop] - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: workflow_call jobs: audit: @@ -27,6 +19,7 @@ jobs: build: name: Lint, Build, and Test + needs: audit runs-on: self-hosted steps: - name: Checkout diff --git a/.github/workflows/typos.yml b/.github/workflows/_typos.yml similarity index 95% rename from .github/workflows/typos.yml rename to .github/workflows/_typos.yml index 323601d696b..266b703a24d 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/_typos.yml @@ -1,6 +1,6 @@ name: Typos -on: [pull_request] +on: workflow_call concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/docs-lint.yml b/.github/workflows/docs-lint.yml deleted file mode 100644 index d282c16b03d..00000000000 --- a/.github/workflows/docs-lint.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Docs spellcheck and linting - -on: - push: - branches: [develop, extensions] - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} - -jobs: - diff: - runs-on: [self-hosted] - outputs: - isDoc: ${{ steps.diff.outputs.isDoc }} - isOldDoc: ${{ steps.diff.outputs.isOldDoc }} - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Detect Changes - uses: "./.github/actions/diffs" - id: diff - - spelling: - name: Lint documentation - needs: diff - if: needs.diff.outputs.isDoc == 'true' - runs-on: [self-hosted] - - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Spell Check Docs - uses: crate-ci/typos@v1.16.11 - with: - files: ./docs/content diff --git a/.github/workflows/dprint.yml b/.github/workflows/dprint.yml deleted file mode 100644 index d0938810ac9..00000000000 --- a/.github/workflows/dprint.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Dprint Format - -on: - push: - branches: - - "develop" - - "devnet" - - "testnet" - - "mainnet" - - "releases/iota-*-release" - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} - -jobs: - dprint-format: - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Check dprint formatting - run: dprint check diff --git a/.github/workflows/external.yml b/.github/workflows/external.yml deleted file mode 100644 index 04c374affd6..00000000000 --- a/.github/workflows/external.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: External crates - -on: - push: - branches: [develop, extensions, devnet] - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} - -env: - CARGO_TERM_COLOR: always - # Disable incremental compilation. - # - # Incremental compilation is useful as part of an edit-build-test-edit cycle, - # as it lets the compiler avoid recompiling code that hasn't changed. However, - # on CI, we're not making small edits; we're almost always building the entire - # project from scratch. Thus, incremental compilation on CI actually - # introduces *additional* overhead to support making future builds - # faster...but no future builds will ever occur in any given CI environment. - # - # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow - # for details. - CARGO_INCREMENTAL: 0 - # Allow more retries for network requests in cargo (downloading crates) and - # rustup (installing toolchains). This should help to reduce flaky CI failures - # from transient network timeouts or other issues. - CARGO_NET_RETRY: 10 - RUSTUP_MAX_RETRIES: 10 - # Don't emit giant backtraces in the CI logs. - RUST_BACKTRACE: short - # RUSTFLAGS: -D warnings - RUSTDOCFLAGS: -D warnings - -jobs: - changes: - runs-on: [self-hosted] - outputs: - components: ${{ steps.filter.outputs.changes }} - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - uses: dorny/paths-filter@v3 - id: filter - with: - list-files: "json" - filters: .github/external-crates-filters.yml - - diff: - runs-on: [self-hosted] - outputs: - isRust: ${{ steps.diff.outputs.isRust }} - isMove: ${{ steps.diff.outputs.isMove }} - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Detect Changes - uses: "./.github/actions/diffs" - id: diff - - test: - name: Test ${{ matrix.components }} - if: needs.diff.outputs.isRust == 'true' - env: - # Tests written with #[sim_test] are often flaky if run as #[tokio::test] - this var - # causes #[sim_test] to only run under the deterministic `simtest` job, and not the - # non-deterministic `test` job. - IOTA_SKIP_SIMTESTS: 1 - strategy: - matrix: - components: ${{ fromJson(needs.changes.outputs.components) }} - fail-fast: false - runs-on: [self-hosted] - needs: changes - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - uses: taiki-e/install-action@nextest - - name: Install python dependencies - run: pip install pyopenssl --upgrade - - name: cargo test - run: > - cargo nextest run - --config-file .config/nextest.toml - --manifest-path external-crates/move/Cargo.toml - -E '!test(prove) and !test(run_all::simple_build_with_docs/args.txt) and !test(run_test::nested_deps_bad_parent/Move.toml) and rdeps(${{matrix.components}})' - -p ${{matrix.components}} - --profile ci - - clippy: - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - run: rustup component add clippy - - # See '.cargo/config' for list of enabled/disabled clippy lints - - name: cargo move-clippy - run: | - cargo ci-clippy-external - - rustfmt: - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - name: Install latest nightly - run: rustup toolchain install nightly --component rustfmt --allow-downgrade - - name: Check Rust formatting - run: | - cargo +nightly ci-fmt-external - - cargo-deny: - name: cargo-deny (advisories, licenses, bans, ...) - needs: diff - if: needs.diff.outputs.isRust == 'true' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 - - uses: EmbarkStudios/cargo-deny-action@v1 diff --git a/.github/workflows/hierarchy.yml b/.github/workflows/hierarchy.yml new file mode 100644 index 00000000000..568862e5551 --- /dev/null +++ b/.github/workflows/hierarchy.yml @@ -0,0 +1,130 @@ +name: Hierarchy + +on: + push: + branches: + - "develop" + - "devnet" + - "testnet" + - "mainnet" + - "releases/iota-*-release" + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} + +jobs: + diff: + runs-on: [self-hosted] + outputs: + isRust: ${{ steps.diff.outputs.isRust }} + isMove: ${{ steps.diff.outputs.isMove }} + isDoc: ${{ steps.diff.outputs.isDoc }} + isReleaseNotesEligible: ${{ steps.diff.outputs.isReleaseNotesEligible }} + isWallet: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), 'iota-wallet')) }} + isExplorer: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), 'iota-explorer')) }} + isTypescriptSDK: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), '@iota/iota.js')) }} + isGraphQlTransport: ${{ (steps.turbo.outputs.packages && contains(fromJson(steps.turbo.outputs.packages), '@iota/graphql-transport')) }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Detect Changes (turbo) + uses: "./.github/actions/turbo-diffs" + id: turbo + - name: Detect Changes (diff) + uses: "./.github/actions/diffs" + id: diff + + dprint-format: + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Check dprint formatting + run: dprint check + + typos: + uses: ./.github/workflows/_typos.yml + + license-check: + name: license-check + needs: diff + if: ${{ needs.diff.outputs.isRust == 'true' }} + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Run license check + run: cargo ci-license + + docusaurus: + needs: diff + if: ${{ needs.diff.outputs.isDoc == 'true' }} + uses: ./.github/workflows/_docusaurus.yml + + docs-lint: + needs: diff + if: ${{ needs.diff.outputs.isDoc == 'true' }} + uses: ./.github/workflows/_docs_lint.yml + + release-notes-description-check: + name: release-notes-check + needs: diff + if: ${{ needs.diff.outputs.isReleaseNotesEligible == 'true' }} + runs-on: [self-hosted] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 + - name: Check Pull Request Description + shell: bash + run: | + export PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") + export DESCRIPTION=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/iotaledger/iota/pulls/${PR_NUMBER}" \ + | jq --raw-output .body) + + if [[ "${DESCRIPTION}" == *"[x]"* ]]; then + if [[ "${DESCRIPTION}" =~ Release[[:space:]]notes[[:space:]]+.* ]]; then + exit 0 + else + echo "At least one '[x]' was checked under 'Type of Change (Check all that apply)', you need to add a blob under the 'Release Notes' section." + exit 1 + fi + fi + + turborepo: + needs: + - dprint-format + - license-check + - typos + uses: ./.github/workflows/_turborepo.yml + + move-tests: + needs: + - diff + - dprint-format + - license-check + - typos + if: ${{ needs.diff.outputs.isRust == 'false' && needs.diff.outputs.isMove == 'true' }} + uses: ./.github/workflows/_move_tests.yml + + rust: + needs: + - diff + - dprint-format + - license-check + - typos + if: ${{ needs.diff.outputs.isRust == 'true' }} + uses: ./.github/workflows/_rust.yml + + e2e: + needs: + - diff + - dprint-format + - license-check + - typos + uses: ./.github/workflows/_e2e.yml + with: + isRust: ${{ needs.diff.outputs.isRust == 'true' }} + isWallet: ${{ needs.diff.outputs.isWallet == 'true' }} + isExplorer: ${{ needs.diff.outputs.isExplorer == 'true' }} + isTypescriptSDK: ${{ needs.diff.outputs.isTypescriptSDK == 'true' }} + isGraphQlTransport: ${{ needs.diff.outputs.isGraphQlTransport == 'true' }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index f5118a644cf..0f9ebd1df92 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,4 +1,4 @@ -name: "Pull Request Labeler" +name: Pull Request Labeler on: - pull_request_target