E2E matrix #594
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E matrix | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| env: | |
| CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress | |
| permissions: {} | |
| jobs: | |
| preinstall: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| node_version: | |
| - 19 | |
| - 18 | |
| - 16 | |
| exclude: | |
| # run just node v18 on macos | |
| - os: macos-latest | |
| node_version: 19 | |
| - os: macos-latest | |
| node_version: 16 | |
| name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node_version }} | |
| - name: Cache node_modules | |
| id: cache-modules | |
| uses: actions/cache@v3 | |
| with: | |
| lookup-only: true | |
| path: '**/node_modules' | |
| key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ github.run_id }} | |
| - name: Install packages | |
| if: steps.cache-modules.outputs.cache-hit != 'true' | |
| run: yarn install --prefer-offline --frozen-lockfile --non-interactive | |
| - name: Homebrew cache directory path | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: homebrew-cache-dir-path | |
| run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT | |
| - name: Cache Homebrew | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.homebrew-cache-dir-path.outputs.dir }} | |
| key: brew-${{ matrix.node_version }} | |
| restore-keys: | | |
| brew- | |
| - name: Cache Cypress | |
| id: cache-cypress | |
| uses: actions/cache@v3 | |
| with: | |
| lookup-only: true | |
| path: '${{ github.workspace }}/.cypress' | |
| key: ${{ runner.os }}-cypress | |
| - name: Install Cypress | |
| if: steps.cache-cypress.outputs.cache-hit != 'true' | |
| run: npx cypress install | |
| e2e: | |
| needs: preinstall | |
| permissions: | |
| contents: read | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 90 | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| node_version: | |
| - 19 | |
| - 18 | |
| - 16 | |
| package_manager: | |
| - npm | |
| - yarn | |
| - pnpm | |
| project: | |
| - e2e-angular-core | |
| - e2e-angular-extensions | |
| - e2e-cypress | |
| - e2e-detox | |
| - e2e-esbuild | |
| - e2e-expo | |
| - e2e-jest | |
| - e2e-js | |
| - e2e-lerna-smoke-tests | |
| - e2e-linter | |
| - e2e-next | |
| - e2e-node | |
| - e2e-nx-init | |
| - e2e-nx-misc | |
| - e2e-nx-plugin | |
| - e2e-nx-run | |
| - e2e-react-core | |
| - e2e-react-extensions | |
| - e2e-react-native | |
| - e2e-web | |
| - e2e-rollup | |
| - e2e-storybook | |
| - e2e-storybook-angular | |
| - e2e-vite | |
| - e2e-webpack | |
| - e2e-workspace-create | |
| - e2e-workspace-create-npm | |
| include: | |
| # os short names | |
| - os: ubuntu-latest | |
| os_name: 'Linux' | |
| - os: macos-latest | |
| os_name: 'MacOS' | |
| # test timeouts | |
| - os: ubuntu-latest | |
| os_timeout: 60 | |
| - os: macos-latest | |
| os_timeout: 90 | |
| # codeowner groups | |
| - project: e2e-angular-core | |
| codeowners: 'S04SS457V38' | |
| - project: e2e-angular-extensions | |
| codeowners: 'S04SS457V38' | |
| - project: e2e-cypress | |
| codeowners: 'S04T16BTJJY' | |
| - project: e2e-detox | |
| codeowners: 'S04TNCNJG5N' | |
| - project: e2e-esbuild | |
| codeowners: 'S04SJ6HHP0X' | |
| - project: e2e-expo | |
| codeowners: 'S04TNCNJG5N' | |
| - project: e2e-jest | |
| codeowners: 'S04T16BTJJY' | |
| - project: e2e-js | |
| codeowners: 'S04SJ6HHP0X' | |
| - project: e2e-lerna-smoke-tests | |
| codeowners: 'S04TNCVEETS' | |
| - project: e2e-linter | |
| codeowners: 'S04SYJGKSCT' | |
| - project: e2e-next | |
| codeowners: 'S04TNCNJG5N' | |
| - project: e2e-node | |
| codeowners: 'S04SJ6HHP0X' | |
| - project: e2e-nx-init | |
| codeowners: 'S04SYHYKGNP' | |
| - project: e2e-nx-misc | |
| codeowners: 'S04SYHYKGNP' | |
| - project: e2e-nx-plugin | |
| codeowners: 'S04SYHYKGNP' | |
| - project: e2e-nx-run | |
| codeowners: 'S04SYHYKGNP' | |
| - project: e2e-react-core | |
| codeowners: 'S04TNCNJG5N' | |
| - project: e2e-react-extensions | |
| codeowners: 'S04TNCNJG5N' | |
| - project: e2e-react-native | |
| codeowners: 'S04TNCNJG5N' | |
| - project: e2e-web | |
| codeowners: 'S04SJ6PL98X' | |
| - project: e2e-rollup | |
| codeowners: 'S04SJ6PL98X' | |
| - project: e2e-storybook | |
| codeowners: 'S04SVQ8H0G5' | |
| - project: e2e-storybook-angular | |
| codeowners: 'S04SVQ8H0G5' | |
| - project: e2e-vite | |
| codeowners: 'S04SJ6PL98X' | |
| - project: e2e-webpack | |
| codeowners: 'S04SJ6PL98X' | |
| - project: e2e-workspace-create | |
| codeowners: 'S04SYHYKGNP' | |
| - project: e2e-workspace-create-npm | |
| codeowners: 'S04SYHYKGNP' | |
| exclude: | |
| # exclude react-native tests from ubuntu | |
| - os: ubuntu-latest | |
| project: e2e-react-native | |
| - os: ubuntu-latest | |
| project: e2e-detox | |
| - os: ubuntu-latest | |
| project: e2e-expo | |
| # exclude non-CNW/Lerna tests from non-LTS node versions | |
| - node_version: 16 | |
| project: e2e-angular-core | |
| - node_version: 16 | |
| project: e2e-angular-extensions | |
| - node_version: 16 | |
| project: e2e-cypress | |
| - node_version: 16 | |
| project: e2e-detox | |
| - node_version: 16 | |
| project: e2e-esbuild | |
| - node_version: 16 | |
| project: e2e-expo | |
| - node_version: 16 | |
| project: e2e-jest | |
| - node_version: 16 | |
| project: e2e-js | |
| - node_version: 16 | |
| project: e2e-linter | |
| - node_version: 16 | |
| project: e2e-next | |
| - node_version: 16 | |
| project: e2e-node | |
| - node_version: 16 | |
| project: e2e-nx-init | |
| - node_version: 16 | |
| project: e2e-nx-misc | |
| - node_version: 16 | |
| project: e2e-nx-plugin | |
| - node_version: 16 | |
| project: e2e-lerna-smoke-tests | |
| - node_version: 16 | |
| project: e2e-react-core | |
| - node_version: 16 | |
| project: e2e-react-extensions | |
| - node_version: 16 | |
| project: e2e-react-native | |
| - node_version: 16 | |
| project: e2e-web | |
| - node_version: 16 | |
| project: e2e-rollup | |
| - node_version: 16 | |
| project: e2e-storybook | |
| - node_version: 16 | |
| project: e2e-storybook-angular | |
| - node_version: 16 | |
| project: e2e-vite | |
| - node_version: 16 | |
| project: e2e-webpack | |
| - node_version: 19 | |
| project: e2e-angular-core | |
| - node_version: 19 | |
| project: e2e-angular-extensions | |
| - node_version: 19 | |
| project: e2e-cypress | |
| - node_version: 19 | |
| project: e2e-detox | |
| - node_version: 19 | |
| project: e2e-esbuild | |
| - node_version: 19 | |
| project: e2e-expo | |
| - node_version: 19 | |
| project: e2e-jest | |
| - node_version: 19 | |
| project: e2e-js | |
| - node_version: 19 | |
| project: e2e-linter | |
| - node_version: 19 | |
| project: e2e-next | |
| - node_version: 19 | |
| project: e2e-node | |
| - node_version: 19 | |
| project: e2e-nx-init | |
| - node_version: 19 | |
| project: e2e-nx-misc | |
| - node_version: 19 | |
| project: e2e-nx-plugin | |
| - node_version: 19 | |
| project: e2e-lerna-smoke-tests | |
| - node_version: 19 | |
| project: e2e-react-core | |
| - node_version: 19 | |
| project: e2e-react-extensions | |
| - node_version: 19 | |
| project: e2e-react-native | |
| - node_version: 19 | |
| project: e2e-web | |
| - node_version: 19 | |
| project: e2e-rollup | |
| - node_version: 19 | |
| project: e2e-storybook | |
| - node_version: 19 | |
| project: e2e-storybook-angular | |
| - node_version: 19 | |
| project: e2e-vite | |
| - node_version: 19 | |
| project: e2e-webpack | |
| # run just npm v18 on macos | |
| - os: macos-latest | |
| package_manager: yarn | |
| - os: macos-latest | |
| package_manager: pnpm | |
| - os: macos-latest | |
| node_version: 16 | |
| - os: macos-latest | |
| node_version: 19 | |
| fail-fast: false | |
| name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Prepare dir for output | |
| run: mkdir -p outputs | |
| - name: Install PNPM | |
| if: ${{ matrix.package_manager == 'pnpm' }} | |
| uses: pnpm/action-setup@v2 | |
| with: | |
| version: 7.30.5 | |
| - name: Use Node.js ${{ matrix.node_version }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node_version }} | |
| registry-url: http://localhost:4872 | |
| - name: Cache node_modules | |
| id: cache-modules | |
| uses: actions/cache@v3 | |
| with: | |
| path: '**/node_modules' | |
| key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ github.run_id }} | |
| - name: Install packages | |
| if: steps.cache-modules.outputs.cache-hit != 'true' | |
| run: yarn install --prefer-offline --frozen-lockfile --non-interactive | |
| - name: Cleanup | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| # Workaround to provide additional free space for testing. | |
| # https://github.com/actions/virtual-environments/issues/2840 | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo apt-get install lsof | |
| echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p | |
| - name: Homebrew cache directory path | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: homebrew-cache-dir-path | |
| run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT | |
| - name: Cache Homebrew | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.homebrew-cache-dir-path.outputs.dir }} | |
| key: brew-${{ matrix.node_version }} | |
| restore-keys: | | |
| brew- | |
| - name: Cache Cypress | |
| id: cache-cypress | |
| uses: actions/cache@v3 | |
| with: | |
| path: '${{ github.workspace }}/.cypress' | |
| key: ${{ runner.os }}-cypress | |
| - name: Install Cypress | |
| if: steps.cache-cypress.outputs.cache-hit != 'true' | |
| run: npx cypress install | |
| - name: Install applesimutils, reset ios simulators | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null | |
| xcrun simctl shutdown all && xcrun simctl erase all | |
| - name: Configure git metadata (needed for lerna smoke tests) | |
| run: | | |
| git config --global user.email [email protected] | |
| git config --global user.name "Test Test" | |
| - name: Set starting timestamp | |
| id: before-e2e | |
| run: | | |
| echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Run e2e tests | |
| id: e2e-run | |
| run: yarn nx run-many -t e2e,e2e-macos -p ${{ matrix.project }} | |
| timeout-minutes: ${{ matrix.os_timeout }} | |
| env: | |
| GIT_AUTHOR_EMAIL: [email protected] | |
| GIT_AUTHOR_NAME: Test | |
| GIT_COMMITTER_EMAIL: [email protected] | |
| GIT_COMMITTER_NAME: Test | |
| NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }} | |
| NODE_OPTIONS: --max_old_space_size=8192 | |
| SELECTED_PM: ${{ matrix.package_manager }} | |
| npm_config_registry: http://localhost:4872 | |
| YARN_REGISTRY: http://localhost:4872 | |
| NX_CACHE_DIRECTORY: 'tmp' | |
| NX_E2E_SKIP_BUILD_CLEANUP: 'true' | |
| NX_E2E_RUN_CYPRESS: 'true' | |
| NX_E2E_VERBOSE_LOGGING: 'true' | |
| NX_PERF_LOGGING: 'false' | |
| - name: Save matrix config in file | |
| if: ${{ always() }} | |
| id: save-matrix | |
| shell: bash | |
| run: | | |
| before=${{ steps.before-e2e.outputs.timestamp }} | |
| now=$(date +%s) | |
| delta=$(($now - $before)) | |
| matrix=$(( | |
| echo '${{ toJSON(matrix) }}' | |
| ) | jq --argjson delta $delta -c '. + { "status": "${{ steps.e2e-run.outcome}}", "duration": $delta }') | |
| echo "$matrix" > matrix | |
| path=outputs/${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }} | |
| echo "path=$path" >> $GITHUB_OUTPUT | |
| echo "$matrix" > $path | |
| - name: Upload matrix config | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: outputs | |
| path: ${{ steps.save-matrix.outputs.path }} | |
| - name: Setup tmate session | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }} | |
| uses: mxschmitt/[email protected] | |
| timeout-minutes: 15 | |
| process-result: | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| needs: e2e | |
| outputs: | |
| message: ${{ steps.process-json.outputs.SLACK_MESSAGE }} | |
| proj-duration: ${{ steps.process-json.outputs.SLACK_PROJ_DURATION }} | |
| pm-duration: ${{ steps.process-json.outputs.SLACK_PM_DURATION }} | |
| codeowners: ${{ steps.process-json.outputs.CODEOWNERS }} | |
| steps: | |
| - name: Load outputs | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: outputs | |
| path: outputs | |
| - name: Join and stringify matrix configs | |
| id: combine-json | |
| run: | | |
| combined=$((jq -s . outputs/*) | jq tostring) | |
| echo "combined=$combined" >> $GITHUB_OUTPUT | |
| - name: Make slack outputs | |
| id: process-json | |
| uses: actions/github-script@v6 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| script: | | |
| const combined = JSON.parse(${{ steps.combine-json.outputs.combined }}); | |
| const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project)); | |
| // codeowners | |
| const codeowners = new Set(); | |
| failedProjects.forEach(c => { | |
| codeowners.add(c.codeowners); | |
| }); | |
| core.setOutput('CODEOWNERS', Array.from(codeowners).join(',')); | |
| function trimSpace(res) { | |
| return res.split('\n').map((l) => l.trim()).join('\n'); | |
| } | |
| // failed message | |
| let lastProject; | |
| let result = ` | |
| \`\`\` | |
| | Failed project | PM | OS | Node | | |
| |--------------------------------|------|-------|------|`; | |
| failedProjects.forEach(matrix => { | |
| const project = matrix.project !== lastProject ? matrix.project : '...'; | |
| result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name} | v${matrix.node_version.toString().padEnd(3)} |` | |
| lastProject = matrix.project; | |
| }); | |
| result += `\`\`\``; | |
| core.setOutput('SLACK_MESSAGE', trimSpace(result)); | |
| function humanizeDuration(num) { | |
| let res = ''; | |
| const hours = Math.floor(num / 3600); | |
| if (hours) { | |
| res += `${hours}h `; | |
| } | |
| const mins = Math.floor((num % 3600) / 60); | |
| if (mins) { | |
| res += `${mins}m `; | |
| } | |
| const sec = num % 60; | |
| if (sec) { | |
| res += `${sec}s` | |
| } | |
| return res; | |
| } | |
| // duration message | |
| const timeReport = {}; | |
| const pmReport = { | |
| npm: 0, | |
| yarn: 0, | |
| pnpm: 0 | |
| }; | |
| combined.forEach((matrix) => { | |
| if (matrix.os_name === 'Linux' && matrix.node_version === 18) { | |
| pmReport[matrix.package_manager] += matrix.duration; | |
| } | |
| if (timeReport[matrix.project]) { | |
| if (matrix.duration > timeReport[matrix.project].max) { | |
| timeReport[matrix.project].max = matrix.duration; | |
| timeReport[ | |
| matrix.project | |
| ].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`; | |
| } | |
| if (matrix.duration < timeReport[matrix.project].min) { | |
| timeReport[matrix.project].min = matrix.duration; | |
| timeReport[ | |
| matrix.project | |
| ].minEnv = `${matrix.os_name}, ${matrix.package_manager}`; | |
| } | |
| } else { | |
| timeReport[matrix.project] = { | |
| min: matrix.duration, | |
| max: matrix.duration, | |
| minEnv: `${matrix.os_name}, ${matrix.package_manager}`, | |
| maxEnv: `${matrix.os_name}, ${matrix.package_manager}`, | |
| }; | |
| } | |
| }); | |
| // project time report | |
| let resultPkg = ` | |
| \`\`\` | |
| | Project | Time | | |
| |--------------------------------|---------------------------|`; | |
| function mapProjectTime(proj, section) { | |
| let res = ''; | |
| res += `${humanizeDuration(timeReport[proj][section])}`; | |
| res += ` (${timeReport[proj][section + 'Env']})` | |
| return res; | |
| } | |
| function durationIcon(proj, section) { | |
| if (timeReport[proj][section] < 12 * 60) { | |
| return `${section} ✅`; | |
| } | |
| if (timeReport[proj][section] < 15 * 60) { | |
| return `${section} ❗`; | |
| } | |
| return `${section} ❌`; | |
| } | |
| Object.keys(timeReport).forEach(proj => { | |
| resultPkg += `\n| ${proj.padEnd(30)} | |`; | |
| resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`; | |
| resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`; | |
| }); | |
| resultPkg += `\`\`\``; | |
| core.setOutput('SLACK_PROJ_DURATION', trimSpace(resultPkg)); | |
| let resultPm = ` | |
| \`\`\` | |
| | PM | Total time | | |
| |------|-------------|`; | |
| Object.keys(pmReport).forEach(pm => { | |
| resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm]).padEnd(11)} |` | |
| }); | |
| resultPm += `\`\`\``; | |
| core.setOutput('SLACK_PM_DURATION', trimSpace(resultPm)); | |
| report-failure: | |
| if: ${{ failure() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report failure | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'failure' | |
| message_format: '{emoji} Workflow has {status_message} ${{ needs.process-result.outputs.message }}' | |
| notification_title: '{workflow}' | |
| footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' | |
| mention_groups: ${{ needs.process-result.outputs.codeowners }} | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-success: | |
| if: ${{ success() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: e2e | |
| runs-on: ubuntu-latest | |
| name: Report status | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ needs.e2e.result }} | |
| message_format: '{emoji} Workflow has {status_message}' | |
| notification_title: '{workflow}' | |
| footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-pm-time: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report duration per package manager | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'skipped' | |
| message_format: '${{ needs.process-result.outputs.pm-duration }}' | |
| notification_title: 'Total duration per package manager (ubuntu only)' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-proj-time: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report duration per package manager | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'skipped' | |
| message_format: '${{ needs.process-result.outputs.proj-duration }}' | |
| notification_title: 'E2E Project duration stats' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} |