From 78ffb9b63af4e8fe0fd93e134c537fff9ba011ac Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 16 Dec 2024 17:20:49 +0100 Subject: [PATCH] chore: enable build caching for playwright tests (#2338) Relates to #2321 - Move build step as precondition for all checks and tests in the pipeline to create cache and avoid downstream build failures. - Extracted shared playwright config into `@sit-onyx/shared` package as `PLAYWRIGHT_BASE_CONFIG` to reduce code duplications. - Pass playwright shard configuration from `playwright.config.ts` file instead of using CLI arguments. See beneath for more details: There was an issue where every shard executed a fresh build of all packages. The cause was the `--shard` cli argument that differs for every shard. Because [turbo considers](https://turbo.build/repo/docs/crafting-your-repository/caching#task-inputs) pass-through arguments in their hashing, there was always a hash mismatch. Unfortunately, turbo doesn't support changing this behavior. Luckily, playwright allows setting the sharding configuration via the `playwright.config.ts` file, where we can use environment variables. And because turbo can be configured to ignore specific environment variables, we can use them instead of CLI arguments. --- .github/workflows/check.yml | 14 ++++ .github/workflows/playwright.yml | 6 +- apps/docs/package.json | 2 +- apps/docs/playwright.config.ts | 28 +------ .../docs/src/principles/contributing/index.md | 2 +- .../src/principles/contributing/testing.md | 2 +- apps/playground/package.json | 2 +- apps/playground/playwright.config.ts | 18 +--- package.json | 6 +- packages/chartjs-plugin/package.json | 2 +- packages/chartjs-plugin/playwright.config.ts | 24 +----- packages/headless/package.json | 3 +- packages/headless/playwright.config.ts | 37 +------- packages/shared/src/playwright.config.base.ts | 84 +++++++++++++++++++ packages/sit-onyx/package.json | 2 +- packages/sit-onyx/playwright.config.ts | 34 +------- pnpm-lock.yaml | 3 + turbo.json | 10 +-- 18 files changed, 135 insertions(+), 144 deletions(-) create mode 100644 packages/shared/src/playwright.config.base.ts diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 25b527261d..40efa1ac25 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -13,9 +13,22 @@ env: TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE__TURBO_TOKEN }} jobs: + # We run the build first standalone to ensure everything is cached for all other jobs + build: + name: Build and Cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/templates/node-setup + + - name: 🛠️ Build packages + run: pnpm run build:all + check: name: Check code quality runs-on: ubuntu-latest + needs: build steps: - uses: actions/checkout@v4 with: @@ -68,6 +81,7 @@ jobs: path: apps/docs/src/.vitepress/dist screenshots: + needs: build name: Component tests uses: ./.github/workflows/playwright.yml secrets: inherit diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index ac75241624..8f1f381ff6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -33,7 +33,11 @@ jobs: run: pnpm exec playwright install-deps - name: 🔎 Run Playwright tests - run: pnpm run test:components:all -- --shard=${{ matrix.shard }}/${{ strategy.job-total }} ${{ inputs.update-snapshots == true && '--update-snapshots' || '' }} + run: pnpm run test:playwright:all + env: + PW_UPDATE_SNAPSHOTS: "${{ inputs.update-snapshots }}" + PW_SHARD: "${{ matrix.shard }}" + PW_TOTAL_SHARDS: "${{ strategy.job-total }}" # we only want to include actual changed screenshots in the artifact to prevent that old/unchanged screenshots # override changed screenshots from other shards when creating the pull request diff --git a/apps/docs/package.json b/apps/docs/package.json index 5046c41e74..973f4a1fff 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -11,7 +11,7 @@ "type-check": "vue-tsc --noEmit", "preview": "vitepress preview src", "storybook": "storybook dev -p 6006 --no-open", - "test:integration": "playwright install && playwright test" + "test:playwright": "playwright install && playwright test" }, "devDependencies": { "@playwright/test": "^1.49.1", diff --git a/apps/docs/playwright.config.ts b/apps/docs/playwright.config.ts index 389d1d6ba3..7799127147 100644 --- a/apps/docs/playwright.config.ts +++ b/apps/docs/playwright.config.ts @@ -1,4 +1,5 @@ -import { defineConfig, devices } from "@playwright/test"; +import { defineConfig } from "@playwright/test"; +import { PLAYWRIGHT_BASE_CONFIG } from "@sit-onyx/shared/playwright.config.base"; // NOTE: You need to run "pnpm build" before running the tests @@ -6,33 +7,10 @@ import { defineConfig, devices } from "@playwright/test"; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ + ...PLAYWRIGHT_BASE_CONFIG, testDir: "./tests", testMatch: `**/*.ct.{ts,tsx}`, - snapshotDir: "./playwright/snapshots", - // custom snapshotPathTemplate to remove the testFileName folder that we don't want - snapshotPathTemplate: "{snapshotDir}/{testFileDir}/{arg}-{projectName}-{platform}{ext}", - // we don't want to update snapshots on the local machine of each developer. - // if you want to update snapshots for your branch, use the corresponding GitHub action: - // https://github.com/SchwarzIT/onyx/actions/workflows/playwright-screenshots.yml - ignoreSnapshots: !process.env.CI, - updateSnapshots: "none", expect: { toHaveScreenshot: { maxDiffPixelRatio: 0.01 } }, - fullyParallel: true, - forbidOnly: !!process.env.CI, // fail build on CI if we left test.only in the source code - retries: process.env.CI ? 1 : 0, // retry on CI only - /* In the CI pipeline it generates dot (for the stdout) and blob reports, locally only a html report is generated */ - reporter: process.env.CI ? [["dot"], ["blob"]] : [["html", { open: "never" }]], - use: { - baseURL: "http://localhost:3200/", - trace: process.env.CI ? "retain-on-failure" : "off", - video: process.env.CI ? "retain-on-failure" : "off", - }, - /* Configure projects for major browsers */ - projects: [ - { name: "edge", use: { ...devices["Desktop Edge"], channel: "msedge" } }, - { name: "firefox", use: { ...devices["Desktop Firefox"] } }, - { name: "webkit", use: { ...devices["Desktop Safari"] } }, - ], /* Run your local dev server before starting the tests */ webServer: { command: "pnpm preview --port 3200 --mode test", diff --git a/apps/docs/src/principles/contributing/index.md b/apps/docs/src/principles/contributing/index.md index fa7a02bb02..16b35972e1 100644 --- a/apps/docs/src/principles/contributing/index.md +++ b/apps/docs/src/principles/contributing/index.md @@ -48,7 +48,7 @@ pnpm format:all # format all files pnpm dev # run Storybook in dev mode when developing components pnpm build # build all onyx components pnpm test # run unit tests -pnpm test:components # run Playwright component tests +pnpm test:playwright # run Playwright component tests ``` ```sh [apps/docs] diff --git a/apps/docs/src/principles/contributing/testing.md b/apps/docs/src/principles/contributing/testing.md index c12cbf5c57..39b1a19cb0 100644 --- a/apps/docs/src/principles/contributing/testing.md +++ b/apps/docs/src/principles/contributing/testing.md @@ -30,7 +30,7 @@ For standalone tests or more complicated setups, [`toHaveScreenshot`](https://pl ### Development -In our monorepo component tests are run non-interactively using the `pnpm test:components` script. +In our monorepo component tests are run non-interactively using the `pnpm test:playwright` script. To use Playwright interactively run `pnpm exec playwright test --ui` (add the `--headed` flag to open the see the- browsers) in the package directory. diff --git a/apps/playground/package.json b/apps/playground/package.json index 2e2b36f9ac..f82cb7732c 100644 --- a/apps/playground/package.json +++ b/apps/playground/package.json @@ -14,7 +14,7 @@ "build": "pnpm run '/type-check|build-only/'", "build-only": "vite build", "type-check": "vue-tsc --noEmit -p tsconfig.app.json --composite false", - "test:integration": "playwright install && playwright test" + "test:playwright": "playwright install && playwright test" }, "dependencies": { "@sit-onyx/icons": "workspace:^", diff --git a/apps/playground/playwright.config.ts b/apps/playground/playwright.config.ts index 4cfa9d6d64..e4da52b292 100644 --- a/apps/playground/playwright.config.ts +++ b/apps/playground/playwright.config.ts @@ -1,4 +1,5 @@ -import { defineConfig, devices } from "@playwright/test"; +import { defineConfig } from "@playwright/test"; +import { PLAYWRIGHT_BASE_CONFIG } from "@sit-onyx/shared/playwright.config.base"; // NOTE: You need to run "pnpm build" before running the tests @@ -6,22 +7,9 @@ import { defineConfig, devices } from "@playwright/test"; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ + ...PLAYWRIGHT_BASE_CONFIG, testDir: "./tests", fullyParallel: true, - forbidOnly: !!process.env.CI, // fail build on CI if we left test.only in the source code - retries: process.env.CI ? 1 : 0, // retry on CI only - /* In the CI pipeline it generates dot (for the stdout) and blob reports, locally only a html report is generated */ - reporter: process.env.CI ? [["dot"], ["blob"]] : [["html", { open: "never" }]], - use: { - trace: process.env.CI ? "retain-on-failure" : "off", - video: process.env.CI ? "retain-on-failure" : "off", - }, - /* Configure projects for major browsers */ - projects: [ - { name: "edge", use: { ...devices["Desktop Edge"], channel: "msedge" } }, - { name: "firefox", use: { ...devices["Desktop Firefox"] } }, - { name: "webkit", use: { ...devices["Desktop Safari"] } }, - ], /* Run your local dev server before starting the tests */ webServer: { command: "pnpm preview --port 3200 --mode test", diff --git a/package.json b/package.json index d738673659..7a92815b75 100644 --- a/package.json +++ b/package.json @@ -5,9 +5,9 @@ "author": "Schwarz IT KG", "license": "Apache-2.0", "scripts": { - "build:all": "turbo build build:storybook", - "test:all": "turbo test:coverage", - "test:components:all": "turbo test:components test:integration --concurrency 1", + "build:all": "turbo run build build:storybook", + "test:all": "turbo run test:coverage", + "test:playwright:all": "turbo run test:playwright --concurrency 1", "format:all": "prettier --write .", "format:check:all": "prettier --check .", "lint:all": "eslint .", diff --git a/packages/chartjs-plugin/package.json b/packages/chartjs-plugin/package.json index 48dc08ea45..9ba6f032a4 100644 --- a/packages/chartjs-plugin/package.json +++ b/packages/chartjs-plugin/package.json @@ -28,7 +28,7 @@ "build": "vue-tsc --noEmit", "test": "vitest", "test:coverage": "vitest run --coverage", - "test:components": "playwright install && playwright test" + "test:playwright": "playwright install && playwright test" }, "peerDependencies": { "chart.js": ">= 4.0.0", diff --git a/packages/chartjs-plugin/playwright.config.ts b/packages/chartjs-plugin/playwright.config.ts index 500ca97131..6ec52e8551 100644 --- a/packages/chartjs-plugin/playwright.config.ts +++ b/packages/chartjs-plugin/playwright.config.ts @@ -1,34 +1,14 @@ import { defineConfig, devices } from "@playwright/experimental-ct-vue"; -import vue from "@vitejs/plugin-vue"; +import { PLAYWRIGHT_BASE_CONFIG } from "@sit-onyx/shared/playwright.config.base"; /** * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ + ...PLAYWRIGHT_BASE_CONFIG, testDir: "./src", testMatch: `**/*.ct.tsx`, - snapshotDir: "./playwright/snapshots", - // custom snapshotPathTemplate to remove the testFileName folder that we don't want - snapshotPathTemplate: "{snapshotDir}/{testFileDir}/{arg}-{projectName}-{platform}{ext}", - // we don't want to update snapshots on the local machine of each developer. - // if you want to update snapshots for your branch, use the corresponding GitHub action: - // https://github.com/SchwarzIT/onyx/actions/workflows/playwright-screenshots.yml - ignoreSnapshots: !process.env.CI, - updateSnapshots: "none", expect: { toHaveScreenshot: { maxDiffPixelRatio: 0.01 } }, - fullyParallel: true, - forbidOnly: !!process.env.CI, // fail build on CI if we left test.only in the source code - retries: process.env.CI ? 2 : 0, // retry on CI only - /* In the CI pipeline it generates dot (for the stdout) and blob reports, locally only a html report is generated */ - reporter: process.env.CI ? [["dot"], ["blob"]] : [["html", { open: "never" }]], - use: { - trace: process.env.CI ? "retain-on-failure" : "off", - video: process.env.CI ? "retain-on-failure" : "off", - ctPort: 3100, - ctViteConfig: { - plugins: [vue()], - }, - }, /* Configure projects for major browsers */ projects: [ // one browser is sufficient for the screenshot tests diff --git a/packages/headless/package.json b/packages/headless/package.json index 360f38ec9e..68558c021d 100644 --- a/packages/headless/package.json +++ b/packages/headless/package.json @@ -25,13 +25,14 @@ "scripts": { "build": "vue-tsc --build --force", "test": "vitest", - "test:components": "playwright install && playwright test" + "test:playwright": "playwright install && playwright test" }, "peerDependencies": { "typescript": ">= 5", "vue": ">= 3.5.0" }, "devDependencies": { + "@sit-onyx/shared": "workspace:^", "@vue/compiler-dom": "catalog:", "vue": "catalog:" } diff --git a/packages/headless/playwright.config.ts b/packages/headless/playwright.config.ts index 7b7c57dbc8..d674582a4c 100644 --- a/packages/headless/playwright.config.ts +++ b/packages/headless/playwright.config.ts @@ -1,42 +1,11 @@ -import { defineConfig, devices } from "@playwright/experimental-ct-vue"; +import { defineConfig } from "@playwright/experimental-ct-vue"; +import { PLAYWRIGHT_BASE_CONFIG } from "@sit-onyx/shared/playwright.config.base"; /** * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ + ...PLAYWRIGHT_BASE_CONFIG, testDir: "./", testMatch: `**/*.ct.tsx`, - /* Maximum time one test can run for. */ - timeout: 10 * 1000, - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, - /* In the CI pipeline it generates dot (for the stdout) and blob reports, locally only a html report is generated */ - reporter: process.env.CI ? [["dot"], ["blob"]] : [["html", { open: "never" }]], - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "on-first-retry", - - /* Port to use for Playwright component endpoint. */ - ctPort: 3100, - }, - - /* Configure projects for major browsers */ - projects: [ - { name: "edge", use: { ...devices["Desktop Edge"], channel: "msedge" } }, - { - name: "firefox", - use: { ...devices["Desktop Firefox"] }, - }, - { - name: "webkit", - use: { ...devices["Desktop Safari"] }, - }, - ], }); diff --git a/packages/shared/src/playwright.config.base.ts b/packages/shared/src/playwright.config.base.ts new file mode 100644 index 0000000000..59316bb51f --- /dev/null +++ b/packages/shared/src/playwright.config.base.ts @@ -0,0 +1,84 @@ +import { devices, PlaywrightTestConfig } from "@playwright/experimental-ct-vue"; +import vue, { Options } from "@vitejs/plugin-vue"; + +export const vuePluginOptions: Options = { + template: { + compilerOptions: { + // comments can cause issues for components where classes + // are not merged correctly, e.g. when using `` + comments: false, + }, + }, +}; + +/** + * Basic, shared playwright configuration + * + * See https://playwright.dev/docs/test-configuration + */ +export const PLAYWRIGHT_BASE_CONFIG = { + /** + * SCREENSHOTS + * + * See: https://playwright.dev/docs/screenshots + */ + snapshotDir: "./playwright/snapshots", + // custom snapshotPathTemplate to remove the testFileName folder that we don't want + snapshotPathTemplate: "{snapshotDir}/{testFileDir}/{arg}-{projectName}-{platform}{ext}", + // we don't want to update snapshots on the local machine of each developer. + // if you want to update snapshots for your branch, use the corresponding GitHub action: + // https://github.com/SchwarzIT/onyx/actions/workflows/playwright-screenshots.yml + ignoreSnapshots: !process.env.CI, + updateSnapshots: process.env.PW_UPDATE_SNAPSHOTS ? "all" : "none", + + /** + * SHARDING + * + * See: https://playwright.dev/docs/test-sharding + */ + fullyParallel: true, + // when (in the pipeline) the sharding environment variables are set, sharding is enabled + shard: + process.env.CI && process.env.PW_SHARD && process.env.PW_TOTAL_SHARDS + ? { + current: +process.env.PW_SHARD, + total: +process.env.PW_TOTAL_SHARDS, + } + : null, + + /** + * FAILURE HANDLING + * + * See: https://playwright.dev/docs/test-retries + */ + timeout: 20 * 1000, + forbidOnly: !!process.env.CI, // fail build on CI if we left test.only in the source code + retries: process.env.CI ? 1 : 0, // retry on CI only + + /** + * REPORTERS + * + * See: https://playwright.dev/docs/test-reporters + */ + /* In the CI pipeline it generates dot (for the stdout) and blob reports, locally only a html report is generated */ + reporter: process.env.CI ? [["dot"], ["blob"]] : [["html", { open: "never" }]], + use: { + trace: process.env.CI ? "retain-on-failure" : "off", + video: process.env.CI ? "retain-on-failure" : "off", + ctPort: 3100, + ctViteConfig: { + plugins: [vue(vuePluginOptions)], + }, + }, + + /** + * BROWSERS + * + * See: https://playwright.dev/docs/test-projects + */ + projects: [ + { name: "edge", use: { ...devices["Desktop Edge"], channel: "msedge" } }, + { name: "firefox", use: { ...devices["Desktop Firefox"] } }, + { name: "webkit", use: { ...devices["Desktop Safari"] } }, + ], +} as const satisfies PlaywrightTestConfig; diff --git a/packages/sit-onyx/package.json b/packages/sit-onyx/package.json index 9a7376507c..e5e46935ea 100644 --- a/packages/sit-onyx/package.json +++ b/packages/sit-onyx/package.json @@ -41,7 +41,7 @@ "preview": "vite serve storybook-static", "test": "vitest", "test:coverage": "vitest run --coverage", - "test:components": "playwright install && playwright test" + "test:playwright": "playwright install && playwright test" }, "peerDependencies": { "@sit-onyx/icons": "workspace:^", diff --git a/packages/sit-onyx/playwright.config.ts b/packages/sit-onyx/playwright.config.ts index 200203ce20..3df6bf3476 100644 --- a/packages/sit-onyx/playwright.config.ts +++ b/packages/sit-onyx/playwright.config.ts @@ -1,5 +1,6 @@ -import { defineConfig, devices } from "@playwright/experimental-ct-vue"; -import vue, { Options } from "@vitejs/plugin-vue"; +import { defineConfig } from "@playwright/experimental-ct-vue"; +import { PLAYWRIGHT_BASE_CONFIG } from "@sit-onyx/shared/playwright.config.base"; +import { Options } from "@vitejs/plugin-vue"; export const vuePluginOptions: Options = { template: { @@ -15,34 +16,7 @@ export const vuePluginOptions: Options = { * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ + ...PLAYWRIGHT_BASE_CONFIG, testDir: "./src", testMatch: `**/*.ct.tsx`, - snapshotDir: "./playwright/snapshots", - // custom snapshotPathTemplate to remove the testFileName folder that we don't want - snapshotPathTemplate: "{snapshotDir}/{testFileDir}/{arg}-{projectName}-{platform}{ext}", - // we don't want to update snapshots on the local machine of each developer. - // if you want to update snapshots for your branch, use the corresponding GitHub action: - // https://github.com/SchwarzIT/onyx/actions/workflows/playwright-screenshots.yml - ignoreSnapshots: !process.env.CI, - updateSnapshots: "none", - timeout: 20 * 1000, - fullyParallel: true, - forbidOnly: !!process.env.CI, // fail build on CI if we left test.only in the source code - retries: process.env.CI ? 1 : 0, // retry on CI only - /* In the CI pipeline it generates dot (for the stdout) and blob reports, locally only a html report is generated */ - reporter: process.env.CI ? [["dot"], ["blob"]] : [["html", { open: "never" }]], - use: { - trace: process.env.CI ? "retain-on-failure" : "off", - video: process.env.CI ? "retain-on-failure" : "off", - ctPort: 3100, - ctViteConfig: { - plugins: [vue(vuePluginOptions)], - }, - }, - /* Configure projects for major browsers */ - projects: [ - { name: "edge", use: { ...devices["Desktop Edge"], channel: "msedge" } }, - { name: "firefox", use: { ...devices["Desktop Firefox"] } }, - { name: "webkit", use: { ...devices["Desktop Safari"] } }, - ], }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e36bb7413c..d23f880804 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -302,6 +302,9 @@ importers: specifier: '>= 5' version: 5.7.2 devDependencies: + '@sit-onyx/shared': + specifier: workspace:^ + version: link:../shared '@vue/compiler-dom': specifier: 3.5.13 version: 3.5.13 diff --git a/turbo.json b/turbo.json index cef8c656bd..bc118269e9 100644 --- a/turbo.json +++ b/turbo.json @@ -5,6 +5,7 @@ "teamSlug": "onyx", "enabled": true }, + "globalPassThroughEnv": ["PW_SHARD", "PW_TOTAL_SHARDS", "PW_UPDATE_SNAPSHOTS"], "globalDependencies": ["pnpm-lock.yaml"], "tasks": { "build": { @@ -19,13 +20,8 @@ "outputs": ["coverage"], "cache": false }, - "test:components": { - "dependsOn": ["^build"], - "outputs": ["playwright-report", "test-results", "blob-reports"], - "cache": false - }, - "test:integration": { - "dependsOn": ["sit-onyx#build", "build"], + "test:playwright": { + "dependsOn": ["build"], "outputs": ["playwright-report", "test-results", "blob-reports"], "cache": false },