[e2e] Steps to reduce flakiness in test reporting

.circleci/config.yml

@@ -123,7 +123,7 @@ jobs:
       suite: #stable or full
         type: string
     executor: pw-focal-development
-    parallelism: 6
+    parallelism: 7
     steps:
       - build_and_install:
           node-version: lts/hydrogen
@@ -132,7 +132,12 @@ jobs:
             equal: ["full", <<parameters.suite>>]
           steps:
             - run: npx playwright install chrome-beta
-      - run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL}
+      - run:
+          command: |
+            mkdir test-results
+            TESTFILES=$(circleci tests glob "e2e/**/*.spec.js")
+            echo "$TESTFILES" | circleci tests run --command="xargs npm run test:e2e:<<parameters.suite>> --verbose --split-by=timings"
+      #- run: SHARD="$((${CIRCLE_NODE_INDEX}+1))"; npm run test:e2e:<<parameters.suite>> -- --shard=${SHARD}/${CIRCLE_NODE_TOTAL}
       - when:
           condition:
             equal: [42, 42] # Always run codecov reports regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
@@ -239,6 +244,7 @@ jobs:
             equal: [42, 42] # Always generate version artifacts regardless of test failure https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957/2
           steps:
             - generate_and_store_version_and_filesystem_artifacts
+
 workflows:
   overall-circleci-commit-status: #These jobs run on every commit
     jobs:

.github/workflows/e2e-flakefinder.yml

@@ -0,0 +1,61 @@
+name: 'pr:e2e:flakefinder'
+
+on:
+  push:
+    branches: master
+  workflow_dispatch:
+  pull_request:
+    types:
+      - labeled
+      - opened
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  e2e-flakefinder:
+    if: contains(github.event.pull_request.labels.*.name, 'pr:e2e:flakefinder') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event.action == 'opened'
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/hydrogen'
+
+      - name: Cache NPM dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npx [email protected] install
+      - run: npm install --cache ~/.npm --no-audit --progress=false
+
+      - name: Run E2E Tests (Repeated 10 Times)
+        run: npm run test:e2e:stable -- --retries=0 --repeat-each=10 --max-failures=50
+
+      - name: Archive test results
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          path: test-results
+
+      - name: Remove pr:e2e:flakefinder label (if present)
+        if: always()
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { owner, repo, number } = context.issue;
+            const labelToRemove = 'pr:e2e:flakefinder';
+            try {
+                await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: number,
+                name: labelToRemove
+                });
+            } catch (error) {
+                core.warning(`Failed to remove ' + labelToRemove + ' label: ${error.message}`);
+            }