refactor test config override validation for experimental retry subkeys

packages/config/src/browser.ts

@@ -197,6 +197,26 @@ export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride:
       return
     }
 
+    // this is unique validation, not applied to the general cy config.
+    // it will be removed when we support defining experimental retries
+    // in test config overrides
+
+    // TODO: remove when experimental retry overriding is supported
+
+    if (configKey === 'retries') {
+      const experimentalRetryCfgKeys = [
+        'experimentalStrategy', 'experimentalOptions',
+      ]
+
+      Object.keys(config.retries || {})
+      .filter((v) => experimentalRetryCfgKeys.includes(v))
+      .forEach((invalidExperimentalCfgOverride) => {
+        onErr({
+          invalidConfigKey: `retries.${invalidExperimentalCfgOverride}`,
+          supportedOverrideLevel: 'global_only',
+        })
+      })
+    }
     // TODO: add a hook to ensure valid testing-type configuration is being set at runtime for all configuration values.
     // https://github.com/cypress-io/cypress/issues/24365
 

packages/driver/src/cy/testConfigOverrides.ts

@@ -52,22 +52,6 @@ function setConfig (testConfig: ResolvedTestConfigOverride, config, localConfigO
 
       try {
         testConfig.applied = overrideLevel
-        // this is unique validation, not applied to the general cy config.
-        // it will be removed when we support defining experimental retries
-        // in test config overrides
-
-        // TODO: remove when experimental overriding is supported
-
-        const experimentalRetryCfgKeys = [
-          'experimentalStrategy', 'experimentalOptions',
-        ]
-
-        const experimentalRetryConfigAttempted = Object.keys(testConfigOverride.retries || {})
-        .filter((v) => experimentalRetryCfgKeys.includes(v))
-
-        if (experimentalRetryConfigAttempted.length) {
-          throw new Error(`The following config keys cannot be set per-${overrideLevel}; they must be set globally: ${experimentalRetryConfigAttempted.join(', ')}`)
-        }
 
         config(testConfigOverride)
       } catch (e: any) {

packages/driver/src/cypress/error_messages.ts

@@ -272,6 +272,10 @@ export default {
         message: `The \`{{invalidConfigKey}}\` configuration can only be overridden from a suite-level override.`,
         docsUrl: 'https://on.cypress.io/config',
       },
+      global_only: {
+        message: `The \`{{invalidConfigKey}}\` configuration can only be set globally.`,
+        docsUrl: 'https://on.cypress.io/config',
+      },
     },
     invalid_test_override: {
       message: `The config passed to your {{overrideLevel}}-level overrides has the following validation error:\n\n{{errMsg}}`,

packages/driver/src/util/config.ts

@@ -103,7 +103,9 @@ export const validateConfig = (state: State, config: Record<string, any>, skipCo
     validateOverridableAtRunTime(config, isSuiteOverride, (validationResult) => {
       let errKey = 'config.cypress_config_api.read_only'
 
-      if (validationResult.supportedOverrideLevel === 'suite') {
+      if (validationResult.supportedOverrideLevel === 'global_only') {
+        errKey = 'config.invalid_mocha_config_override.global_only'
+      } else if (validationResult.supportedOverrideLevel === 'suite') {
         errKey = 'config.invalid_mocha_config_override.suite_only'
       } else if (mochaOverrideLevel) {
         errKey = 'config.invalid_mocha_config_override.read_only'

system-tests/__snapshots__/testConfigOverrides_spec.ts.js

@@ -1377,20 +1377,31 @@ exports['testConfigOverrides / experimental retries specific behavior / fails wh
   overriding legacy retries with experimental retries
     at the describe level
       1) sets the config
+    at the test level
+      2) sets the config
 
 
   0 passing
-  1 failing
+  2 failing
 
   1) overriding legacy retries with experimental retries
        at the describe level
          sets the config:
      CypressError: The config passed to your suite-level overrides has the following validation error:
 
-The following config keys cannot be set per-suite; they must be set globally: experimentalStrategy, experimentalOptions
+CypressError: The \`retries.experimentalStrategy\` configuration can only be set globally.
+
+https://on.cypress.io/config
+      [stack trace lines]
+
+  2) overriding legacy retries with experimental retries
+       at the test level
+         sets the config:
+     CypressError: The config passed to your test-level overrides has the following validation error:
+
+CypressError: The \`retries.experimentalStrategy\` configuration can only be set globally.
 
 https://on.cypress.io/config
-  Error
       [stack trace lines]
 
 
@@ -1399,9 +1410,9 @@ https://on.cypress.io/config
   (Results)
 
   ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
-  │ Tests:        1                                                                                │
+  │ Tests:        2                                                                                │
   │ Passing:      0                                                                                │
-  │ Failing:      1                                                                                │
+  │ Failing:      2                                                                                │
   │ Pending:      0                                                                                │
   │ Skipped:      0                                                                                │
   │ Screenshots:  0                                                                                │
@@ -1418,10 +1429,10 @@ https://on.cypress.io/config
 
        Spec                                              Tests  Passing  Failing  Pending  Skipped  
   ┌────────────────────────────────────────────────────────────────────────────────────────────────┐
-  │ ✖  override-with-experimental-retries.      XX:XX        1        -        1        -        - │
+  │ ✖  override-with-experimental-retries.      XX:XX        2        -        2        -        - │
   │    cy.js                                                                                       │
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
-    ✖  1 of 1 failed (100%)                     XX:XX        1        -        1        -        -  
+    ✖  1 of 1 failed (100%)                     XX:XX        2        -        2        -        -  
 
 
 `

system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js

@@ -26,4 +26,26 @@ describe('overriding legacy retries with experimental retries', () => {
       expect(retries.openMode).to.eq(openMode)
     })
   })
+
+  describe('at the test level', () => {
+    it('sets the config', {
+      retries: {
+        experimentalStrategy,
+        openMode,
+        runMode,
+        experimentalOptions: {
+          maxRetries,
+          passesRequired,
+        },
+      },
+    }, () => {
+      const retries = Cypress.config('retries')
+
+      expect(retries.experimentalStrategy).to.eq(experimentalStrategy)
+      expect(retries.experimentalOptions?.maxRetries).to.eq(maxRetries)
+      expect(retries.experimentalOptions?.passesRequired).to.eq(passesRequired)
+      expect(retries.runMode).to.eq(runMode)
+      expect(retries.openMode).to.eq(openMode)
+    })
+  })
 })

system-tests/test/testConfigOverrides_spec.ts

@@ -89,7 +89,7 @@ describe('testConfigOverrides', () => {
       spec: 'override-with-experimental-retries.cy.js',
       project: 'experimental-retries',
       configFile: 'cypress-legacy-retries.config.js',
-      expectedExitCode: 1,
+      expectedExitCode: 2,
       browser: '!webkit',
       snapshot: true,
     })