From 577ae4938cf245d95987ae4f9a5eadaec4b5d49d Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Tue, 24 Oct 2023 10:22:20 -0400 Subject: [PATCH] refactor test config override validation for experimental retry subkeys --- packages/config/src/browser.ts | 20 +++++++++++++++ packages/driver/src/cy/testConfigOverrides.ts | 16 ------------ packages/driver/src/cypress/error_messages.ts | 4 +++ packages/driver/src/util/config.ts | 4 ++- .../testConfigOverrides_spec.ts.js | 25 +++++++++++++------ .../override-with-experimental-retries.cy.js | 22 ++++++++++++++++ system-tests/test/testConfigOverrides_spec.ts | 2 +- 7 files changed, 68 insertions(+), 25 deletions(-) diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts index 10bcbbd33588..07018ab7c3c7 100644 --- a/packages/config/src/browser.ts +++ b/packages/config/src/browser.ts @@ -197,6 +197,26 @@ export const validateOverridableAtRunTime = (config: any, isSuiteLevelOverride: return } + // this is unique validation, not applied to the general cy config. + // it will be removed when we support defining experimental retries + // in test config overrides + + // TODO: remove when experimental retry overriding is supported + + if (configKey === 'retries') { + const experimentalRetryCfgKeys = [ + 'experimentalStrategy', 'experimentalOptions', + ] + + Object.keys(config.retries || {}) + .filter((v) => experimentalRetryCfgKeys.includes(v)) + .forEach((invalidExperimentalCfgOverride) => { + onErr({ + invalidConfigKey: `retries.${invalidExperimentalCfgOverride}`, + supportedOverrideLevel: 'global_only', + }) + }) + } // TODO: add a hook to ensure valid testing-type configuration is being set at runtime for all configuration values. // https://github.com/cypress-io/cypress/issues/24365 diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts index f399a12d109c..92097243f2aa 100644 --- a/packages/driver/src/cy/testConfigOverrides.ts +++ b/packages/driver/src/cy/testConfigOverrides.ts @@ -52,22 +52,6 @@ function setConfig (testConfig: ResolvedTestConfigOverride, config, localConfigO try { testConfig.applied = overrideLevel - // this is unique validation, not applied to the general cy config. - // it will be removed when we support defining experimental retries - // in test config overrides - - // TODO: remove when experimental overriding is supported - - const experimentalRetryCfgKeys = [ - 'experimentalStrategy', 'experimentalOptions', - ] - - const experimentalRetryConfigAttempted = Object.keys(testConfigOverride.retries || {}) - .filter((v) => experimentalRetryCfgKeys.includes(v)) - - if (experimentalRetryConfigAttempted.length) { - throw new Error(`The following config keys cannot be set per-${overrideLevel}; they must be set globally: ${experimentalRetryConfigAttempted.join(', ')}`) - } config(testConfigOverride) } catch (e: any) { diff --git a/packages/driver/src/cypress/error_messages.ts b/packages/driver/src/cypress/error_messages.ts index cd65fc93b9a8..eafe770c003f 100644 --- a/packages/driver/src/cypress/error_messages.ts +++ b/packages/driver/src/cypress/error_messages.ts @@ -272,6 +272,10 @@ export default { message: `The \`{{invalidConfigKey}}\` configuration can only be overridden from a suite-level override.`, docsUrl: 'https://on.cypress.io/config', }, + global_only: { + message: `The \`{{invalidConfigKey}}\` configuration can only be set globally.`, + docsUrl: 'https://on.cypress.io/config', + }, }, invalid_test_override: { message: `The config passed to your {{overrideLevel}}-level overrides has the following validation error:\n\n{{errMsg}}`, diff --git a/packages/driver/src/util/config.ts b/packages/driver/src/util/config.ts index 7c622a011672..950a41fdf130 100644 --- a/packages/driver/src/util/config.ts +++ b/packages/driver/src/util/config.ts @@ -103,7 +103,9 @@ export const validateConfig = (state: State, config: Record, skipCo validateOverridableAtRunTime(config, isSuiteOverride, (validationResult) => { let errKey = 'config.cypress_config_api.read_only' - if (validationResult.supportedOverrideLevel === 'suite') { + if (validationResult.supportedOverrideLevel === 'global_only') { + errKey = 'config.invalid_mocha_config_override.global_only' + } else if (validationResult.supportedOverrideLevel === 'suite') { errKey = 'config.invalid_mocha_config_override.suite_only' } else if (mochaOverrideLevel) { errKey = 'config.invalid_mocha_config_override.read_only' diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 37fc63f715c3..70267c1ed1c2 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -1377,20 +1377,31 @@ exports['testConfigOverrides / experimental retries specific behavior / fails wh overriding legacy retries with experimental retries at the describe level 1) sets the config + at the test level + 2) sets the config 0 passing - 1 failing + 2 failing 1) overriding legacy retries with experimental retries at the describe level sets the config: CypressError: The config passed to your suite-level overrides has the following validation error: -The following config keys cannot be set per-suite; they must be set globally: experimentalStrategy, experimentalOptions +CypressError: The \`retries.experimentalStrategy\` configuration can only be set globally. + +https://on.cypress.io/config + [stack trace lines] + + 2) overriding legacy retries with experimental retries + at the test level + sets the config: + CypressError: The config passed to your test-level overrides has the following validation error: + +CypressError: The \`retries.experimentalStrategy\` configuration can only be set globally. https://on.cypress.io/config - Error [stack trace lines] @@ -1399,9 +1410,9 @@ https://on.cypress.io/config (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 1 │ + │ Tests: 2 │ │ Passing: 0 │ - │ Failing: 1 │ + │ Failing: 2 │ │ Pending: 0 │ │ Skipped: 0 │ │ Screenshots: 0 │ @@ -1418,10 +1429,10 @@ https://on.cypress.io/config Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ override-with-experimental-retries. XX:XX 1 - 1 - - │ + │ ✖ override-with-experimental-retries. XX:XX 2 - 2 - - │ │ cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + ✖ 1 of 1 failed (100%) XX:XX 2 - 2 - - ` diff --git a/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js index bd38d32d2a16..d5e10ae32341 100644 --- a/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js +++ b/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js @@ -26,4 +26,26 @@ describe('overriding legacy retries with experimental retries', () => { expect(retries.openMode).to.eq(openMode) }) }) + + describe('at the test level', () => { + it('sets the config', { + retries: { + experimentalStrategy, + openMode, + runMode, + experimentalOptions: { + maxRetries, + passesRequired, + }, + }, + }, () => { + const retries = Cypress.config('retries') + + expect(retries.experimentalStrategy).to.eq(experimentalStrategy) + expect(retries.experimentalOptions?.maxRetries).to.eq(maxRetries) + expect(retries.experimentalOptions?.passesRequired).to.eq(passesRequired) + expect(retries.runMode).to.eq(runMode) + expect(retries.openMode).to.eq(openMode) + }) + }) }) diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index eee826b02a3f..f16d6e311b10 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -89,7 +89,7 @@ describe('testConfigOverrides', () => { spec: 'override-with-experimental-retries.cy.js', project: 'experimental-retries', configFile: 'cypress-legacy-retries.config.js', - expectedExitCode: 1, + expectedExitCode: 2, browser: '!webkit', snapshot: true, })