From c42844307946f438a7369b381a702f121ff17657 Mon Sep 17 00:00:00 2001 From: Bill Glesias Date: Wed, 26 Jul 2023 16:00:56 -0400 Subject: [PATCH] feat: add burnIn Configuration option (currently a no-op) (#27377) * feat: add the burnIn Configuration to the config package. Option currently is a no-op * chore: make burn in experimental * chore: set experimentalBurnIn to false by default --- cli/types/cypress.d.ts | 12 ++ .../config/__snapshots__/index.spec.ts.js | 3 + .../__snapshots__/validation.spec.ts.js | 71 +++++++++++ packages/config/src/options.ts | 5 + packages/config/src/validation.ts | 22 ++++ packages/config/test/project/utils.spec.ts | 2 + packages/config/test/validation.spec.ts | 115 ++++++++++++++++++ .../frontend-shared/src/locales/en-US.json | 4 + 8 files changed, 234 insertions(+) diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index b2b1c3272ab4..914d03ffa658 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -3055,6 +3055,18 @@ declare namespace Cypress { * @default 'top' */ scrollBehavior: scrollBehaviorOptions + /** + * Enables flake detection on risky passing tests, such as for new or modified tests or tests which failed or flaked on previous runs. + * Note: This feature is only available for Team, Business, and Enterprise plans on Cypress Cloud. + * Furthermore, only Business and Enterprise plans can override the `experimentalBurnIn.default` or `experimentalBurnIn.flaky` settings. + * TODO: make sure experimental link is available in documentation before experimentalBurnIn release! + * @see https://on.cypress.io/experiments#Experimental-Burn-In + * @default false + */ + experimentalBurnIn: boolean | { + default: number + flaky: number + }, /** * Indicates whether Cypress should allow CSP header directives from the application under test. * - When this option is set to `false`, Cypress will strip the entire CSP header. diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js index ee999fa83308..27a34d91f4c3 100644 --- a/packages/config/__snapshots__/index.spec.ts.js +++ b/packages/config/__snapshots__/index.spec.ts.js @@ -21,6 +21,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1 'animationDistanceThreshold': 5, 'baseUrl': null, 'blockHosts': null, + 'experimentalBurnIn': false, 'chromeWebSecurity': true, 'clientCertificates': [], 'component': { @@ -109,6 +110,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f 'animationDistanceThreshold': 5, 'baseUrl': null, 'blockHosts': null, + 'experimentalBurnIn': false, 'chromeWebSecurity': true, 'clientCertificates': [], 'component': { @@ -198,6 +200,7 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key 'arch', 'baseUrl', 'blockHosts', + 'experimentalBurnIn', 'chromeWebSecurity', 'clientCertificates', 'component', diff --git a/packages/config/__snapshots__/validation.spec.ts.js b/packages/config/__snapshots__/validation.spec.ts.js index ace0c97230bd..dbc4461334c4 100644 --- a/packages/config/__snapshots__/validation.spec.ts.js +++ b/packages/config/__snapshots__/validation.spec.ts.js @@ -261,3 +261,74 @@ exports['invalid upper bound'] = { 'value': 52, 'type': 'a valid CRF number between 1 & 51, 0 or false to disable compression, or true to use the default compression of 32', } + +exports['missing key "flaky"'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': 1, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['missing key "default"'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'flaky': 1, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['keys cannot be zero key'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': 0, + 'flaky': 0, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['keys cannot be zero'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': 0, + 'flaky': 0, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['keys cannot be negative'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': -3, + 'flaky': -40, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['keys cannot be floating point numbers'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': 5.7, + 'flaky': 8.22, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['keys cannot be Infinity'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': null, + 'flaky': null, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} + +exports['extraneous keys'] = { + 'key': 'experimentalBurnIn', + 'value': { + 'default': 3, + 'flaky': 5, + 'notflaky': null, + }, + 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', +} diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 3128d671f7f4..2cef1c2a3688 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -152,6 +152,11 @@ const driverConfigOptions: Array = [ validation: validate.isStringOrArrayOfStrings, overrideLevel: 'any', requireRestartOnChange: 'server', + }, { + // experimentalBurnIn config is only considered in run-mode while recording to cypress-cloud + name: 'experimentalBurnIn', + defaultValue: false, + validation: validate.isValidBurnInConfig, }, { name: 'chromeWebSecurity', defaultValue: true, diff --git a/packages/config/src/validation.ts b/packages/config/src/validation.ts index 1cbcb34c71b1..180d59086105 100644 --- a/packages/config/src/validation.ts +++ b/packages/config/src/validation.ts @@ -384,3 +384,25 @@ export function isNullOrArrayOfStrings (key: string, value: any): ErrResult | tr return errMsg(key, value, 'an array of strings or null') } + +export function isValidBurnInConfig (key: string, value: any): ErrResult | true { + // false can be provided to turn off test burn in + if (_.isBoolean(value)) { + return true + } + + if (_.isPlainObject(value)) { + const { default: defaultKey, flaky: flakyKey, ...extraneousKeys } = value + + if (defaultKey !== undefined && defaultKey !== undefined && _.isEmpty(extraneousKeys)) { + const isDefaultKeyValid = _.isInteger(defaultKey) && _.inRange(defaultKey, 1, Infinity) + const isFlakyKeyValid = _.isInteger(flakyKey) && _.inRange(flakyKey, 1, Infinity) + + if (isDefaultKeyValid && isFlakyKeyValid) { + return true + } + } + } + + return errMsg(key, value, 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.') +} diff --git a/packages/config/test/project/utils.spec.ts b/packages/config/test/project/utils.spec.ts index d76d8f239aa2..7342c6557527 100644 --- a/packages/config/test/project/utils.spec.ts +++ b/packages/config/test/project/utils.spec.ts @@ -1073,6 +1073,7 @@ describe('config/src/project/utils', () => { baseUrl: { value: null, from: 'default' }, blockHosts: { value: null, from: 'default' }, browsers: { value: [], from: 'default' }, + experimentalBurnIn: { value: false, from: 'default' }, chromeWebSecurity: { value: true, from: 'default' }, clientCertificates: { value: [], from: 'default' }, defaultCommandTimeout: { value: 4000, from: 'default' }, @@ -1172,6 +1173,7 @@ describe('config/src/project/utils', () => { baseUrl: { value: 'http://localhost:8080', from: 'config' }, blockHosts: { value: null, from: 'default' }, browsers: { value: [], from: 'default' }, + experimentalBurnIn: { value: false, from: 'default' }, chromeWebSecurity: { value: true, from: 'default' }, clientCertificates: { value: [], from: 'default' }, defaultCommandTimeout: { value: 4000, from: 'default' }, diff --git a/packages/config/test/validation.spec.ts b/packages/config/test/validation.spec.ts index 05af3d8d8d44..df538d799da8 100644 --- a/packages/config/test/validation.spec.ts +++ b/packages/config/test/validation.spec.ts @@ -509,4 +509,119 @@ describe('config/src/validation', () => { return snapshot('invalid upper bound', upperBoundMsg) }) }) + + describe('.isValidBurnInConfig', () => { + it('validates default config if enabled', () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: 3, + flaky: 5, + }) + + expect(validate).to.be.true + }) + + it('validates false', () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', false) + + expect(validate).to.be.true + }) + + it('validates true', () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', true) + + expect(validate).to.be.true + }) + + it('does not allow extraneous keys', () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: 3, + flaky: 5, + // extraneous key + notflaky: null, + }) + + expect(validate).to.not.be.true + + return snapshot(`extraneous keys`, validate) + }) + + const possibleConfigKeys = ['default', 'flaky'] + + possibleConfigKeys.forEach((burnInConfigKey) => { + it(`Does not populate missing config value with default values (key "${burnInConfigKey}")`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + [burnInConfigKey]: 1, + }) + + expect(validate).to.not.be.true + + const missingKey = possibleConfigKeys.find((key) => key !== burnInConfigKey) + + return snapshot(`missing key "${missingKey}"`, validate) + }) + }) + + it(`does not allow keys to be zero`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: 0, + flaky: 0, + }) + + expect(validate).to.not.be.true + + return snapshot(`keys cannot be zero`, validate) + }) + + it(`does not allow keys to be negative`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: -3, + flaky: -40, + }) + + expect(validate).to.not.be.true + + return snapshot(`keys cannot be negative`, validate) + }) + + it(`does not allow keys to be floating point numbers`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: 5.7, + flaky: 8.22, + }) + + expect(validate).to.not.be.true + + return snapshot(`keys cannot be floating point numbers`, validate) + }) + + it(`does not allow keys to be Infinity`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: Infinity, + flaky: Infinity, + }) + + expect(validate).to.not.be.true + + return snapshot(`keys cannot be Infinity`, validate) + }) + + it(`tests lower bound`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: 1, + flaky: 1, + }) + + expect(validate).to.be.true + }) + + // TODO: revisit limit on 'default' and 'flaky' keys to set sane limits + it(`tests upper bound (currently no limit and is subject to change)`, () => { + const validate = validation.isValidBurnInConfig('experimentalBurnIn', { + default: 100000, + flaky: 142342342, + }) + + expect(validate).to.be.true + }) + }) }) diff --git a/packages/frontend-shared/src/locales/en-US.json b/packages/frontend-shared/src/locales/en-US.json index 5e8cd31b41cf..da26aa6f1dc9 100644 --- a/packages/frontend-shared/src/locales/en-US.json +++ b/packages/frontend-shared/src/locales/en-US.json @@ -570,6 +570,10 @@ "experiments": { "title": "Experiments", "description": "If you'd like to try out new features that we're working on, you can enable beta features for your project by turning on the experimental features you'd like to try. {0}", + "experimentalBurnIn": { + "name": "Burn-in", + "description": "Enables flake detection on risky passing tests, such as for new or modified tests or tests which failed or flaked on previous runs. Note: This feature is only available for Team, Business, and Enterprise plans on Cypress Cloud. Furthermore, only Business and Enterprise plans can override the `experimentalBurnIn.default` or `experimentalBurnIn.flaky` settings." + }, "experimentalCspAllowList": { "name": "CSP Allow List", "description": "Enables Cypress to selectively permit Content-Security-Policy and Content-Security-Policy-Report-Only header directives, including those that might otherwise block Cypress from running."