From beaa0090ef40535d41fdd5f5b4fa4225a8b02299 Mon Sep 17 00:00:00 2001 From: Navateja Alagam Date: Thu, 31 Oct 2024 19:31:16 +0530 Subject: [PATCH] feat: throw axe Error in case of axe not running (#753) Co-authored-by: Navateja Alagam --- packages/format/src/format.ts | 16 +++++++ packages/format/src/index.ts | 2 +- ...roupViolationResultsProcessor.test.ts.snap | 42 ++++++++++++++++--- packages/jest/__tests__/automatic.test.ts | 11 +++++ .../groupViolationResultsProcessor.test.ts | 2 + packages/jest/src/automatic.ts | 4 +- .../src/groupViolationResultsProcessor.ts | 37 +++++++++++----- .../__data__/sa11y-custom-rules-invalid.json | 3 ++ packages/test-utils/src/index.ts | 1 + packages/test-utils/src/test-data.ts | 1 + 10 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 packages/test-utils/__data__/sa11y-custom-rules-invalid.json diff --git a/packages/format/src/format.ts b/packages/format/src/format.ts index 8ecb49fe..d5bc116d 100644 --- a/packages/format/src/format.ts +++ b/packages/format/src/format.ts @@ -46,6 +46,22 @@ const defaultOptions: Options = { deduplicate: false, }; +export class AxeError extends Error { + /** + * Throw error with Axe error + */ + + constructor(message: string) { + super(message); + this.name = AxeError.name; + this.message = message; + } + + static throwAxeError(e: Error): void { + throw new AxeError(`${e.message}`); + } +} + /** * Custom error object to represent a11y violations */ diff --git a/packages/format/src/index.ts b/packages/format/src/index.ts index 2589e507..598f1586 100644 --- a/packages/format/src/index.ts +++ b/packages/format/src/index.ts @@ -5,6 +5,6 @@ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -export { A11yError, Options } from './format'; +export { A11yError, AxeError, Options } from './format'; export { exceptionListFilter, exceptionListFilterSelectorKeywords } from './filter'; export { A11yResult, A11yResults, appendWcag } from './result'; diff --git a/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap b/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap index 71e081e1..c6246d92 100644 --- a/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap +++ b/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap @@ -34,7 +34,7 @@ none" exports[`Group Violation Results Processor should process test results as expected 1`] = ` Object { "numFailedTestSuites": 1, - "numFailedTests": 6, + "numFailedTests": 7, "numPassedTestSuites": 0, "numPassedTests": 0, "numPendingTestSuites": 0, @@ -42,7 +42,7 @@ Object { "numRuntimeErrorTestSuites": 0, "numTodoTests": 0, "numTotalTestSuites": 0, - "numTotalTests": 6, + "numTotalTests": 7, "openHandles": Array [], "snapshot": Object { "added": 0, @@ -65,7 +65,7 @@ Object { "testResults": Array [ Object { "leaks": false, - "numFailingTests": 6, + "numFailingTests": 7, "numPassingTests": 0, "numPendingTests": 0, "numTodoTests": 0, @@ -2603,6 +2603,18 @@ Fix any of the following: "fullName": "Error@5", "status": "failed", }, + Object { + "ancestorTitles": Array [ + "sa11y", + ], + "failureDetails": Array [ + Object { + "name": "AxeError", + }, + ], + "fullName": "AxeError@6", + "status": "failed", + }, ], }, ], @@ -2613,7 +2625,7 @@ Fix any of the following: exports[`Group Violation Results Processor should process test results as expected 2`] = ` Object { "numFailedTestSuites": 1, - "numFailedTests": 4, + "numFailedTests": 5, "numPassedTestSuites": 0, "numPassedTests": 0, "numPendingTestSuites": 0, @@ -2621,7 +2633,7 @@ Object { "numRuntimeErrorTestSuites": 0, "numTodoTests": 0, "numTotalTestSuites": 0, - "numTotalTests": 6, + "numTotalTests": 7, "openHandles": Array [], "snapshot": Object { "added": 0, @@ -2644,7 +2656,7 @@ Object { "testResults": Array [ Object { "leaks": false, - "numFailingTests": 4, + "numFailingTests": 5, "numPassingTests": 0, "numPendingTests": 0, "numTodoTests": 0, @@ -4622,6 +4634,24 @@ For guidance on accessibility related specifics, contact our A11y team: http://s "fullName": "Error@5", "status": "passed", }, + Object { + "ancestorTitles": Array [ + "sa11y", + ], + "failureDetails": Array [ + Object { + "name": "AxeError", + }, + ], + "failureMessages": Array [ + " + The test has failed to execute the accessibility check. + Please contact our Sa11y team: http://sfdc.co/sa11y-users + ", + ], + "fullName": "AxeError@6", + "status": "failed", + }, ], }, ], diff --git a/packages/jest/__tests__/automatic.test.ts b/packages/jest/__tests__/automatic.test.ts index e4cfb1ad..f6510844 100644 --- a/packages/jest/__tests__/automatic.test.ts +++ b/packages/jest/__tests__/automatic.test.ts @@ -17,6 +17,7 @@ import { customRulesFilePath, domWithA11yCustomIssues, domWithA11yIncompleteIssues, + customRulesFilePathInvalid, } from '@sa11y/test-utils'; import * as Sa11yCommon from '@sa11y/common'; import { expect, jest } from '@jest/globals'; @@ -234,6 +235,16 @@ describe('automatic checks call', () => { delete process.env.SELECTOR_FILTER_KEYWORDS; }); + it('should throw an Axe error for axe related issues', async () => { + document.body.innerHTML = domWithNoA11yIssues; + // expect(document).toBeAccessible(); + process.env.SA11Y_CUSTOM_RULES = customRulesFilePathInvalid; + await expect(automaticCheck({ cleanupAfterEach: true })).rejects.toThrow( + 'Error running accessibility checks using axe' + ); + delete process.env.SA11Y_CUSTOM_RULES; + }); + it('should pass when run in DOM Mutation Observer mode', async () => { document.body.innerHTML = domWithA11yIssues; await expect( diff --git a/packages/jest/__tests__/groupViolationResultsProcessor.test.ts b/packages/jest/__tests__/groupViolationResultsProcessor.test.ts index 33c56b0c..1daa6de4 100644 --- a/packages/jest/__tests__/groupViolationResultsProcessor.test.ts +++ b/packages/jest/__tests__/groupViolationResultsProcessor.test.ts @@ -16,6 +16,7 @@ import { ErrorElement, createA11yErrorElements, } from '../src/groupViolationResultsProcessor'; +import { AxeError } from '@sa11y/format/src'; const a11yResults: A11yResult[] = []; const aggregatedResults = makeEmptyAggregatedTestResult(); @@ -71,6 +72,7 @@ beforeAll(async () => { addTestFailure(testSuite, new A11yError(combinedViolations, a11yResults)); // Add non-a11y test failure addTestFailure(testSuite, new Error('foo')); + addTestFailure(testSuite, new AxeError('Axe is running')); testSuite.testFilePath = '/test/data/sa11y-auto-checks.js'; addResult(aggregatedResults, testSuite); }); diff --git a/packages/jest/src/automatic.ts b/packages/jest/src/automatic.ts index de811bea..f9d05cb7 100644 --- a/packages/jest/src/automatic.ts +++ b/packages/jest/src/automatic.ts @@ -7,7 +7,7 @@ import { AxeResults, log, useCustomRules } from '@sa11y/common'; import { getA11yResultsJSDOM } from '@sa11y/assert'; -import { A11yError, exceptionListFilterSelectorKeywords } from '@sa11y/format'; +import { A11yError, AxeError, exceptionListFilterSelectorKeywords } from '@sa11y/format'; import { isTestUsingFakeTimer } from './matcher'; import { expect } from '@jest/globals'; import { @@ -124,6 +124,8 @@ export async function automaticCheck(opts: AutoCheckOpts = defaultAutoCheckOpts) } } } + } catch (e) { + AxeError.throwAxeError(e as Error); } finally { if (opts.runDOMMutationObserver) { mutatedNodes = []; diff --git a/packages/jest/src/groupViolationResultsProcessor.ts b/packages/jest/src/groupViolationResultsProcessor.ts index 6df59296..d60ec7f2 100644 --- a/packages/jest/src/groupViolationResultsProcessor.ts +++ b/packages/jest/src/groupViolationResultsProcessor.ts @@ -6,11 +6,16 @@ */ import { AggregatedResult, AssertionResult, TestResult } from '@jest/test-result'; import { log } from '@sa11y/common'; -import { A11yError } from '@sa11y/format'; +import { A11yError, AxeError } from '@sa11y/format'; -type FailureDetail = { +type a11yFailureDetail = { error?: A11yError; }; + +type axeFailureDetail = { + error?: AxeError; +}; + interface FailureMatcherDetail { error?: { matcherResult?: { @@ -175,30 +180,40 @@ For guidance on accessibility related specifics, contact our A11y team: http://s * Convert any a11y errors from test failures into their own test suite, results */ function processA11yErrors(results: AggregatedResult, testSuite: TestResult, testResult: AssertionResult) { - const a11yFailureDetails: FailureDetail[] = []; + const a11yFailureDetails: a11yFailureDetail[] = []; + const axeFailureDetails: axeFailureDetail[] = []; const a11yFailureMessages: string[] = []; - let a11yErrorsExist = false; + const axeFailureMessages: string[] = []; + let a11yAxeErrorsExist = false; testResult.failureDetails.forEach((failure) => { - let error = (failure as FailureDetail).error; + let error = (failure as a11yFailureDetail).error; // If using circus test runner https://github.com/facebook/jest/issues/11405#issuecomment-843549606 // TODO (code cov): Add test data covering the case for circus test runner /* istanbul ignore next */ if (error === undefined) error = failure as A11yError; if (error.name === A11yError.name) { - a11yErrorsExist = true; - a11yFailureDetails.push({ ...(failure as FailureDetail) } as FailureDetail); + a11yAxeErrorsExist = true; + a11yFailureDetails.push({ ...(failure as a11yFailureDetail) } as a11yFailureDetail); processA11yDetailsAndMessages(error, a11yFailureMessages); } + if (error.name === AxeError.name) { + a11yAxeErrorsExist = true; + axeFailureDetails.push({ ...(failure as axeFailureDetail) } as axeFailureDetail); + axeFailureMessages.push(` + The test has failed to execute the accessibility check. + Please contact our Sa11y team: http://sfdc.co/sa11y-users + `); + } }); - if (!a11yErrorsExist) { + if (!a11yAxeErrorsExist) { testSuite.numFailingTests -= 1; results.numFailedTests -= 1; if (testSuite.numFailingTests === 0) results.numFailedTestSuites -= 1; } - testResult.failureDetails = [...a11yFailureDetails]; - testResult.failureMessages = [...a11yFailureMessages]; - testResult.status = a11yFailureDetails.length === 0 ? 'passed' : testResult.status; + testResult.failureDetails = [...a11yFailureDetails, ...axeFailureDetails]; + testResult.failureMessages = [...a11yFailureMessages, ...axeFailureMessages]; + testResult.status = testResult.failureDetails.length === 0 ? 'passed' : testResult.status; } function processA11yMatcherErrors(results: AggregatedResult, testSuite: TestResult, testResult: AssertionResult) { diff --git a/packages/test-utils/__data__/sa11y-custom-rules-invalid.json b/packages/test-utils/__data__/sa11y-custom-rules-invalid.json new file mode 100644 index 00000000..e01f2e89 --- /dev/null +++ b/packages/test-utils/__data__/sa11y-custom-rules-invalid.json @@ -0,0 +1,3 @@ +{ + "rules": ["non-existent-rule"] +} diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index adf08710..19989fd5 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -22,6 +22,7 @@ export { shadowDomID, videoURL, customRulesFilePath, + customRulesFilePathInvalid, domWithA11yCustomIssues, domWithA11yIncompleteIssues, } from './test-data'; diff --git a/packages/test-utils/src/test-data.ts b/packages/test-utils/src/test-data.ts index 1686fae9..3429f403 100644 --- a/packages/test-utils/src/test-data.ts +++ b/packages/test-utils/src/test-data.ts @@ -14,6 +14,7 @@ const dataDir = path.resolve(__dirname, '../__data__/'); export const domWithA11yIssuesBodyID = 'dom-with-issues'; const fileWithA11yIssues = path.resolve(dataDir, 'a11yIssues.html'); export const customRulesFilePath = path.resolve(dataDir, 'sa11y-custom-rules.json'); +export const customRulesFilePathInvalid = path.resolve(dataDir, 'sa11y-custom-rules-invalid.json'); export const domWithA11yCustomIssuesPath = path.resolve(dataDir, 'a11yCustomIssues.html'); export const domWithA11yIncompleteIssuesPath = path.resolve(dataDir, 'a11yIncompleteIssues.html');