feat: throw axe Error in case of axe not running (#753)

Co-authored-by: Navateja Alagam <[email protected]>

packages/format/src/format.ts

@@ -46,6 +46,22 @@ const defaultOptions: Options = {
     deduplicate: false,
 };
 
+export class AxeError extends Error {
+    /**
+     * Throw error with Axe error
+     */
+
+    constructor(message: string) {
+        super(message);
+        this.name = AxeError.name;
+        this.message = message;
+    }
+
+    static throwAxeError(e: Error): void {
+        throw new AxeError(`${e.message}`);
+    }
+}
+
 /**
  *  Custom error object to represent a11y violations
  */

packages/format/src/index.ts

@@ -5,6 +5,6 @@
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-export { A11yError, Options } from './format';
+export { A11yError, AxeError, Options } from './format';
 export { exceptionListFilter, exceptionListFilterSelectorKeywords } from './filter';
 export { A11yResult, A11yResults, appendWcag } from './result';

packages/jest/__tests__/automatic.test.ts

@@ -17,6 +17,7 @@ import {
     customRulesFilePath,
     domWithA11yCustomIssues,
     domWithA11yIncompleteIssues,
+    customRulesFilePathInvalid,
 } from '@sa11y/test-utils';
 import * as Sa11yCommon from '@sa11y/common';
 import { expect, jest } from '@jest/globals';
@@ -234,6 +235,16 @@ describe('automatic checks call', () => {
         delete process.env.SELECTOR_FILTER_KEYWORDS;
     });
 
+    it('should throw an Axe error for axe related issues', async () => {
+        document.body.innerHTML = domWithNoA11yIssues;
+        // expect(document).toBeAccessible();
+        process.env.SA11Y_CUSTOM_RULES = customRulesFilePathInvalid;
+        await expect(automaticCheck({ cleanupAfterEach: true })).rejects.toThrow(
+            'Error running accessibility checks using axe'
+        );
+        delete process.env.SA11Y_CUSTOM_RULES;
+    });
+
     it('should pass when run in DOM Mutation Observer mode', async () => {
         document.body.innerHTML = domWithA11yIssues;
         await expect(

packages/jest/__tests__/groupViolationResultsProcessor.test.ts

@@ -16,6 +16,7 @@ import {
     ErrorElement,
     createA11yErrorElements,
 } from '../src/groupViolationResultsProcessor';
+import { AxeError } from '@sa11y/format/src';
 
 const a11yResults: A11yResult[] = [];
 const aggregatedResults = makeEmptyAggregatedTestResult();
@@ -71,6 +72,7 @@ beforeAll(async () => {
     addTestFailure(testSuite, new A11yError(combinedViolations, a11yResults));
     // Add non-a11y test failure
     addTestFailure(testSuite, new Error('foo'));
+    addTestFailure(testSuite, new AxeError('Axe is running'));
     testSuite.testFilePath = '/test/data/sa11y-auto-checks.js';
     addResult(aggregatedResults, testSuite);
 });

packages/jest/src/automatic.ts

@@ -7,7 +7,7 @@
 
 import { AxeResults, log, useCustomRules } from '@sa11y/common';
 import { getA11yResultsJSDOM } from '@sa11y/assert';
-import { A11yError, exceptionListFilterSelectorKeywords } from '@sa11y/format';
+import { A11yError, AxeError, exceptionListFilterSelectorKeywords } from '@sa11y/format';
 import { isTestUsingFakeTimer } from './matcher';
 import { expect } from '@jest/globals';
 import {
@@ -124,6 +124,8 @@ export async function automaticCheck(opts: AutoCheckOpts = defaultAutoCheckOpts)
                 }
             }
         }
+    } catch (e) {
+        AxeError.throwAxeError(e as Error);
     } finally {
         if (opts.runDOMMutationObserver) {
             mutatedNodes = [];

packages/jest/src/groupViolationResultsProcessor.ts

@@ -6,11 +6,16 @@
  */
 import { AggregatedResult, AssertionResult, TestResult } from '@jest/test-result';
 import { log } from '@sa11y/common';
-import { A11yError } from '@sa11y/format';
+import { A11yError, AxeError } from '@sa11y/format';
 
-type FailureDetail = {
+type a11yFailureDetail = {
     error?: A11yError;
 };
+
+type axeFailureDetail = {
+    error?: AxeError;
+};
+
 interface FailureMatcherDetail {
     error?: {
         matcherResult?: {
@@ -175,30 +180,40 @@ For guidance on accessibility related specifics, contact our A11y team: http://s
  * Convert any a11y errors from test failures into their own test suite, results
  */
 function processA11yErrors(results: AggregatedResult, testSuite: TestResult, testResult: AssertionResult) {
-    const a11yFailureDetails: FailureDetail[] = [];
+    const a11yFailureDetails: a11yFailureDetail[] = [];
+    const axeFailureDetails: axeFailureDetail[] = [];
     const a11yFailureMessages: string[] = [];
-    let a11yErrorsExist = false;
+    const axeFailureMessages: string[] = [];
+    let a11yAxeErrorsExist = false;
 
     testResult.failureDetails.forEach((failure) => {
-        let error = (failure as FailureDetail).error;
+        let error = (failure as a11yFailureDetail).error;
         // If using circus test runner https://github.com/facebook/jest/issues/11405#issuecomment-843549606
         // TODO (code cov): Add test data covering the case for circus test runner
         /* istanbul ignore next */
         if (error === undefined) error = failure as A11yError;
         if (error.name === A11yError.name) {
-            a11yErrorsExist = true;
-            a11yFailureDetails.push({ ...(failure as FailureDetail) } as FailureDetail);
+            a11yAxeErrorsExist = true;
+            a11yFailureDetails.push({ ...(failure as a11yFailureDetail) } as a11yFailureDetail);
             processA11yDetailsAndMessages(error, a11yFailureMessages);
         }
+        if (error.name === AxeError.name) {
+            a11yAxeErrorsExist = true;
+            axeFailureDetails.push({ ...(failure as axeFailureDetail) } as axeFailureDetail);
+            axeFailureMessages.push(`
+            The test has failed to execute the accessibility check.
+            Please contact our Sa11y team: http://sfdc.co/sa11y-users
+            `);
+        }
     });
-    if (!a11yErrorsExist) {
+    if (!a11yAxeErrorsExist) {
         testSuite.numFailingTests -= 1;
         results.numFailedTests -= 1;
         if (testSuite.numFailingTests === 0) results.numFailedTestSuites -= 1;
     }
-    testResult.failureDetails = [...a11yFailureDetails];
-    testResult.failureMessages = [...a11yFailureMessages];
-    testResult.status = a11yFailureDetails.length === 0 ? 'passed' : testResult.status;
+    testResult.failureDetails = [...a11yFailureDetails, ...axeFailureDetails];
+    testResult.failureMessages = [...a11yFailureMessages, ...axeFailureMessages];
+    testResult.status = testResult.failureDetails.length === 0 ? 'passed' : testResult.status;
 }
 
 function processA11yMatcherErrors(results: AggregatedResult, testSuite: TestResult, testResult: AssertionResult) {

packages/test-utils/__data__/sa11y-custom-rules-invalid.json

@@ -0,0 +1,3 @@
+{
+    "rules": ["non-existent-rule"]
+}

packages/test-utils/src/index.ts

@@ -22,6 +22,7 @@ export {
     shadowDomID,
     videoURL,
     customRulesFilePath,
+    customRulesFilePathInvalid,
     domWithA11yCustomIssues,
     domWithA11yIncompleteIssues,
 } from './test-data';

packages/test-utils/src/test-data.ts

@@ -14,6 +14,7 @@ const dataDir = path.resolve(__dirname, '../__data__/');
 export const domWithA11yIssuesBodyID = 'dom-with-issues';
 const fileWithA11yIssues = path.resolve(dataDir, 'a11yIssues.html');
 export const customRulesFilePath = path.resolve(dataDir, 'sa11y-custom-rules.json');
+export const customRulesFilePathInvalid = path.resolve(dataDir, 'sa11y-custom-rules-invalid.json');
 export const domWithA11yCustomIssuesPath = path.resolve(dataDir, 'a11yCustomIssues.html');
 export const domWithA11yIncompleteIssuesPath = path.resolve(dataDir, 'a11yIncompleteIssues.html');