diff --git a/axe.d.ts b/axe.d.ts index 5940029b8e..7796ad4ca1 100644 --- a/axe.d.ts +++ b/axe.d.ts @@ -155,7 +155,7 @@ declare namespace axe { toolOptions: RunOptions; passes: Result[]; violations: Result[]; - incomplete: Result[]; + incomplete: IncompleteResult[]; inapplicable: Result[]; } interface Result { @@ -167,6 +167,9 @@ declare namespace axe { tags: TagValue[]; nodes: NodeResult[]; } + interface IncompleteResult extends Result { + error?: Omit; + } interface NodeResult { html: string; impact?: ImpactValue; @@ -204,6 +207,21 @@ declare namespace axe { fail: string | { [key: string]: string }; incomplete?: string | { [key: string]: string }; } + interface SupportError { + name: string; + message: string; + stack: string; + ruleId?: string; + method?: string; + cause?: SerialError; + errorNode?: DqElement; + } + interface SerialError { + message: string; + stack: string; + name: string; + cause?: SerialError; + } interface CheckLocale { [key: string]: CheckMessages; } @@ -461,7 +479,13 @@ declare namespace axe { isLabelledShadowDomSelector: ( selector: unknown ) => selector is LabelledShadowDomSelector; - + SupportError: ( + error: Error, + ruleId?: string, + method?: string, + errorNode?: DqElement + ) => SupportError; + serializeError: (error: Error) => SerialError; DqElement: DqElementConstructor; uuid: ( options?: { random?: Uint8Array | Array }, diff --git a/doc/rule-descriptions.md b/doc/rule-descriptions.md index f1e84652ca..bfb58893b4 100644 --- a/doc/rule-descriptions.md +++ b/doc/rule-descriptions.md @@ -61,7 +61,7 @@ | [listitem](https://dequeuniversity.com/rules/axe/4.10/listitem?application=RuleDescription) | Ensure <li> elements are used semantically | Serious | cat.structure, wcag2a, wcag131, EN-301-549, EN-9.1.3.1, RGAAv4, RGAA-9.3.1 | failure | | | [marquee](https://dequeuniversity.com/rules/axe/4.10/marquee?application=RuleDescription) | Ensure <marquee> elements are not used | Serious | cat.parsing, wcag2a, wcag222, TTv5, TT2.b, EN-301-549, EN-9.2.2.2, RGAAv4, RGAA-13.8.1 | failure | | | [meta-refresh](https://dequeuniversity.com/rules/axe/4.10/meta-refresh?application=RuleDescription) | Ensure <meta http-equiv="refresh"> is not used for delayed refresh | Critical | cat.time-and-media, wcag2a, wcag221, TTv5, TT8.a, EN-301-549, EN-9.2.2.1, RGAAv4, RGAA-13.1.2 | failure | [bc659a](https://act-rules.github.io/rules/bc659a), [bisz58](https://act-rules.github.io/rules/bisz58) | -| [meta-viewport](https://dequeuniversity.com/rules/axe/4.10/meta-viewport?application=RuleDescription) | Ensure <meta name="viewport"> does not disable text scaling and zooming | Critical | cat.sensory-and-visual-cues, wcag2aa, wcag144, EN-301-549, EN-9.1.4.4, ACT, RGAAv4, RGAA-10.4.2 | failure | [b4f0c3](https://act-rules.github.io/rules/b4f0c3) | +| [meta-viewport](https://dequeuniversity.com/rules/axe/4.10/meta-viewport?application=RuleDescription) | Ensure <meta name="viewport"> does not disable text scaling and zooming | Moderate | cat.sensory-and-visual-cues, wcag2aa, wcag144, EN-301-549, EN-9.1.4.4, ACT, RGAAv4, RGAA-10.4.2 | failure | [b4f0c3](https://act-rules.github.io/rules/b4f0c3) | | [nested-interactive](https://dequeuniversity.com/rules/axe/4.10/nested-interactive?application=RuleDescription) | Ensure interactive controls are not nested as they are not always announced by screen readers or can cause focus problems for assistive technologies | Serious | cat.keyboard, wcag2a, wcag412, TTv5, TT6.a, EN-301-549, EN-9.4.1.2, RGAAv4, RGAA-7.1.1 | failure, needs review | [307n5z](https://act-rules.github.io/rules/307n5z) | | [no-autoplay-audio](https://dequeuniversity.com/rules/axe/4.10/no-autoplay-audio?application=RuleDescription) | Ensure <video> or <audio> elements do not autoplay audio for more than 3 seconds without a control mechanism to stop or mute the audio | Moderate | cat.time-and-media, wcag2a, wcag142, TTv5, TT2.a, EN-301-549, EN-9.1.4.2, ACT, RGAAv4, RGAA-4.10.1 | needs review | [80f0bf](https://act-rules.github.io/rules/80f0bf) | | [object-alt](https://dequeuniversity.com/rules/axe/4.10/object-alt?application=RuleDescription) | Ensure <object> elements have alternative text | Serious | cat.text-alternatives, wcag2a, wcag111, section508, section508.22.a, EN-301-549, EN-9.1.1.1, RGAAv4, RGAA-1.1.6 | failure, needs review | [8fc3b6](https://act-rules.github.io/rules/8fc3b6) | diff --git a/eslint.config.js b/eslint.config.js index db39951cb6..203edc2778 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -122,6 +122,188 @@ module.exports = [ 'no-use-before-define': 'off' } }, + { + // disallow imports from node modules + ignores: ['lib/core/imports/**/*.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + regex: '^[^.]', + message: 'Only core/imports files should import from node modules' + } + ] + } + ] + } + }, + { + // disallow imports in standards + files: ['lib/standards/**/*.js'], + // index file can import other standards and from utils + ignores: ['lib/standards/index.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + group: ['*'], + message: + "Standard files shouldn't use imports as they are just hard coded data objects" + } + ] + } + ] + } + }, + { + // restrict imports to core/utils files to other core/utils, core, core/base, standards, imports, or reporters/helpers + files: ['lib/core/utils/**/*.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + // e.g. "../commons/aria/" or "../public/" + regex: + '.*\\.\\.\\/(commons|public|checks|rules)(\\/|$)|.*\\.\\.\\/reporters\\/.*?\\.js', + message: + 'Util files should only import from other utils, core, or standard files' + }, + // disallow imports from node modules + // seems only 1 regex pattern is allowed to match as not having this allows node module imports even while having the general rule above for all files) + { + regex: '^[^.]', + message: 'Only core/imports files should import from node modules' + } + ] + } + ] + } + }, + { + // restrict imports to core/public files to other core/public, or imports allowed by core/utils + files: ['lib/core/public/**/*.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + // e.g. "../commons/aria/" or "../checks/" + regex: + '.*\\.\\.\\/(commons|checks|rules)(\\/|$)|.*\\.\\.\\/reporters\\/.*?\\.js', + message: + 'Public files should only import from other public, util, core, or standard files' + }, + // disallow imports from node modules + { + regex: '^[^.]', + message: 'Only core/imports files should import from node modules' + } + ] + } + ] + } + }, + { + // disallow imports in core/imports files to any non-node module + files: ['lib/core/imports/**/*.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + // relative file paths + regex: '\\\.\\\.\\/', + message: 'Import files should only import from node modules' + } + ] + } + ] + } + }, + { + // disallow imports in core/reporters files to any non-util file + files: ['lib/core/reporters/**/*.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + // e.g. "../commons/aria/" or "../checks/" + regex: '.*\\.\\.\\/(commons|base|public|checks|rules)(\\/|$)', + message: 'Reporter files should only import util functions' + }, + // disallow imports from node modules + { + regex: '^[^.]', + message: 'Only core/imports files should import from node modules' + } + ] + } + ] + } + }, + { + // disallow imports in commons files to any check or rule + files: ['lib/commons/**/*.js'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + // e.g. ../checks/" + regex: '.*\\.\\.\\/(checks|rules)(\\/|$)', + message: 'Commons files cannot import from checks and rules' + }, + // disallow imports from node modules + { + regex: '^[^.]', + message: 'Only core/imports files should import from node modules' + } + ] + } + ] + } + }, + { + // Utils should be functions that can be used without setting up the virtual tree, as opposed to commons which require the virtual tree + files: ['lib/core/utils/**/*.js'], + ignores: [ + // these are files with known uses of virtual node that are legacy before this rule was enforced + 'lib/core/utils/closest.js', + 'lib/core/utils/contains.js', + 'lib/core/utils/query-selector-all-filter.js', + 'lib/core/utils/selector-cache.js', + // this will create a virtual node if one doesn't exist already in order to truncate the html output properly + 'lib/core/utils/dq-element.js', + // this sets up the virtual tree so is allowed vNode + 'lib/core/utils/get-flattened-tree.js' + ], + rules: { + 'no-restricted-syntax': [ + 'error', + { + selector: 'MemberExpression[object.name=vNode]', + message: + "Utils is meant for utility functions that work independently of axe's state; utilities that require the virtual tree to be set up should go in commons, not utils." + }, + { + selector: 'MemberExpression[object.name=virtualNode]', + message: + "Utils is meant for utility functions that work independently of axe's state; utilities that require the virtual tree to be set up should go in commons, not utils." + } + ] + } + }, { files: ['doc/examples/chrome-debugging-protocol/axe-cdp.js'], languageOptions: { @@ -189,7 +371,7 @@ module.exports = [ 'build/tasks/aria-supported.js', 'doc/api/*', 'doc/examples/jest_react/*.js', - 'lib/core/imports/*.js', + 'lib/core/imports/polyfills.js', 'lib/core/utils/uuid.js', 'axe.js', 'axe.min.js' diff --git a/lib/checks/shared/error-occurred.json b/lib/checks/shared/error-occurred.json new file mode 100644 index 0000000000..7d8c614463 --- /dev/null +++ b/lib/checks/shared/error-occurred.json @@ -0,0 +1,10 @@ +{ + "id": "error-occurred", + "evaluate": "exists-evaluate", + "metadata": { + "messages": { + "pass": "", + "incomplete": "Axe encountered an error; test the page for this type of problem manually" + } + } +} diff --git a/lib/commons/text/has-unicode.js b/lib/commons/text/has-unicode.js index a7ac371906..710d5fa39e 100644 --- a/lib/commons/text/has-unicode.js +++ b/lib/commons/text/has-unicode.js @@ -4,7 +4,7 @@ import { getPunctuationRegExp, getCategoryFormatRegExp } from './unicode'; -import emojiRegexText from 'emoji-regex'; +import { emojiRegexText } from '../../core/imports'; /** * Determine if a given string contains unicode characters, specified in options diff --git a/lib/commons/text/remove-unicode.js b/lib/commons/text/remove-unicode.js index 4527cfc871..d455bc4136 100644 --- a/lib/commons/text/remove-unicode.js +++ b/lib/commons/text/remove-unicode.js @@ -4,7 +4,7 @@ import { getPunctuationRegExp, getCategoryFormatRegExp } from './unicode.js'; -import emojiRegexText from 'emoji-regex'; +import { emojiRegexText } from '../../core/imports'; /** * Remove specified type(s) unicode characters diff --git a/lib/core/base/audit.js b/lib/core/base/audit.js index 36bcb325ff..3353545af1 100644 --- a/lib/core/base/audit.js +++ b/lib/core/base/audit.js @@ -9,9 +9,10 @@ import { preload, findBy, ruleShouldRun, - performanceTimer + performanceTimer, + serializeError } from '../utils'; -import doT from '@deque/dot'; +import { doT } from '../imports'; import constants from '../constants'; const dotRegex = /\{\{.+?\}\}/g; @@ -368,6 +369,9 @@ export default class Audit { after(results, options) { const rules = this.rules; return results.map(ruleResult => { + if (ruleResult.error) { + return ruleResult; + } const rule = findBy(rules, 'id', ruleResult.id); if (!rule) { // If you see this, you're probably running the Mocha tests with the axe extension installed @@ -375,7 +379,14 @@ export default class Audit { 'Result for unknown rule. You may be running mismatch axe-core versions' ); } - return rule.after(ruleResult, options); + try { + return rule.after(ruleResult, options); + } catch (err) { + if (options.debug) { + throw err; + } + return createIncompleteErrorResult(rule, err); + } }); } /** @@ -732,36 +743,37 @@ function getDefferedRule(rule, context, options) { rule.run( context, options, - // resolve callback for rule `run` - ruleResult => { - // resolve - resolve(ruleResult); - }, - // reject callback for rule `run` + ruleResult => resolve(ruleResult), err => { - // if debug - construct error details - if (!options.debug) { - const errResult = Object.assign(new RuleResult(rule), { - result: constants.CANTTELL, - description: 'An error occured while running this rule', - message: err.message, - stack: err.stack, - error: err, - // Add a serialized reference to the node the rule failed on for easier debugging. - // See https://github.com/dequelabs/axe-core/issues/1317. - errorNode: err.errorNode - }); - // resolve - resolve(errResult); - } else { - // reject + if (options.debug) { reject(err); + } else { + resolve(createIncompleteErrorResult(rule, err)); } } ); }; } +function createIncompleteErrorResult(rule, error) { + const { errorNode } = error; + const serialError = serializeError(error); + const none = [ + { + id: 'error-occurred', + result: undefined, + data: serialError, + relatedNodes: [] + } + ]; + const node = errorNode || new DqElement(document.documentElement); + return Object.assign(new RuleResult(rule), { + error: serialError, + result: constants.CANTTELL, + nodes: [{ any: [], all: [], none, node }] + }); +} + /** * For all the rules, create the helpUrl and add it to the data for that rule */ diff --git a/lib/core/base/rule.js b/lib/core/base/rule.js index 95d68ec2d5..84ef29cfba 100644 --- a/lib/core/base/rule.js +++ b/lib/core/base/rule.js @@ -1,4 +1,3 @@ -/*global SupportError */ import { createExecutionContext } from './check'; import RuleResult from './rule-result'; import { @@ -8,7 +7,8 @@ import { queue, DqElement, select, - assert + assert, + RuleError } from '../utils'; import { isVisibleToScreenReaders } from '../../commons/dom'; import constants from '../constants'; @@ -181,7 +181,16 @@ Rule.prototype.runChecks = function runChecks( const check = self._audit.checks[c.id || c]; const option = getCheckOption(check, self.id, options); checkQueue.defer((res, rej) => { - check.run(node, option, context, res, rej); + check.run(node, option, context, res, error => { + rej( + new RuleError({ + ruleId: self.id, + method: `${check.id}#evaluate`, + errorNode: new DqElement(node), + error + }) + ); + }); }); }); @@ -235,8 +244,7 @@ Rule.prototype.run = function run(context, options = {}, resolve, reject) { // Matches throws an error when it lacks support for document methods nodes = this.gatherAndMatchNodes(context, options); } catch (error) { - // Exit the rule execution if matches fails - reject(new SupportError({ cause: error, ruleId: this.id })); + reject(error); return; } @@ -312,15 +320,7 @@ Rule.prototype.runSync = function runSync(context, options = {}) { } const ruleResult = new RuleResult(this); - let nodes; - - try { - nodes = this.gatherAndMatchNodes(context, options); - } catch (error) { - // Exit the rule execution if matches fails - throw new SupportError({ cause: error, ruleId: this.id }); - } - + const nodes = this.gatherAndMatchNodes(context, options); if (options.performanceTimer) { this._logGatherPerformance(nodes); } @@ -451,7 +451,18 @@ Rule.prototype.gatherAndMatchNodes = function gatherAndMatchNodes( performanceTimer.mark(markMatchesStart); } - nodes = nodes.filter(node => this.matches(node.actualNode, node, context)); + nodes = nodes.filter(node => { + try { + return this.matches(node.actualNode, node, context); + } catch (error) { + throw new RuleError({ + ruleId: this.id, + method: `#matches`, + errorNode: new DqElement(node), + error + }); + } + }); if (options.performanceTimer) { performanceTimer.mark(markMatchesEnd); @@ -542,12 +553,20 @@ function sanitizeNodes(result) { */ Rule.prototype.after = function after(result, options) { const afterChecks = findAfterChecks(this); - const ruleID = this.id; afterChecks.forEach(check => { const beforeResults = findCheckResults(result.nodes, check.id); - const checkOption = getCheckOption(check, ruleID, options); - - const afterResults = check.after(beforeResults, checkOption.options); + const checkOption = getCheckOption(check, this.id, options); + let afterResults; + try { + afterResults = check.after(beforeResults, checkOption.options); + } catch (error) { + throw new RuleError({ + ruleId: this.id, + method: `${check.id}#after`, + errorNode: result.nodes?.[0]?.node, + error + }); + } if (this.reviewOnFail) { afterResults.forEach(checkResult => { diff --git a/lib/core/constants.js b/lib/core/constants.js index ecd01c8aba..c5b9fd2b97 100644 --- a/lib/core/constants.js +++ b/lib/core/constants.js @@ -47,7 +47,15 @@ const constants = { timeout: 10000 }), allOrigins: '', - sameOrigin: '' + sameOrigin: '', + serializableErrorProps: Object.freeze([ + 'message', + 'stack', + 'name', + 'code', + 'ruleId', + 'method' + ]) }; definitions.forEach(definition => { diff --git a/lib/core/index.js b/lib/core/index.js index 2d15d52824..9529707189 100644 --- a/lib/core/index.js +++ b/lib/core/index.js @@ -25,16 +25,3 @@ if (typeof window.getComputedStyle === 'function') { } // local namespace for common functions let commons; - -function SupportError(error) { - this.name = 'SupportError'; - this.cause = error.cause; - this.message = `\`${error.cause}\` - feature unsupported in your environment.`; - if (error.ruleId) { - this.ruleId = error.ruleId; - this.message += ` Skipping ${this.ruleId} rule.`; - } - this.stack = new Error().stack; -} -SupportError.prototype = Object.create(Error.prototype); -SupportError.prototype.constructor = SupportError; diff --git a/lib/core/utils/css-parser.js b/lib/core/utils/css-parser.js index 355a40236d..b76ca1349f 100644 --- a/lib/core/utils/css-parser.js +++ b/lib/core/utils/css-parser.js @@ -1,4 +1,4 @@ -import { CssSelectorParser } from 'css-selector-parser'; +import { CssSelectorParser } from '../imports'; const parser = new CssSelectorParser(); parser.registerSelectorPseudos('not'); diff --git a/lib/core/utils/dq-element.js b/lib/core/utils/dq-element.js index bfec1a9353..556d625aa9 100644 --- a/lib/core/utils/dq-element.js +++ b/lib/core/utils/dq-element.js @@ -212,6 +212,7 @@ DqElement.prototype = { } }; +/** @deprecated */ DqElement.fromFrame = function fromFrame(node, options, frame) { const spec = DqElement.mergeSpecs(node, frame); return new DqElement(frame.element, options, spec); diff --git a/lib/core/utils/index.js b/lib/core/utils/index.js index 81164587e2..01ce2fa085 100644 --- a/lib/core/utils/index.js +++ b/lib/core/utils/index.js @@ -87,6 +87,8 @@ export { default as ruleShouldRun } from './rule-should-run'; export { default as filterHtmlAttrs } from './filter-html-attrs'; export { default as select } from './select'; export { default as sendCommandToFrame } from './send-command-to-frame'; +export { default as serializeError } from './serialize-error'; +export { default as RuleError } from './rule-error'; export { default as setScrollState } from './set-scroll-state'; export { default as shadowSelect } from './shadow-select'; export { default as shadowSelectAll } from './shadow-select-all'; diff --git a/lib/core/utils/memoize.js b/lib/core/utils/memoize.js index 88ec2f4729..d8b6a30686 100644 --- a/lib/core/utils/memoize.js +++ b/lib/core/utils/memoize.js @@ -1,4 +1,4 @@ -import memoize from 'memoizee'; +import { memoize } from '../imports'; // FYI: memoize does not always play nice with esbuild // and sometimes is built out of order. diff --git a/lib/core/utils/merge-results.js b/lib/core/utils/merge-results.js index 717eeb198a..0064623d82 100644 --- a/lib/core/utils/merge-results.js +++ b/lib/core/utils/merge-results.js @@ -95,6 +95,9 @@ function mergeResults(frameResults, options) { if (ruleResult.nodes.length) { spliceNodes(res.nodes, ruleResult.nodes); } + if (ruleResult.error) { + res.error ??= ruleResult.error; + } } }); }); diff --git a/lib/core/utils/rule-error.js b/lib/core/utils/rule-error.js new file mode 100644 index 0000000000..eb8147f481 --- /dev/null +++ b/lib/core/utils/rule-error.js @@ -0,0 +1,23 @@ +import serializeError from './serialize-error'; + +export default class RuleError extends Error { + constructor({ error, ruleId, method, errorNode }) { + super(); + this.name = error.name ?? 'RuleError'; + this.message = error.message; + this.stack = error.stack; + if (error.cause) { + this.cause = serializeError(error.cause); + } + if (ruleId) { + this.ruleId = ruleId; + this.message += ` Skipping ${this.ruleId} rule.`; + } + if (method) { + this.method = method; + } + if (errorNode) { + this.errorNode = errorNode; + } + } +} diff --git a/lib/core/utils/serialize-error.js b/lib/core/utils/serialize-error.js new file mode 100644 index 0000000000..5da40c4b8d --- /dev/null +++ b/lib/core/utils/serialize-error.js @@ -0,0 +1,24 @@ +import constants from '../constants'; + +/** + * Serializes an error to a JSON object + * @param e - The error to serialize + * @returns A JSON object representing the error + */ +export default function serializeError(err, iteration = 0) { + if (typeof err !== 'object' || err === null) { + return { message: String(err) }; + } + const serial = {}; + for (const prop of constants.serializableErrorProps) { + if (['string', 'number', 'boolean'].includes(typeof err[prop])) { + serial[prop] = err[prop]; + } + } + // Recursively serialize cause up to 10 levels deep + if (err.cause) { + serial.cause = + iteration < 10 ? serializeError(err.cause, iteration + 1) : '...'; + } + return serial; +} diff --git a/lib/rules/meta-viewport.json b/lib/rules/meta-viewport.json index c0995a8d1f..0f08d59daf 100644 --- a/lib/rules/meta-viewport.json +++ b/lib/rules/meta-viewport.json @@ -1,6 +1,6 @@ { "id": "meta-viewport", - "impact": "critical", + "impact": "moderate", "selector": "meta[name=\"viewport\"]", "matches": "is-initiator-matches", "excludeHidden": false, diff --git a/locales/_template.json b/locales/_template.json index 05ff09b929..f6239a1f92 100644 --- a/locales/_template.json +++ b/locales/_template.json @@ -984,6 +984,10 @@ "pass": "Document has a non-empty element", "fail": "Document does not have a non-empty <title> element" }, + "error-occurred": { + "pass": "", + "incomplete": "Axe encountered an error; test the page for this type of problem manually" + }, "exists": { "pass": "Element does not exist", "incomplete": "Element exists" diff --git a/test/core/base/audit.js b/test/core/base/audit.js index b04002874d..12c3644104 100644 --- a/test/core/base/audit.js +++ b/test/core/base/audit.js @@ -2,37 +2,54 @@ describe('Audit', () => { const Audit = axe._thisWillBeDeletedDoNotUse.base.Audit; const Rule = axe._thisWillBeDeletedDoNotUse.base.Rule; const ver = axe.version.substring(0, axe.version.lastIndexOf('.')); - const { fixtureSetup } = axe.testUtils; + const { fixtureSetup, captureError } = axe.testUtils; let audit; - const isNotCalled = function (err) { + const isNotCalled = err => { throw err || new Error('Reject should not be called'); }; const noop = () => {}; + const assertEqualRuleError = (actual, expect) => { + assert.include(actual.message, expect.message); + assert.equal(actual.stack, expect.stack); + assert.equal(actual.name, expect.name); + }; + + const assertErrorResults = (result, error, selector) => { + assert.equal(result.result, 'cantTell'); + assertEqualRuleError(result.error, error); + + assert.lengthOf(result.nodes, 1); + const node1 = result.nodes[0]; + assert.isEmpty(node1.any); + assert.isEmpty(node1.all); + assert.include(node1.node.selector, selector); + + assert.lengthOf(node1.none, 1); + const none = node1.none[0]; + assert.equal(none.id, 'error-occurred'); + assert.equal(none.result, undefined); + assert.isDefined(none.data); + assertEqualRuleError(none.data, error); + assert.lengthOf(none.relatedNodes, 0); + }; + const mockChecks = [ { id: 'positive1-check1', - evaluate: () => { - return true; - } + evaluate: () => true }, { id: 'positive2-check1', - evaluate: () => { - return true; - } + evaluate: () => true }, { id: 'negative1-check1', - evaluate: () => { - return true; - } + evaluate: () => true }, { id: 'positive3-check1', - evaluate: () => { - return true; - } + evaluate: () => true } ]; @@ -68,9 +85,7 @@ describe('Audit', () => { ]; const fixture = document.getElementById('fixture'); - let origAuditRun; - beforeEach(() => { audit = new Audit(); mockRules.forEach(function (r) { @@ -1139,45 +1154,6 @@ describe('Audit', () => { ); }); - it('catches errors and passes them as a cantTell result', done => { - const err = new Error('Launch the super sheep!'); - audit.addRule({ - id: 'throw1', - selector: '*', - any: [ - { - id: 'throw1-check1' - } - ] - }); - audit.addCheck({ - id: 'throw1-check1', - evaluate: () => { - throw err; - } - }); - axe._tree = axe.utils.getFlattenedTree(fixture); - axe._selectorData = axe.utils.getSelectorData(axe._tree); - audit.run( - { include: [axe._tree[0]] }, - { - runOnly: { - type: 'rule', - values: ['throw1'] - } - }, - function (results) { - assert.lengthOf(results, 1); - assert.equal(results[0].result, 'cantTell'); - assert.equal(results[0].message, err.message); - assert.equal(results[0].stack, err.stack); - assert.equal(results[0].error, err); - done(); - }, - isNotCalled - ); - }); - it('should not halt if errors occur', done => { audit.addRule({ id: 'throw1', @@ -1230,43 +1206,6 @@ describe('Audit', () => { assert.equal(checked, 'options validated'); }); - it('should halt if an error occurs when debug is set', done => { - audit.addRule({ - id: 'throw1', - selector: '*', - any: [ - { - id: 'throw1-check1' - } - ] - }); - audit.addCheck({ - id: 'throw1-check1', - evaluate: () => { - throw new Error('Launch the super sheep!'); - } - }); - - // check error node requires _selectorCache to be setup - axe.setup(); - - audit.run( - { include: [axe.utils.getFlattenedTree(fixture)[0]] }, - { - debug: true, - runOnly: { - type: 'rule', - values: ['throw1'] - } - }, - noop, - function (err) { - assert.equal(err.message, 'Launch the super sheep!'); - done(); - } - ); - }); - it('propagates DqElement options', async () => { fixtureSetup('<input id="input">'); const results = await new Promise((resolve, reject) => { @@ -1281,6 +1220,59 @@ describe('Audit', () => { assert.equal(node.element, fixture.firstChild); assert.equal(node.selector, 'html > body > #fixture > #input'); }); + + describe('when an error occurs', () => { + let err; + beforeEach(() => { + err = new Error('Launch the super sheep!'); + audit.addRule({ + id: 'throw1', + selector: '#fixture', + any: [ + { + id: 'throw1-check1' + } + ] + }); + audit.addCheck({ + id: 'throw1-check1', + evaluate: () => { + throw err; + } + }); + axe.setup(); + }); + + it('catches errors and resolves them as a cantTell result', done => { + audit.run( + { include: [axe._tree[0]] }, + { runOnly: { type: 'rule', values: ['throw1'] } }, + captureError(results => { + assert.lengthOf(results, 1); + assertErrorResults(results[0], err, '#fixture'); + done(); + }, done), + isNotCalled + ); + }); + + it('should halt if an error occurs when debug is set', done => { + const context = { include: [axe.utils.getFlattenedTree(fixture)[0]] }; + const options = { + debug: true, + runOnly: { type: 'rule', values: ['throw1'] } + }; + audit.run( + context, + options, + noop, + captureError(reject => { + assert.include(reject.message, err.message); + done(); + }, done) + ); + }); + }); }); describe('Audit#after', () => { @@ -1288,7 +1280,7 @@ describe('Audit', () => { /*eslint no-unused-vars:0*/ audit = new Audit(); let success = false; - const options = [{ id: 'hehe', enabled: true, monkeys: 'bananas' }]; + const options = { runOnly: 'hehe' }; const results = [ { id: 'hehe', @@ -1310,6 +1302,91 @@ describe('Audit', () => { }; audit.after(results, options); + assert.isTrue(success); + }); + + it('does not run Rule#after if the result has an error', () => { + audit = new Audit(); + const results = [{ id: 'throw1', error: new Error('La la la!') }]; + let success = true; + audit.rules.push(new Rule({ id: 'throw1' })); + audit.rules[0].after = () => (success = false); + audit.after(results, {}); + assert.lengthOf(results, 1); + assert.equal(results[0].error.message, 'La la la!'); + assert.isTrue(success, 'Rule#after should not be called'); + }); + + it('catches errors and passes them as a cantTell result', () => { + audit = new Audit(); + const err = new SyntaxError('La la la!'); + const results = [ + { + id: 'throw1', + nodes: [ + { + id: 'throw1-check1-after', + node: new axe.utils.DqElement(fixture), + any: [{ id: 'throw1-check1-after', result: false }], + all: [], + none: [] + } + ] + } + ]; + audit.addRule({ + id: 'throw1', + selector: '#fixture', + any: [{ id: 'throw1-check1-after' }] + }); + audit.addCheck({ + id: 'throw1-check1-after', + after: () => { + throw err; + } + }); + axe.setup(); + const result = audit.after(results, {}); + assert.lengthOf(result, 1); + assertErrorResults(result[0], err, '#fixture'); + }); + + it('throws errors when debug is set', () => { + audit = new Audit(); + const err = new SyntaxError('La la la!'); + const options = { debug: true }; + const results = [ + { + id: 'throw1', + nodes: [ + { + id: 'throw1-check1-after', + node: new axe.utils.DqElement(fixture), + any: [{ id: 'throw1-check1-after', result: false }], + all: [], + none: [] + } + ] + } + ]; + audit.addRule({ + id: 'throw1', + selector: '#fixture', + any: [{ id: 'throw1-check1-after' }] + }); + audit.addCheck({ + id: 'throw1-check1-after', + after: () => { + throw err; + } + }); + axe.setup(); + try { + audit.after(results, options); + assert.fail('Should have thrown'); + } catch (actual) { + assertEqualRuleError(actual, err); + } }); }); diff --git a/test/core/base/rule.js b/test/core/base/rule.js index 893fa92da2..7809dfad45 100644 --- a/test/core/base/rule.js +++ b/test/core/base/rule.js @@ -4,7 +4,7 @@ describe('Rule', () => { const metadataFunctionMap = axe._thisWillBeDeletedDoNotUse.base.metadataFunctionMap; const fixture = document.getElementById('fixture'); - const { fixtureSetup } = axe.testUtils; + const { fixtureSetup, captureError } = axe.testUtils; const noop = () => {}; const isNotCalled = function (err) { throw err || new Error('Reject should not be called'); @@ -233,30 +233,6 @@ describe('Rule', () => { ); }); - it('should handle an error in #matches', done => { - const div = document.createElement('div'); - div.setAttribute('style', '#fff'); - fixture.appendChild(div); - let success = false, - rule = new Rule({ - matches: () => { - throw new Error('this is an error'); - } - }); - - rule.run( - { - include: [axe.utils.getFlattenedTree(div)[0]] - }, - {}, - isNotCalled, - () => { - assert.isFalse(success); - done(); - } - ); - }); - it('should execute Check#run on its child checks - any', done => { fixtureSetup('<blink>Hi</blink>'); let success = false; @@ -618,9 +594,7 @@ describe('Rule', () => { it('should pass thrown errors to the reject param', done => { fixtureSetup('<blink>Hi</blink>'); const rule = new Rule( - { - none: ['cats'] - }, + { none: ['cats'] }, { checks: { cats: { @@ -719,6 +693,62 @@ describe('Rule', () => { ); }); + describe('error handling', () => { + it('should return a RuleError if #matches throws', done => { + const rule = new Rule({ + id: 'fizz', + matches: () => { + throw new Error('this is an error'); + } + }); + axe.setup(); + rule.run( + { include: [axe.utils.getFlattenedTree(fixture)[0]] }, + {}, + isNotCalled, + captureError(err => { + assert.instanceOf(err, axe.utils.RuleError); + assert.include(err.message, 'this is an error'); + assert.equal(err.ruleId, 'fizz'); + assert.equal(err.method, '#matches'); + assert.deepEqual(err.errorNode.selector, ['#fixture']); + done(); + }, done) + ); + }); + + it('should return a RuleError if check.evaluate throws', done => { + const rule = new Rule( + { id: 'garden', any: ['plants'] }, + { + checks: { + plants: new Check({ + id: 'plants', + enabled: true, + evaluate: () => { + throw new Error('zombies ate my pants'); + } + }) + } + } + ); + axe.setup(); + rule.run( + { include: axe.utils.getFlattenedTree(fixture) }, + {}, + isNotCalled, + captureError(err => { + assert.instanceOf(err, axe.utils.RuleError); + assert.include(err.message, 'zombies ate my pants'); + assert.equal(err.ruleId, 'garden'); + assert.equal(err.method, 'plants#evaluate'); + assert.deepEqual(err.errorNode.selector, ['#fixture']); + done(); + }, done) + ); + }); + }); + describe('NODE rule', () => { it('should create a RuleResult', () => { axe.setup(); @@ -1660,9 +1690,50 @@ describe('Rule', () => { assert.lengthOf(result.nodes, 1); }); + + it('should throw a RuleError if check.after throws', () => { + const rule = new Rule( + { id: 'dogs', any: ['cats'] }, + { + checks: { + cats: { + id: 'cats', + enabled: true, + after: () => { + throw new Error('this is an error'); + } + } + } + } + ); + axe.setup(); + try { + rule.after( + { + id: 'cats', + nodes: [ + { + all: [], + none: [], + any: [{ id: 'cats', result: true }], + node: new axe.utils.DqElement(fixture) + } + ] + }, + {} + ); + assert.fail('Should have thrown'); + } catch (err) { + assert.instanceOf(err, axe.utils.RuleError); + assert.include(err.message, 'this is an error'); + assert.equal(err.ruleId, 'dogs'); + assert.equal(err.method, 'cats#after'); + assert.deepEqual(err.errorNode.selector, ['#fixture']); + } + }); }); - describe('after', () => { + describe('reviewOnFail', () => { it('should mark checks as incomplete if reviewOnFail is set to true for all', () => { axe.setup(); const rule = new Rule( diff --git a/test/core/constants.js b/test/core/constants.js index 23f05a5333..31520cb8a3 100644 --- a/test/core/constants.js +++ b/test/core/constants.js @@ -40,4 +40,11 @@ describe('axe.constants', function () { it('should have a selectorSimilarFilterLimit', function () { assert.equal(axe.constants.selectorSimilarFilterLimit, 700); }); + + it('has a serializableErrorProps array', function () { + assert.isArray(axe.constants.serializableErrorProps); + axe.constants.serializableErrorProps.forEach(prop => { + assert.typeOf(prop, 'string', `prop ${prop} is not a string`); + }); + }); }); diff --git a/test/core/public/run-rules.js b/test/core/public/run-rules.js index 394093ee73..e89ead1db8 100644 --- a/test/core/public/run-rules.js +++ b/test/core/public/run-rules.js @@ -1,10 +1,10 @@ -describe('runRules', function () { - 'use strict'; - var ver = axe.version.substring(0, axe.version.lastIndexOf('.')); +describe('runRules', () => { + let ver = axe.version.substring(0, axe.version.lastIndexOf('.')); + const { captureError } = axe.testUtils; function iframeReady(src, context, id, cb) { - var i = document.createElement('iframe'); - i.addEventListener('load', function () { + let i = document.createElement('iframe'); + i.addEventListener('load', () => { cb(); }); i.src = src; @@ -13,9 +13,9 @@ describe('runRules', function () { } function createFrames(url, callback) { - var frame, + let frame, num = 2; - var loaded = 0; + let loaded = 0; if (typeof url === 'function') { callback = url; @@ -42,22 +42,22 @@ describe('runRules', function () { return frame; } - var fixture = document.getElementById('fixture'); + let fixture = document.getElementById('fixture'); - var isNotCalled; - beforeEach(function () { - isNotCalled = function (err) { + let isNotCalled; + beforeEach(() => { + isNotCalled = err => { throw err || new Error('Reject should not be called'); }; }); - afterEach(function () { + afterEach(() => { fixture.innerHTML = ''; axe._audit = null; axe.teardown(); }); - it('should work', function (done) { + it('should work', done => { axe._load({ rules: [ { @@ -66,29 +66,22 @@ describe('runRules', function () { any: ['html'] } ], - checks: [ - { - id: 'html', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'html', evaluate: () => true }], messages: {} }); - var frame = document.createElement('iframe'); + let frame = document.createElement('iframe'); frame.src = '../mock/frames/frame-frame.html'; - frame.addEventListener('load', function () { - setTimeout(function () { + frame.addEventListener('load', () => { + setTimeout(() => { axe._runRules( document, {}, - function (r) { + captureError(r => { assert.lengthOf(r[0].passes, 3); done(); - }, + }, done), err => done(err) ); }, 500); @@ -96,7 +89,7 @@ describe('runRules', function () { fixture.appendChild(frame); }); - it('should properly order iframes', function (done) { + it('should properly order iframes', done => { axe._load({ rules: [ { @@ -105,39 +98,28 @@ describe('runRules', function () { any: ['iframe'] } ], - checks: [ - { - id: 'iframe', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'iframe', evaluate: () => true }], messages: {} }); - var frame = document.createElement('iframe'); - frame.addEventListener('load', function () { - setTimeout(function () { + let frame = document.createElement('iframe'); + frame.addEventListener('load', () => { + setTimeout(() => { axe._runRules( document, {}, - function (r) { - var nodes = r[0].passes.map(function (detail) { + captureError(r => { + let nodes = r[0].passes.map(detail => { return detail.node.selector; }); - try { - assert.deepEqual(nodes, [ - ['#level0'], - ['#level0', '#level1'], - ['#level0', '#level1', '#level2a'], - ['#level0', '#level1', '#level2b'] - ]); - done(); - } catch (e) { - done(e); - } - }, + assert.deepEqual(nodes, [ + ['#level0'], + ['#level0', '#level1'], + ['#level0', '#level1', '#level2a'], + ['#level0', '#level1', '#level2b'] + ]); + done(); + }, done), isNotCalled ); }, 500); @@ -147,7 +129,7 @@ describe('runRules', function () { fixture.appendChild(frame); }); - it('should properly calculate context and return results from matching frames', function (done) { + it('should properly calculate context and return results from matching frames', done => { axe._load({ rules: [ { @@ -162,19 +144,14 @@ describe('runRules', function () { } ], checks: [ - { - id: 'has-target', - evaluate: function () { - return true; - } - }, + { id: 'has-target', evaluate: () => true }, { id: 'first-div', - evaluate: function (node) { + evaluate(node) { this.relatedNodes([node]); return false; }, - after: function (results) { + after(results) { if (results.length) { results[0].result = true; } @@ -185,136 +162,124 @@ describe('runRules', function () { messages: {} }); - iframeReady( - '../mock/frames/context.html', - fixture, - 'context-test', - function () { - var div = document.createElement('div'); - fixture.appendChild(div); + iframeReady('../mock/frames/context.html', fixture, 'context-test', () => { + let div = document.createElement('div'); + fixture.appendChild(div); - axe._runRules( - '#fixture', - {}, - function (results) { - try { - assert.deepEqual(JSON.parse(JSON.stringify(results)), [ + axe._runRules( + '#fixture', + {}, + captureError(results => { + assert.deepEqual(JSON.parse(JSON.stringify(results)), [ + { + id: 'div#target', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/div#target?application=axeAPI', + pageLevel: false, + impact: null, + inapplicable: [], + incomplete: [], + violations: [], + passes: [ { - id: 'div#target', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/div#target?application=axeAPI', - pageLevel: false, + result: 'passed', impact: null, - inapplicable: [], - incomplete: [], - violations: [], - passes: [ + node: { + selector: ['#context-test', '#target'], + ancestry: [ + 'html > body > div:nth-child(1) > iframe:nth-child(1)', + 'html > body > div:nth-child(2)' + ], + xpath: [ + "//iframe[@id='context-test']", + "//div[@id='target']" + ], + source: '<div id="target"></div>', + nodeIndexes: [12, 14], + fromFrame: true + }, + any: [ { - result: 'passed', - impact: null, - node: { - selector: ['#context-test', '#target'], - ancestry: [ - 'html > body > div:nth-child(1) > iframe:nth-child(1)', - 'html > body > div:nth-child(2)' - ], - xpath: [ - "//iframe[@id='context-test']", - "//div[@id='target']" - ], - source: '<div id="target"></div>', - nodeIndexes: [12, 14], - fromFrame: true - }, - any: [ - { - id: 'has-target', - data: null, - relatedNodes: [] - } - ], - all: [], - none: [] + id: 'has-target', + data: null, + relatedNodes: [] } ], - result: 'passed', - tags: [] - }, + all: [], + none: [] + } + ], + result: 'passed', + tags: [] + }, + { + id: 'first-div', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/first-div?application=axeAPI', + pageLevel: false, + impact: null, + inapplicable: [], + incomplete: [], + violations: [], + passes: [ { - id: 'first-div', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/first-div?application=axeAPI', - pageLevel: false, + result: 'passed', impact: null, - inapplicable: [], - incomplete: [], - violations: [], - passes: [ + node: { + selector: ['#context-test', '#foo'], + ancestry: [ + 'html > body > div:nth-child(1) > iframe:nth-child(1)', + 'html > body > div:nth-child(1)' + ], + xpath: ["//iframe[@id='context-test']", "//div[@id='foo']"], + source: + '<div id="foo">\n <div id="bar"></div>\n </div>', + nodeIndexes: [12, 9], + fromFrame: true + }, + any: [ { - result: 'passed', - impact: null, - node: { - selector: ['#context-test', '#foo'], - ancestry: [ - 'html > body > div:nth-child(1) > iframe:nth-child(1)', - 'html > body > div:nth-child(1)' - ], - xpath: [ - "//iframe[@id='context-test']", - "//div[@id='foo']" - ], - source: - '<div id="foo">\n <div id="bar"></div>\n </div>', - nodeIndexes: [12, 9], - fromFrame: true - }, - any: [ + id: 'first-div', + data: null, + relatedNodes: [ { - id: 'first-div', - data: null, - relatedNodes: [ - { - selector: ['#context-test', '#foo'], - ancestry: [ - 'html > body > div:nth-child(1) > iframe:nth-child(1)', - 'html > body > div:nth-child(1)' - ], - xpath: [ - "//iframe[@id='context-test']", - "//div[@id='foo']" - ], - source: - '<div id="foo">\n <div id="bar"></div>\n </div>', - nodeIndexes: [12, 9], - fromFrame: true - } - ] + selector: ['#context-test', '#foo'], + ancestry: [ + 'html > body > div:nth-child(1) > iframe:nth-child(1)', + 'html > body > div:nth-child(1)' + ], + xpath: [ + "//iframe[@id='context-test']", + "//div[@id='foo']" + ], + source: + '<div id="foo">\n <div id="bar"></div>\n </div>', + nodeIndexes: [12, 9], + fromFrame: true } - ], - all: [], - none: [] + ] } ], - result: 'passed', - tags: [] + all: [], + none: [] } - ]); - done(); - } catch (e) { - done(e); + ], + result: 'passed', + tags: [] } - }, - isNotCalled - ); - } - ); + ]); + done(); + }, done), + isNotCalled + ); + }); }); - it('should reject if the context is invalid', function (done) { + it('should reject if the context is invalid', done => { axe._load({ rules: [ { @@ -326,32 +291,26 @@ describe('runRules', function () { messages: {} }); - iframeReady( - '../mock/frames/context.html', - fixture, - 'context-test', - function () { - axe._runRules( - '#not-happening', - {}, - function () { - assert.fail('This selector should not exist.'); - }, - function (error) { - assert.isOk(error); - assert.equal( - error.message, - 'No elements found for include in page Context' - ); - - done(); - } - ); - } - ); + iframeReady('../mock/frames/context.html', fixture, 'context-test', () => { + axe._runRules( + '#not-happening', + {}, + () => { + assert.fail('This selector should not exist.'); + }, + captureError(error => { + assert.isOk(error); + assert.equal( + error.message, + 'No elements found for include in page Context' + ); + done(); + }, done) + ); + }); }); - it('should accept a jQuery-like object', function (done) { + it('should accept a jQuery-like object', done => { axe._load({ rules: [ { @@ -360,38 +319,34 @@ describe('runRules', function () { none: ['bob'] } ], - checks: [ - { - id: 'bob', - evaluate: function () { - return true; - } - } - ] + checks: [{ id: 'bob', evaluate: () => true }] }); fixture.innerHTML = '<div id="t1"><span></span></div><div id="t2"><em></em></div>'; - var $test = { + let $test = { 0: fixture.querySelector('#t1'), 1: fixture.querySelector('#t2'), length: 2 }; - axe.run($test, function (err, results) { - assert.isNull(err); - assert.lengthOf(results.violations, 1); - assert.lengthOf(results.violations[0].nodes, 4); - assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); - // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); - assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); - // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); - done(); - }); + axe.run( + $test, + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.violations, 1); + assert.lengthOf(results.violations[0].nodes, 4); + assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); + // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); + assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); + // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); + done(); + }, done) + ); }); - it('should accept a NodeList', function (done) { + it('should accept a NodeList', done => { axe._load({ rules: [ { @@ -400,33 +355,29 @@ describe('runRules', function () { none: ['fred'] } ], - checks: [ - { - id: 'fred', - evaluate: function () { - return true; - } - } - ] + checks: [{ id: 'fred', evaluate: () => true }] }); fixture.innerHTML = '<div class="foo" id="t1"><span></span></div><div class="foo" id="t2"><em></em></div>'; - var test = fixture.querySelectorAll('.foo'); - axe.run(test, function (err, results) { - assert.isNull(err); - assert.lengthOf(results.violations, 1); - assert.lengthOf(results.violations[0].nodes, 4); - assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); - // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); - assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); - // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); - done(); - }); + let test = fixture.querySelectorAll('.foo'); + axe.run( + test, + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.violations, 1); + assert.lengthOf(results.violations[0].nodes, 4); + assert.deepEqual(results.violations[0].nodes[0].target, ['#t1']); + // assert.deepEqual(results.violations[0].nodes[1].target, ['span']); + assert.deepEqual(results.violations[0].nodes[2].target, ['#t2']); + // assert.deepEqual(results.violations[0].nodes[3].target, ['em']); + done(); + }, done) + ); }); - it('should pull metadata from configuration', function (done) { + it('should pull metadata from configuration', done => { axe._load({ rules: [ { @@ -441,19 +392,14 @@ describe('runRules', function () { } ], checks: [ - { - id: 'has-target', - evaluate: function () { - return false; - } - }, + { id: 'has-target', evaluate: () => false }, { id: 'first-div', - evaluate: function (node) { + evaluate(node) { this.relatedNodes([node]); return false; }, - after: function (results) { + after(results) { if (results.length) { results[0].result = true; } @@ -477,12 +423,12 @@ describe('runRules', function () { thingy: true, impact: 'serious', messages: { - fail: function (checkResult) { + fail(checkResult) { return checkResult.id === 'first-div' ? 'failing is not good' : 'y u wrong rule?'; }, - pass: function (checkResult) { + pass(checkResult) { return checkResult.id === 'first-div' ? 'passing is good' : 'y u wrong rule?'; @@ -493,12 +439,12 @@ describe('runRules', function () { otherThingy: true, impact: 'moderate', messages: { - fail: function (checkResult) { + fail(checkResult) { return checkResult.id === 'has-target' ? 'failing is not good' : 'y u wrong rule?'; }, - pass: function (checkResult) { + pass(checkResult) { return checkResult.id === 'has-target' ? 'passing is good' : 'y u wrong rule?'; @@ -512,119 +458,115 @@ describe('runRules', function () { axe._runRules( '#fixture', {}, - function (results) { - try { - assert.deepEqual(JSON.parse(JSON.stringify(results)), [ - { - id: 'div#target', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/div#target?application=axeAPI', - pageLevel: false, - foo: 'bar', - stuff: 'blah', - impact: 'moderate', - passes: [], - inapplicable: [], - incomplete: [], - violations: [ - { - result: 'failed', - node: { - selector: ['#target'], - ancestry: [ - 'html > body > div:nth-child(1) > div:nth-child(1)' - ], - xpath: ["//div[@id='target']"], - source: '<div id="target">Target!</div>', - nodeIndexes: [12], - fromFrame: false - }, - impact: 'moderate', - any: [ - { - impact: 'moderate', - otherThingy: true, - message: 'failing is not good', - id: 'has-target', - data: null, - relatedNodes: [] - } + captureError(results => { + assert.deepEqual(JSON.parse(JSON.stringify(results)), [ + { + id: 'div#target', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/div#target?application=axeAPI', + pageLevel: false, + foo: 'bar', + stuff: 'blah', + impact: 'moderate', + passes: [], + inapplicable: [], + incomplete: [], + violations: [ + { + result: 'failed', + node: { + selector: ['#target'], + ancestry: [ + 'html > body > div:nth-child(1) > div:nth-child(1)' ], - all: [], - none: [] - } - ], - result: 'failed', - tags: [] - }, - { - id: 'first-div', - helpUrl: - 'https://dequeuniversity.com/rules/axe/' + - ver + - '/first-div?application=axeAPI', - pageLevel: false, - bar: 'foo', - stuff: 'no', - impact: null, - inapplicable: [], - incomplete: [], - violations: [], - passes: [ - { - result: 'passed', - impact: null, - node: { - selector: ['#target'], - xpath: ["//div[@id='target']"], - ancestry: [ - 'html > body > div:nth-child(1) > div:nth-child(1)' - ], - source: '<div id="target">Target!</div>', - nodeIndexes: [12], - fromFrame: false - }, - any: [ - { - impact: 'serious', - id: 'first-div', - thingy: true, - message: 'passing is good', - data: null, - relatedNodes: [ - { - selector: ['#target'], - ancestry: [ - 'html > body > div:nth-child(1) > div:nth-child(1)' - ], - xpath: ["//div[@id='target']"], - source: '<div id="target">Target!</div>', - nodeIndexes: [12], - fromFrame: false - } - ] - } + xpath: ["//div[@id='target']"], + source: '<div id="target">Target!</div>', + nodeIndexes: [12], + fromFrame: false + }, + impact: 'moderate', + any: [ + { + impact: 'moderate', + otherThingy: true, + message: 'failing is not good', + id: 'has-target', + data: null, + relatedNodes: [] + } + ], + all: [], + none: [] + } + ], + result: 'failed', + tags: [] + }, + { + id: 'first-div', + helpUrl: + 'https://dequeuniversity.com/rules/axe/' + + ver + + '/first-div?application=axeAPI', + pageLevel: false, + bar: 'foo', + stuff: 'no', + impact: null, + inapplicable: [], + incomplete: [], + violations: [], + passes: [ + { + result: 'passed', + impact: null, + node: { + selector: ['#target'], + xpath: ["//div[@id='target']"], + ancestry: [ + 'html > body > div:nth-child(1) > div:nth-child(1)' ], - all: [], - none: [] - } - ], - result: 'passed', - tags: [] - } - ]); - done(); - } catch (e) { - done(e); - } - }, + source: '<div id="target">Target!</div>', + nodeIndexes: [12], + fromFrame: false + }, + any: [ + { + impact: 'serious', + id: 'first-div', + thingy: true, + message: 'passing is good', + data: null, + relatedNodes: [ + { + selector: ['#target'], + ancestry: [ + 'html > body > div:nth-child(1) > div:nth-child(1)' + ], + xpath: ["//div[@id='target']"], + source: '<div id="target">Target!</div>', + nodeIndexes: [12], + fromFrame: false + } + ] + } + ], + all: [], + none: [] + } + ], + result: 'passed', + tags: [] + } + ]); + done(); + }, done), isNotCalled ); }); - it('should call the reject argument if an error occurs', function (done) { + it('should call the reject argument if an error occurs', done => { axe._load({ rules: [ { @@ -635,25 +577,24 @@ describe('runRules', function () { messages: {} }); - createFrames(function () { - setTimeout(function () { + createFrames(() => { + setTimeout(() => { axe._runRules( document, {}, - function () { - assert.ok(false, 'You shall not pass!'); - done(); + () => { + done(new Error('You shall not pass!')); }, - function (err) { + captureError(err => { assert.instanceOf(err, Error); done(); - } + }, done) ); }, 100); }); }); - it('should resolve to cantTell when a rule fails', function (done) { + it('should resolve to cantTell when a rule fails', done => { axe._load({ rules: [ { @@ -668,15 +609,10 @@ describe('runRules', function () { } ], checks: [ - { - id: 'undeffed', - evaluate: function () { - return undefined; - } - }, + { id: 'undeffed', evaluate: () => undefined }, { id: 'thrower', - evaluate: function () { + evaluate: () => { throw new Error('Check failed to complete'); } } @@ -685,21 +621,20 @@ describe('runRules', function () { fixture.innerHTML = '<div></div>'; - axe.run('#fixture', function (err, results) { - assert.isNull(err); - assert.lengthOf(results.incomplete, 2); - assert.equal(results.incomplete[0].id, 'incomplete-1'); - assert.equal(results.incomplete[1].id, 'incomplete-2'); - - assert.include( - results.incomplete[1].description, - 'An error occured while running this rule' - ); - done(); - }); + axe.run( + '#fixture', + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.incomplete, 2); + assert.equal(results.incomplete[0].id, 'incomplete-1'); + assert.equal(results.incomplete[1].id, 'incomplete-2'); + assert.isNotNull(results.incomplete[1].error); + done(); + }, done) + ); }); - it('should resolve to cantTell if an error occurs inside frame rules', function (done) { + it('should resolve to cantTell if an error occurs inside frame rules', done => { axe._load({ rules: [ { @@ -714,18 +649,8 @@ describe('runRules', function () { } ], checks: [ - { - id: 'undeffed', - evaluate: function () { - return false; - } - }, - { - id: 'thrower', - evaluate: function () { - return false; - } - } + { id: 'undeffed', evaluate: () => false }, + { id: 'thrower', evaluate: () => false } ] }); @@ -733,24 +658,23 @@ describe('runRules', function () { '../mock/frames/rule-error.html', fixture, 'context-test', - function () { - axe.run('#fixture', function (err, results) { - assert.isNull(err); - assert.lengthOf(results.incomplete, 2); - assert.equal(results.incomplete[0].id, 'incomplete-1'); - assert.equal(results.incomplete[1].id, 'incomplete-2'); - - assert.include( - results.incomplete[1].description, - 'An error occured while running this rule' - ); - done(); - }); + () => { + axe.run( + '#fixture', + captureError((err, results) => { + assert.isNull(err); + assert.lengthOf(results.incomplete, 2); + assert.equal(results.incomplete[0].id, 'incomplete-1'); + assert.equal(results.incomplete[1].id, 'incomplete-2'); + assert.isNotNull(results.incomplete[1].error); + done(); + }, done) + ); } ); }); - it('should cascade `no elements found` errors in frames to reject run_rules', function (done) { + it('should cascade `no elements found` errors in frames to reject run_rules', done => { axe._load({ rules: [ { @@ -761,29 +685,29 @@ describe('runRules', function () { messages: {} }); fixture.innerHTML = '<div id="outer"></div>'; - var outer = document.getElementById('outer'); + let outer = document.getElementById('outer'); - iframeReady('../mock/frames/context.html', outer, 'target', function () { + iframeReady('../mock/frames/context.html', outer, 'target', () => { axe._runRules( [['#target', '#elementNotFound']], {}, function resolve() { - assert.ok(false, 'frame should have thrown an error'); + done(new Error('frame should have thrown an error')); }, - function reject(err) { + captureError(function reject(err) { assert.instanceOf(err, Error); assert.include( err.message, 'No elements found for include in frame Context' ); done(); - } + }, done) ); }); }); - it('should not call reject when the resolve throws', function (done) { - var rejectCalled = false; + it('should not call reject when the resolve throws', done => { + let rejectCalled = false; axe._load({ rules: [ { @@ -792,19 +716,12 @@ describe('runRules', function () { any: ['html'] } ], - checks: [ - { - id: 'html', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'html', evaluate: () => true }], messages: {} }); function resolve() { - setTimeout(function () { + setTimeout(() => { assert.isFalse(rejectCalled); axe.log = log; done(); @@ -815,15 +732,15 @@ describe('runRules', function () { rejectCalled = true; } - var log = axe.log; - axe.log = function (e) { + let log = axe.log; + axe.log = e => { assert.equal(e.message, 'err'); axe.log = log; }; axe._runRules(document, {}, resolve, reject); }); - it('should ignore iframes if `iframes` === false', function (done) { + it('should ignore iframes if `iframes` === false', done => { axe._load({ rules: [ { @@ -832,26 +749,19 @@ describe('runRules', function () { any: ['html'] } ], - checks: [ - { - id: 'html', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'html', evaluate: () => true }], messages: {} }); - var frame = document.createElement('iframe'); + let frame = document.createElement('iframe'); frame.src = '../mock/frames/frame-frame.html'; - frame.addEventListener('load', function () { - setTimeout(function () { + frame.addEventListener('load', () => { + setTimeout(() => { axe._runRules( document, { iframes: false, elementRef: true }, - function (r) { + captureError(r => { assert.lengthOf(r[0].passes, 1); assert.equal( r[0].passes[0].node.element.ownerDocument, @@ -859,7 +769,7 @@ describe('runRules', function () { 'Result should not be in an iframe' ); done(); - }, + }, done), isNotCalled ); }, 500); @@ -867,7 +777,7 @@ describe('runRules', function () { fixture.appendChild(frame); }); - it('should not fail if `include` / `exclude` is overwritten', function (done) { + it('should not fail if `include` / `exclude` is overwritten', done => { function invalid() { throw new Error('nope!'); } @@ -882,32 +792,25 @@ describe('runRules', function () { any: ['html'] } ], - checks: [ - { - id: 'html', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'html', evaluate: () => true }], messages: {} }); axe._runRules( [document], {}, - function (r) { + captureError(r => { assert.lengthOf(r[0].passes, 1); delete Array.prototype.include; delete Array.prototype.exclude; done(); - }, + }, done), isNotCalled ); }); - it('should return a cleanup method', function (done) { + it('should return a cleanup method', done => { axe._load({ rules: [ { @@ -916,21 +819,14 @@ describe('runRules', function () { any: ['html'] } ], - checks: [ - { - id: 'html', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'html', evaluate: () => true }], messages: {} }); axe._runRules( document, {}, - function resolve(out, cleanup) { + captureError(function resolve(out, cleanup) { assert.isDefined(axe._tree); assert.isDefined(axe._selectorData); @@ -938,12 +834,12 @@ describe('runRules', function () { assert.isUndefined(axe._tree); assert.isUndefined(axe._selectorData); done(); - }, + }, done), isNotCalled ); }); - it('should clear up axe._tree / axe._selectorData after an error', function (done) { + it('should clear up axe._tree / axe._selectorData after an error', done => { axe._load({ rules: [ { @@ -954,19 +850,24 @@ describe('runRules', function () { messages: {} }); - createFrames(function () { - setTimeout(function () { - axe._runRules(document, {}, isNotCalled, function () { - assert.isUndefined(axe._tree); - assert.isUndefined(axe._selectorData); - done(); - }); + createFrames(() => { + setTimeout(() => { + axe._runRules( + document, + {}, + isNotCalled, + captureError(() => { + assert.isUndefined(axe._tree); + assert.isUndefined(axe._selectorData); + done(); + }, done) + ); }, 100); }); }); // todo: see issue - https://github.com/dequelabs/axe-core/issues/2168 - it.skip('should clear the memoized cache for each function', function (done) { + it.skip('should clear the memoized cache for each function', done => { axe._load({ rules: [ { @@ -975,25 +876,18 @@ describe('runRules', function () { any: ['html'] } ], - checks: [ - { - id: 'html', - evaluate: function () { - return true; - } - } - ], + checks: [{ id: 'html', evaluate: () => true }], messages: {} }); axe._runRules( document, {}, - function resolve(out, cleanup) { - var called = false; + captureError(function resolve(out, cleanup) { + let called = false; axe._memoizedFns = [ { - clear: function () { + clear: () => { called = true; } } @@ -1001,9 +895,8 @@ describe('runRules', function () { cleanup(); assert.isTrue(called); - done(); - }, + }, done), isNotCalled ); }); diff --git a/test/core/utils/merge-results.js b/test/core/utils/merge-results.js index f7c758c099..b5a5539b32 100644 --- a/test/core/utils/merge-results.js +++ b/test/core/utils/merge-results.js @@ -1,8 +1,9 @@ -describe('axe.utils.mergeResults', function () { +describe('axe.utils.mergeResults', () => { 'use strict'; var queryFixture = axe.testUtils.queryFixture; + var RuleError = axe.utils.RuleError; - it('should normalize empty results', function () { + it('should normalize empty results', () => { var result = axe.utils.mergeResults([ { results: [] }, { results: [{ id: 'a', result: 'b' }] } @@ -15,7 +16,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('merges frame content, including all selector types', function () { + it('merges frame content, including all selector types', () => { var iframe = queryFixture('<iframe id="target"></iframe>').actualNode; var node = { selector: ['#foo'], @@ -49,7 +50,7 @@ describe('axe.utils.mergeResults', function () { assert.deepEqual(node.nodeIndexes, [1, 123]); }); - it('merges frame specs', function () { + it('merges frame specs', () => { var iframe = queryFixture('<iframe id="target"></iframe>').actualNode; var frameSpec = new axe.utils.DqElement(iframe).toJSON(); var node = { @@ -84,7 +85,7 @@ describe('axe.utils.mergeResults', function () { assert.deepEqual(node.nodeIndexes, [1, 123]); }); - it('sorts results from iframes into their correct DOM position', function () { + it('sorts results from iframes into their correct DOM position', () => { var result = axe.utils.mergeResults([ { results: [ @@ -148,7 +149,7 @@ describe('axe.utils.mergeResults', function () { assert.deepEqual(ids, ['h1', 'iframe1 >> h2', 'iframe1 >> h3', 'h4']); }); - it('sorts nested iframes', function () { + it('sorts nested iframes', () => { var result = axe.utils.mergeResults([ { results: [ @@ -219,7 +220,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('sorts results even if nodeIndexes are empty', function () { + it('sorts results even if nodeIndexes are empty', () => { var result = axe.utils.mergeResults([ { results: [ @@ -296,7 +297,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('sorts results even if nodeIndexes are undefined', function () { + it('sorts results even if nodeIndexes are undefined', () => { var result = axe.utils.mergeResults([ { results: [ @@ -370,7 +371,7 @@ describe('axe.utils.mergeResults', function () { ]); }); - it('sorts nodes all placed on the same result', function () { + it('sorts nodes all placed on the same result', () => { var result = axe.utils.mergeResults([ { results: [ @@ -419,4 +420,57 @@ describe('axe.utils.mergeResults', function () { '#level0 >> #level1 >> #level2b' ]); }); + + describe('errors', () => { + it('sets error if it is present', () => { + const result = axe.utils.mergeResults([ + { results: [{ id: 'a', result: 'b', error: new Error('test') }] } + ]); + assert.equal(result[0].error.message, 'test'); + }); + + it('picks the first error if there are multiple', () => { + const result = axe.utils.mergeResults([ + { + results: [ + { + id: 'error-occurred', + result: undefined, + nodes: [{ node: { selector: ['h1'], nodeIndexes: [1] } }] + }, + { + id: 'error-occurred', + result: undefined, + error: new RuleError({ error: new Error('test 1') }), + nodes: [ + { + node: { + selector: ['iframe1', 'h2'], + nodeIndexes: [2, 1], + fromFrame: true + } + } + ] + }, + { + id: 'error-occurred', + result: undefined, + error: new RuleError({ error: new Error('test 2') }), + nodes: [ + { + node: { + selector: ['iframe2', 'h3'], + nodeIndexes: [3, 1], + fromFrame: true + } + } + ] + } + ] + } + ]); + + assert.equal(result[0].error.message, 'test 1'); + }); + }); }); diff --git a/test/core/utils/performance-timer.js b/test/core/utils/performance-timer.js index 49da815df3..66af18f582 100644 --- a/test/core/utils/performance-timer.js +++ b/test/core/utils/performance-timer.js @@ -46,14 +46,31 @@ describe('performance timer', () => { it('measures time elapsed between marks', async () => { performanceTimer.start(); + + const timestampPreMarkStart = performance.now(); performanceTimer.mark('foo_start'); + const timestampPostMarkStart = performance.now(); + await sleep(100); + + const timestampPreMarkEnd = performance.now(); performanceTimer.mark('foo_end'); + const timestampPostMarkEnd = performance.now(); + performanceTimer.measure('foo', 'foo_start', 'foo_end'); performanceTimer.logMeasures('foo'); assert.equal(messages.length, 1); - assert.closeTo(getNumber(messages[0]), 100, 10); + const actual = getNumber(messages[0]); + + assert.isAtLeast(actual, 100); + + // The actual value might be significantly >100ms in a browser, especially on a + // CI agent with a slow CPU, but should be consistent with performance.now() + const maxExpected = timestampPostMarkEnd - timestampPreMarkStart; + const minExpected = timestampPreMarkEnd - timestampPostMarkStart; + assert.isAtLeast(actual, minExpected); + assert.isAtMost(actual, maxExpected); }); it('measures time elapsed if auditStart() was called', () => { @@ -144,23 +161,58 @@ describe('performance timer', () => { describe('timeElapsed', () => { it('returns the time elapsed since axe started', async () => { + const timestampPreStart = performance.now(); performanceTimer.start(); + const timestampPostStart = performance.now(); + await sleep(100); - assert.closeTo(performanceTimer.timeElapsed(), 100, 10); + + const timestampPreTimeElapsed = performance.now(); + const actual = performanceTimer.timeElapsed(); + const timestampPostTimeElapsed = performance.now(); + + assert.isAtLeast(actual, 100); + const maxExpected = timestampPostTimeElapsed - timestampPreStart; + const minExpected = timestampPreTimeElapsed - timestampPostStart; + assert.isAtLeast(actual, minExpected); + assert.isAtMost(actual, maxExpected); }); it('returns the time elapsed since auditStart() was called', async () => { + const timestampPreStart = performance.now(); performanceTimer.auditStart(); + const timestampPostStart = performance.now(); + await sleep(100); - assert.closeTo(performanceTimer.timeElapsed(), 100, 10); + + const timestampPreTimeElapsed = performance.now(); + const actual = performanceTimer.timeElapsed(); + const timestampPostTimeElapsed = performance.now(); + + assert.isAtLeast(actual, 100); + const maxExpected = timestampPostTimeElapsed - timestampPreStart; + const minExpected = timestampPreTimeElapsed - timestampPostStart; + assert.isAtLeast(actual, minExpected); + assert.isAtMost(actual, maxExpected); }); it('does not use auditStart if axe started', async () => { + const timestampPreStart = performance.now(); performanceTimer.start(); + const timestampPostStart = performance.now(); + await sleep(100); + performanceTimer.auditStart(); // Should be ignored - performanceTimer.auditStart(); - assert.closeTo(performanceTimer.timeElapsed(), 100, 10); + const timestampPreTimeElapsed = performance.now(); + const actual = performanceTimer.timeElapsed(); + const timestampPostTimeElapsed = performance.now(); + + assert.isAtLeast(actual, 100); + const maxExpected = timestampPostTimeElapsed - timestampPreStart; + const minExpected = timestampPreTimeElapsed - timestampPostStart; + assert.isAtLeast(actual, minExpected); + assert.isAtMost(actual, maxExpected); }); }); }); diff --git a/test/core/utils/rule-error.js b/test/core/utils/rule-error.js new file mode 100644 index 0000000000..3ebd318a2d --- /dev/null +++ b/test/core/utils/rule-error.js @@ -0,0 +1,45 @@ +describe('utils.RuleError', () => { + const RuleError = axe.utils.RuleError; + + it('returns a serializable error', () => { + const error = new Error('test'); + const ruleError = new RuleError({ error }); + assert.ownInclude(ruleError, { + message: error.message, + stack: error.stack, + name: error.name + }); + }); + + it('returns a instanceof Error', () => { + const error = new Error('test'); + const ruleError = new RuleError({ error }); + assert.instanceOf(ruleError, Error); + }); + + it('includes the ruleId if provided', () => { + const error = new Error('test'); + const ruleError = new RuleError({ error, ruleId: 'aria' }); + assert.equal(ruleError.ruleId, 'aria'); + assert.include(ruleError.message, 'Skipping aria rule.'); + }); + + it('includes the method if provided', () => { + const error = new Error('test'); + const ruleError = new RuleError({ error, method: '#matches' }); + assert.equal(ruleError.method, '#matches'); + }); + + it('includes the errorNode if provided', () => { + const error = new Error('test'); + const ruleError = new RuleError({ error, errorNode: 'err' }); + assert.equal(ruleError.errorNode, 'err'); + }); + + it('includes a serialized cause if provided', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + const ruleError = new RuleError({ error }); + assert.deepEqual(ruleError.cause, axe.utils.serializeError(error.cause)); + }); +}); diff --git a/test/core/utils/serialize-error.js b/test/core/utils/serialize-error.js new file mode 100644 index 0000000000..550fad1a76 --- /dev/null +++ b/test/core/utils/serialize-error.js @@ -0,0 +1,86 @@ +describe('utils.serializeError', function () { + const serializeError = axe.utils.serializeError; + + it('should serialize an error', () => { + const error = new Error('test'); + const serialized = serializeError(error); + assert.ownInclude(serialized, { + message: error.message, + stack: error.stack, + name: error.name + }); + }); + + it('should serialize known serializable properties', () => { + const error = new Error('test'); + error.code = 3; + error.ruleId = 'test1'; + error.method = 'test2'; + const serialized = serializeError(error); + assert.ownInclude(serialized, { + code: error.code, + ruleId: error.ruleId, + method: error.method + }); + }); + + it('should not include nullish properties', () => { + const error = new Error('test'); + + // Neither an explicitly undefined nor an omitted property should be included + error.code = null; + error.method = undefined; + // error.ruleId = undefined; + + const serialized = serializeError(error); + assert.doesNotHaveAnyKeys(serialized, ['code', 'method', 'ruleId']); + }); + + it('should not include non-scalar values even in allow-listed properties', () => { + const error = new Error('test'); + error.code = { foo: 'bar' }; + error.ruleId = ['baz', 'qux']; + const serialized = serializeError(error); + assert.doesNotHaveAnyKeys(serialized, ['method', 'ruleId']); + }); + + it('should not include non-allow-listed properties', () => { + const error = new Error('test'); + error.someUnknownProp = 'test'; + error.errorNode = 'test'; + const serialized = serializeError(error); + assert.doesNotHaveAnyKeys(serialized, ['someUnknownProp', 'errorNode']); + }); + + it('should serialize an error with a cause', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + const serialized = serializeError(error); + assert.ownInclude(serialized.cause, { + message: error.cause.message, + stack: error.cause.stack, + name: error.cause.name + }); + }); + + it('should serialize recursively', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + error.cause.cause = new Error('cause2'); + const serialized = serializeError(error); + assert.ownInclude(serialized.cause.cause, { + message: error.cause.cause.message, + stack: error.cause.cause.stack, + name: error.cause.cause.name + }); + }); + + it('should not serialize the cause if the stack exceeds 10 levels', () => { + const error = new Error('test'); + error.cause = new Error('cause'); + error.cause.cause = new Error('cause2'); + error.cause.cause.cause = new Error('cause3'); + const serialized = serializeError(error, 9); + assert.equal(serialized.cause.cause, '...'); + }); +}); diff --git a/test/integration/full/error-occurred/error-frame.html b/test/integration/full/error-occurred/error-frame.html new file mode 100644 index 0000000000..dbe5eafb48 --- /dev/null +++ b/test/integration/full/error-occurred/error-frame.html @@ -0,0 +1,30 @@ +<!doctype html> +<html lang="en"> + <head> + <title>error-occurred in frame test + + + + + + + + + +
+ + + + + + diff --git a/test/integration/full/error-occurred/error-frame.js b/test/integration/full/error-occurred/error-frame.js new file mode 100644 index 0000000000..f94b3476b5 --- /dev/null +++ b/test/integration/full/error-occurred/error-frame.js @@ -0,0 +1,118 @@ +describe('error-occurred test', () => { + const { runPartialRecursive } = axe.testUtils; + let results; + + describe('axe.run()', () => { + before(done => { + axe.testUtils.awaitNestedLoad(() => { + axe.run( + { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }, + function (err, r) { + assert.isNull(err); + results = r; + done(); + } + ); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#frame', '#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#frame', '#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#frame', '#target'] + }); + }); + }); + }); + + describe('axe.runPartial() + axe.finishRun()', () => { + before(() => { + return new Promise(resolve => { + axe.testUtils.awaitNestedLoad(async () => { + const runOptions = { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }; + const partialResults = await Promise.all( + runPartialRecursive(document, runOptions) + ); + results = await axe.finishRun(partialResults, runOptions); + resolve(); + }); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#frame', '#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#frame', '#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#frame', '#target'] + }); + }); + }); + }); +}); diff --git a/test/integration/full/error-occurred/error-occurred.html b/test/integration/full/error-occurred/error-occurred.html new file mode 100644 index 0000000000..bbec76bb3b --- /dev/null +++ b/test/integration/full/error-occurred/error-occurred.html @@ -0,0 +1,30 @@ + + + + error-occurred test + + + + + + + + +
+
+ + + + + + diff --git a/test/integration/full/error-occurred/error-occurred.js b/test/integration/full/error-occurred/error-occurred.js new file mode 100644 index 0000000000..9f87380942 --- /dev/null +++ b/test/integration/full/error-occurred/error-occurred.js @@ -0,0 +1,118 @@ +describe('error-occurred test', () => { + const { runPartialRecursive } = axe.testUtils; + let results; + + describe('axe.run()', () => { + before(done => { + axe.testUtils.awaitNestedLoad(() => { + axe.run( + { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }, + function (err, r) { + assert.isNull(err); + results = r; + done(); + } + ); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#target'] + }); + }); + }); + }); + + describe('axe.runPartial() + axe.finishRun()', () => { + before(() => { + return new Promise(resolve => { + axe.testUtils.awaitNestedLoad(async () => { + const runOptions = { + runOnly: ['matches-error', 'evaluate-error', 'after-error'] + }; + const partialResults = await Promise.all( + runPartialRecursive(document, runOptions) + ); + results = await axe.finishRun(partialResults, runOptions); + resolve(); + }); + }); + }); + + it('should find 0 violations', () => { + assert.lengthOf(results.violations, 0); + }); + + it('should find 0 passes', () => { + assert.lengthOf(results.passes, 0); + }); + + describe('incomplete', () => { + it('should find matches-error', () => { + const matchesError = results.incomplete.find( + result => result.id === 'matches-error' + ); + window.assertIsErrorOccurred(matchesError, { + message: 'matches error', + target: ['#target'] + }); + }); + + it('should find evaluate-error', () => { + const evaluateError = results.incomplete.find( + result => result.id === 'evaluate-error' + ); + window.assertIsErrorOccurred(evaluateError, { + message: 'evaluate error', + target: ['#target'] + }); + }); + + it('should find after-error', () => { + const afterError = results.incomplete.find( + result => result.id === 'after-error' + ); + window.assertIsErrorOccurred(afterError, { + message: 'after error', + target: ['#target'] + }); + }); + }); + }); +}); diff --git a/test/integration/full/error-occurred/error-ruleset.js b/test/integration/full/error-occurred/error-ruleset.js new file mode 100644 index 0000000000..67b83ca772 --- /dev/null +++ b/test/integration/full/error-occurred/error-ruleset.js @@ -0,0 +1,61 @@ +window.assertIsErrorOccurred = function (result, { message, target }) { + assert.isDefined(result); + assert.isDefined(result.error); + assert.include(result.error.message, message); + assert.isDefined(result.error.method); + // errorNode is not included as it can be unsafe to serialize + assert.isUndefined(result.error.errorNode); + assert.isUndefined(result.errorNode); + + assert.lengthOf(result.nodes, 1); + const node = result.nodes[0]; + assert.lengthOf(node.any, 0); + assert.lengthOf(node.all, 0); + assert.lengthOf(node.none, 1); + assert.equal(node.none[0].id, 'error-occurred'); + assert.include(node.none[0].message, 'Axe encountered an error'); + assert.deepEqual(node.none[0].data, result.error); + assert.deepEqual(node.target, target); + assert.isDefined(node.html); +}; + +axe.configure({ + rules: [ + { + id: 'matches-error', + selector: '#target', + matches: () => { + throw new Error('matches error'); + }, + any: ['exists'] + }, + { + id: 'evaluate-error', + selector: '#target', + any: ['check-evaluate-error'] + }, + { + id: 'after-error', + selector: '#target', + any: ['check-after-error'] + } + ], + checks: [ + { + id: 'check-evaluate-error', + evaluate: () => { + throw new Error('evaluate error'); + }, + after: () => { + throw new Error('I should not be seen'); + } + }, + { + id: 'check-after-error', + evaluate: () => true, + after: () => { + throw new Error('after error'); + } + } + ] +}); diff --git a/test/integration/full/error-occurred/frames/error.html b/test/integration/full/error-occurred/frames/error.html new file mode 100644 index 0000000000..81a004af01 --- /dev/null +++ b/test/integration/full/error-occurred/frames/error.html @@ -0,0 +1,4 @@ +
+ + + diff --git a/typings/axe-core/axe-core-tests.ts b/typings/axe-core/axe-core-tests.ts index 34cba1e8dc..68c9b087fd 100644 --- a/typings/axe-core/axe-core-tests.ts +++ b/typings/axe-core/axe-core-tests.ts @@ -16,6 +16,13 @@ axe.run(context, {}, (error: Error, results: axe.AxeResults) => { } console.log(results.passes.length); console.log(results.incomplete.length); + const errors = results.incomplete.map(result => result.error); + console.log( + errors.map( + ({ message, stack, ruleId, method }) => + `${message} ${ruleId} ${method}\n\n${stack}` + ) + ); console.log(results.inapplicable.length); console.log(results.violations.length); console.log(results.violations[0].nodes[0].failureSummary);