Merge pull request #87 from mfrances17/fail-a11y-on-ci

Ensure CI fails if there are a11y violations or incomplete test runs

Author: nicolethoen

lib/reporter.js

@@ -5,21 +5,20 @@ const { getHTMLReport } = require("./htmlReport");
 
 const report = {};
 let completedUrls = 0;
+let exitCode = 0;
 
 function makeReportMessage(type, error) {
-  const messages = error.nodes.map((node) => node.any[0]);
+  const messages = error.nodes?.map((node) => node.any?.[0]) ?? [];
 
   return {
     message: JSON.stringify(messages, null, 2),
     type: `${type}: ${error.id}`,
   };
 }
+const getIdReport = (errors = []) => errors.map((err) => err.id).sort().join(", ") || "None";
 
 function makeReport(aggregate) {
-  const totalTime = Object.values(report).reduce(
-    (prev, cur) => (prev += cur.time),
-    0
-  );
+  const totalTime = Object.values(report).reduce((prev, cur) => prev + cur.time, 0);
   const suites = [];
 
   if (aggregate) {
@@ -41,7 +40,7 @@ function makeReport(aggregate) {
         suites.push({
           name: key,
           timestamp: new Date(),
-          time: val.reduce((prev, cur) => (prev += cur.time), 0),
+          time: val.reduce((prev, cur) => prev + cur.time, 0),
           testCases: Object.values(val).map((testCase) => ({
             name: testCase.name,
             failures: testCase.violations.map((error) =>
@@ -81,37 +80,20 @@ function makeReport(aggregate) {
 }
 
 function writeConsoleError(suites) {
-  const failedSuites = suites.filter((suite) => {
-    for (let testCase of suite.testCases) {
-      if (testCase.failures.length > 0) {
-        return true;
-      }
-      if (testCase.incomplete && testCase.incomplete.length > 0) {
-        return true;
-      }
-    }
-    return false;
-  });
+  const failedSuites = suites.filter(suite => 
+    suite.testCases.some(testCase => testCase.failures.length > 0 || testCase.incomplete?.length > 0)
+  );
   if (failedSuites.length > 0) {
     console.log("\nReport summary");
-    failedSuites.forEach((suite) => {
+    failedSuites.forEach(suite => {
       console.error(`================= ${suite.name} ===================`);
-      suite.testCases.forEach((testCase) => {
-        const hasFailures = testCase.failures.length > 0;
-        const hasIncomplete =
-          testCase.incomplete && testCase.incomplete.length > 0;
-        if (hasFailures || hasIncomplete) {
+      suite.testCases
+      .filter(testCase => testCase.failures.length || testCase.incomplete?.length)
+      .forEach(testCase => {
           console.error(`     ${testCase.name}`);
-        }
-        if (hasFailures) {
-          console.error(`         Violations`);
-          testCase.failures.forEach((error) => console.error(error.message));
-        }
-        if (hasIncomplete) {
-          console.error(`         Incomplete`);
-          testCase.incomplete.forEach((error) => console.error(error.message));
-        }
-      });
+          if (testCase.failures.length) console.error("         Violations", testCase.failures.map(e => e.message));
+          if (testCase.incomplete?.length) console.error("         Incomplete", testCase.incomplete.map(e => e.message));
+    });
     });
   }
 }
@@ -124,44 +106,29 @@ function writeCoverage(aggregate, exitCodeOnFailure) {
     reportTitle: "Patternfly axe report",
   });
 
-  // writeConsoleError(suites);
-
   fs.copySync(path.join(__dirname, '../report/dist'), "coverage", { recursive: true });
   fs.writeFileSync("coverage/results.json", JSON.stringify(report, null, 2));
   fs.writeFileSync("coverage/coverage.xml", junitXml);
   fs.writeFileSync("coverage/report.html", htmlReport);
 
-  // Exit code
-  if (exitCodeOnFailure) {
-    if (
-      suites.find((suite) =>
-        suite.testCases.find((test) => test.failures.length > 0)
-      )
-    ) {
-      return 1;
-    }
-    if (
-      suites.find((suite) =>
-        suite.testCases.find(
-          (test) => test.incomplete && test.incomplete.length > 0
-        )
-      )
-    ) {
-      return 2;
-    }
+  // Only force exit code here so fail occurs after all files are tested, otherwise will fail on first violation
+  if (exitCode > 0) {
+    console.error(`\nExiting with code ${exitCode}`);
+    process.exit(exitCode);
   }
-
-  return 0;
 }
 
-const addAnyField = (error) =>
-  error.nodes.forEach((node) => {
+const addAnyField = (error) => {
+  if (!error.nodes) return;
+
+  error.nodes.forEach(node => {
     // Usually the any field is prefilled with some useful stuff, but if not, add it
-    node.any = node.any || [];
+    node.any = Array.isArray(node.any) ? node.any : [];
     node.any[0] = node.any[0] || { relatedNodes: [] };
-    // Add the html that most erorrs report
+    // Add the html that most errors report
     node.any[0].relatedNodes.push({ html: node.html });
   });
+};
 
 function recordPage(
   prefix,
@@ -174,23 +141,20 @@ function recordPage(
   index,
   screenshotFile,
   axeOptions,
-  context
+  context,
 ) {
-  const elapsed = process.hrtime(startTime);
   // Report program is doing something
   console.log(`\n${++completedUrls}/${numUrls}`.padEnd(10, " "), url);
 
-  if (axeResults.incomplete) {
-    axeResults.incomplete.forEach(addAnyField);
-  }
-  if (axeResults.violations) {
-    axeResults.violations.forEach(addAnyField);
-  }
+  axeResults?.incomplete?.forEach(addAnyField);
+  axeResults?.violations?.forEach(addAnyField);
 
+  const elapsedTime = process.hrtime(startTime).reduce((s, ns) => s + ns / 1e9);
+  
   const urlReport = {
-    incomplete: axeResults.incomplete,
-    violations: axeResults.violations,
-    time: elapsed[0] + elapsed[1] / 1000000000,
+    incomplete: axeResults.incomplete || [],
+    violations: axeResults.violations || [],
+    time: elapsedTime,
     screenshotFile,
     prefix,
     url,
@@ -202,19 +166,16 @@ function recordPage(
 
   report[`${index}_${url}`] = urlReport;
 
-  const getIdReport = (errors) =>
-    errors
-      .map((err) => err.id)
-      .sort()
-      .join(", ");
-
-  if (urlReport.violations.length > 0) {
+  const hasViolations = urlReport.violations.length > 0;
+  const hasIncomplete = !ignoreIncomplete && urlReport.incomplete.length > 0;
+  
+  if (hasViolations) {
     console.log("violations:", getIdReport(urlReport.violations));
+    exitCode = 1;
   }
-  const numIncomplete =
-    !ignoreIncomplete && urlReport.incomplete && urlReport.incomplete.length;
-  if (numIncomplete) {
+  if (hasIncomplete) {
     console.error("incomplete:", getIdReport(urlReport.incomplete));
+    exitCode = exitCode === 1 ? 1 : 2;
   }
 }