Eleventy build: Support generating WCAG 2.1 docs from main branch (#4007)

## Background

There has long been interest in re-generating updated versions of the
informative docs for WCAG 2.1. Previously, this relied upon a separate
`WCAG-2.1` branch, but the ideal outcome is to be able to build directly
from `main`, altering/pruning content as appropriate.

## Changes

This adds more functionality to the Eleventy-based build system to
support building content targeting WCAG 2.1 in addition to 2.2,
specifically when `WCAG_VERSION=21` is set in environment variables:

- Guidelines are pulled from the web so as to not pick up information
and alterations from newer WCAG versions (e.g. addition of new SC and
removal of 4.1.1)
- Parsing/processing is augmented as necessary to support both source
and pre-processed guidelines
- Acknowledgements are also pulled from the web, pinned to 2.1
- `target-size-enhanced` is output to `target-size` for 2.1
- Detects what techniques only apply to later WCAG versions, and prunes
them from associations
- Avoids emitting pages for techniques and understanding documents that
exclusively pertain to later WCAG versions
- Further updates hard-coded version numbers in titles and level-1
headings, as well as throughout pages (especially `refer-to-wcag`) via
Liquid expressions
- Adds support for setting both `note` and `wcagXY` class on the same
element
- Conditionalizes a couple of 2.2-specific notes that were not within
elements with `wcag22` class
- Unchanged but worth noting: Obsolete technique data includes version
information, so anything marked obsolete as of 2.2 will not contain an
obsolete banner when publishing for 2.1

Author: kfranqueiro

11ty/CustomLiquid.ts

@@ -1,4 +1,3 @@
-import type { Cheerio, Element } from "cheerio";
 import { Liquid, type Template } from "liquidjs";
 import type { RenderOptions } from "liquidjs/dist/liquid-options";
 import compact from "lodash-es/compact";
@@ -9,9 +8,9 @@ import { basename } from "path";
 import type { GlobalData } from "eleventy.config";
 
 import { biblioPattern, getBiblio } from "./biblio";
-import { flattenDom, load } from "./cheerio";
+import { flattenDom, load, type CheerioAnyNode } from "./cheerio";
 import { generateId } from "./common";
-import { getTermsMap } from "./guidelines";
+import { getAcknowledgementsForVersion, getTermsMap } from "./guidelines";
 import { resolveTechniqueIdFromHref, understandingToTechniqueLinkSelector } from "./techniques";
 import { techniqueToUnderstandingLinkSelector } from "./understanding";
 
@@ -63,7 +62,7 @@ const normalizeTocLabel = (label: string) =>
  * expand to a link with the full technique ID and title.
  * @param $el a $()-wrapped link element
  */
-function expandTechniqueLink($el: Cheerio<Element>) {
+function expandTechniqueLink($el: CheerioAnyNode) {
   const href = $el.attr("href");
   if (!href) throw new Error("expandTechniqueLink: non-link element encountered");
   const id = resolveTechniqueIdFromHref(href);
@@ -308,6 +307,14 @@ export class CustomLiquid extends Liquid {
     if (indexPattern.test(scope.page.inputPath)) {
       // Remove empty list items due to obsolete technique link removal
       if (scope.isTechniques) $("ul.toc-wcag-docs li:empty").remove();
+
+      // Replace acknowledgements with pinned content for older versions
+      if (process.env.WCAG_VERSION && $("section#acknowledgements").length) {
+        const pinnedAcknowledgements = await getAcknowledgementsForVersion(scope.version);
+        for (const [id, content] of Object.entries(pinnedAcknowledgements)) {
+          $(`#${id} h3 +`).html(content);
+        }
+      }
     } else {
       const $title = $("title");
 
@@ -401,7 +408,7 @@ export class CustomLiquid extends Liquid {
       // Process defined terms within #render,
       // where we have access to global data and the about box's HTML
       const $termLinks = $(termLinkSelector);
-      const extractTermName = ($el: Cheerio<Element>) => {
+      const extractTermName = ($el: CheerioAnyNode) => {
         const name = $el
           .text()
           .toLowerCase()
@@ -426,7 +433,7 @@ export class CustomLiquid extends Liquid {
         });
       } else if (scope.isUnderstanding) {
         const $termsList = $("section#key-terms dl");
-        const extractTermNames = ($links: Cheerio<Element>) =>
+        const extractTermNames = ($links: CheerioAnyNode) =>
           compact(uniq($links.toArray().map((el) => extractTermName($(el)))));
 
         if ($termLinks.length) {
@@ -496,15 +503,17 @@ export class CustomLiquid extends Liquid {
     // (This is also needed for techniques/about)
     $("div.note").each((_, el) => {
       const $el = $(el);
-      $el.replaceWith(`<div class="note">
+      const classes = el.attribs.class;
+      $el.replaceWith(`<div class="${classes}">
 				<p class="note-title marker">Note</p>
 				<div>${$el.html()}</div>
 			</div>`);
     });
     // Handle p variant after div (the reverse would double-process)
     $("p.note").each((_, el) => {
       const $el = $(el);
-      $el.replaceWith(`<div class="note">
+      const classes = el.attribs.class;
+      $el.replaceWith(`<div class="${classes}">
 				<p class="note-title marker">Note</p>
 				<p>${$el.html()}</p>
 			</div>`);
@@ -522,13 +531,19 @@ export class CustomLiquid extends Liquid {
     // Handle new-in-version content
     $("[class^='wcag']").each((_, el) => {
       // Just like the XSLT process, this naively assumes that version numbers are the same length
-      const classVersion = +el.attribs.class.replace(/^wcag/, "");
-      const buildVersion = +scope.version;
+      const classMatch = el.attribs.class.match(/\bwcag(\d\d)\b/);
+      if (!classMatch) throw new Error(`Invalid wcagXY class found: ${el.attribs.class}`);
+      const classVersion = +classMatch[1];
       if (isNaN(classVersion)) throw new Error(`Invalid wcagXY class found: ${el.attribs.class}`);
+      const buildVersion = +scope.version;
+
       if (classVersion > buildVersion) {
         $(el).remove();
       } else if (classVersion === buildVersion) {
-        $(el).prepend(`<span class="new-version">New in WCAG ${scope.versionDecimal}: </span>`);
+        if (/\bnote\b/.test(el.attribs.class))
+          $(el).find(".marker").append(` (new in WCAG ${scope.versionDecimal})`);
+        else
+          $(el).prepend(`<span class="new-version">New in WCAG ${scope.versionDecimal}: </span>`);
       }
       // Output as-is if content pertains to a version older than what's being built
     });

11ty/README.md

@@ -49,12 +49,17 @@ but may be useful if you're not seeing what you expect in the output files.
 
 ### `WCAG_VERSION`
 
-**Usage context:** currently this should not be changed, pending future improvements to `21` support
+**Usage context:** for building older versions of techniques and understanding docs
 
 Indicates WCAG version being built, in `XY` format (i.e. no `.`).
-Influences base URLs for links to guidelines, techniques, and understanding pages.
-
-**Default:** `22`
+Influences which pages get included, guideline/SC content,
+and a few details within pages (e.g. titles/URLs, "New in ..." content).
+Also influences base URLs for links to guidelines, techniques, and understanding pages.
+Explicitly setting this causes the build to reference guidelines and acknowledgements
+published under `w3.org/TR/WCAG{version}`, rather than using the local checkout
+(which is effectively the 2.2 Editors' Draft).
+
+Possible values: `22`, `21`
 
 ### `WCAG_MODE`
 

11ty/cheerio.ts

@@ -5,6 +5,9 @@ import { dirname, resolve } from "path";
 
 export { load } from "cheerio";
 
+/** Superset of the type returned by any Cheerio $() call. */
+export type CheerioAnyNode = ReturnType<ReturnType<typeof load>>;
+
 /** Convenience function that combines readFile and load. */
 export const loadFromFile = async (
   inputPath: string,

11ty/guidelines.ts

@@ -1,10 +1,11 @@
-import type { Cheerio, Element } from "cheerio";
+import axios from "axios";
+import type { CheerioAPI } from "cheerio";
 import { glob } from "glob";
 
 import { readFile } from "fs/promises";
 import { basename } from "path";
 
-import { flattenDomFromFile, load } from "./cheerio";
+import { flattenDomFromFile, load, type CheerioAnyNode } from "./cheerio";
 import { generateId } from "./common";
 
 export type WcagVersion = "20" | "21" | "22";
@@ -34,40 +35,21 @@ export const actRules = (
 )["act-rules"];
 
 /**
- * Returns an object with keys for each existing WCAG 2 version,
- * each mapping to an array of basenames of HTML files under understanding/<version>
- * (Functionally equivalent to "guidelines-versions" target in build.xml)
+ * Flattened object hash, mapping each WCAG 2 SC slug to the earliest WCAG version it applies to.
+ * (Functionally equivalent to "guidelines-versions" target in build.xml; structurally inverted)
  */
-export async function getGuidelinesVersions() {
+const scVersions = await (async function () {
   const paths = await glob("*/*.html", { cwd: "understanding" });
-  const versions: Record<WcagVersion, string[]> = { "20": [], "21": [], "22": [] };
+  const map: Record<string, WcagVersion> = {};
 
   for (const path of paths) {
-    const [version, filename] = path.split("/");
-    assertIsWcagVersion(version);
-    versions[version].push(basename(filename, ".html"));
+    const [fileVersion, filename] = path.split("/");
+    assertIsWcagVersion(fileVersion);
+    map[basename(filename, ".html")] = fileVersion;
   }
 
-  for (const version of Object.keys(versions)) {
-    assertIsWcagVersion(version);
-    versions[version].sort();
-  }
-  return versions;
-}
-
-/**
- * Like getGuidelinesVersions, but mapping each basename to the version it appears in
- */
-export async function getInvertedGuidelinesVersions() {
-  const versions = await getGuidelinesVersions();
-  const invertedVersions: Record<string, string> = {};
-  for (const [version, basenames] of Object.entries(versions)) {
-    for (const basename of basenames) {
-      invertedVersions[basename] = version;
-    }
-  }
-  return invertedVersions;
-}
+  return map;
+})();
 
 export interface DocNode {
   id: string;
@@ -79,15 +61,15 @@ export interface DocNode {
 export interface Principle extends DocNode {
   content: string;
   num: `${number}`; // typed as string for consistency with guidelines/SC
-  version: "WCAG20";
+  version: "20";
   guidelines: Guideline[];
   type: "Principle";
 }
 
 export interface Guideline extends DocNode {
   content: string;
   num: `${Principle["num"]}.${number}`;
-  version: `WCAG${"20" | "21"}`;
+  version: "20" | "21";
   successCriteria: SuccessCriterion[];
   type: "Guideline";
 }
@@ -97,50 +79,63 @@ export interface SuccessCriterion extends DocNode {
   num: `${Guideline["num"]}.${number}`;
   /** Level may be empty for obsolete criteria */
   level: "A" | "AA" | "AAA" | "";
-  version: `WCAG${WcagVersion}`;
+  version: WcagVersion;
   type: "SC";
 }
 
 export function isSuccessCriterion(criterion: any): criterion is SuccessCriterion {
   return !!(criterion?.type === "SC" && "level" in criterion);
 }
 
+/** Version-dependent overrides of SC shortcodes for older versions */
+export const scSlugOverrides: Record<string, (version: WcagVersion) => string> = {
+  "target-size-enhanced": (version) => (version < "22" ? "target-size" : "target-size-enhanced"),
+};
+
+/** Selectors ignored when capturing content of each Principle / Guideline / SC */
+const contentIgnores = [
+  "h1, h2, h3, h4, h5, h6",
+  "section",
+  ".change",
+  ".conformance-level",
+  // Selectors below are specific to pre-published guidelines (for previous versions)
+  ".header-wrapper",
+  ".doclinks",
+];
+
 /**
- * Returns HTML content used for Understanding guideline/SC boxes.
+ * Returns HTML content used for Understanding guideline/SC boxes and term definitions.
  * @param $el Cheerio element of the full section from flattened guidelines/index.html
  */
-const getContentHtml = ($el: Cheerio<Element>) => {
+const getContentHtml = ($el: CheerioAnyNode) => {
   // Load HTML into a new instance, remove elements we don't want, then return the remainder
   const $ = load($el.html()!, null, false);
-  $("h1, h2, h3, h4, h5, h6, section, .change, .conformance-level").remove();
-  return $.html();
+  $(contentIgnores.join(", ")).remove();
+  return $.html().trim();
 };
 
-/**
- * Resolves information from guidelines/index.html;
- * comparable to the principles section of wcag.xml from the guidelines-xml Ant task.
- */
-export async function getPrinciples() {
-  const versions = await getInvertedGuidelinesVersions();
-  const $ = await flattenDomFromFile("guidelines/index.html");
-
+/** Performs processing common across WCAG versions */
+function processPrinciples($: CheerioAPI) {
   const principles: Principle[] = [];
   $(".principle").each((i, el) => {
     const guidelines: Guideline[] = [];
-    $(".guideline", el).each((j, guidelineEl) => {
+    $("> .guideline", el).each((j, guidelineEl) => {
       const successCriteria: SuccessCriterion[] = [];
-      $(".sc", guidelineEl).each((k, scEl) => {
-        const resolvedVersion = versions[scEl.attribs.id];
-        assertIsWcagVersion(resolvedVersion);
-
+      // Source uses sc class, published uses guideline class (again)
+      $("> .guideline, > .sc", guidelineEl).each((k, scEl) => {
+        const scId = scEl.attribs.id;
         successCriteria.push({
           content: getContentHtml($(scEl)),
-          id: scEl.attribs.id,
+          id: scId,
           name: $("h4", scEl).text().trim(),
           num: `${i + 1}.${j + 1}.${k + 1}`,
-          level: $("p.conformance-level", scEl).text().trim() as SuccessCriterion["level"],
+          // conformance-level contains only letters in source, full (Level ...) in publish
+          level: $("p.conformance-level", scEl)
+            .text()
+            .trim()
+            .replace(/^\(Level (.*)\)$/, "$1") as SuccessCriterion["level"],
           type: "SC",
-          version: `WCAG${resolvedVersion}`,
+          version: scVersions[scId],
         });
       });
 
@@ -150,7 +145,7 @@ export async function getPrinciples() {
         name: $("h3", guidelineEl).text().trim(),
         num: `${i + 1}.${j + 1}`,
         type: "Guideline",
-        version: guidelineEl.attribs.id === "input-modalities" ? "WCAG21" : "WCAG20",
+        version: guidelineEl.attribs.id === "input-modalities" ? "21" : "20",
         successCriteria,
       });
     });
@@ -161,14 +156,21 @@ export async function getPrinciples() {
       name: $("h2", el).text().trim(),
       num: `${i + 1}`,
       type: "Principle",
-      version: "WCAG20",
+      version: "20",
       guidelines,
     });
   });
 
   return principles;
 }
 
+/**
+ * Resolves information from guidelines/index.html;
+ * comparable to the principles section of wcag.xml from the guidelines-xml Ant task.
+ */
+export const getPrinciples = async () =>
+  processPrinciples(await flattenDomFromFile("guidelines/index.html"));
+
 /**
  * Returns a flattened object hash, mapping shortcodes to each principle/guideline/SC.
  */
@@ -225,3 +227,62 @@ export async function getTermsMap() {
 
   return terms;
 }
+
+// Version-specific APIs
+
+const remoteGuidelines$: Partial<Record<WcagVersion, CheerioAPI>> = {};
+
+/** Loads guidelines from TR space for specific version, caching for future calls. */
+const loadRemoteGuidelines = async (version: WcagVersion) => {
+  if (!remoteGuidelines$[version]) {
+    const $ = load(
+      (await axios.get(`https://www.w3.org/TR/WCAG${version}/`, { responseType: "text" })).data
+    );
+
+    // Re-collapse definition links and notes, to be processed by this build system
+    $(".guideline a.internalDFN").removeAttr("class data-link-type id href title");
+    $(".guideline [role='note'] .marker").remove();
+    $(".guideline [role='note']").find("> div, > p").addClass("note").unwrap();
+
+    // Bibliography references are not processed in Understanding SC boxes
+    $(".guideline cite:has(a.bibref:only-child)").each((_, el) => {
+      const $el = $(el);
+      const $parent = $el.parent();
+      $el.remove();
+      // Remove surrounding square brackets (which aren't in a dedicated element)
+      $parent.html($parent.html()!.replace(/ \[\]/g, ""));
+    });
+
+    // Remove extra markup from headings so they can be parsed for names
+    $("bdi").remove();
+
+    // Remove abbr elements which exist only in TR, not in informative docs
+    $("#acknowledgements li abbr").each((_, abbrEl) => {
+      $(abbrEl).replaceWith($(abbrEl).text());
+    });
+
+    remoteGuidelines$[version] = $;
+  }
+  return remoteGuidelines$[version]!;
+};
+
+/**
+ * Retrieves heading and content information for acknowledgement subsections,
+ * for preserving the section in About pages for earlier versions.
+ */
+export const getAcknowledgementsForVersion = async (version: WcagVersion) => {
+  const $ = await loadRemoteGuidelines(version);
+  const subsections: Record<string, string> = {};
+
+  $("section#acknowledgements section").each((_, el) => {
+    subsections[el.attribs.id] = $(".header-wrapper + *", el).html()!;
+  });
+
+  return subsections;
+};
+
+/**
+ * Retrieves and processes a pinned WCAG version using published guidelines.
+ */
+export const getPrinciplesForVersion = async (version: WcagVersion) =>
+  processPrinciples(await loadRemoteGuidelines(version));