Automate generating techniques changelog

11ty/CustomLiquid.ts

@@ -17,7 +17,7 @@ import { techniqueToUnderstandingLinkSelector } from "./understanding";
 const titleSuffix = " | WAI | W3C";
 
 /** Matches index and about pages, traditionally processed differently than individual pages */
-const indexPattern = /(techniques|understanding)\/(index|about)\.html$/;
+const indexPattern = /(techniques|understanding)\/(index|about|changelog)\.html$/;
 const techniquesPattern = /\btechniques\//;
 const understandingPattern = /\bunderstanding\//;
 

11ty/README.md

@@ -23,11 +23,25 @@ Common tasks:
   - http://localhost:8080/techniques
   - http://localhost:8080/understanding
 
-Maintenance tasks (for working with Eleventy config and supporting files under this subdirectory):
+Maintenance tasks (for working with Eleventy config and supporting files used by the build):
 
 - `npm run check` checks for TypeScript errors
 - `npm run fmt` formats all TypeScript files
 
+## Publishing to WAI website
+
+The following npm scripts can be used to assist with publishing updates to the WAI website:
+
+- `npm run publish-w3c` to publish 2.2
+- `npm run publish-w3c:21` to publish 2.1
+
+Each of these scripts performs the following steps:
+
+1. Updates the data used for the Techniques Change Log page
+   - Note that this step may result in changes to `techniques/changelog.11tydata.json`, which should be committed to `main`
+2. Runs the build for the appropriate WCAG version, generating pages and `wcag.json` under `_site`
+3. Copies the built files from `_site` to the CVS checkout (see [`WCAG_CVSDIR`](#wcag_cvsdir))
+
 ## Environment Variables
 
 ### `WCAG_CVSDIR`

11ty/data-dependencies.ts

@@ -0,0 +1,111 @@
+import {
+  assertIsWcagVersion,
+  getFlatGuidelines,
+  getPrinciples,
+  getPrinciplesForVersion,
+  type FlatGuidelinesMap,
+} from "./guidelines";
+import {
+  getTechniquesByTechnology,
+  getFlatTechniques,
+  getTechniqueAssociations,
+  isTechniqueObsolete,
+  type Technology,
+} from "./techniques";
+import { getUnderstandingDocs, generateUnderstandingNavMap } from "./understanding";
+
+/**
+ * Central function that loads data necessary for the Eleventy build process,
+ * exposed for reuse in separate scripts (e.g. techniques changelog generation).
+ * @param version A version string, e.g. from process.env.WCAG_VERSION; may be undefined
+ */
+export async function loadDataDependencies(version?: string) {
+  const definedVersion = version ?? "22";
+  assertIsWcagVersion(definedVersion);
+
+  /** Tree of Principles/Guidelines/SC across all versions (including later than selected) */
+  const allPrinciples = await getPrinciples();
+  const allFlatGuidelines = getFlatGuidelines(allPrinciples);
+
+  async function resolveRelevantPrinciples() {
+    // Resolve from local filesystem if input version was unset (e.g. no explicit WCAG_VERSION)
+    if (!version) return allPrinciples;
+    // Also resolve from local filesystem if explicit path was given via environment variable
+    if (process.env.WCAG_FORCE_LOCAL_GUIDELINES)
+      return await getPrinciples(process.env.WCAG_FORCE_LOCAL_GUIDELINES);
+    // Otherwise, resolve from published guidelines (requires internet connection)
+    assertIsWcagVersion(definedVersion);
+    return await getPrinciplesForVersion(definedVersion);
+  }
+
+  // Note: many of these variables are documented in the return value instead of up-front,
+  // in order to provide docs to consumers of this function
+
+  const principles = await resolveRelevantPrinciples();
+  const flatGuidelines = getFlatGuidelines(principles);
+  /** Flattened Principles/Guidelines/SC that only exist in later versions (to filter techniques) */
+  const futureGuidelines: FlatGuidelinesMap = {};
+  for (const [key, value] of Object.entries(allFlatGuidelines)) {
+    if (value.version > definedVersion) futureGuidelines[key] = value;
+  }
+
+  const techniques = await getTechniquesByTechnology(flatGuidelines);
+  const flatTechniques = getFlatTechniques(techniques);
+
+  const techniqueAssociations = await getTechniqueAssociations(flatGuidelines);
+  const futureTechniqueAssociations = await getTechniqueAssociations(futureGuidelines);
+  const futureExclusiveTechniqueAssociations: typeof techniqueAssociations = {};
+  for (const [id, associations] of Object.entries(futureTechniqueAssociations)) {
+    if (!techniqueAssociations[id]) futureExclusiveTechniqueAssociations[id] = associations;
+  }
+
+  for (const [id, associations] of Object.entries(techniqueAssociations)) {
+    // Prune associations from non-obsolete techniques to obsolete SCs
+    techniqueAssociations[id] = associations.filter(
+      ({ criterion }) =>
+        criterion.level !== "" || isTechniqueObsolete(flatTechniques[id], definedVersion)
+    );
+  }
+
+  for (const [technology, list] of Object.entries(techniques)) {
+    // Prune techniques that are obsolete or associated with SCs from later versions
+    // (only prune hierarchical structure for ToC; keep all in flatTechniques for lookups)
+    techniques[technology as Technology] = list.filter(
+      (technique) =>
+        (!technique.obsoleteSince || technique.obsoleteSince > definedVersion) &&
+        !futureExclusiveTechniqueAssociations[technique.id]
+    );
+  }
+
+  const understandingDocs = getUnderstandingDocs(definedVersion);
+  const understandingNav = generateUnderstandingNavMap(principles, understandingDocs);
+
+  return {
+    /** Tree of Principles/Guidelines/SC relevant to selected version */
+    principles,
+    /** Flattened Principles/Guidelines/SC relevant to selected version */
+    flatGuidelines,
+    /** Flattened Principles/Guidelines/SC across all versions (including later than selected) */
+    allFlatGuidelines,
+    /**
+     * Techniques organized hierarchically per technology;
+     * excludes obsolete techniques and techniques that only reference SCs of future WCAG versions
+     **/
+    techniques,
+    /**
+     * Techniques organized in a flat map indexed by ID;
+     * _does not_ exclude obsolete/future techniques (so that cross-page links can be resolved)
+     */
+    flatTechniques,
+    /** Maps technique IDs to SCs found in target version */
+    techniqueAssociations,
+    /** Only maps techniques that only reference SCs introduced in future WCAG versions */
+    futureExclusiveTechniqueAssociations,
+    /** Information for other top-level understanding pages */
+    understandingDocs,
+    /** Contains next/previous/parent information for each understanding page, for rendering nav */
+    understandingNav,
+    // Pass back version so the consumer doesn't need to re-assert
+    version: definedVersion,
+  };
+}

11ty/generate-techniques-changelog-data.ts

@@ -0,0 +1,187 @@
+import { exec } from "child_process";
+import { access, readFile, writeFile } from "fs/promises";
+import { glob } from "glob";
+import { basename, join } from "path";
+import { promisify } from "util";
+
+import type { WcagVersion } from "./guidelines";
+import { technologies, technologyTitles } from "./techniques";
+import { loadDataDependencies } from "./data-dependencies";
+
+const execAsync = promisify(exec);
+
+// Only list changes since initial 2.1 rec publication.
+// Note: this script intentionally searches the full commit list,
+// then later filters by date, rather than truncating the list of commits.
+// The latter would cause additional edge cases
+// (e.g. due to a few files being re-added after deletion)
+// and would only save ~10% of total run time.
+const sinceDate = new Date(2018, 5, 6);
+
+interface Entry {
+  date: Date;
+  hash: string;
+  technique: string;
+  type: "added" | "removed";
+}
+
+interface CompoundEntry extends Omit<Entry, "technique"> {
+  techniques: string[];
+}
+
+const excludes = {
+  G222: true, // Was added then removed, related to a SC that was never published
+};
+
+/** Checks for existence and non-obsolescene of technique. */
+const checkTechnique = async (path: string) =>
+  access(path).then(
+    () => readFile(path, "utf8").then((content) => !/^obsoleteSince:/m.test(content)),
+    () => false
+  );
+
+function resolveEarliest(a: EarliestInfo | null, b: EarliestInfo | null) {
+  if (a && b) {
+    if (a.date > b.date) return b;
+    return a;
+  }
+  if (a || b) return a || b;
+  return null;
+}
+
+interface EarliestInfo {
+  date: Date;
+  hash: string;
+}
+
+// Cache getEarliestDate results to save time processing across versions
+const earliestCache: Record<string, EarliestInfo | null> = {};
+
+// Common function called by getEarlest...Date functions below
+async function getEarliest(path: string, logArgs: string) {
+  const cacheKey = `${path} ${logArgs}`;
+  if (!(cacheKey in earliestCache)) {
+    const command = `git log ${logArgs} --format='%h %ad' --date=iso-strict -- ${path} | tail -1`;
+    const output = (await execAsync(command)).stdout.trim();
+    if (output) {
+      const match = /^(\w+) (.+)$/.exec(output);
+      if (!match) throw new Error("Unexpected git log output format");
+      earliestCache[cacheKey] = { date: new Date(match[2]), hash: match[1] };
+    } else earliestCache[cacheKey] = null;
+  }
+  return earliestCache[cacheKey];
+}
+
+// --diff-filter=A is NOT used for additions because it misses odd merge commit cases e.g. F99.
+// (Not using it seems to have negligible performance difference and causes no other changes.)
+const getEarliestAddition = (path: string) => getEarliest(path, "");
+const getEarliestRemoval = (path: string) => getEarliest(path, "--diff-filter=D");
+const getEarliestObsolescence = (path: string, version: WcagVersion) => {
+  const valuePattern = `2[0-${version[1]}]`;
+  return getEarliest(
+    path,
+    `-G'${
+      path.endsWith(".json")
+        ? `"obsoleteSince": "${valuePattern}"`
+        : `obsoleteSince: ${valuePattern}`
+    }'`
+  );
+};
+
+async function generateChangelog(version: WcagVersion) {
+  const { futureExclusiveTechniqueAssociations } = await loadDataDependencies(version);
+  const entries: Entry[] = [];
+
+  for (const technology of technologies) {
+    const techniquesPath = join("techniques", technology);
+    const techniquesFiles = (await glob("*.html", { cwd: techniquesPath })).filter((filename) =>
+      /^[A-Z]+\d+\.html$/.test(filename)
+    );
+    if (!techniquesFiles.length) continue;
+
+    const code = techniquesFiles[0].replace(/\d+\.html$/, "");
+    const technologyObsolescence = await getEarliestObsolescence(
+      join(techniquesPath, `${technology}.11tydata.json`),
+      version
+    );
+    if (technologyObsolescence && technologyObsolescence.date > sinceDate) {
+      // The above check can handle future cases of marking an entire folder as obsolete;
+      // for legacy cases (Flash/SL), check the first technique file for removal
+      const technologyRemoval = await getEarliestRemoval(join(techniquesPath, `${code}1.html`));
+      entries.push({
+        date: (technologyRemoval || technologyObsolescence).date,
+        hash: (technologyRemoval || technologyObsolescence).hash,
+        technique: `all ${technologyTitles[technology]}`,
+        type: "removed",
+      });
+      continue;
+    }
+
+    const max = techniquesFiles.reduce((currentMax, filename) => {
+      const num = +basename(filename, ".html").replace(/^[A-Z]+/, "");
+      return Math.max(num, currentMax);
+    }, 0);
+
+    for (let i = 1; i <= max; i++) {
+      const techniquePath = join(techniquesPath, `${code}${i}.html`);
+      const technique = basename(techniquePath, ".html");
+      if (technique in excludes || technique in futureExclusiveTechniqueAssociations) continue;
+
+      const addition = await getEarliestAddition(techniquePath);
+      if (!addition) continue; // Skip if added prior to start date
+      if (addition.date > sinceDate)
+        entries.push({
+          ...addition,
+          technique,
+          type: "added",
+        });
+
+      const removal = (await checkTechnique(techniquePath))
+        ? null
+        : resolveEarliest(
+            await getEarliestRemoval(techniquePath),
+            await getEarliestObsolescence(techniquePath, version)
+          );
+      if (removal && removal.date > sinceDate)
+        entries.push({
+          ...removal,
+          technique,
+          type: "removed",
+        });
+    }
+  }
+
+  // Sort most-recent-first, tie-breaking by additions before removals
+  entries.sort((a, b) => {
+    if (a.date > b.date) return -1;
+    if (a.date < b.date) return 1;
+    if (a.type < b.type) return -1;
+    if (a.type > b.type) return 1;
+    return 0;
+  });
+
+  return entries.reduce((collectedEntries, { date, hash, technique, type }) => {
+    const previousEntry = collectedEntries[collectedEntries.length - 1];
+    if (previousEntry && previousEntry.type === type && +previousEntry.date === +date)
+      previousEntry.techniques.push(technique);
+    else collectedEntries.push({ date, hash, techniques: [technique], type });
+    return collectedEntries;
+  }, [] as CompoundEntry[]);
+}
+
+const startTime = Date.now();
+await writeFile(
+  join("techniques", `changelog.11tydata.json`),
+  JSON.stringify(
+    {
+      "//": "DO NOT EDIT THIS FILE; it is automatically generated",
+      changelog: {
+        "21": await generateChangelog("21"),
+        "22": await generateChangelog("22"),
+      },
+    },
+    null,
+    "  "
+  )
+);
+console.log(`Wrote changelog in ${Date.now() - startTime}ms`);

11ty/techniques.ts

@@ -41,6 +41,13 @@ function assertIsTechnology(
   if (!(technology in technologyTitles)) throw new Error(`Invalid technology name: ${technology}`);
 }
 
+/**
+ * Returns boolean indicating whether a technique is obsolete for the given version.
+ * Tolerates undefined for use with hash lookups.
+ */
+export const isTechniqueObsolete = (technique: Technique | undefined, version: WcagVersion) =>
+  !!technique?.obsoleteSince && technique.obsoleteSince <= version;
+
 export const techniqueAssociationTypes = ["sufficient", "advisory", "failure"] as const;
 export type TechniqueAssociationType = (typeof techniqueAssociationTypes)[number];
 

11ty/understanding.ts

@@ -15,7 +15,7 @@ export const techniqueToUnderstandingLinkSelector = [
  * Resolves information for top-level understanding pages;
  * ported from generate-structure-xml.xslt
  */
-export async function getUnderstandingDocs(version: WcagVersion): Promise<DocNode[]> {
+export function getUnderstandingDocs(version: WcagVersion): DocNode[] {
   const decimalVersion = resolveDecimalVersion(version);
   return [
     {

README.md

@@ -219,9 +219,8 @@ appended to the "Note" title in applicable versions, and the note will be hidden
 
 ### Techniques Change Log
 
-At the time of writing (November 2024), the Change Log in the Techniques index is identical between WCAG 2.1 and 2.2.
-These have been split out into separate version-specific includes under `_includes/techniques/changelog/*.html`
-for future-proofing in support of building multiple versions of informative documents from the same branch.
+Data for the Techniques Change Log is now generated automatically by a script that reads git history;
+see [Eleventy Usage](11ty/README.md#usage).
 
 ## Working Examples