Add "freeze patch" mechanism, add one for filter-effects-1 (#1505)

This creates a new way to curate the results of a crawl: a freeze patch
allows to freeze the results of a crawl to a specific commit ID in Webref. This
is meant for specs that are temporarily broken beyond repair.

First such patch is for the filter-effects-1 spec.

Each freeze patch is implemented as a JSON file named after the spec's
shortname and that contains the commit ID and a `pending` key that links to the
issue that tracks the problem in some GitHub repository. The `clean-patches`
script proposes to drop the patch when the issue gets closed.

Author: tidoust

.github/workflows/test.yml

@@ -7,6 +7,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        # Need to checkout all history for curation job
+        fetch-depth: 0
     - uses: actions/setup-node@v4
       with:
         node-version: 20

ed/freezepatches/README.md

@@ -0,0 +1,8 @@
+# Freeze patches
+
+These are patches applied to specs to freeze all the extracts for that spec to a past result, identified by a commit ID.
+
+Each patch should be a JSON file named after the spec's shortname that defines a JSON object with two keys:
+
+- `commit`: The full commit ID in Webref that identifies the crawl results to use for the spec.
+- `pending`: The URL of an issue that tracks the problem. This allows to detect when the patch is no longer needed.

ed/freezepatches/filter-effects-1.json

@@ -0,0 +1,4 @@
+{
+  "commit": "647faa1643647b66b31a505e5b247a8695955352",
+  "pending": "https://github.com/w3c/fxtf-drafts/issues/591"
+}

tools/apply-patches.js

@@ -19,7 +19,7 @@ import path from 'node:path';
 import util from 'node:util';
 import { fileURLToPath } from 'node:url';
 import { execFile as execCb } from 'node:child_process';
-import { createFolderIfNeeded } from './utils.js';
+import { createFolderIfNeeded, loadJSON, getTargetedExtracts } from './utils.js';
 const execFile = util.promisify(execCb);
 
 async function applyPatches(rawFolder, outputFolder, type) {
@@ -61,6 +61,7 @@ async function applyPatches(rawFolder, outputFolder, type) {
   ];
 
   await createFolderIfNeeded(outputFolder);
+  await applyFreezePatches(rawFolder, outputFolder);
 
   for (const { name, srcDir, dstDir, patchDir, fileExt } of packages) {
     if (!type.includes(name)) {
@@ -100,6 +101,66 @@ async function applyPatches(rawFolder, outputFolder, type) {
 }
 
 
+/**
+ * Apply "freeze" patches, which freeze curation data for a spec to the results
+ * of a previous crawl result, identified by a commit ID.
+ *
+ * Freeze patches are meant to be used for specs that are (hopefully
+ * temporarily) severely broken.
+ */
+async function applyFreezePatches(rawFolder, outputFolder) {
+  const patchDir = path.join(rawFolder, 'freezepatches');
+  const patchFiles = await fs.readdir(patchDir);
+
+  const outputIndex = await loadJSON(path.join(outputFolder, 'index.json'));
+  let patchApplied = false;
+
+  for (const file of patchFiles) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
+    const shortname = file.replace(/\.json$/, '');
+    const patch = path.join(patchDir, file);
+    const json = await loadJSON(patch);
+
+    console.log(`Applying ${path.relative(rawFolder, patch)}`);
+    const outputSpecPos = outputIndex.results.findIndex(spec => spec.shortname === shortname);
+
+    // Get back to the patch commit
+    // (note this does not touch the `curated` folder because it is in
+    // the `.gitignore` file)
+    await execFile('git', ['checkout', json.commit]);
+
+    const crawlIndex = await loadJSON(path.join(rawFolder, 'index.json'));
+    const crawlSpec = crawlIndex.results.find(spec => spec.shortname === shortname);
+
+    for (const propValue of Object.values(crawlSpec)) {
+      const extractFiles = getTargetedExtracts(propValue);
+      for (const extractFile of extractFiles) {
+        await fs.copyFile(
+          path.join(rawFolder, extractFile),
+          path.join(outputFolder, extractFile)
+        );
+      }
+      outputIndex.results.splice(outputSpecPos, 1, crawlSpec);
+    }
+
+    await execFile('git', ['checkout', 'main']);
+    patchApplied = true;
+  }
+
+  // Update curated version of the index.json file
+  if (patchApplied) {
+    await fs.writeFile(
+      path.join(outputFolder, 'index.json'),
+      JSON.stringify(outputIndex, null, 2),
+      'utf8'
+    );
+  }
+}
+
+
 /**************************************************
 Export methods for use as module
 **************************************************/

tools/clean-patches.js

@@ -30,7 +30,7 @@ async function dropPatchesWhenPossible() {
     if (subDir.endsWith("patches")) {
       const files = fs.readdirSync(path.join(rootDir, subDir));
       for (const file of files) {
-        if (file.endsWith(".patch")) {
+        if (file.endsWith(".patch") || file.endsWith(".json")) {
           const patch = path.join(subDir, file);
           console.log(`- add "${patch}"`);
           patches.push({ name: patch });
@@ -44,9 +44,16 @@ async function dropPatchesWhenPossible() {
   const diffStart = /^---$/m;
   const issueUrl = /(?<=^|\s)https:\/\/github\.com\/([^\/]+)\/([^\/]+)\/(issues|pull)\/(\d+)(?=\s|$)/g;
   for (const patch of patches) {
+    let patchIssues;
     const contents = fs.readFileSync(path.join(rootDir, patch.name), "utf8");
-    const desc = contents.substring(0, contents.match(diffStart)?.index);
-    const patchIssues = [...desc.matchAll(issueUrl)];
+    if (patch.name.endsWith(".json")) {
+      const json = JSON.parse(contents);
+      patchIssues = [...(json.pending ?? '').matchAll(issueUrl)];
+    }
+    else {
+      const desc = contents.substring(0, contents.match(diffStart)?.index);
+      patchIssues = [...desc.matchAll(issueUrl)];
+    }
     for (const patchIssue of patchIssues) {
       if (!patch.issues) {
         patch.issues = [];

tools/prepare-curated.js

@@ -23,7 +23,8 @@ import { rimraf } from 'rimraf';
 import {
   createFolderIfNeeded,
   loadJSON,
-  copyFolder } from './utils.js';
+  copyFolder,
+  getTargetedExtracts } from './utils.js';
 import { applyPatches } from './apply-patches.js';
 import { dropCSSPropertyDuplicates } from './drop-css-property-duplicates.js';
 import { curateEvents } from './amend-event-data.js';
@@ -35,10 +36,9 @@ import { crawlSpecs } from 'reffy';
 */
 async function removeFromCuration(spec, curatedFolder) {
   for (const property of ['cddl', 'css', 'elements', 'events', 'idl']) {
-    if (spec[property] &&
-        (typeof spec[property] === 'string') &&
-        spec[property].match(/^[^\/]+\/[^\/]+\.(json|idl|cddl)$/)) {
-      const filename = path.join(curatedFolder, spec[property]);
+    const extractFiles = getTargetedExtracts(spec[property]);
+    for (const extractFile of extractFiles) {
+      const filename = path.join(curatedFolder, extractFile);
       console.log(`Removing ${spec.standing} ${spec.title} from curation: del ${filename}`);
       await fs.unlink(filename);
     }
@@ -56,16 +56,15 @@ async function removeFromCuration(spec, curatedFolder) {
 async function cleanCrawlOutcome(spec) {
   for (const property of Object.keys(spec)) {
     // Only consider properties that link to an extract
-    if (spec[property] &&
-        (typeof spec[property] === 'string') &&
-        spec[property].match(/^[^\/]+\/[^\/]+\.(json|idl|cddl)$/)) {
-      try {
-        await fs.lstat(path.join(curatedFolder, spec[property]));
-      }
-      catch (err) {
-        delete spec[property];
+    const extractFiles = getTargetedExtracts(spec[property]);
+    try {
+      for (const extractFile of extractFiles) {
+        await fs.lstat(path.join(curatedFolder, extractFile));
       }
     }
+    catch (err) {
+      delete spec[property];
+    }
   }
 }
 

tools/utils.js

@@ -68,8 +68,30 @@ async function copyFolder(source, target, { excludeRoot = false } = {}) {
 };
 
 
+/**
+ * Return the list of extract files that the given value targets.
+ *
+ * Note: The `cddl` property value targets an array of extracts, the actual
+ * extract being under the `file` key each time.
+ */
+function getTargetedExtracts(value) {
+  const reExtractFile = /^[^\/]+\/[^\/]+\.(json|idl|cddl)$/;
+  if (Array.isArray(value)) {
+    return value
+      .filter(v => typeof v.file === 'string' && v.file.match(reExtractFile))
+      .map(v => v.file);
+  }
+  else if (typeof value === 'string' && value.match(reExtractFile)) {
+    return [value];
+  }
+  else {
+    return [];
+  }
+}
+
 export {
   createFolderIfNeeded,
   loadJSON,
-  copyFolder
+  copyFolder,
+  getTargetedExtracts
 };