Add "freeze patch" mechanism, add one for filter-effects-1

.github/workflows/test.yml

@@ -7,6 +7,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        # Need to checkout all history for curation job
+        fetch-depth: 0
     - uses: actions/setup-node@v4
       with:
         node-version: 20

ed/freezepatches/README.md

@@ -0,0 +1,8 @@
+# Freeze patches
+
+These are patches applied to specs to freeze all the extracts for that spec to a past result, identified by a commit ID.
+
+Each patch should be a JSON file named after the spec's shortname that defines a JSON object with two keys:
+
+- `commit`: The full commit ID in Webref that identifies the crawl results to use for the spec.
+- `pending`: The URL of an issue that tracks the problem. This allows to detect when the patch is no longer needed.

ed/freezepatches/filter-effects-1.json

@@ -0,0 +1,4 @@
+{
+  "commit": "647faa1643647b66b31a505e5b247a8695955352",
+  "pending": "https://github.com/w3c/fxtf-drafts/issues/591"
+}

tools/apply-patches.js

@@ -19,7 +19,7 @@ import path from 'node:path';
 import util from 'node:util';
 import { fileURLToPath } from 'node:url';
 import { execFile as execCb } from 'node:child_process';
-import { createFolderIfNeeded } from './utils.js';
+import { createFolderIfNeeded, loadJSON } from './utils.js';
 const execFile = util.promisify(execCb);
 
 async function applyPatches(rawFolder, outputFolder, type) {
@@ -61,6 +61,7 @@ async function applyPatches(rawFolder, outputFolder, type) {
   ];
 
   await createFolderIfNeeded(outputFolder);
+  await applyFreezePatches(rawFolder, outputFolder);
 
   for (const { name, srcDir, dstDir, patchDir, fileExt } of packages) {
     if (!type.includes(name)) {
@@ -100,6 +101,82 @@ async function applyPatches(rawFolder, outputFolder, type) {
 }
 
 
+/**
+ * Apply "freeze" patches, which freeze curation data for a spec to the results
+ * of a previous crawl result, identified by a commit ID.
+ *
+ * Freeze patches are meant to be used for specs that are (hopefully
+ * temporarily) severely broken.
+ */
+async function applyFreezePatches(rawFolder, outputFolder) {
+  const patchDir = path.join(rawFolder, 'freezepatches');
+  const patchFiles = await fs.readdir(patchDir);
+
+  const outputIndex = await loadJSON(path.join(outputFolder, 'index.json'));
+  let patchApplied = false;
+
+  for (const file of patchFiles) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
+    const shortname = file.replace(/\.json$/, '');
+    const branchName = `freezepatch-${shortname}`;
+    const patch = path.join(patchDir, file);
+    const json = await loadJSON(patch);
+
+    console.log(`Applying ${path.relative(rawFolder, patch)}`);
+    const outputSpecPos = outputIndex.results.findIndex(spec => spec.shortname === shortname);
+
+    // Get back to the patch commit
+    // (note this does not touch the `curated` folder because it is in
+    // the `.gitignore` file)
+    await execFile('git', ['checkout', '-B', branchName, json.commit]);
+
+    const crawlIndex = await loadJSON(path.join(rawFolder, 'index.json'));
+    const crawlSpec = crawlIndex.results.find(spec => spec.shortname === shortname);
+
+    for (const [extractType, extractFile] of Object.entries(crawlSpec)) {
+      if (extractType === 'cddl') {
+        // Handle CDDL extracts separately, it's an array of extracts
+        for (const { file: cddlFile } of extractFile) {
+          await fs.copyFile(
+            path.join(rawFolder, cddlFile),
+            path.join(outputFolder, cddlFile)
+          );
+        }
+      }
+      else if (!extractFile ||
+          (typeof extractFile !== 'string') ||
+          !extractFile.match(/^[^\/]+\/[^\/]+\.(json|idl)$/)) {
+        // Skip properties that do not link to an extract
+        continue;
+      }
+      else {
+        await fs.copyFile(
+          path.join(rawFolder, extractFile),
+          path.join(outputFolder, extractFile)
+        );
+      }
+      outputIndex.results.splice(outputSpecPos, 1, crawlSpec);
+    }
+
+    await execFile('git', ['checkout', 'main']);
+    await execFile('git', ['branch', '-D', branchName]);
+    patchApplied = true;
+  }
+
+  // Update curated version of the index.json file
+  if (patchApplied) {
+    await fs.writeFile(
+      path.join(outputFolder, 'index.json'),
+      JSON.stringify(outputIndex, null, 2),
+      'utf8'
+    );
+  }
+}
+
+
 /**************************************************
 Export methods for use as module
 **************************************************/

tools/clean-patches.js

@@ -30,7 +30,7 @@ async function dropPatchesWhenPossible() {
     if (subDir.endsWith("patches")) {
       const files = fs.readdirSync(path.join(rootDir, subDir));
       for (const file of files) {
-        if (file.endsWith(".patch")) {
+        if (file.endsWith(".patch") || file.endsWith(".json")) {
           const patch = path.join(subDir, file);
           console.log(`- add "${patch}"`);
           patches.push({ name: patch });
@@ -44,9 +44,16 @@ async function dropPatchesWhenPossible() {
   const diffStart = /^---$/m;
   const issueUrl = /(?<=^|\s)https:\/\/github\.com\/([^\/]+)\/([^\/]+)\/(issues|pull)\/(\d+)(?=\s|$)/g;
   for (const patch of patches) {
+    let patchIssues;
     const contents = fs.readFileSync(path.join(rootDir, patch.name), "utf8");
-    const desc = contents.substring(0, contents.match(diffStart)?.index);
-    const patchIssues = [...desc.matchAll(issueUrl)];
+    if (patch.name.endsWith(".json")) {
+      const json = JSON.parse(contents);
+      patchIssues = [...(json.pending ?? '').matchAll(issueUrl)];
+    }
+    else {
+      const desc = contents.substring(0, contents.match(diffStart)?.index);
+      patchIssues = [...desc.matchAll(issueUrl)];
+    }
     for (const patchIssue of patchIssues) {
       if (!patch.issues) {
         patch.issues = [];