Add suggested changes, fix removeFromCuration for CDDL

This generalizes the logic that parses property values of a crawn index to
gather extract files, and applies it throughout. This actually fixes the
`removeFromCuration` logic that was broken for CDDL.

Author: tidoust

tools/apply-patches.js

@@ -19,7 +19,7 @@ import path from 'node:path';
 import util from 'node:util';
 import { fileURLToPath } from 'node:url';
 import { execFile as execCb } from 'node:child_process';
-import { createFolderIfNeeded, loadJSON } from './utils.js';
+import { createFolderIfNeeded, loadJSON, getTargetedExtracts } from './utils.js';
 const execFile = util.promisify(execCb);
 
 async function applyPatches(rawFolder, outputFolder, type) {
@@ -121,7 +121,6 @@ async function applyFreezePatches(rawFolder, outputFolder) {
     }
 
     const shortname = file.replace(/\.json$/, '');
-    const branchName = `freezepatch-${shortname}`;
     const patch = path.join(patchDir, file);
     const json = await loadJSON(patch);
 
@@ -131,28 +130,14 @@ async function applyFreezePatches(rawFolder, outputFolder) {
     // Get back to the patch commit
     // (note this does not touch the `curated` folder because it is in
     // the `.gitignore` file)
-    await execFile('git', ['checkout', '-B', branchName, json.commit]);
+    await execFile('git', ['checkout', json.commit]);
 
     const crawlIndex = await loadJSON(path.join(rawFolder, 'index.json'));
     const crawlSpec = crawlIndex.results.find(spec => spec.shortname === shortname);
 
-    for (const [extractType, extractFile] of Object.entries(crawlSpec)) {
-      if (extractType === 'cddl') {
-        // Handle CDDL extracts separately, it's an array of extracts
-        for (const { file: cddlFile } of extractFile) {
-          await fs.copyFile(
-            path.join(rawFolder, cddlFile),
-            path.join(outputFolder, cddlFile)
-          );
-        }
-      }
-      else if (!extractFile ||
-          (typeof extractFile !== 'string') ||
-          !extractFile.match(/^[^\/]+\/[^\/]+\.(json|idl)$/)) {
-        // Skip properties that do not link to an extract
-        continue;
-      }
-      else {
+    for (const propValue of Object.values(crawlSpec)) {
+      const extractFiles = getTargetedExtracts(propValue);
+      for (const extractFile of extracFiles) {
         await fs.copyFile(
           path.join(rawFolder, extractFile),
           path.join(outputFolder, extractFile)
@@ -162,7 +147,6 @@ async function applyFreezePatches(rawFolder, outputFolder) {
     }
 
     await execFile('git', ['checkout', 'main']);
-    await execFile('git', ['branch', '-D', branchName]);
     patchApplied = true;
   }
 

tools/prepare-curated.js

@@ -23,7 +23,8 @@ import { rimraf } from 'rimraf';
 import {
   createFolderIfNeeded,
   loadJSON,
-  copyFolder } from './utils.js';
+  copyFolder,
+  getTargetedExtracts } from './utils.js';
 import { applyPatches } from './apply-patches.js';
 import { dropCSSPropertyDuplicates } from './drop-css-property-duplicates.js';
 import { curateEvents } from './amend-event-data.js';
@@ -35,10 +36,9 @@ import { crawlSpecs } from 'reffy';
 */
 async function removeFromCuration(spec, curatedFolder) {
   for (const property of ['cddl', 'css', 'elements', 'events', 'idl']) {
-    if (spec[property] &&
-        (typeof spec[property] === 'string') &&
-        spec[property].match(/^[^\/]+\/[^\/]+\.(json|idl|cddl)$/)) {
-      const filename = path.join(curatedFolder, spec[property]);
+    const extractFiles = getTargetedExtracts(spec[property]);
+    for (const extractFile of extractFiles) {
+      const filename = path.join(curatedFolder, extractFile);
       console.log(`Removing ${spec.standing} ${spec.title} from curation: del ${filename}`);
       await fs.unlink(filename);
     }
@@ -56,16 +56,15 @@ async function removeFromCuration(spec, curatedFolder) {
 async function cleanCrawlOutcome(spec) {
   for (const property of Object.keys(spec)) {
     // Only consider properties that link to an extract
-    if (spec[property] &&
-        (typeof spec[property] === 'string') &&
-        spec[property].match(/^[^\/]+\/[^\/]+\.(json|idl|cddl)$/)) {
-      try {
-        await fs.lstat(path.join(curatedFolder, spec[property]));
-      }
-      catch (err) {
-        delete spec[property];
+    const extractFiles = getTargetedExtracts(spec[property]);
+    try {
+      for (const extractFile of extractFiles) {
+        await fs.lstat(path.join(curatedFolder, extractFile));
       }
     }
+    catch (err) {
+      delete spec[property];
+    }
   }
 }
 

tools/utils.js

@@ -68,8 +68,30 @@ async function copyFolder(source, target, { excludeRoot = false } = {}) {
 };
 
 
+/**
+ * Return the list of extract files that the given value targets.
+ *
+ * Note: The `cddl` property value targets an array of extracts, the actual
+ * extract being under the `file` key each time.
+ */
+function getTargetedExtracts(value) {
+  const reExtractFile = /^[^\/]+\/[^\/]+\.(json|idl|cddl)$/;
+  if (Array.isArray(value)) {
+    return value
+      .filter(v => typeof v.file === 'string' && v.file.match(reExtractFile))
+      .map(v => v.file);
+  }
+  else if (typeof value === 'string') {
+    return [value];
+  }
+  else {
+    return [];
+  }
+}
+
 export {
   createFolderIfNeeded,
   loadJSON,
-  copyFolder
+  copyFolder,
+  getTargetedExtracts
 };