Improve separation of concerns

Extract the technical upgrade functionality into a standalone method

Author: Ndpnt

CHANGELOG.md

@@ -6,10 +6,14 @@ All changes that impact users of this module are documented in this file, in the
 
 > Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
 
+### Added
+
+- Add `applyTechnicalUpgrades()` function as a standalone method to apply technical upgrades without regular tracking
+- Add `ota apply-technical-upgrades` CLI command to apply technical upgrades independently
+
 ### Changed
 
-- **Breaking:** Rename `extractOnly` parameter to `technicalUpgradeOnly` in `track()` function to clarify that only the technical upgrade pass will run; ifyou are using the `track()` function directly with the `extractOnly` parameter, rename it to `technicalUpgradeOnly`
-- **Breaking:** Rename CLI flag from `--extract-only` to `--technical-upgrade-only` for consistency and clarity; if you are using the CLI with the `--extract-only` flag, rename it to `--technical-upgrade-only`
+- **Breaking:** Remove `--extract-only` option from `ota track` command; use the new `ota apply-technical-upgrades` command instead
 - Enhance technical upgrade pass to fetch missing snapshots for newly added source documents in combined terms declarations
 
 ## 9.2.0 - 2025-11-05

bin/ota-apply-technical-upgrades.js

@@ -0,0 +1,19 @@
+#! /usr/bin/env node
+import './env.js';
+
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+
+import { program } from 'commander';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const { applyTechnicalUpgrades } = await import(pathToFileURL(path.resolve(__dirname, '../src/index.js'))); // load asynchronously to ensure env.js is loaded before
+
+program
+  .name('ota apply-technical-upgrades')
+  .description('Apply technical upgrades: regenerate versions from existing snapshots with updated declarations/engine, and fetch missing snapshots for newly added source documents')
+  .option('-s, --services [serviceId...]', 'service IDs to apply technical upgrades to')
+  .option('-t, --types [termsType...]', 'terms types to apply technical upgrades to');
+
+applyTechnicalUpgrades(program.parse(process.argv).opts());

bin/ota-track.js

@@ -15,7 +15,6 @@ program
   .description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
   .option('-s, --services [serviceId...]', 'service IDs of services to track')
   .option('-t, --types [termsType...]', 'terms types to track')
-  .option('-u, --technical-upgrade-only', 'only apply technical upgrades: regenerate versions from existing snapshots with updated declarations/engine, and fetch missing snapshots for newly added source documents; skip regular tracking')
   .option('--schedule', 'track automatically at a regular interval');
 
 track(program.parse(process.argv).opts());

bin/ota.js

@@ -11,6 +11,7 @@ program
   .description(description)
   .version(version)
   .command('track', 'Track the current terms of services according to provided declarations')
+  .command('apply-technical-upgrades', 'Apply technical upgrades: regenerate versions from existing snapshots with updated declarations/engine')
   .command('validate', 'Run a series of tests to check the validity of terms declarations')
   .command('lint', 'Check format and stylistic errors in declarations and auto fix them')
   .command('dataset', 'Export the versions dataset into a ZIP file and optionally publish it to GitHub releases')

src/archivist/index.js

@@ -20,7 +20,7 @@ const { version: PACKAGE_VERSION } = require('../../package.json');
 // - too many requests on the same endpoint yield 403
 // - sometimes when creating a commit no SHA are returned for unknown reasons
 const MAX_PARALLEL_TRACKING = 1;
-const MAX_PARALLEL_EXTRACTING = 10;
+const MAX_PARALLEL_TECHNICAL_UPGRADES = 10;
 
 export const EVENTS = [
   'snapshotRecorded',
@@ -128,22 +128,22 @@ export default class Archivist extends events.EventEmitter {
     });
   }
 
-  async track({ services: servicesIds = this.servicesIds, types: termsTypes = [], technicalUpgradeOnly = false } = {}) {
+  async track({ services: servicesIds = this.servicesIds, types: termsTypes = [] } = {}) {
     const numberOfTerms = Service.getNumberOfTerms(this.services, servicesIds, termsTypes);
 
-    this.emit('trackingStarted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
+    this.emit('trackingStarted', servicesIds.length, numberOfTerms, false);
 
     await Promise.all([ launchHeadlessBrowser(), this.recorder.initialize() ]);
 
-    this.trackingQueue.concurrency = technicalUpgradeOnly ? MAX_PARALLEL_EXTRACTING : MAX_PARALLEL_TRACKING;
+    this.trackingQueue.concurrency = MAX_PARALLEL_TRACKING;
 
     servicesIds.forEach(serviceId => {
       this.services[serviceId].getTermsTypes().forEach(termsType => {
         if (termsTypes.length && !termsTypes.includes(termsType)) {
           return;
         }
 
-        this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), technicalUpgradeOnly });
+        this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), technicalUpgradeOnly: false });
       });
     });
 
@@ -153,7 +153,35 @@ export default class Archivist extends events.EventEmitter {
 
     await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize() ]);
 
-    this.emit('trackingCompleted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
+    this.emit('trackingCompleted', servicesIds.length, numberOfTerms, false);
+  }
+
+  async applyTechnicalUpgrades({ services: servicesIds = this.servicesIds, types: termsTypes = [] } = {}) {
+    const numberOfTerms = Service.getNumberOfTerms(this.services, servicesIds, termsTypes);
+
+    this.emit('trackingStarted', servicesIds.length, numberOfTerms, true);
+
+    await Promise.all([ launchHeadlessBrowser(), this.recorder.initialize() ]);
+
+    this.trackingQueue.concurrency = MAX_PARALLEL_TECHNICAL_UPGRADES;
+
+    servicesIds.forEach(serviceId => {
+      this.services[serviceId].getTermsTypes().forEach(termsType => {
+        if (termsTypes.length && !termsTypes.includes(termsType)) {
+          return;
+        }
+
+        this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), technicalUpgradeOnly: true });
+      });
+    });
+
+    if (this.trackingQueue.length()) {
+      await this.trackingQueue.drain();
+    }
+
+    await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize() ]);
+
+    this.emit('trackingCompleted', servicesIds.length, numberOfTerms, true);
   }
 
   async trackTermsChanges({ terms, technicalUpgradeOnly = false }) {

src/archivist/index.test.js

@@ -167,7 +167,7 @@ describe('Archivist', function () {
 
             app.services[SERVICE_A_ID].getTerms({ type: SERVICE_A_TYPE }).sourceDocuments[0].contentSelectors = 'h1';
 
-            await app.track({ services: [ 'service·A', 'Service B!' ], technicalUpgradeOnly: true });
+            await app.applyTechnicalUpgrades({ services: [ 'service·A', 'Service B!' ] });
 
             const [reExtractedVersionCommit] = await gitVersion.log({ file: SERVICE_A_EXPECTED_VERSION_FILE_PATH });
 
@@ -229,7 +229,7 @@ describe('Archivist', function () {
               }
               versionNotChangedSpy(record);
             });
-            await app.track({ services, technicalUpgradeOnly: true });
+            await app.applyTechnicalUpgrades({ services });
           });
 
           after(resetGitRepositories);

src/index.js

@@ -13,7 +13,7 @@ import Reporter from './reporter/index.js';
 const require = createRequire(import.meta.url);
 const { version: PACKAGE_VERSION } = require('../package.json');
 
-export default async function track({ services, types, technicalUpgradeOnly, schedule }) {
+export default async function track({ services, types, schedule }) {
   const archivist = new Archivist({
     recorderConfig: config.get('@opentermsarchive/engine.recorder'),
     fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
@@ -44,11 +44,7 @@ export default async function track({ services, types, technicalUpgradeOnly, sch
   // This regenerates versions from existing snapshots with updated extraction logic.
   // For terms with combined source documents, if a new document was added to the declaration, it will be fetched and combined with existing snapshots to regenerate the complete version.
   // All versions from this pass are labeled as technical upgrades to avoid false notifications about content changes.
-  await archivist.track({ services, types, technicalUpgradeOnly: true });
-
-  if (technicalUpgradeOnly) {
-    return;
-  }
+  await archivist.applyTechnicalUpgrades({ services, types });
 
   if (process.env.OTA_ENGINE_SENDINBLUE_API_KEY) {
     try {
@@ -91,3 +87,33 @@ export default async function track({ services, types, technicalUpgradeOnly, sch
     () => archivist.track({ services, types }),
   );
 }
+
+export async function applyTechnicalUpgrades({ services, types }) {
+  const archivist = new Archivist({
+    recorderConfig: config.get('@opentermsarchive/engine.recorder'),
+    fetcherConfig: config.get('@opentermsarchive/engine.fetcher'),
+  });
+
+  archivist.attach(logger);
+
+  await archivist.initialize();
+
+  const collection = await getCollection();
+  const collectionName = collection?.name ? ` with ${collection.name} collection` : '';
+
+  logger.info(`Start engine v${PACKAGE_VERSION}${collectionName}\n`);
+
+  if (services?.length) {
+    services = services.filter(serviceId => {
+      const isServiceDeclared = archivist.services[serviceId];
+
+      if (!isServiceDeclared) {
+        logger.warn(`Parameter "${serviceId}" was interpreted as a service ID to update, but no matching declaration was found; it will be ignored`);
+      }
+
+      return isServiceDeclared;
+    });
+  }
+
+  await archivist.applyTechnicalUpgrades({ services, types });
+}