Fix extractOnly step

Fetch missing snapshots for newly added source documents in combined terms declarations

Author: Ndpnt

src/archivist/index.js

@@ -159,6 +159,10 @@ export default class Archivist extends events.EventEmitter {
   async trackTermsChanges({ terms, extractOnly = false }) {
     if (!extractOnly) {
       await this.fetchAndRecordSnapshots(terms);
+    } else {
+      // In extractOnly mode (technical upgrade pass), fetch and record snapshots only for new source documents
+      // that don't have existing snapshots yet (e.g., when a declaration is updated to add a new source document)
+      await this.fetchAndRecordMissingSnapshots(terms);
     }
 
     const contents = await this.extractContentsFromSnapshots(terms);
@@ -190,6 +194,50 @@ export default class Archivist extends events.EventEmitter {
     }
   }
 
+  async fetchAndRecordMissingSnapshots(terms) {
+    if (!terms.hasMultipleSourceDocuments) { // If the terms has only one source document, there is nothing to do
+      return;
+    }
+
+    const existingVersion = await this.recorder.versionsRepository.findLatest(terms.service.id, terms.type);
+
+    if (!existingVersion) { // If the terms already has a version recorded, skip this step as the next version will be tagged as "First record…" anyway
+      return;
+    }
+
+    const missingSourceDocuments = [];
+
+    for (const sourceDocument of terms.sourceDocuments) {
+      const snapshot = await this.recorder.getLatestSnapshot(terms, sourceDocument.id);
+
+      if (!snapshot) {
+        missingSourceDocuments.push(sourceDocument);
+      }
+    }
+
+    if (missingSourceDocuments.length) {
+      return;
+    }
+
+    terms.fetchDate = new Date();
+    const fetchDocumentErrors = [];
+
+    for (const sourceDocument of missingSourceDocuments) {
+      const error = await this.fetchSourceDocument(sourceDocument);
+
+      if (error) {
+        fetchDocumentErrors.push(error);
+      } else {
+        await this.recordSnapshot(terms, sourceDocument);
+        sourceDocument.clearContent(); // Reduce memory usage by clearing no longer needed large content strings
+      }
+    }
+
+    if (fetchDocumentErrors.length) {
+      throw new InaccessibleContentError(fetchDocumentErrors);
+    }
+  }
+
   async fetchSourceDocument(sourceDocument) {
     const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
 

src/index.js

@@ -40,8 +40,11 @@ export default async function track({ services, types, extractOnly, schedule })
     });
   }
 
-  // The result of the extraction step that generates the version from the snapshots may depend on changes to the engine or its dependencies.
-  // The process thus starts by only performing the extraction process so that any version following such changes can be labelled (to avoid sending notifications, for example)
+  // Technical upgrade pass: apply changes from engine, dependency, or declaration upgrades.
+  // This regenerates versions from existing snapshots with updated extraction logic.
+  // For terms with combined source documents, if a new document was added to the declaration,
+  // it will be fetched and combined with existing snapshots to regenerate the complete version.
+  // All versions from this pass are labeled as technical upgrades to avoid false notifications about content changes.
   await archivist.track({ services, types, extractOnly: true });
 
   if (extractOnly) {