Fetch multiple source documents sequentially to prevent bot detection (#1176)

Author: Ndpnt

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [minor]
+
+> Development of this release was supported by the [Lab Platform Governance, Media and Technology](https://platform-governance.org) (PGMT), Centre for Media, Communication and Information Research (ZeMKI), University of Bremen as part of the project [Governance: Private ordering of ComAI through corporate communication and policies](https://comai.space/en/projects/p4-governance-private-ordering-of-comai-through-corporate-communication-and-policies/) in the research unit [Communicative AI](https://comai.space/en/), funded by the German Research Foundation (DFG) ([Grant No. 516511468)](https://gepris.dfg.de/gepris/projekt/544643936?language=en).
+
+### Changed
+
+- Fetch multiple source documents sequentially to prevent bot detection and improve tracking success rate
+
 ## 6.0.1 - 2025-07-07
 
 _Full changeset and discussions: [#1171](https://github.com/OpenTermsArchive/engine/pull/1171)._

src/archivist/index.js

@@ -183,11 +183,11 @@ export default class Archivist extends events.EventEmitter {
 
     const fetchDocumentErrors = [];
 
-    await Promise.all(terms.sourceDocuments.map(async sourceDocument => {
+    for (const sourceDocument of terms.sourceDocuments) {
       const { location: url, executeClientScripts, cssSelectors } = sourceDocument;
 
       try {
-        const { mimeType, content, fetcher } = await this.fetch({ url, executeClientScripts, cssSelectors });
+        const { mimeType, content, fetcher } = await this.fetch({ url, executeClientScripts, cssSelectors }); // eslint-disable-line no-await-in-loop
 
         sourceDocument.content = content;
         sourceDocument.mimeType = mimeType;
@@ -199,7 +199,7 @@ export default class Archivist extends events.EventEmitter {
 
         fetchDocumentErrors.push(error);
       }
-    }));
+    }
 
     if (fetchDocumentErrors.length) {
       throw new InaccessibleContentError(fetchDocumentErrors);