Treat bot blocking errors as potentially transient

Author: Ndpnt

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [minor]
+
+> Development of this release was supported by the [Lab Platform Governance, Media and Technology](https://platform-governance.org) (PGMT), Centre for Media, Communication and Information Research (ZeMKI), University of Bremen as part of the project [Governance: Private ordering of ComAI through corporate communication and policies](https://comai.space/en/projects/p4-governance-private-ordering-of-comai-through-corporate-communication-and-policies/) in the research unit [Communicative AI](https://comai.space/en/), funded by the German Research Foundation (DFG) ([Grant No. 516511468)](https://gepris.dfg.de/gepris/projekt/544643936?language=en).
+
+### Added
+
+- Extend automatic retry mechanism for failed tracking attempts due to likely bot blocking errors to improve tracking success rate
+
 ## 5.5.0 - 2025-06-04
 
 _Full changeset and discussions: [#1159](https://github.com/OpenTermsArchive/engine/pull/1159)._

src/archivist/fetcher/errors.js

@@ -1,18 +1,25 @@
 export class FetchDocumentError extends Error {
+  static LIKELY_BOT_BLOCKING_ERRORS = [
+    'HTTP code 403',
+    'HTTP code 406',
+    'HTTP code 502',
+    'ECONNRESET',
+  ];
+
   static LIKELY_TRANSIENT_ERRORS = [
     'EAI_AGAIN', // DNS lookup temporary failure - DNS server is temporarily unavailable or overloaded
     'ETIMEDOUT', // Connection timeout - network latency or server load issues
-    'ECONNRESET', // Connection reset - connection was forcibly closed, often due to network issues
     'ERR_NAME_NOT_RESOLVED', // DNS lookup temporary failure - DNS server is temporarily unavailable or overloaded
     'HTTP code 500', // Internal Server Error - server encountered an error while processing the request
-    'HTTP code 502', // Bad Gateway - upstream server returned invalid response, often temporary
     'HTTP code 503', // Service Unavailable - server is temporarily overloaded or down for maintenance
     'HTTP code 504', // Gateway Timeout - upstream server took too long to respond, might be temporary
+    ...FetchDocumentError.LIKELY_BOT_BLOCKING_ERRORS,
   ];
 
   constructor(message) {
     super(`Fetch failed: ${message}`);
     this.name = 'FetchDocumentError';
     this.mayBeTransient = FetchDocumentError.LIKELY_TRANSIENT_ERRORS.some(err => message.includes(err));
+    this.mayBeBotBlocking = FetchDocumentError.LIKELY_BOT_BLOCKING_ERRORS.some(err => message.includes(err));
   }
 }

src/archivist/fetcher/index.js

@@ -12,13 +12,6 @@ export const FETCHER_TYPES = {
   HTML_ONLY: 'htmlOnly',
 };
 
-const LIKELY_BOT_BLOCKING_ERRORS = [
-  'HTTP code 403',
-  'HTTP code 406',
-  'HTTP code 502',
-  'ECONNRESET',
-];
-
 /**
  * Fetch a resource from the network, returning a promise which is fulfilled once the response is available
  * @function fetch
@@ -70,9 +63,7 @@ async function fetchWithFallback(url, cssSelectors, fetcherConfig) {
   try {
     return await fetchWithHtmlOnly(url, fetcherConfig);
   } catch (error) {
-    const isBotBlockingError = LIKELY_BOT_BLOCKING_ERRORS.some(code => error.message.includes(code));
-
-    if (!isBotBlockingError || fetcherConfig.executeClientScripts === false) {
+    if (!error.mayBeBotBlocking || fetcherConfig.executeClientScripts === false) {
       throw error;
     }