Merge pull request #934 from firefoxNX/httpDownload_retry_issue_929

fix: add retry logic for downloads with configurable retries and backoff

Author: hasezoey

docs/api/config-options.md

@@ -269,6 +269,19 @@ Also see [ARCHIVE_NAME](#archive_name).
 Keep in mind that downloaded binaries will never be automatically deleted.
 :::
 
+### MAX_RETRIES
+
+| Environment Variable | PackageJson |
+| :------------------: | :---------: |
+|  `MONGOMS_MAX_RETRIES`  |  `maxRetries`  |
+
+Option `MAX_RETRIES` is used to set the maximum number of retry attempts for downloading binaries when a retryable error occurs.
+
+Default: `3`
+
+Set this to control how many times the downloader will attempt to recover from transient errors (like network issues) before failing.
+
+
 ## How to use them in the package.json
 
 To use the config options in the `package.json`, they need to be camelCased (and without `_`), and need to be in the property `config.mongodbMemoryServer`

packages/mongodb-memory-server-core/src/util/MongoBinaryDownload.ts

@@ -19,6 +19,9 @@ import { RequestOptions } from 'https';
 
 const log = debug('MongoMS:MongoBinaryDownload');
 
+const retryableStatusCodes = [503, 500];
+const retryableErrorCodes = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED'];
+
 export interface MongoBinaryDownloadProgress {
   current: number;
   length: number;
@@ -353,126 +356,199 @@ export class MongoBinaryDownload {
   }
 
   /**
-   * Downlaod given httpOptions to tempDownloadLocation, then move it to downloadLocation
+   * Download given httpOptions to tempDownloadLocation, then move it to downloadLocation
+   * @param url The URL to download the file from
    * @param httpOptions The httpOptions directly passed to https.get
    * @param downloadLocation The location the File should be after the download
    * @param tempDownloadLocation The location the File should be while downloading
+   * @param maxRetries Maximum number of retries on download failure
+   * @param baseDelay Base delay in milliseconds for retrying the download
    */
   async httpDownload(
     url: URL,
     httpOptions: RequestOptions,
     downloadLocation: string,
-    tempDownloadLocation: string
+    tempDownloadLocation: string,
+    maxRetries?: number,
+    baseDelay: number = 1000
   ): Promise<string> {
     log('httpDownload');
     const downloadUrl = this.assignDownloadingURL(url);
 
-    const maxRedirects = parseInt(resolveConfig(ResolveConfigVariables.MAX_REDIRECTS) || '');
+    const maxRedirects = parseInt(resolveConfig(ResolveConfigVariables.MAX_REDIRECTS) ?? '');
     const useHttpsOptions: Parameters<typeof https.get>[1] = {
       maxRedirects: Number.isNaN(maxRedirects) ? 2 : maxRedirects,
       ...httpOptions,
     };
 
+    // Get maxRetries from config if not provided
+    const retriesFromConfig = parseInt(resolveConfig(ResolveConfigVariables.MAX_RETRIES) ?? '');
+    const retries =
+      typeof maxRetries === 'number'
+        ? maxRetries
+        : !Number.isNaN(retriesFromConfig)
+          ? retriesFromConfig
+          : 3;
+
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      try {
+        return await this.attemptDownload(
+          url,
+          useHttpsOptions,
+          downloadLocation,
+          tempDownloadLocation,
+          downloadUrl,
+          httpOptions
+        );
+      } catch (error: any) {
+        const shouldRetry =
+          (error instanceof DownloadError &&
+            retryableStatusCodes.some((code) => error.message.includes(code.toString()))) ||
+          (error?.code && retryableErrorCodes.includes(error.code));
+
+        if (!shouldRetry || attempt === retries) {
+          throw error;
+        }
+
+        const base = baseDelay * Math.pow(2, attempt);
+        const jitter = Math.floor(Math.random() * 1000);
+        const delay = base + jitter;
+        log(
+          `httpDownload: attempt ${attempt + 1} failed with ${error.message}, retrying in ${delay}ms...`
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+
+    throw new DownloadError(downloadUrl, 'Max retries exceeded');
+  }
+
+  /**
+   * Attempt to download the file from the given URL
+   * This function is used internally by `httpDownload`
+   * @param url
+   * @param useHttpsOptions
+   * @param downloadLocation
+   * @param tempDownloadLocation
+   * @param downloadUrl
+   * @param httpOptions
+   * @private
+   */
+  private async attemptDownload(
+    url: URL,
+    useHttpsOptions: Parameters<typeof https.get>[1],
+    downloadLocation: string,
+    tempDownloadLocation: string,
+    downloadUrl: string,
+    httpOptions: RequestOptions
+  ): Promise<string> {
     return new Promise((resolve, reject) => {
       log(`httpDownload: trying to download "${downloadUrl}"`);
-      https
-        .get(url, useHttpsOptions, (response) => {
-          if (response.statusCode != 200) {
-            if (response.statusCode === 403) {
-              reject(
-                new DownloadError(
-                  downloadUrl,
-                  "Status Code is 403 (MongoDB's 404)\n" +
-                    "This means that the requested version-platform combination doesn't exist\n" +
-                    "Try to use different version 'new MongoMemoryServer({ binary: { version: 'X.Y.Z' } })'\n" +
-                    'List of available versions can be found here: ' +
-                    'https://www.mongodb.com/download-center/community/releases/archive'
-                )
-              );
-
-              return;
-            }
 
+      const request = https.get(url, useHttpsOptions, (response) => {
+        if (response.statusCode != 200) {
+          if (response.statusCode === 403) {
             reject(
-              new DownloadError(downloadUrl, `Status Code isnt 200! (it is ${response.statusCode})`)
+              new DownloadError(
+                downloadUrl,
+                "Status Code is 403 (MongoDB's 404)\n" +
+                  "This means that the requested version-platform combination doesn't exist\n" +
+                  "Try to use different version 'new MongoMemoryServer({ binary: { version: 'X.Y.Z' } })'\n" +
+                  'List of available versions can be found here: ' +
+                  'https://www.mongodb.com/download-center/community/releases/archive'
+              )
             );
 
             return;
           }
 
-          // content-length, otherwise 0
-          let contentLength: number;
+          reject(
+            new DownloadError(downloadUrl, `Status Code isn't 200! (it is ${response.statusCode})`)
+          );
 
-          if (typeof response.headers['content-length'] != 'string') {
-            log('Response header "content-lenght" is empty!');
+          return;
+        }
 
-            contentLength = 0;
-          } else {
-            contentLength = parseInt(response.headers['content-length'], 10);
+        // content-length, otherwise 0
+        let contentLength: number;
 
-            if (Number.isNaN(contentLength)) {
-              log('Response header "content-lenght" resolved to NaN!');
+        if (typeof response.headers['content-length'] != 'string') {
+          log('Response header "content-length" is empty!');
+          contentLength = 0;
+        } else {
+          contentLength = parseInt(response.headers['content-length'], 10);
 
-              contentLength = 0;
-            }
+          if (Number.isNaN(contentLength)) {
+            log('Response header "content-length" resolved to NaN!');
+            contentLength = 0;
           }
+        }
+
+        // error if the content-length header is missing or is 0 if config option "DOWNLOAD_IGNORE_MISSING_HEADER" is not set to "true"
+        if (
+          !envToBool(resolveConfig(ResolveConfigVariables.DOWNLOAD_IGNORE_MISSING_HEADER)) &&
+          contentLength <= 0
+        ) {
+          reject(
+            new DownloadError(
+              downloadUrl,
+              'Response header "content-length" does not exist or resolved to NaN'
+            )
+          );
+
+          return;
+        }
+
+        this.dlProgress.current = 0;
+        this.dlProgress.length = contentLength;
+        this.dlProgress.totalMb = Math.round((this.dlProgress.length / 1048576) * 10) / 10;
+
+        const fileStream = createWriteStream(tempDownloadLocation);
 
-          // error if the content-length header is missing or is 0 if config option "DOWNLOAD_IGNORE_MISSING_HEADER" is not set to "true"
+        response.pipe(fileStream);
+
+        fileStream.on('finish', async () => {
           if (
-            !envToBool(resolveConfig(ResolveConfigVariables.DOWNLOAD_IGNORE_MISSING_HEADER)) &&
-            contentLength <= 0
+            this.dlProgress.current < this.dlProgress.length &&
+            !httpOptions.path?.endsWith('.md5')
           ) {
             reject(
               new DownloadError(
                 downloadUrl,
-                'Response header "content-length" does not exist or resolved to NaN'
+                `Too small (${this.dlProgress.current} bytes) mongod binary downloaded.`
               )
             );
 
             return;
           }
 
-          this.dlProgress.current = 0;
-          this.dlProgress.length = contentLength;
-          this.dlProgress.totalMb = Math.round((this.dlProgress.length / 1048576) * 10) / 10;
-
-          const fileStream = createWriteStream(tempDownloadLocation);
-
-          response.pipe(fileStream);
-
-          fileStream.on('finish', async () => {
-            if (
-              this.dlProgress.current < this.dlProgress.length &&
-              !httpOptions.path?.endsWith('.md5')
-            ) {
-              reject(
-                new DownloadError(
-                  downloadUrl,
-                  `Too small (${this.dlProgress.current} bytes) mongod binary downloaded.`
-                )
-              );
-
-              return;
-            }
+          this.printDownloadProgress({ length: 0 }, true);
 
-            this.printDownloadProgress({ length: 0 }, true);
+          fileStream.close();
+          await fspromises.rename(tempDownloadLocation, downloadLocation);
+          log(`httpDownload: moved "${tempDownloadLocation}" to "${downloadLocation}"`);
 
-            fileStream.close();
-            await fspromises.rename(tempDownloadLocation, downloadLocation);
-            log(`httpDownload: moved "${tempDownloadLocation}" to "${downloadLocation}"`);
+          resolve(downloadLocation);
+        });
 
-            resolve(downloadLocation);
-          });
+        response.on('data', (chunk: any) => {
+          this.printDownloadProgress(chunk);
+        });
 
-          response.on('data', (chunk: any) => {
-            this.printDownloadProgress(chunk);
-          });
-        })
-        .on('error', (err: Error) => {
-          // log it without having debug enabled
-          console.error(`Couldnt download "${downloadUrl}"!`, err.message);
+        response.on('error', (err: Error) => {
           reject(new DownloadError(downloadUrl, err.message));
         });
+      });
+
+      request.on('error', (err: Error) => {
+        console.error(`Could NOT download "${downloadUrl}"!`, err.message);
+        reject(new DownloadError(downloadUrl, err.message));
+      });
+
+      request.setTimeout(60000, () => {
+        request.destroy();
+        reject(new DownloadError(downloadUrl, 'Request timeout after 60 seconds'));
+      });
     });
   }
 

packages/mongodb-memory-server-core/src/util/resolveConfig.ts

@@ -33,6 +33,7 @@ export enum ResolveConfigVariables {
   SYSTEM_BINARY_VERSION_CHECK = 'SYSTEM_BINARY_VERSION_CHECK',
   USE_ARCHIVE_NAME_FOR_BINARY_NAME = 'USE_ARCHIVE_NAME_FOR_BINARY_NAME',
   MAX_REDIRECTS = 'MAX_REDIRECTS',
+  MAX_RETRIES = 'MAX_RETRIES', // Added for download retry configuration
   DISTRO = 'DISTRO',
 }
 
@@ -51,6 +52,7 @@ export const defaultValues = new Map<ResolveConfigVariables, string>([
   [ResolveConfigVariables.USE_ARCHIVE_NAME_FOR_BINARY_NAME, 'false'],
   [ResolveConfigVariables.MD5_CHECK, 'true'],
   [ResolveConfigVariables.MAX_REDIRECTS, '2'],
+  [ResolveConfigVariables.MAX_RETRIES, '3'], // Default maxRetries for downloads
 ]);
 
 /** Interface for storing information about the found package.json from `findPackageJson` */