fix(MongoBinaryDownload::attempDownload): add experimental feature "EXP_RESUME_DOWNLOAD"

Author: hasezoey

docs/api/config-options.md

@@ -146,8 +146,8 @@ Format: `https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-4.0.20
 
 ### DOWNLOAD_IGNORE_MISSING_HEADER
 
-|  Environment Variable  |  PackageJson  |
-| :--------------------: | :-----------: |
+|       Environment Variable       |          PackageJson          |
+| :------------------------------: | :---------------------------: |
 | `DOWNLOAD_IGNORE_MISSING_HEADER` | `downloadIgnoreMissingHeader` |
 
 Option `DOWNLOAD_IGNORE_MISSING_HEADER` can be set to `true` to ignore missing response headers like `content-length`.
@@ -239,9 +239,9 @@ Default: `false`
 
 ### MAX_REDIRECTS
 
-| Environment Variable | PackageJson |
-| :------------------: | :---------: |
-|  `MONGOMS_MAX_REDIRECTS`  |  `maxRedirects`  |
+|  Environment Variable   |  PackageJson   |
+| :---------------------: | :------------: |
+| `MONGOMS_MAX_REDIRECTS` | `maxRedirects` |
 
 Option `MAX_REDIRECTS` is used to set the maximal amount of redirects to follow
 
@@ -271,16 +271,27 @@ Keep in mind that downloaded binaries will never be automatically deleted.
 
 ### MAX_RETRIES
 
-| Environment Variable | PackageJson |
-| :------------------: | :---------: |
-|  `MONGOMS_MAX_RETRIES`  |  `maxRetries`  |
+| Environment Variable  | PackageJson  |
+| :-------------------: | :----------: |
+| `MONGOMS_MAX_RETRIES` | `maxRetries` |
 
 Option `MAX_RETRIES` is used to set the maximum number of retry attempts for downloading binaries when a retryable error occurs.
 
 Default: `3`
 
 Set this to control how many times the downloader will attempt to recover from transient errors (like network issues) before failing.
 
+### EXP_RESUME_DOWNLOAD
+
+|     Environment Variable      |     PackageJson     |
+| :---------------------------: | :-----------------: |
+| `MONGOMS_EXP_RESUME_DOWNLOAD` | `expResumeDownload` |
+
+Option `EXP_RESUME_DOWNLOAD` is used to enable / disable resuming a download of a binary instead of starting over on retries or interruptions.
+
+This is a experimental option, it maybe removed, renamed or have changed behavior in the future and any version.
+
+Default: `false`
 
 ## How to use them in the package.json
 

packages/mongodb-memory-server-core/src/util/MongoBinaryDownload.ts

@@ -10,7 +10,7 @@ import MongoBinaryDownloadUrl from './MongoBinaryDownloadUrl';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import resolveConfig, { envToBool, ResolveConfigVariables } from './resolveConfig';
 import debug from 'debug';
-import { assertion, mkdir, pathExists, md5FromFile } from './utils';
+import { assertion, mkdir, pathExists, md5FromFile, statPath } from './utils';
 import { DryMongoBinary } from './DryMongoBinary';
 import { MongoBinaryOpts } from './MongoBinary';
 import { clearLine } from 'readline';
@@ -19,7 +19,7 @@ import { RequestOptions } from 'https';
 
 const log = debug('MongoMS:MongoBinaryDownload');
 
-const retryableStatusCodes = [429, 500, 503];
+const retryableStatusCodes = [416, 429, 500, 503];
 
 const retryableErrorCodes = [
   'ECONNRESET',
@@ -460,17 +460,31 @@ export class MongoBinaryDownload {
    */
   private async attemptDownload(
     url: URL,
-    useHttpsOptions: Parameters<typeof https.get>[1],
+    useHttpsOptions: RequestOptions,
     downloadLocation: string,
     tempDownloadLocation: string,
     downloadUrl: string,
     httpOptions: RequestOptions
   ): Promise<string> {
+    /** Offset to resume from; for now a non-0 value indicates to use file "append" mode */
+    let offset = 0;
+
+    if (envToBool(resolveConfig(ResolveConfigVariables.EXP_RESUME_DOWNLOAD))) {
+      const stat = await statPath(tempDownloadLocation);
+
+      if (stat && stat.size != 0) {
+        useHttpsOptions.headers = useHttpsOptions.headers ?? {};
+        useHttpsOptions.headers['Range'] = `bytes=${stat.size}-`;
+        offset = stat.size;
+        log(`httpDownload: resuming download at ${offset}`);
+      }
+    }
+
     return new Promise((resolve, reject) => {
       log(`httpDownload: trying to download "${downloadUrl}"`);
 
-      const request = https.get(url, useHttpsOptions, (response) => {
-        if (response.statusCode != 200) {
+      const request = https.get(url, useHttpsOptions, async (response) => {
+        if (response.statusCode != 200 && response.statusCode != 206) {
           if (response.statusCode === 403) {
             reject(
               new DownloadError(
@@ -487,6 +501,11 @@ export class MongoBinaryDownload {
             return;
           }
 
+          // delete temporary file on "416: Range Not Satisfiable" and let it re-try
+          if (response.statusCode === 416) {
+            await fspromises.rm(tempDownloadLocation, { force: true });
+          }
+
           reject(
             new DownloadError(
               downloadUrl,
@@ -499,6 +518,7 @@ export class MongoBinaryDownload {
         }
 
         // content-length, otherwise 0
+        // note that content-length will not be the full size when resuming a download via "Range"
         let contentLength: number;
 
         if (typeof response.headers['content-length'] != 'string') {
@@ -533,7 +553,12 @@ export class MongoBinaryDownload {
         this.dlProgress.length = contentLength;
         this.dlProgress.totalMb = Math.round((this.dlProgress.length / 1048576) * 10) / 10;
 
-        const fileStream = createWriteStream(tempDownloadLocation);
+        // "w" opens for write, creates the file if it does not exist, or truncates if it does
+        // "a" opens for append, creates the file if it does not exist
+        const flags = offset === 0 ? 'w' : 'a';
+
+        // not using option "start" as we already open in "append" mode
+        const fileStream = createWriteStream(tempDownloadLocation, { /* start: offset, */ flags });
 
         response.pipe(fileStream);
 

packages/mongodb-memory-server-core/src/util/resolveConfig.ts

@@ -35,6 +35,7 @@ export enum ResolveConfigVariables {
   MAX_REDIRECTS = 'MAX_REDIRECTS',
   MAX_RETRIES = 'MAX_RETRIES', // Added for download retry configuration
   DISTRO = 'DISTRO',
+  EXP_RESUME_DOWNLOAD = 'EXP_RESUME_DOWNLOAD',
 }
 
 /** The Prefix for Environmental values */