Merge pull request #274 from jeroen-plug/fix271

refactor(MongoBinaryDownloads): Replace decompress

Author: nodkz

.eslintrc.js

@@ -48,8 +48,4 @@ module.exports = {
     jasmine: true,
     jest: true,
   },
-  globals: {
-    Class: true,
-    Iterator: true,
-  },
 };

packages/mongodb-memory-server-core/package.json

@@ -33,6 +33,8 @@
   "devDependencies": {
     "@types/jest": "25.1.4",
     "@types/mongodb": "3.5.2",
+    "@types/tar-stream": "^2.1.0",
+    "@types/yauzl": "^2.9.1",
     "@typescript-eslint/eslint-plugin": "2.23.0",
     "@typescript-eslint/parser": "2.23.0",
     "cross-env": "^7.0.2",
@@ -48,7 +50,6 @@
   "dependencies": {
     "@types/cross-spawn": "^6.0.1",
     "@types/debug": "^4.1.5",
-    "@types/decompress": "^4.2.3",
     "@types/dedent": "^0.7.0",
     "@types/find-cache-dir": "^3.2.0",
     "@types/find-package-json": "^1.1.1",
@@ -61,7 +62,6 @@
     "camelcase": "^5.3.1",
     "cross-spawn": "^7.0.1",
     "debug": "^4.1.1",
-    "decompress": "^4.2.0",
     "dedent": "^0.7.0",
     "find-cache-dir": "3.3.1",
     "find-package-json": "^1.2.0",
@@ -70,8 +70,10 @@
     "lockfile": "^1.0.4",
     "md5-file": "^4.0.0",
     "mkdirp": "^1.0.3",
-    "tmp": "^0.1.0",
-    "uuid": "^7.0.2"
+    "tar-stream": "^2.1.1",
+    "uuid": "^7.0.2",
+    "yauzl": "^2.10.0",
+    "tmp": "^0.1.0"
   },
   "optionalDependencies": {
     "mongodb": "^3.5.4"

packages/mongodb-memory-server-core/src/__tests__/replset-single-restart-test.ts

@@ -1,5 +1,6 @@
-import MongoMemoryReplSet from '../MongoMemoryReplSet';
+import MongoMemoryReplSet, { MongoMemoryReplSetOptsT } from '../MongoMemoryReplSet';
 import * as tmp from 'tmp';
+import getPort from 'get-port';
 
 let tmpDir: tmp.DirResult;
 beforeEach(() => {
@@ -16,7 +17,7 @@ const sleep = (ms: number) => {
 
 describe('single-member replica set', () => {
   it('should start multiple times', async () => {
-    const opts: any = {
+    const opts = {
       replSet: {
         storageEngine: 'wiredTiger',
       },
@@ -26,11 +27,17 @@ describe('single-member replica set', () => {
           dbPath: tmpDir.name,
         },
       ],
-    };
+    } as MongoMemoryReplSetOptsT;
 
     const replSetBefore = new MongoMemoryReplSet(opts);
     await replSetBefore.waitUntilRunning();
+
+    // Write real port to config (because 27017 may be busy, we need to get real port)
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    opts.instanceOpts![0].port = await replSetBefore.servers[0].getPort();
+
     await replSetBefore.stop();
+
     /**
      * get-port has a portlocking-feature that keeps ports locked for
      * "a minimum of 15 seconds and a maximum of 30 seconds before being released again"

packages/mongodb-memory-server-core/src/util/MongoBinaryDownload.ts

@@ -4,14 +4,17 @@ import path from 'path';
 import fs from 'fs';
 import md5File from 'md5-file';
 import https from 'https';
-import decompress from 'decompress'; // 💩💩💩 this package does not work with Node@11+Jest+Babel
+import { createUnzip } from 'zlib';
+import tar from 'tar-stream';
+import yauzl from 'yauzl';
 import MongoBinaryDownloadUrl from './MongoBinaryDownloadUrl';
 import { DownloadProgressT } from '../types';
 import { LATEST_VERSION } from './MongoBinary';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import { promisify } from 'util';
 import resolveConfig, { envToBool } from './resolve-config';
 import debug from 'debug';
+import dedent from 'dedent';
 
 const log = debug('MongoMS:MongoBinaryDownload');
 
@@ -186,24 +189,25 @@ export default class MongoBinaryDownload {
       fs.mkdirSync(extractDir);
     }
 
-    let filter;
+    let filter: (file: string) => boolean;
     if (this.platform === 'win32') {
-      filter = (file: any) => {
-        return /bin\/mongod.exe$/.test(file.path) || /.dll$/.test(file.path);
+      filter = (file: string) => {
+        return /bin\/mongod.exe$/.test(file) || /.dll$/.test(file);
       };
     } else {
-      filter = (file: any) => /bin\/mongod$/.test(file.path);
+      filter = (file: string) => /bin\/mongod$/.test(file);
     }
 
-    await decompress(mongoDBArchive, extractDir, {
-      // extract only `bin/mongod` file
-      filter,
-      // extract to root folder
-      map: (file) => {
-        file.path = path.basename(file.path);
-        return file;
-      },
-    });
+    if (/(.tar.gz|.tgz)$/.test(path.extname(mongoDBArchive))) {
+      await this.extractTarGz(mongoDBArchive, extractDir, filter);
+    } else if (/.zip$/.test(path.extname(mongoDBArchive))) {
+      await this.extractZip(mongoDBArchive, extractDir, filter);
+    } else {
+      throw new Error(
+        `MongoBinaryDownload: unsupported archive ${mongoDBArchive} (downloaded from ${this
+          ._downloadingUrl ?? 'unkown'}). Broken archive from MongoDB Provider?`
+      );
+    }
 
     if (!(await this.locationExists(path.resolve(this.downloadDir, this.version, binaryName)))) {
       throw new Error(
@@ -214,6 +218,83 @@ export default class MongoBinaryDownload {
     return extractDir;
   }
 
+  /**
+   * Extract a .tar.gz archive
+   * @param mongoDBArchive Archive location
+   * @param extractDir Directory to extract to
+   * @param filter Method to determine which files to extract
+   */
+  async extractTarGz(
+    mongoDBArchive: string,
+    extractDir: string,
+    filter: (file: string) => boolean
+  ): Promise<void> {
+    const extract = tar.extract();
+    extract.on('entry', (header, stream, next) => {
+      if (filter(header.name)) {
+        stream.pipe(
+          fs.createWriteStream(path.resolve(extractDir, path.basename(header.name)), {
+            mode: 0o775,
+          })
+        );
+      }
+      stream.on('end', () => next());
+      stream.resume();
+    });
+
+    return new Promise((resolve, reject) => {
+      fs.createReadStream(mongoDBArchive)
+        .on('error', (err) => {
+          reject('Unable to open tarball ' + mongoDBArchive + ': ' + err);
+        })
+        .pipe(createUnzip())
+        .on('error', (err) => {
+          reject('Error during unzip for ' + mongoDBArchive + ': ' + err);
+        })
+        .pipe(extract)
+        .on('error', (err) => {
+          reject('Error during untar for ' + mongoDBArchive + ': ' + err);
+        })
+        .on('finish', (result) => {
+          resolve(result);
+        });
+    });
+  }
+
+  /**
+   * Extract a .zip archive
+   * @param mongoDBArchive Archive location
+   * @param extractDir Directory to extract to
+   * @param filter Method to determine which files to extract
+   */
+  async extractZip(
+    mongoDBArchive: string,
+    extractDir: string,
+    filter: (file: string) => boolean
+  ): Promise<void> {
+    return new Promise((resolve, reject) => {
+      yauzl.open(mongoDBArchive, { lazyEntries: true }, (e, zipfile) => {
+        if (e || !zipfile) return reject(e);
+        zipfile.readEntry();
+
+        zipfile.on('end', () => resolve());
+
+        zipfile.on('entry', (entry) => {
+          if (!filter(entry.fileName)) return zipfile.readEntry();
+          zipfile.openReadStream(entry, (e, r) => {
+            if (e || !r) return reject(e);
+            r.on('end', () => zipfile.readEntry());
+            r.pipe(
+              fs.createWriteStream(path.resolve(extractDir, path.basename(entry.fileName)), {
+                mode: 0o775,
+              })
+            );
+          });
+        });
+      });
+    });
+  }
+
   /**
    * Downlaod given httpOptions to tempDownloadLocation, then move it to downloadLocation
    * @param httpOptions The httpOptions directly passed to https.get
@@ -233,10 +314,10 @@ export default class MongoBinaryDownload {
           if (response.statusCode != 200) {
             if (response.statusCode === 403) {
               reject(
-                new Error(
-                  "Status Code is 403 (MongoDB's 404)\n" +
-                    'This means that the requested version-platform combination dosnt exist'
-                )
+                new Error(dedent`
+                  Status Code is 403 (MongoDB's 404)\n
+                  This means that the requested version-platform combination dosnt exist
+                `)
               );
               return;
             }
@@ -304,8 +385,7 @@ export default class MongoBinaryDownload {
 
     const crReturn = this.platform === 'win32' ? '\x1b[0G' : '\r';
     process.stdout.write(
-      `Downloading MongoDB ${this.version}: ${percentComplete} % (${mbComplete}mb ` +
-        `/ ${this.dlProgress.totalMb}mb)${crReturn}`
+      `Downloading MongoDB ${this.version}: ${percentComplete} % (${mbComplete}mb / ${this.dlProgress.totalMb}mb)${crReturn}`
     );
   }