feat: added a new CLI arg --merge-async to asynchronously and incrementally merge process coverage files to avoid OOM due to heap exhaustion (#469)

Author: bizob2828

lib/commands/report.js

@@ -34,7 +34,8 @@ exports.outputReport = async function (argv) {
     allowExternal: argv.allowExternal,
     src: argv.src,
     skipFull: argv.skipFull,
-    excludeNodeModules: argv.excludeNodeModules
+    excludeNodeModules: argv.excludeNodeModules,
+    mergeAsync: argv.mergeAsync
   })
   await report.run()
   if (argv.checkCoverage) await checkCoverages(argv, report)

lib/parse-args.js

@@ -152,6 +152,12 @@ function buildYargs (withCommands = false) {
       describe: 'supplying --allowExternal will cause c8 to allow files from outside of your cwd. This applies both to ' +
         'files discovered in coverage temp files and also src files discovered if using the --all flag.'
     })
+    .options('merge-async', {
+      default: false,
+      type: 'boolean',
+      describe: 'supplying --merge-async will merge all v8 coverage reports asynchronously and incrementally. ' +
+        'This is to avoid OOM issues with Node.js runtime.'
+    })
     .pkgConf('c8')
     .demandCommand(1)
     .check((argv) => {

lib/report.js

@@ -2,6 +2,12 @@ const Exclude = require('test-exclude')
 const libCoverage = require('istanbul-lib-coverage')
 const libReport = require('istanbul-lib-report')
 const reports = require('istanbul-reports')
+let readFile
+try {
+  ;({ readFile } = require('fs/promises'))
+} catch (err) {
+  ;({ readFile } = require('fs').promises)
+}
 const { readdirSync, readFileSync, statSync } = require('fs')
 const { isAbsolute, resolve, extname } = require('path')
 const { pathToFileURL, fileURLToPath } = require('url')
@@ -30,7 +36,8 @@ class Report {
     src,
     allowExternal = false,
     skipFull,
-    excludeNodeModules
+    excludeNodeModules,
+    mergeAsync
   }) {
     this.reporter = reporter
     this.reporterOptions = reporterOptions || {}
@@ -53,6 +60,7 @@ class Report {
     this.all = all
     this.src = this._getSrc(src)
     this.skipFull = skipFull
+    this.mergeAsync = mergeAsync
   }
 
   _getSrc (src) {
@@ -90,7 +98,13 @@ class Report {
     if (this._allCoverageFiles) return this._allCoverageFiles
 
     const map = libCoverage.createCoverageMap()
-    const v8ProcessCov = this._getMergedProcessCov()
+    let v8ProcessCov
+
+    if (this.mergeAsync) {
+      v8ProcessCov = await this._getMergedProcessCovAsync()
+    } else {
+      v8ProcessCov = this._getMergedProcessCov()
+    }
     const resultCountPerPath = new Map()
     const possibleCjsEsmBridges = new Map()
 
@@ -188,43 +202,106 @@ class Report {
     }
 
     if (this.all) {
-      const emptyReports = []
+      const emptyReports = this._includeUncoveredFiles(fileIndex)
       v8ProcessCovs.unshift({
         result: emptyReports
       })
-      const workingDirs = this.src
-      const { extension } = this.exclude
-      for (const workingDir of workingDirs) {
-        this.exclude.globSync(workingDir).forEach((f) => {
-          const fullPath = resolve(workingDir, f)
-          if (!fileIndex.has(fullPath)) {
-            const ext = extname(fullPath)
-            if (extension.includes(ext)) {
-              const stat = statSync(fullPath)
-              const sourceMap = getSourceMapFromFile(fullPath)
-              if (sourceMap) {
-                this.sourceMapCache[pathToFileURL(fullPath)] = { data: sourceMap }
-              }
-              emptyReports.push({
-                scriptId: 0,
-                url: resolve(fullPath),
-                functions: [{
-                  functionName: '(empty-report)',
-                  ranges: [{
-                    startOffset: 0,
-                    endOffset: stat.size,
-                    count: 0
-                  }],
-                  isBlockCoverage: true
-                }]
-              })
-            }
+    }
+
+    return mergeProcessCovs(v8ProcessCovs)
+  }
+
+  /**
+   * Returns the merged V8 process coverage.
+   *
+   * It asynchronously and incrementally reads and merges individual process coverages
+   * generated by Node. This can be used via the `--merge-async` CLI arg.  It's intended
+   * to be used across a large multi-process test run.
+   *
+   * @return {ProcessCov} Merged V8 process coverage.
+   * @private
+   */
+  async _getMergedProcessCovAsync () {
+    const { mergeProcessCovs } = require('@bcoe/v8-coverage')
+    const fileIndex = new Set() // Set<string>
+    let mergedCov = null
+    for (const file of readdirSync(this.tempDirectory)) {
+      try {
+        const rawFile = await readFile(
+          resolve(this.tempDirectory, file),
+          'utf8'
+        )
+        let report = JSON.parse(rawFile)
+
+        if (this._isCoverageObject(report)) {
+          if (report['source-map-cache']) {
+            Object.assign(this.sourceMapCache, this._normalizeSourceMapCache(report['source-map-cache']))
           }
-        })
+          report = this._normalizeProcessCov(report, fileIndex)
+          if (mergedCov) {
+            mergedCov = mergeProcessCovs([mergedCov, report])
+          } else {
+            mergedCov = mergeProcessCovs([report])
+          }
+        }
+      } catch (err) {
+        debuglog(`${err.stack}`)
       }
     }
 
-    return mergeProcessCovs(v8ProcessCovs)
+    if (this.all) {
+      const emptyReports = this._includeUncoveredFiles(fileIndex)
+      const emptyReport = {
+        result: emptyReports
+      }
+
+      mergedCov = mergeProcessCovs([emptyReport, mergedCov])
+    }
+
+    return mergedCov
+  }
+
+  /**
+   * Adds empty coverage reports to account for uncovered/untested code.
+   * This is only done when the `--all` flag is present.
+   *
+   * @param {Set} fileIndex list of files that have coverage
+   * @returns {Array} list of empty coverage reports
+   */
+  _includeUncoveredFiles (fileIndex) {
+    const emptyReports = []
+    const workingDirs = this.src
+    const { extension } = this.exclude
+    for (const workingDir of workingDirs) {
+      this.exclude.globSync(workingDir).forEach((f) => {
+        const fullPath = resolve(workingDir, f)
+        if (!fileIndex.has(fullPath)) {
+          const ext = extname(fullPath)
+          if (extension.includes(ext)) {
+            const stat = statSync(fullPath)
+            const sourceMap = getSourceMapFromFile(fullPath)
+            if (sourceMap) {
+              this.sourceMapCache[pathToFileURL(fullPath)] = { data: sourceMap }
+            }
+            emptyReports.push({
+              scriptId: 0,
+              url: resolve(fullPath),
+              functions: [{
+                functionName: '(empty-report)',
+                ranges: [{
+                  startOffset: 0,
+                  endOffset: stat.size,
+                  count: 0
+                }],
+                isBlockCoverage: true
+              }]
+            })
+          }
+        }
+      })
+    }
+
+    return emptyReports
   }
 
   /**