feat: use separate workers for dependencies and issues (#636)

In order to not block the next iteration on the getDependencies call, we use a separate worker for dependencies calculations (so getIssues from the previous compilation will not block the next getDependencies call)

✅ Closes: #612, #634

Author: piotr-oles

src/ForkTsCheckerWebpackPlugin.ts

@@ -29,9 +29,18 @@ class ForkTsCheckerWebpackPlugin implements webpack.Plugin {
    */
   static readonly version: string = '{{VERSION}}'; // will be replaced by the @semantic-release/exec
   /**
-   * Default pool for the plugin concurrency limit
+   * Default pools for the plugin concurrency limit
    */
-  static readonly pool: Pool = createPool(Math.max(1, os.cpus().length));
+  static readonly issuesPool: Pool = createPool(Math.max(1, os.cpus().length));
+  static readonly dependenciesPool: Pool = createPool(Math.max(1, os.cpus().length));
+
+  /**
+   * @deprecated Use ForkTsCheckerWebpackPlugin.issuesPool instead
+   */
+  static get pool(): Pool {
+    // for backward compatibility
+    return ForkTsCheckerWebpackPlugin.issuesPool;
+  }
 
   private readonly options: ForkTsCheckerWebpackPluginOptions;
 
@@ -56,25 +65,37 @@ class ForkTsCheckerWebpackPlugin implements webpack.Plugin {
   apply(compiler: webpack.Compiler) {
     const configuration = createForkTsCheckerWebpackPluginConfiguration(compiler, this.options);
     const state = createForkTsCheckerWebpackPluginState();
-    const reporters: ReporterRpcClient[] = [];
+    const issuesReporters: ReporterRpcClient[] = [];
+    const dependenciesReporters: ReporterRpcClient[] = [];
 
     if (configuration.typescript.enabled) {
       assertTypeScriptSupport(configuration.typescript);
-      reporters.push(createTypeScriptReporterRpcClient(configuration.typescript));
+      issuesReporters.push(createTypeScriptReporterRpcClient(configuration.typescript));
+      dependenciesReporters.push(createTypeScriptReporterRpcClient(configuration.typescript));
     }
 
     if (configuration.eslint.enabled) {
       assertEsLintSupport(configuration.eslint);
-      reporters.push(createEsLintReporterRpcClient(configuration.eslint));
+      issuesReporters.push(createEsLintReporterRpcClient(configuration.eslint));
+      dependenciesReporters.push(createEsLintReporterRpcClient(configuration.eslint));
     }
 
-    if (reporters.length) {
-      const reporter = createAggregatedReporter(composeReporterRpcClients(reporters));
+    if (issuesReporters.length) {
+      const issuesReporter = createAggregatedReporter(composeReporterRpcClients(issuesReporters));
+      const dependenciesReporter = createAggregatedReporter(
+        composeReporterRpcClients(dependenciesReporters)
+      );
 
       tapAfterEnvironmentToPatchWatching(compiler, state);
-      tapStartToConnectAndRunReporter(compiler, reporter, configuration, state);
+      tapStartToConnectAndRunReporter(
+        compiler,
+        issuesReporter,
+        dependenciesReporter,
+        configuration,
+        state
+      );
       tapAfterCompileToAddDependencies(compiler, configuration, state);
-      tapStopToDisconnectReporter(compiler, reporter, state);
+      tapStopToDisconnectReporter(compiler, issuesReporter, dependenciesReporter, state);
       tapErrorToLogMessage(compiler, configuration);
     } else {
       throw new Error(

src/ForkTsCheckerWebpackPluginState.ts

@@ -3,7 +3,8 @@ import { FilesMatch, Report } from './reporter';
 import { Issue } from './issue';
 
 interface ForkTsCheckerWebpackPluginState {
-  reportPromise: Promise<Report | undefined>;
+  issuesReportPromise: Promise<Report | undefined>;
+  dependenciesReportPromise: Promise<Report | undefined>;
   issuesPromise: Promise<Issue[] | undefined>;
   dependenciesPromise: Promise<FilesMatch | undefined>;
   lastDependencies: FilesMatch | undefined;
@@ -14,7 +15,8 @@ interface ForkTsCheckerWebpackPluginState {
 
 function createForkTsCheckerWebpackPluginState(): ForkTsCheckerWebpackPluginState {
   return {
-    reportPromise: Promise.resolve(undefined),
+    issuesReportPromise: Promise.resolve(undefined),
+    dependenciesReportPromise: Promise.resolve(undefined),
     issuesPromise: Promise.resolve(undefined),
     dependenciesPromise: Promise.resolve(undefined),
     lastDependencies: undefined,

src/hooks/tapDoneToAsyncGetIssues.ts

@@ -22,7 +22,7 @@ function tapDoneToAsyncGetIssues(
       return;
     }
 
-    const reportPromise = state.reportPromise;
+    const reportPromise = state.issuesReportPromise;
     const issuesPromise = state.issuesPromise;
     let issues: Issue[] | undefined;
 
@@ -46,7 +46,7 @@ function tapDoneToAsyncGetIssues(
       return;
     }
 
-    if (reportPromise !== state.reportPromise) {
+    if (reportPromise !== state.issuesReportPromise) {
       // there is a newer report - ignore this one
       return;
     }

src/hooks/tapStartToConnectAndRunReporter.ts

@@ -12,7 +12,8 @@ import { ForkTsCheckerWebpackPlugin } from '../ForkTsCheckerWebpackPlugin';
 
 function tapStartToConnectAndRunReporter(
   compiler: webpack.Compiler,
-  reporter: ReporterRpcClient,
+  issuesReporter: ReporterRpcClient,
+  dependenciesReporter: ReporterRpcClient,
   configuration: ForkTsCheckerWebpackPluginConfiguration,
   state: ForkTsCheckerWebpackPluginState
 ) {
@@ -64,43 +65,70 @@ function tapStartToConnectAndRunReporter(
     }
 
     let resolveDependencies: (dependencies: FilesMatch | undefined) => void;
-    let rejectedDependencies: (error: Error) => void;
+    let rejectDependencies: (error: Error) => void;
     let resolveIssues: (issues: Issue[] | undefined) => void;
     let rejectIssues: (error: Error) => void;
 
     state.dependenciesPromise = new Promise((resolve, reject) => {
       resolveDependencies = resolve;
-      rejectedDependencies = reject;
+      rejectDependencies = reject;
     });
     state.issuesPromise = new Promise((resolve, reject) => {
       resolveIssues = resolve;
       rejectIssues = reject;
     });
-    const previousReportPromise = state.reportPromise;
-    state.reportPromise = ForkTsCheckerWebpackPlugin.pool.submit(
+    const previousIssuesReportPromise = state.issuesReportPromise;
+    const previousDependenciesReportPromise = state.dependenciesReportPromise;
+
+    change = await hooks.start.promise(change, compilation);
+
+    state.issuesReportPromise = ForkTsCheckerWebpackPlugin.issuesPool.submit(
       (done) =>
         new Promise(async (resolve) => {
-          change = await hooks.start.promise(change, compilation);
+          try {
+            await issuesReporter.connect();
+
+            const previousReport = await previousIssuesReportPromise;
+            if (previousReport) {
+              await previousReport.close();
+            }
 
+            const report = await issuesReporter.getReport(change);
+            resolve(report);
+
+            report.getIssues().then(resolveIssues).catch(rejectIssues).finally(done);
+          } catch (error) {
+            if (error instanceof OperationCanceledError) {
+              hooks.canceled.call(compilation);
+            } else {
+              hooks.error.call(error, compilation);
+            }
+
+            resolve(undefined);
+            resolveIssues(undefined);
+            done();
+          }
+        })
+    );
+    state.dependenciesReportPromise = ForkTsCheckerWebpackPlugin.dependenciesPool.submit(
+      (done) =>
+        new Promise(async (resolve) => {
           try {
-            await reporter.connect();
+            await dependenciesReporter.connect();
 
-            const previousReport = await previousReportPromise;
+            const previousReport = await previousDependenciesReportPromise;
             if (previousReport) {
               await previousReport.close();
             }
 
-            const report = await reporter.getReport(change);
+            const report = await dependenciesReporter.getReport(change);
             resolve(report);
 
             report
               .getDependencies()
               .then(resolveDependencies)
-              .catch(rejectedDependencies)
-              .finally(() => {
-                // get issues after dependencies are resolved as it can be blocking
-                report.getIssues().then(resolveIssues).catch(rejectIssues).finally(done);
-              });
+              .catch(rejectDependencies)
+              .finally(done);
           } catch (error) {
             if (error instanceof OperationCanceledError) {
               hooks.canceled.call(compilation);
@@ -110,7 +138,6 @@ function tapStartToConnectAndRunReporter(
 
             resolve(undefined);
             resolveDependencies(undefined);
-            resolveIssues(undefined);
             done();
           }
         })

src/hooks/tapStopToDisconnectReporter.ts

@@ -4,22 +4,25 @@ import { ReporterRpcClient } from '../reporter';
 
 function tapStopToDisconnectReporter(
   compiler: webpack.Compiler,
-  reporter: ReporterRpcClient,
+  issuesReporter: ReporterRpcClient,
+  dependenciesReporter: ReporterRpcClient,
   state: ForkTsCheckerWebpackPluginState
 ) {
   compiler.hooks.watchClose.tap('ForkTsCheckerWebpackPlugin', () => {
-    reporter.disconnect();
+    issuesReporter.disconnect();
+    dependenciesReporter.disconnect();
   });
 
   compiler.hooks.done.tap('ForkTsCheckerWebpackPlugin', async () => {
     if (!state.watching) {
-      await reporter.disconnect();
+      await Promise.all([issuesReporter.disconnect(), dependenciesReporter.disconnect()]);
     }
   });
 
   compiler.hooks.failed.tap('ForkTsCheckerWebpackPlugin', () => {
     if (!state.watching) {
-      reporter.disconnect();
+      issuesReporter.disconnect();
+      dependenciesReporter.disconnect();
     }
   });
 }