fix: enhance process terminated errors and handle more edge cases in critial errors

Author: shadowusr

src/cli/index.ts

@@ -22,6 +22,9 @@ process.on("uncaughtException", err => {
 });
 
 process.on("unhandledRejection", (reason, p) => {
+    // This flag lets other unhandledRejection handlers know that we already processed it on Testplane side.
+    // Currently we use this to avoid duplicate error logging and force shutdown in HTML Reporter.
+    (global as Record<string, unknown>)["__TESTPLANE_INTERNAL_UNHANDLED_REJECTION_PROCESSED"] = true;
     if (shouldIgnoreUnhandledRejection(reason as Error)) {
         logger.warn(`Unhandled Rejection "${reason}" in testplane:master:${process.pid} was ignored`);
         return;

src/runner/index.ts

@@ -208,7 +208,9 @@ export class MainRunner extends RunnableEmitter {
 
         this.activeBrowserRunners.forEach(runner => runner.cancel(error));
 
-        this.workers?.cancel();
+        this.workers?.cancel().catch(() => {
+            /* we can just ignore the error thrown, because we don't care about cleanup at this point */
+        });
     }
 
     registerWorkers<T extends ReadonlyArray<string>>(workerFilepath: string, exportedMethods: T): RegisterWorkers<T> {

src/utils/processor.js

@@ -2,12 +2,24 @@
 
 const _ = require("lodash");
 const { WORKER_UNHANDLED_REJECTION } = require("../constants/process-messages");
+const debug = require("debug")("testplane:worker:processor");
 const logger = require("./logger");
 const ipc = require("./ipc");
 const { shouldIgnoreUnhandledRejection } = require("./errors");
 const { utilInspectSafe } = require("./secret-replacer");
 const { preloadWebdriverIO, preloadMochaReader } = require("./preload-utils.js");
 
+process.on("uncaughtException", err => {
+    if (err.code === "EPIPE" || err.code === "ERR_IPC_CHANNEL_CLOSED") {
+        debug(
+            "The following error was ignored in worker, because we tried to send message to master process, but it has already exited",
+        );
+        debug(err);
+        return;
+    }
+    throw err;
+});
+
 process.on("unhandledRejection", (reason, p) => {
     if (shouldIgnoreUnhandledRejection(reason)) {
         logger.warn(`Unhandled Rejection "${reason}" in testplane:worker:${process.pid} was ignored`);

src/utils/workers-registry.js

@@ -17,6 +17,8 @@ const {
     TEST_ASSIGNED_TO_WORKER,
 } = require("../constants/process-messages");
 const { isRunInNodeJsEnv } = require("./config");
+const { utilInspectSafe } = require("./secret-replacer");
+const { NEW_ISSUE_LINK } = require("../constants/help");
 
 const extractErrorFromWorkerMessage = data => {
     if (data.error) {
@@ -76,7 +78,27 @@ module.exports = class WorkersRegistry extends EventEmitter {
                 if (this._ended) {
                     return Promise.reject(new Error(`Can't execute method '${methodName}' because worker farm ended.`));
                 }
-                return promisify(this._workerFarm.execute)(workerFilepath, methodName, args);
+                const stack = new Error().stack;
+                return promisify(this._workerFarm.execute)(workerFilepath, methodName, args).catch(error => {
+                    if (error.name === "ProcessTerminatedError") {
+                        const workerCallError = new Error(
+                            `Testplane tried to run method '${methodName}' with args ${utilInspectSafe(
+                                args,
+                            )} in worker, but failed to do so.\n` +
+                                `Most likely this happened due to a critical error in the worker like unhandled promise rejection or the worker process was terminated unexpectedly.\n` +
+                                `Check surrounding logs for more details on the cause. If you believe this should not have happened, let us know: ${NEW_ISSUE_LINK}\n\n`,
+                        );
+                        try {
+                            workerCallError.stack = workerCallError.name + stack.split("\n").slice(1).join("\n");
+                        } catch {
+                            /* */
+                        }
+                        workerCallError.cause = error;
+
+                        throw workerCallError;
+                    }
+                    throw error;
+                });
             };
         }