implement graceful shutdown at node renderer workers

Author: AbanoubGhadban

react_on_rails_pro/packages/node-renderer/src/master.ts

@@ -48,9 +48,15 @@ export = function masterRun(runningConfig?: Partial<Config>) {
       allWorkersRestartInterval,
       delayBetweenIndividualWorkerRestarts,
     );
-    setInterval(() => {
-      restartWorkers(delayBetweenIndividualWorkerRestarts);
-    }, allWorkersRestartInterval * MILLISECONDS_IN_MINUTE);
+
+    const allWorkersRestartIntervalMS = allWorkersRestartInterval * MILLISECONDS_IN_MINUTE;
+    const scheduleWorkersRestart = () => {
+      void restartWorkers(delayBetweenIndividualWorkerRestarts).finally(() => {
+        setTimeout(scheduleWorkersRestart, allWorkersRestartIntervalMS);
+      });
+    };
+
+    setTimeout(scheduleWorkersRestart, allWorkersRestartIntervalMS);
   } else if (allWorkersRestartInterval || delayBetweenIndividualWorkerRestarts) {
     log.error(
       "Misconfiguration, please provide both 'allWorkersRestartInterval' and " +

react_on_rails_pro/packages/node-renderer/src/master/restartWorkers.ts

@@ -5,6 +5,7 @@
 
 import cluster from 'cluster';
 import log from '../shared/log';
+import { SHUTDOWN_WORKER_MESSAGE } from '../shared/utils';
 
 const MILLISECONDS_IN_MINUTE = 60000;
 
@@ -14,26 +15,42 @@ declare module 'cluster' {
   }
 }
 
-export = function restartWorkers(delayBetweenIndividualWorkerRestarts: number) {
+export = async function restartWorkers(delayBetweenIndividualWorkerRestarts: number) {
   log.info('Started scheduled restart of workers');
 
-  let delay = 0;
   if (!cluster.workers) {
     throw new Error('No workers to restart');
   }
-  Object.values(cluster.workers).forEach((worker) => {
-    const killWorker = () => {
-      if (!worker) return;
-      log.debug('Kill worker #%d', worker.id);
-      // eslint-disable-next-line no-param-reassign -- necessary change
-      worker.isScheduledRestart = true;
-      worker.destroy();
-    };
-    setTimeout(killWorker, delay);
-    delay += delayBetweenIndividualWorkerRestarts * MILLISECONDS_IN_MINUTE;
-  });
-
-  setTimeout(() => {
-    log.info('Finished scheduled restart of workers');
-  }, delay);
+  for (const worker of Object.values(cluster.workers)) {
+    if (!worker) return;
+    log.debug('Kill worker #%d', worker.id);
+    worker.isScheduledRestart = true;
+
+    worker.send(SHUTDOWN_WORKER_MESSAGE);
+
+    // It's inteded to restart worker in sequence, it shouldn't happens in parallel
+    // eslint-disable-next-line no-await-in-loop
+    await new Promise<void>((resolve) => {
+      let timeout: NodeJS.Timeout;
+
+      const onExit = () => {
+        clearTimeout(timeout);
+        resolve();
+      };
+      worker.on('exit', onExit);
+
+      timeout = setTimeout(() => {
+        log.debug('Worker #%d timed out, forcing kill it', worker.id);
+        worker.destroy();
+        worker.off('exit', onExit);
+        resolve();
+      }, 100_000);
+    });
+    // eslint-disable-next-line no-await-in-loop
+    await new Promise((resolve) => {
+      setTimeout(resolve, delayBetweenIndividualWorkerRestarts * MILLISECONDS_IN_MINUTE);
+    });
+  }
+
+  log.info('Finished scheduled restart of workers');
 };

react_on_rails_pro/packages/node-renderer/src/shared/utils.ts

@@ -11,6 +11,8 @@ import type { RenderResult } from '../worker/vm';
 
 export const TRUNCATION_FILLER = '\n... TRUNCATED ...\n';
 
+export const SHUTDOWN_WORKER_MESSAGE = 'NODE_RENDERER_SHUTDOWN_WORKER';
+
 export function workerIdLabel() {
   // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- worker is nullable in the primary process
   return cluster?.worker?.id || 'NO WORKER ID';

react_on_rails_pro/packages/node-renderer/src/worker.ts

@@ -17,6 +17,7 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from './worker/typ
 import checkProtocolVersion from './worker/checkProtocolVersionHandler';
 import authenticate from './worker/authHandler';
 import { handleRenderRequest, type ProvidedNewBundle } from './worker/handleRenderRequest';
+import handleGracefulShutdown from './worker/handleGracefulShutdown';
 import {
   errorResponseResult,
   formatExceptionMessage,
@@ -127,6 +128,8 @@ export default function run(config: Partial<Config>) {
     ...fastifyServerOptions,
   });
 
+  handleGracefulShutdown(app);
+
   // We shouldn't have unhandled errors here, but just in case
   app.addHook('onError', (req, res, err, done) => {
     // Not errorReporter.error so that integrations can decide how to log the errors.

react_on_rails_pro/packages/node-renderer/src/worker/handleGracefulShutdown.ts

@@ -0,0 +1,49 @@
+import cluster from 'cluster';
+import { FastifyInstance } from './types';
+import { SHUTDOWN_WORKER_MESSAGE } from '../shared/utils';
+import log from '../shared/log';
+
+const handleGracefulShutdown = (app: FastifyInstance) => {
+  const { worker } = cluster;
+  if (!worker) {
+    log.error('handleGracefulShutdown is called on master, expected to call it on worker only');
+    return;
+  }
+
+  let activeRequestsCount = 0;
+  let isShuttingDown = false;
+
+  process.on('message', (msg) => {
+    if (msg === SHUTDOWN_WORKER_MESSAGE) {
+      log.debug('Worker #%d received graceful shutdown message', worker.id);
+      isShuttingDown = true;
+      if (activeRequestsCount === 0) {
+        log.debug('Worker #%d has no active requests, killing the worker', worker.id);
+        worker.destroy();
+      } else {
+        log.debug(
+          'Worker #%d has "%d" active requests, disconnecting the worker',
+          worker.id,
+          activeRequestsCount,
+        );
+        worker.disconnect();
+      }
+    }
+  });
+
+  app.addHook('onRequest', (_req, _reply, done) => {
+    activeRequestsCount += 1;
+    done();
+  });
+
+  app.addHook('onResponse', (_req, _reply, done) => {
+    activeRequestsCount -= 1;
+    if (isShuttingDown && activeRequestsCount === 0) {
+      log.debug('Worker #%d served all active requests and going to be killed', worker.id);
+      worker.destroy();
+    }
+    done();
+  });
+};
+
+export default handleGracefulShutdown;