refactor(daemon): stuck-processing only alerts; default 1 hour

Author: andreabadesso

packages/daemon/__tests__/actors/MonitoringActor.test.ts

@@ -173,18 +173,17 @@ describe('MonitoringActor', () => {
     expect(clearTimeout).toHaveBeenCalledTimes(1);
   });
 
-  it('should fire a CRITICAL alert and call back MONITORING_STUCK_PROCESSING when stuck', async () => {
+  it('should fire a MAJOR alert when stuck', async () => {
     MonitoringActor(mockCallback, mockReceive, config);
     sendEvent('PROCESSING_STARTED');
 
     jest.advanceTimersByTime(config['STUCK_PROCESSING_TIMEOUT_MS'] + 1);
-    // Let the async addAlert inside the timeout resolve
     await Promise.resolve();
     await Promise.resolve();
 
     expect(mockAddAlert).toHaveBeenCalledTimes(1);
     expect(mockAddAlert.mock.calls[0][0]).toBe('Daemon Stuck In Processing State');
-    expect(mockCallback).toHaveBeenCalledWith({ type: EventTypes.MONITORING_STUCK_PROCESSING });
+    expect(mockCallback).not.toHaveBeenCalled();
   });
 
   it('should NOT fire the stuck alert when PROCESSING_COMPLETED arrives in time', async () => {

packages/daemon/src/actors/MonitoringActor.ts

@@ -35,7 +35,7 @@ import { Event, EventTypes } from '../types';
  *    RECONNECTION_STORM_WINDOW_MS.
  */
 const DEFAULT_IDLE_EVENT_TIMEOUT_MS = 5 * 60 * 1000;
-const DEFAULT_STUCK_PROCESSING_TIMEOUT_MS = 5 * 60 * 1000;
+const DEFAULT_STUCK_PROCESSING_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
 const DEFAULT_RECONNECTION_STORM_THRESHOLD = 10;
 const DEFAULT_RECONNECTION_STORM_WINDOW_MS = 5 * 60 * 1000;
 
@@ -93,21 +93,17 @@ export default (callback: any, receive: any, config = getConfig()) => {
   const startStuckTimer = () => {
     clearStuckTimer();
     stuckTimer = setTimeout(async () => {
-      logger.error('[monitoring] State machine stuck in processing state — forcing reconnection');
-      try {
-        await addAlert(
-          'Daemon Stuck In Processing State',
-          `The state machine has been processing a single event for more than ` +
-            `${Math.round(stuckTimeoutMs / 60000)} minute(s). ` +
-            'Forcing a reconnection.',
-          Severity.MAJOR,
-          { timeoutMs: String(stuckTimeoutMs) },
-          logger,
-        );
-      } catch (err) {
-        logger.error(`[monitoring] Failed to send stuck-processing alert: ${err}`);
-      }
-      callback({ type: EventTypes.MONITORING_STUCK_PROCESSING });
+      logger.error('[monitoring] State machine stuck in processing state');
+      addAlert(
+        'Daemon Stuck In Processing State',
+        `The state machine has been processing a single event for more than ` +
+          `${Math.round(stuckTimeoutMs / 60000)} minute(s).`,
+        Severity.MAJOR,
+        { timeoutMs: String(stuckTimeoutMs) },
+        logger,
+      ).catch((err: Error) =>
+        logger.error(`[monitoring] Failed to send stuck-processing alert: ${err}`),
+      );
     }, stuckTimeoutMs);
   };
 

packages/daemon/src/config.ts

@@ -90,8 +90,8 @@ export const ACK_TIMEOUT_MS = parseInt(process.env.ACK_TIMEOUT_MS ?? '20000', 10
 // Monitoring configuration
 // Timeout (ms) before alerting when no fullnode events received while WebSocket connected
 export const IDLE_EVENT_TIMEOUT_MS = parseInt(process.env.IDLE_EVENT_TIMEOUT_MS ?? String(5 * 60 * 1000), 10);  // 5 minutes
-// Timeout (ms) before auto-restarting when stuck in a single processing state
-export const STUCK_PROCESSING_TIMEOUT_MS = parseInt(process.env.STUCK_PROCESSING_TIMEOUT_MS ?? String(5 * 60 * 1000), 10);  // 5 minutes
+// Timeout (ms) before alerting when stuck in a single processing state
+export const STUCK_PROCESSING_TIMEOUT_MS = parseInt(process.env.STUCK_PROCESSING_TIMEOUT_MS ?? String(60 * 60 * 1000), 10);  // 1 hour
 // Number of reconnections within RECONNECTION_STORM_WINDOW_MS to trigger a storm alert
 export const RECONNECTION_STORM_THRESHOLD = parseInt(process.env.RECONNECTION_STORM_THRESHOLD ?? '10', 10);
 // Time window (ms) for reconnection storm detection

packages/daemon/src/machines/SyncMachine.ts

@@ -15,7 +15,6 @@ import { WebSocketActor, HealthCheckActor, MonitoringActor } from '../actors';
 import {
   Context,
   Event,
-  EventTypes,
 } from '../types';
 import {
   handleVertexAccepted,
@@ -392,10 +391,6 @@ export const SyncMachine = Machine<Context, any, Event>({
           cond: 'websocketDisconnected',
           target: SYNC_MACHINE_STATES.RECONNECTING,
         }],
-        // Sent by MonitoringActor when a processing state has been active for too long
-        [EventTypes.MONITORING_STUCK_PROCESSING]: {
-          target: `#SyncMachine.${SYNC_MACHINE_STATES.RECONNECTING}`,
-        },
       },
     },
     [SYNC_MACHINE_STATES.ERROR]: {

packages/daemon/src/types/event.ts

@@ -42,7 +42,6 @@ export enum EventTypes {
   WEBSOCKET_SEND_EVENT = 'WEBSOCKET_SEND_EVENT',
   HEALTHCHECK_EVENT = 'HEALTHCHECK_EVENT',
   MONITORING_EVENT = 'MONITORING_EVENT',
-  MONITORING_STUCK_PROCESSING = 'MONITORING_STUCK_PROCESSING',
 }
 
 export enum FullNodeEventTypes {
@@ -83,10 +82,7 @@ export type Event =
   | { type: EventTypes.FULLNODE_EVENT, event: FullNodeEvent }
   | { type: EventTypes.WEBSOCKET_SEND_EVENT, event: WebSocketSendEvent }
   | { type: EventTypes.HEALTHCHECK_EVENT, event: HealthCheckEvent }
-  | { type: EventTypes.MONITORING_EVENT, event: MonitoringEvent }
-  // Machine-internal signal sent by MonitoringActor when a processing state is stuck.
-  // `event: never` keeps the union consistent (all members have an `event` field).
-  | { type: EventTypes.MONITORING_STUCK_PROCESSING, event: never };
+  | { type: EventTypes.MONITORING_EVENT, event: MonitoringEvent };
 
 
 export interface VertexRemovedEventData {