chore(webapp): More false errors turned to logs (#2422)

matt-aitken · web-flow · commit 4264fc066e4a · 2025-08-20T11:44:26.000+01:00
* Remove variables from dequeue log message

* Continue snapshot throw json

* Waiting for deploy error removed

* Realtime ECONNRESET is expected

* Redis worker logErrors changes, removed ids

* Preview branch without a branch shouldn't log an error, it's a user provided issue

* "Task run is not in a cancellable state" isn't an error, it's expected

* "CreateCheckpointService: Child run already resumed" is expected

* "CreateCheckpointService: Batch already resumed" is expected

* "Failed to insert events, will attempt bisection" changed to info, we have errors for complete failures

* Ignore "PrismaClient error"

* Don't log Redis worker DLQ errors if we're ignoring

* "Failed to parse machine config" is fine, sometimes a config is null or undefined

* "Failed to parse machine config" for v3

* MetadataTooLargeError shouldn't log an error
diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts
@@ -201,6 +201,7 @@ function getClient() {
           message: log.message,
           target: log.target,
         },
+        ignoreError: true,
       });
     });
   }
diff --git a/apps/webapp/app/models/runtimeEnvironment.server.ts b/apps/webapp/app/models/runtimeEnvironment.server.ts
@@ -37,7 +37,7 @@ export async function findEnvironmentByApiKey(
 
   if (environment.type === "PREVIEW") {
     if (!branchName) {
-      logger.error("findEnvironmentByApiKey(): Preview env with no branch name provided", {
+      logger.warn("findEnvironmentByApiKey(): Preview env with no branch name provided", {
         environmentId: environment.id,
       });
       return null;
diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts
@@ -27,8 +27,12 @@ export const loader = createLoaderWorkerApiRoute(
 
       return json(continuationResult);
     } catch (error) {
-      logger.error("Failed to suspend run", { runFriendlyId, snapshotFriendlyId, error });
-      throw error;
+      logger.warn("Failed to suspend run", { runFriendlyId, snapshotFriendlyId, error });
+      if (error instanceof Error) {
+        throw json({ error: error.message }, { status: 422 });
+      }
+
+      throw json({ error: "Failed to continue run execution" }, { status: 422 });
     }
   }
 );
diff --git a/apps/webapp/app/services/realtime/redisRealtimeStreams.server.ts b/apps/webapp/app/services/realtime/redisRealtimeStreams.server.ts
@@ -184,6 +184,15 @@ export class RedisRealtimeStreams implements StreamIngestor, StreamResponder {
 
       return new Response(null, { status: 200 });
     } catch (error) {
+      if (error instanceof Error) {
+        if ("code" in error && error.code === "ECONNRESET") {
+          logger.info("[RealtimeStreams][ingestData] Connection reset during ingestData:", {
+            error,
+          });
+          return new Response(null, { status: 500 });
+        }
+      }
+
       logger.error("[RealtimeStreams][ingestData] Error in ingestData:", { error });
 
       return new Response(null, { status: 500 });
diff --git a/apps/webapp/app/v3/alertsWorker.server.ts b/apps/webapp/app/v3/alertsWorker.server.ts
@@ -33,6 +33,7 @@ function initializeWorker() {
         retry: {
           maxAttempts: 3,
         },
+        logErrors: false,
       },
       "v3.performDeploymentAlerts": {
         schema: z.object({
@@ -42,6 +43,7 @@ function initializeWorker() {
         retry: {
           maxAttempts: 3,
         },
+        logErrors: false,
       },
       "v3.deliverAlert": {
         schema: z.object({
@@ -51,6 +53,7 @@ function initializeWorker() {
         retry: {
           maxAttempts: 3,
         },
+        logErrors: false,
       },
     },
     concurrency: {
diff --git a/apps/webapp/app/v3/eventRepository.server.ts b/apps/webapp/app/v3/eventRepository.server.ts
@@ -1246,7 +1246,7 @@ export class EventRepository {
             span.setAttribute("prisma_error_code", errorDetails.code);
           }
 
-          logger.error("Failed to insert events, will attempt bisection", {
+          logger.info("Failed to insert events, will attempt bisection", {
             error: errorDetails,
           });
 
diff --git a/apps/webapp/app/v3/machinePresets.server.ts b/apps/webapp/app/v3/machinePresets.server.ts
@@ -6,7 +6,7 @@ export function machinePresetFromConfig(config: unknown): MachinePreset {
   const parsedConfig = MachineConfig.safeParse(config);
 
   if (!parsedConfig.success) {
-    logger.error("Failed to parse machine config", { config });
+    logger.info("Failed to parse machine config", { config });
 
     return machinePresetFromName("small-1x");
   }
diff --git a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts
@@ -621,7 +621,8 @@ export class SharedQueueConsumer {
     const worker = deployment?.worker;
 
     if (!deployment || !worker) {
-      logger.error("No matching deployment found for task run", {
+      // This happens when a run is "WAITING_FOR_DEPLOY" and is expected
+      logger.info("No matching deployment found for task run", {
         queueMessage: message.data,
         messageId: message.messageId,
       });
diff --git a/apps/webapp/app/v3/runEngineHandlers.server.ts b/apps/webapp/app/v3/runEngineHandlers.server.ts
@@ -17,6 +17,7 @@ import { updateMetadataService } from "~/services/metadata/updateMetadataInstanc
 import { findEnvironmentFromRun } from "~/models/runtimeEnvironment.server";
 import { env } from "~/env.server";
 import { getTaskEventStoreTableForRun } from "./taskEventStore.server";
+import { MetadataTooLargeError } from "~/utils/packets";
 
 export function registerRunEngineEventBusHandlers() {
   engine.eventBus.on("runSucceeded", async ({ time, run }) => {
@@ -381,17 +382,31 @@ export function registerRunEngineEventBusHandlers() {
     try {
       await updateMetadataService.call(run.id, run.metadata, env);
     } catch (e) {
-      logger.error("[runMetadataUpdated] Failed to update metadata", {
-        taskRun: run.id,
-        error:
-          e instanceof Error
-            ? {
-                name: e.name,
-                message: e.message,
-                stack: e.stack,
-              }
-            : e,
-      });
+      if (e instanceof MetadataTooLargeError) {
+        logger.warn("[runMetadataUpdated] Failed to update metadata, too large", {
+          taskRun: run.id,
+          error:
+            e instanceof Error
+              ? {
+                  name: e.name,
+                  message: e.message,
+                  stack: e.stack,
+                }
+              : e,
+        });
+      } else {
+        logger.error("[runMetadataUpdated] Failed to update metadata", {
+          taskRun: run.id,
+          error:
+            e instanceof Error
+              ? {
+                  name: e.name,
+                  message: e.message,
+                  stack: e.stack,
+                }
+              : e,
+        });
+      }
     }
   });
 
diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts
@@ -926,7 +926,7 @@ export class DeliverAlertService extends BaseService {
     });
 
     if (!response.ok) {
-      logger.error("[DeliverAlert] Failed to send alert webhook", {
+      logger.info("[DeliverAlert] Failed to send alert webhook", {
         status: response.status,
         statusText: response.statusText,
         url: webhook.url,
diff --git a/apps/webapp/app/v3/services/cancelTaskRunV1.server.ts b/apps/webapp/app/v3/services/cancelTaskRunV1.server.ts
@@ -43,7 +43,7 @@ export class CancelTaskRunServiceV1 extends BaseService {
 
     // Make sure the task run is in a cancellable state
     if (!isCancellableRunStatus(taskRun.status)) {
-      logger.error("Task run is not in a cancellable state", {
+      logger.info("Task run is not in a cancellable state", {
         runId: taskRun.id,
         status: taskRun.status,
       });
diff --git a/apps/webapp/app/v3/services/createCheckpoint.server.ts b/apps/webapp/app/v3/services/createCheckpoint.server.ts
@@ -132,7 +132,7 @@ export class CreateCheckpointService extends BaseService {
         }
 
         if (childRun.dependency?.resumedAt) {
-          logger.error("CreateCheckpointService: Child run already resumed", {
+          logger.info("CreateCheckpointService: Child run already resumed", {
             childRun,
             params,
           });
@@ -168,7 +168,7 @@ export class CreateCheckpointService extends BaseService {
         }
 
         if (batchRun.resumedAt) {
-          logger.error("CreateCheckpointService: Batch already resumed", {
+          logger.info("CreateCheckpointService: Batch already resumed", {
             batchRun,
             params,
           });
diff --git a/internal-packages/run-engine/src/engine/machinePresets.ts b/internal-packages/run-engine/src/engine/machinePresets.ts
@@ -26,7 +26,7 @@ export function getMachinePreset({
   const parsedConfig = MachineConfig.safeParse(config);
 
   if (!parsedConfig.success) {
-    logger.error("Failed to parse machine config", { config });
+    logger.info("Failed to parse machine config", { config });
 
     return machinePresetFromName(machines, "small-1x");
   }
diff --git a/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts b/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts
@@ -135,11 +135,21 @@ export class DequeueSystem {
 
                 if (isExecuting(snapshot.executionStatus)) {
                   this.$.logger.error(
-                    `RunEngine.dequeueFromWorkerQueue(): Run is not in a valid state to be dequeued: ${runId}\n ${snapshot.id}:${snapshot.executionStatus}`
+                    `RunEngine.dequeueFromWorkerQueue(): Run is not in a valid state to be dequeued`,
+                    {
+                      runId,
+                      snapshotId: snapshot.id,
+                      executionStatus: snapshot.executionStatus,
+                    }
                   );
                 } else {
                   this.$.logger.warn(
-                    `RunEngine.dequeueFromWorkerQueue(): Run is in an expected not valid state to be dequeued: ${runId}\n ${snapshot.id}:${snapshot.executionStatus}`
+                    `RunEngine.dequeueFromWorkerQueue(): Run is in an expected not valid state to be dequeued`,
+                    {
+                      runId,
+                      snapshotId: snapshot.id,
+                      executionStatus: snapshot.executionStatus,
+                    }
                   );
                 }
 
diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts
@@ -71,7 +71,9 @@ export class Logger {
 
     this.#structuredLog(console.error, message, "error", ...args);
 
-    if (Logger.onError) {
+    const ignoreError = args.some((arg) => arg?.ignoreError);
+
+    if (Logger.onError && !ignoreError) {
       Logger.onError(message, ...args);
     }
   }
diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
@@ -36,6 +36,8 @@ export type WorkerCatalog = {
     retry?: RetryOptions;
     cron?: string;
     jitterInMs?: number;
+    /** Defaults to true. If false, errors will not be logged. */
+    logErrors?: boolean;
   };
 };
 
@@ -541,12 +543,12 @@ class Worker<TCatalog extends WorkerCatalog> {
     const catalogItem = this.options.catalog[job as any];
     const handler = this.jobs[job as any];
     if (!handler) {
-      this.logger.error(`No handler found for job type: ${job}`);
+      this.logger.error(`Worker no handler found for job type: ${job}`);
       return;
     }
 
     if (!catalogItem) {
-      this.logger.error(`No catalog item found for job type: ${job}`);
+      this.logger.error(`Worker no catalog item found for job type: ${job}`);
       return;
     }
 
@@ -590,15 +592,25 @@ class Worker<TCatalog extends WorkerCatalog> {
       }
     ).catch(async (error) => {
       const errorMessage = error instanceof Error ? error.message : String(error);
-      this.logger.error(`Error processing item:`, {
+
+      const shouldLogError = catalogItem.logErrors ?? true;
+
+      const logAttributes = {
         name: this.options.name,
         id,
         job,
         item,
         visibilityTimeoutMs,
         error,
         errorMessage,
-      });
+      };
+
+      if (shouldLogError) {
+        this.logger.error(`Worker error processing item`, logAttributes);
+      } else {
+        this.logger.info(`Worker failed to process item`, logAttributes);
+      }
+
       // Attempt requeue logic.
       try {
         const newAttempt = attempt + 1;
@@ -609,15 +621,17 @@ class Worker<TCatalog extends WorkerCatalog> {
         const retryDelay = calculateNextRetryDelay(retrySettings, newAttempt);
 
         if (!retryDelay) {
-          this.logger.error(`Item ${id} reached max attempts. Moving to DLQ.`, {
-            name: this.options.name,
-            id,
-            job,
-            item,
-            visibilityTimeoutMs,
-            attempt: newAttempt,
-            errorMessage,
-          });
+          if (shouldLogError) {
+            this.logger.error(`Worker item reached max attempts. Moving to DLQ.`, {
+              ...logAttributes,
+              attempt: newAttempt,
+            });
+          } else {
+            this.logger.info(`Worker item reached max attempts. Moving to DLQ.`, {
+              ...logAttributes,
+              attempt: newAttempt,
+            });
+          }
 
           await this.queue.moveToDeadLetterQueue(id, errorMessage);
 
@@ -629,7 +643,7 @@ class Worker<TCatalog extends WorkerCatalog> {
         }
 
         const retryDate = new Date(Date.now() + retryDelay);
-        this.logger.info(`Requeuing failed item ${id} with delay`, {
+        this.logger.info(`Worker requeuing failed item with delay`, {
           name: this.options.name,
           id,
           job,
@@ -649,7 +663,7 @@ class Worker<TCatalog extends WorkerCatalog> {
         });
       } catch (requeueError) {
         this.logger.error(
-          `Failed to requeue item ${id}. It will be retried after the visibility timeout.`,
+          `Worker failed to requeue item. It will be retried after the visibility timeout.`,
           {
             name: this.options.name,
             id,

Original file line number	Diff line number	Diff line change
`@@ -201,6 +201,7 @@ function getClient() {`
`201`	`201`	`message: log.message,`
`202`	`202`	`target: log.target,`
`203`	`203`	`},`
	`204`	`+ ignoreError: true,`
`204`	`205`	`});`
`205`	`206`	`});`
`206`	`207`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,8 +27,12 @@ export const loader = createLoaderWorkerApiRoute(`
`27`	`27`
`28`	`28`	`return json(continuationResult);`
`29`	`29`	`} catch (error) {`
`30`		`- logger.error("Failed to suspend run", { runFriendlyId, snapshotFriendlyId, error });`
`31`		`- throw error;`
	`30`	`+ logger.warn("Failed to suspend run", { runFriendlyId, snapshotFriendlyId, error });`
	`31`	`+ if (error instanceof Error) {`
	`32`	`+ throw json({ error: error.message }, { status: 422 });`
	`33`	`+ }`
	`34`	`+`
	`35`	`+ throw json({ error: "Failed to continue run execution" }, { status: 422 });`
`32`	`36`	`}`
`33`	`37`	`}`
`34`	`38`	`);`
Original file line number	Diff line number	Diff line change
`@@ -1246,7 +1246,7 @@ export class EventRepository {`
`1246`	`1246`	`span.setAttribute("prisma_error_code", errorDetails.code);`
`1247`	`1247`	`}`
`1248`	`1248`
`1249`		`- logger.error("Failed to insert events, will attempt bisection", {`
	`1249`	`+ logger.info("Failed to insert events, will attempt bisection", {`
`1250`	`1250`	`error: errorDetails,`
`1251`	`1251`	`});`
`1252`	`1252`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ export function machinePresetFromConfig(config: unknown): MachinePreset {`
`6`	`6`	`const parsedConfig = MachineConfig.safeParse(config);`
`7`	`7`
`8`	`8`	`if (!parsedConfig.success) {`
`9`		`- logger.error("Failed to parse machine config", { config });`
	`9`	`+ logger.info("Failed to parse machine config", { config });`
`10`	`10`
`11`	`11`	`return machinePresetFromName("small-1x");`
`12`	`12`	`}`