chore: Add spans for alert processing (#1219)

Closes HDX-2506

# Summary

This PR adds a span for each task (of any kind) and each alert processing loop (one per connection), with provider, team, and connection span attributes. When ported to Cloud, this will allow us to more easily determine which team logs are coming from.

<img width="1492" height="105" alt="Screenshot 2025-09-29 at 1 04 20 PM" src="https://github.com/user-attachments/assets/2eed25fc-32b3-48b2-b7f0-24ffe0beab18" />
<img width="1449" height="148" alt="Screenshot 2025-09-29 at 1 04 17 PM" src="https://github.com/user-attachments/assets/1b8d6acc-df90-4048-96db-2b2dc2bc2d63" />
<img width="515" height="213" alt="Screenshot 2025-09-29 at 1 04 12 PM" src="https://github.com/user-attachments/assets/061f366f-24ac-4195-9b72-3f1b5a871e36" />

Author: pulpdrew

.changeset/dry-wombats-join.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+chore: Add spans for alert processing

packages/api/src/tasks/checkAlerts.ts

@@ -9,6 +9,7 @@ import {
   ChartConfigWithOptDateRange,
   DisplayType,
 } from '@hyperdx/common-utils/dist/types';
+import { setTraceAttributes } from '@hyperdx/node-opentelemetry';
 import * as fns from 'date-fns';
 import { chunk, isString } from 'lodash';
 import { ObjectId } from 'mongoose';
@@ -39,6 +40,8 @@ import { CheckAlertsTaskArgs, HdxTask } from '@/tasks/types';
 import { roundDownToXMinutes, unflattenObject } from '@/tasks/util';
 import logger from '@/utils/logger';
 
+import { tasksTracer } from './tracer';
+
 export const doesExceedThreshold = (
   thresholdType: AlertThresholdType,
   threshold: number,
@@ -432,26 +435,37 @@ export default class CheckAlertTask implements HdxTask<CheckAlertsTaskArgs> {
     alertTask: AlertTask,
     teamWebhooksById: Map<string, IWebhook>,
   ) {
-    const { alerts, conn } = alertTask;
-    logger.info({
-      message: 'Processing alerts in batch',
-      alertCount: alerts.length,
-    });
+    await tasksTracer.startActiveSpan('processAlertTask', async span => {
+      setTraceAttributes({
+        'hyperdx.alerts.team.id': alertTask.conn.team.toString(),
+        'hyperdx.alerts.connection.id': alertTask.conn.id,
+      });
 
-    const clickhouseClient = await this.provider.getClickHouseClient(conn);
+      try {
+        const { alerts, conn } = alertTask;
+        logger.info({
+          message: 'Processing alerts in batch',
+          alertCount: alerts.length,
+        });
 
-    for (const alert of alerts) {
-      await this.task_queue.add(() =>
-        processAlert(
-          alertTask.now,
-          alert,
-          clickhouseClient,
-          conn.id,
-          this.provider,
-          teamWebhooksById,
-        ),
-      );
-    }
+        const clickhouseClient = await this.provider.getClickHouseClient(conn);
+
+        for (const alert of alerts) {
+          await this.task_queue.add(() =>
+            processAlert(
+              alertTask.now,
+              alert,
+              clickhouseClient,
+              conn.id,
+              this.provider,
+              teamWebhooksById,
+            ),
+          );
+        }
+      } finally {
+        span.end();
+      }
+    });
   }
 
   async execute(): Promise<void> {
@@ -469,6 +483,10 @@ export default class CheckAlertTask implements HdxTask<CheckAlertsTaskArgs> {
       args: this.args,
     });
 
+    setTraceAttributes({
+      'hyperdx.alerts.provider': this.provider.constructor.name,
+    });
+
     const alertTasks = await this.provider.getAlertTasks();
     const taskCount = alertTasks.length;
     logger.debug({

packages/api/src/tasks/index.ts

@@ -9,6 +9,8 @@ import PingPongTask from '@/tasks/pingPongTask';
 import { asTaskArgs, HdxTask, TaskArgs } from '@/tasks/types';
 import logger from '@/utils/logger';
 
+import { tasksTracer } from './tracer';
+
 function createTask(argv: TaskArgs): HdxTask<TaskArgs> {
   const taskName = argv.taskName;
   switch (taskName) {
@@ -22,23 +24,26 @@ function createTask(argv: TaskArgs): HdxTask<TaskArgs> {
 }
 
 const main = async (argv: TaskArgs) => {
-  const task: HdxTask<TaskArgs> = createTask(argv);
-  try {
-    const t0 = performance.now();
-    logger.info(`Task [${task.name()}] started at ${new Date()}`);
-    await task.execute();
-    logger.info(
-      `Task [${task.name()}] finished in ${(performance.now() - t0).toFixed(2)} ms`,
-    );
-  } catch (e: unknown) {
-    logger.error({
-      message: `Task [${task.name()}] failed: ${serializeError(e)}`,
-      cause: e,
-      task,
-    });
-  } finally {
-    await task.asyncDispose();
-  }
+  await tasksTracer.startActiveSpan(argv.taskName || 'task', async span => {
+    const task: HdxTask<TaskArgs> = createTask(argv);
+    try {
+      const t0 = performance.now();
+      logger.info(`Task [${task.name()}] started at ${new Date()}`);
+      await task.execute();
+      logger.info(
+        `Task [${task.name()}] finished in ${(performance.now() - t0).toFixed(2)} ms`,
+      );
+    } catch (e: unknown) {
+      logger.error({
+        message: `Task [${task.name()}] failed: ${serializeError(e)}`,
+        cause: e,
+        task,
+      });
+    } finally {
+      await task.asyncDispose();
+      span.end();
+    }
+  });
 };
 
 // Entry point

packages/api/src/tasks/tracer.ts

@@ -0,0 +1,8 @@
+import opentelemetry from '@opentelemetry/api';
+
+import { CODE_VERSION } from '@/config';
+
+export const tasksTracer = opentelemetry.trace.getTracer(
+  'hyperdx-tasks',
+  CODE_VERSION,
+);