feat: Implement alerting for Raw SQL-based dashboard tiles

Author: pulpdrew

.changeset/short-badgers-applaud.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: Implement alerting for Raw SQL-based dashboard tiles

packages/api/openapi.json

@@ -162,7 +162,7 @@
           },
           "tileId": {
             "type": "string",
-            "description": "Tile ID for tile-based alerts. May not be a Raw-SQL-based tile.",
+            "description": "Tile ID for tile-based alerts. Must be a builder-type line/bar/number tile or a SQL-type line/bar tile.",
             "nullable": true,
             "example": "65f5e4a3b9e77c001a901234"
           },

packages/api/src/controllers/alerts.ts

@@ -1,3 +1,7 @@
+import {
+  displayTypeSupportsRawSqlAlerts,
+  isSqlTemplateValidForAlert,
+} from '@hyperdx/common-utils/dist/core/utils';
 import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
 import { sign, verify } from 'jsonwebtoken';
 import { groupBy } from 'lodash';
@@ -82,7 +86,17 @@ export const validateAlertInput = async (
     }
 
     if (tile.config != null && isRawSqlSavedChartConfig(tile.config)) {
-      throw new Api400Error('Cannot create an alert on a raw SQL tile');
+      if (!displayTypeSupportsRawSqlAlerts(tile.config.displayType)) {
+        throw new Api400Error(
+          'Alerts on Raw SQL tiles are only supported for Line or Stacked Bar display types',
+        );
+      }
+
+      if (!isSqlTemplateValidForAlert(tile.config)) {
+        throw new Api400Error(
+          'Raw SQL alert queries must include time filters and interval parameters to be valid for alerts.',
+        );
+      }
     }
   }
 

packages/api/src/controllers/dashboard.ts

@@ -1,7 +1,11 @@
-import { isBuilderSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
+import {
+  isBuilderSavedChartConfig,
+  isRawSqlSavedChartConfig,
+} from '@hyperdx/common-utils/dist/guards';
 import {
   BuilderSavedChartConfig,
   DashboardWithoutIdSchema,
+  RawSqlSavedChartConfig,
   Tile,
 } from '@hyperdx/common-utils/dist/types';
 import { map, partition, uniq } from 'lodash';
@@ -19,7 +23,7 @@ import Dashboard from '@/models/dashboard';
 
 function pickAlertsByTile(tiles: Tile[]) {
   return tiles.reduce((acc, tile) => {
-    if (isBuilderSavedChartConfig(tile.config) && tile.config.alert) {
+    if (tile.config.alert) {
       acc[tile.id] = tile.config.alert;
     }
     return acc;
@@ -29,6 +33,7 @@ function pickAlertsByTile(tiles: Tile[]) {
 type TileForAlertSync = Pick<Tile, 'id'> & {
   config?:
     | Pick<BuilderSavedChartConfig, 'alert'>
+    | Pick<RawSqlSavedChartConfig, 'alert'>
     | { alert?: IAlert | AlertDocument };
 };
 
@@ -55,9 +60,7 @@ async function syncDashboardAlerts(
 
   const newTilesForAlertSync: TileForAlertSync[] = newTiles.map(t => ({
     id: t.id,
-    config: isBuilderSavedChartConfig(t.config)
-      ? { alert: t.config.alert }
-      : {},
+    config: { alert: t.config.alert },
   }));
   const { tileIds: newTileIds, tileIdsWithAlerts: newTileIdsWithAlerts } =
     extractTileAlertData(newTilesForAlertSync);

packages/api/src/fixtures.ts

@@ -501,7 +501,39 @@ export const makeExternalTile = (opts?: {
   },
 });
 
-export const makeRawSqlTile = (opts?: { id?: string }): Tile => ({
+export const makeRawSqlTile = (opts?: {
+  id?: string;
+  displayType?: DisplayType;
+  sqlTemplate?: string;
+  connectionId?: string;
+}): Tile => ({
+  id: opts?.id ?? randomMongoId(),
+  x: 1,
+  y: 1,
+  w: 1,
+  h: 1,
+  config: {
+    configType: 'sql',
+    displayType: opts?.displayType ?? DisplayType.Line,
+    sqlTemplate: opts?.sqlTemplate ?? 'SELECT 1',
+    connection: opts?.connectionId ?? 'test-connection',
+  } satisfies RawSqlSavedChartConfig,
+});
+
+export const RAW_SQL_ALERT_TEMPLATE = [
+  'SELECT toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts,',
+  ' count() AS cnt',
+  ' FROM default.otel_logs',
+  ' WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})',
+  ' AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})',
+  ' GROUP BY ts ORDER BY ts',
+].join('');
+
+export const makeRawSqlAlertTile = (opts?: {
+  id?: string;
+  connectionId?: string;
+  sqlTemplate?: string;
+}): Tile => ({
   id: opts?.id ?? randomMongoId(),
   x: 1,
   y: 1,
@@ -510,8 +542,8 @@ export const makeRawSqlTile = (opts?: { id?: string }): Tile => ({
   config: {
     configType: 'sql',
     displayType: DisplayType.Line,
-    sqlTemplate: 'SELECT 1',
-    connection: 'test-connection',
+    sqlTemplate: opts?.sqlTemplate ?? RAW_SQL_ALERT_TEMPLATE,
+    connection: opts?.connectionId ?? 'test-connection',
   } satisfies RawSqlSavedChartConfig,
 });
 

packages/api/src/routers/api/__tests__/alerts.test.ts

@@ -1,10 +1,14 @@
+import { DisplayType } from '@hyperdx/common-utils/dist/types';
+
 import {
   getLoggedInAgent,
   getServer,
   makeAlertInput,
+  makeRawSqlAlertTile,
   makeRawSqlTile,
   makeTile,
   randomMongoId,
+  RAW_SQL_ALERT_TEMPLATE,
 } from '@/fixtures';
 import Alert, { AlertSource, AlertThresholdType } from '@/models/alert';
 import Webhook, { WebhookDocument, WebhookService } from '@/models/webhook';
@@ -550,8 +554,36 @@ describe('alerts router', () => {
     await agent.delete(`/alerts/${fakeId}/silenced`).expect(404); // Should fail
   });
 
-  it('rejects creating an alert on a raw SQL tile', async () => {
-    const rawSqlTile = makeRawSqlTile();
+  it('allows creating an alert on a raw SQL line tile', async () => {
+    const rawSqlTile = makeRawSqlAlertTile();
+    const dashboard = await agent
+      .post('/dashboards')
+      .send({
+        name: 'Test Dashboard',
+        tiles: [rawSqlTile],
+        tags: [],
+      })
+      .expect(200);
+
+    const alert = await agent
+      .post('/alerts')
+      .send(
+        makeAlertInput({
+          dashboardId: dashboard.body.id,
+          tileId: rawSqlTile.id,
+          webhookId: webhook._id.toString(),
+        }),
+      )
+      .expect(200);
+    expect(alert.body.data.dashboard).toBe(dashboard.body.id);
+    expect(alert.body.data.tileId).toBe(rawSqlTile.id);
+  });
+
+  it('rejects creating an alert on a raw SQL number tile', async () => {
+    const rawSqlTile = makeRawSqlTile({
+      displayType: DisplayType.Number,
+      sqlTemplate: RAW_SQL_ALERT_TEMPLATE,
+    });
     const dashboard = await agent
       .post('/dashboards')
       .send({
@@ -573,9 +605,37 @@ describe('alerts router', () => {
       .expect(400);
   });
 
-  it('rejects updating an alert to reference a raw SQL tile', async () => {
+  it('rejects creating an alert on a raw SQL tile without interval params', async () => {
+    const rawSqlTile = makeRawSqlTile({
+      sqlTemplate: 'SELECT count() FROM otel_logs',
+    });
+    const dashboard = await agent
+      .post('/dashboards')
+      .send({
+        name: 'Test Dashboard',
+        tiles: [rawSqlTile],
+        tags: [],
+      })
+      .expect(200);
+
+    await agent
+      .post('/alerts')
+      .send(
+        makeAlertInput({
+          dashboardId: dashboard.body.id,
+          tileId: rawSqlTile.id,
+          webhookId: webhook._id.toString(),
+        }),
+      )
+      .expect(400);
+  });
+
+  it('rejects updating an alert to reference a raw SQL number tile', async () => {
     const regularTile = makeTile();
-    const rawSqlTile = makeRawSqlTile();
+    const rawSqlTile = makeRawSqlTile({
+      displayType: DisplayType.Number,
+      sqlTemplate: RAW_SQL_ALERT_TEMPLATE,
+    });
     const dashboard = await agent
       .post('/dashboards')
       .send({

packages/api/src/routers/external-api/__tests__/alerts.test.ts

@@ -2,7 +2,11 @@ import _ from 'lodash';
 import { ObjectId } from 'mongodb';
 import request from 'supertest';
 
-import { getLoggedInAgent, getServer } from '../../../fixtures';
+import {
+  getLoggedInAgent,
+  getServer,
+  RAW_SQL_ALERT_TEMPLATE,
+} from '../../../fixtures';
 import { AlertSource, AlertThresholdType } from '../../../models/alert';
 import Alert from '../../../models/alert';
 import Dashboard from '../../../models/dashboard';
@@ -83,8 +87,13 @@ describe('External API Alerts', () => {
   };
 
   // Helper to create a dashboard with a raw SQL tile for testing
+  // Uses Number display type by default (not alertable) for rejection tests
   const createTestDashboardWithRawSqlTile = async (
-    options: { teamId?: any } = {},
+    options: {
+      teamId?: any;
+      displayType?: string;
+      sqlTemplate?: string;
+    } = {},
   ) => {
     const tileId = new ObjectId().toString();
     const tiles = [
@@ -97,8 +106,8 @@ describe('External API Alerts', () => {
         h: 3,
         config: {
           configType: 'sql',
-          displayType: 'line',
-          sqlTemplate: 'SELECT 1',
+          displayType: options.displayType ?? 'number',
+          sqlTemplate: options.sqlTemplate ?? 'SELECT 1',
           connection: 'test-connection',
         },
       },
@@ -716,7 +725,34 @@ describe('External API Alerts', () => {
           .expect(400);
       });
 
-      it('should reject creating an alert on a raw SQL tile', async () => {
+      it('should allow creating an alert on a raw SQL line tile', async () => {
+        const webhook = await createTestWebhook();
+        const { dashboard, tileId } = await createTestDashboardWithRawSqlTile({
+          displayType: 'line',
+          sqlTemplate: RAW_SQL_ALERT_TEMPLATE,
+        });
+
+        const alertInput = {
+          dashboardId: dashboard._id.toString(),
+          tileId,
+          threshold: 100,
+          interval: '1h',
+          source: AlertSource.TILE,
+          thresholdType: AlertThresholdType.ABOVE,
+          channel: {
+            type: 'webhook',
+            webhookId: webhook._id.toString(),
+          },
+        };
+
+        const res = await authRequest('post', ALERTS_BASE_URL)
+          .send(alertInput)
+          .expect(200);
+        expect(res.body.data.dashboardId).toBe(dashboard._id.toString());
+        expect(res.body.data.tileId).toBe(tileId);
+      });
+
+      it('should reject creating an alert on a raw SQL number tile', async () => {
         const webhook = await createTestWebhook();
         const { dashboard, tileId } = await createTestDashboardWithRawSqlTile();
 
@@ -736,7 +772,30 @@ describe('External API Alerts', () => {
         await authRequest('post', ALERTS_BASE_URL).send(alertInput).expect(400);
       });
 
-      it('should reject updating an alert to reference a raw SQL tile', async () => {
+      it('should reject creating an alert on a raw SQL tile without interval params', async () => {
+        const webhook = await createTestWebhook();
+        const { dashboard, tileId } = await createTestDashboardWithRawSqlTile({
+          displayType: 'line',
+          sqlTemplate: 'SELECT count() FROM otel_logs',
+        });
+
+        const alertInput = {
+          dashboardId: dashboard._id.toString(),
+          tileId,
+          threshold: 100,
+          interval: '1h',
+          source: AlertSource.TILE,
+          thresholdType: AlertThresholdType.ABOVE,
+          channel: {
+            type: 'webhook',
+            webhookId: webhook._id.toString(),
+          },
+        };
+
+        await authRequest('post', ALERTS_BASE_URL).send(alertInput).expect(400);
+      });
+
+      it('should reject updating an alert to reference a raw SQL number tile', async () => {
         const { alert, webhook } = await createTestAlert();
         const { dashboard: rawSqlDashboard, tileId: rawSqlTileId } =
           await createTestDashboardWithRawSqlTile();

packages/api/src/routers/external-api/v2/alerts.ts

@@ -95,7 +95,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod';
  *           example: "65f5e4a3b9e77c001a567890"
  *         tileId:
  *           type: string
- *           description: Tile ID for tile-based alerts. May not be a Raw-SQL-based tile.
+ *           description: Tile ID for tile-based alerts. Must be a builder-type line/bar/number tile or a SQL-type line/bar tile.
  *           nullable: true
  *           example: "65f5e4a3b9e77c001a901234"
  *         savedSearchId:

packages/api/src/routers/external-api/v2/dashboards.ts

@@ -1,4 +1,7 @@
-import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
+import {
+  displayTypeSupportsBuilderAlerts,
+  displayTypeSupportsRawSqlAlerts,
+} from '@hyperdx/common-utils/dist/core/utils';
 import { SearchConditionLanguageSchema as whereLanguageSchema } from '@hyperdx/common-utils/dist/types';
 import express from 'express';
 import { uniq } from 'lodash';
@@ -2053,11 +2056,15 @@ router.put(
         return res.sendStatus(404);
       }
 
-      // Delete alerts for tiles that are now raw SQL (unsupported) or were removed
+      // Delete alerts for tiles that now do not support alerts
       const newTileIdSet = new Set(internalTiles.map(t => t.id));
       const tileIdsToDeleteAlerts = [
         ...internalTiles
-          .filter(tile => isRawSqlSavedChartConfig(tile.config))
+          .filter(tile =>
+            'configType' in tile.config && tile.config.configType === 'sql'
+              ? !displayTypeSupportsRawSqlAlerts(tile.config.displayType)
+              : !displayTypeSupportsBuilderAlerts(tile.config.displayType),
+          )
           .map(tile => tile.id),
         ...[...existingTileIds].filter(id => !newTileIdSet.has(id)),
       ];