diff --git a/.changeset/polite-grapes-cross.md b/.changeset/polite-grapes-cross.md new file mode 100644 index 0000000000..e3781c6d6d --- /dev/null +++ b/.changeset/polite-grapes-cross.md @@ -0,0 +1,8 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +"@hyperdx/cli": patch +--- + +feat: Add additional alert threshold types diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 4b6dc21307..0f946aee01 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -70,7 +70,11 @@ "type": "string", "enum": [ "above", - "below" + "below", + "above_exclusive", + "below_or_equal", + "equal", + "not_equal" ], "description": "Threshold comparison direction." }, diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 48ba7602e0..edb52a3cf3 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -3,6 +3,7 @@ import { validateRawSqlForAlert, } from '@hyperdx/common-utils/dist/core/utils'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; import { sign, verify } from 'jsonwebtoken'; import { groupBy } from 'lodash'; import ms from 'ms'; @@ -13,7 +14,6 @@ import Alert, { AlertChannel, AlertInterval, AlertSource, - AlertThresholdType, IAlert, } from '@/models/alert'; import Dashboard, { IDashboard } from '@/models/dashboard'; diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 42dc07fa38..ffe0137870 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -1,5 +1,6 @@ import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { + AlertThresholdType, BuilderSavedChartConfig, DisplayType, RawSqlSavedChartConfig, @@ -15,7 +16,7 @@ import { AlertInput } from '@/controllers/alerts'; import { getTeam } from '@/controllers/team'; import { findUserByEmail } from '@/controllers/user'; import { mongooseConnection } from '@/models'; -import { AlertInterval, AlertSource, AlertThresholdType } from '@/models/alert'; +import { AlertInterval, AlertSource } from '@/models/alert'; import Server from '@/server'; import logger from '@/utils/logger'; import { MetricModel } from '@/utils/logParser'; diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index b0b1415af0..ce1a81abe4 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -1,14 +1,13 @@ -import { ALERT_INTERVAL_TO_MINUTES } from '@hyperdx/common-utils/dist/types'; +import { + ALERT_INTERVAL_TO_MINUTES, + AlertThresholdType, +} from '@hyperdx/common-utils/dist/types'; +export { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; import mongoose, { Schema } from 'mongoose'; import type { ObjectId } from '.'; import Team from './team'; -export enum AlertThresholdType { - ABOVE = 'above', - BELOW = 'below', -} - export enum AlertState { ALERT = 'ALERT', DISABLED = 'DISABLED', diff --git a/packages/api/src/routers/api/__tests__/alerts.test.ts b/packages/api/src/routers/api/__tests__/alerts.test.ts index 06ac4e1aa1..7dace1c7d5 100644 --- a/packages/api/src/routers/api/__tests__/alerts.test.ts +++ b/packages/api/src/routers/api/__tests__/alerts.test.ts @@ -1,4 +1,7 @@ -import { DisplayType } from '@hyperdx/common-utils/dist/types'; +import { + AlertThresholdType, + DisplayType, +} from '@hyperdx/common-utils/dist/types'; import { getLoggedInAgent, @@ -11,11 +14,7 @@ import { randomMongoId, RAW_SQL_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { - AlertSource, - AlertState, - AlertThresholdType, -} from '@/models/alert'; +import Alert, { AlertSource, AlertState } from '@/models/alert'; import AlertHistory from '@/models/alertHistory'; import Webhook, { WebhookDocument, WebhookService } from '@/models/webhook'; diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index 067b153968..2c9a8aac36 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -34,7 +34,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * description: Evaluation interval. * AlertThresholdType: * type: string - * enum: [above, below] + * enum: [above, below, above_exclusive, below_or_equal, equal, not_equal] * description: Threshold comparison direction. * AlertSource: * type: string diff --git a/packages/api/src/tasks/checkAlerts/__tests__/__snapshots__/renderAlertTemplate.test.ts.snap b/packages/api/src/tasks/checkAlerts/__tests__/__snapshots__/renderAlertTemplate.test.ts.snap new file mode 100644 index 0000000000..66833b94ba --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/__snapshots__/renderAlertTemplate.test.ts.snap @@ -0,0 +1,284 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state above threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state below threshold=5 alertValue=2 1`] = `"🚨 Alert for "My Search" - 2 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `"🚨 Alert for "My Search" - 3 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state equal threshold=5 alertValue=5 1`] = `"🚨 Alert for "My Search" - 5 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `"🚨 Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) above threshold=5 okValue=3 1`] = `"✅ Alert for "My Search" - 3 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `"✅ Alert for "My Search" - 3 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) below threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `"✅ Alert for "My Search" - 10 lines found"`; + +exports[`buildAlertMessageTemplateTitle saved search alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `"✅ Alert for "My Search" - 5 lines found"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state above threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 meets or exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state below threshold=5 alertValue=2 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 2 falls below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 3 falls to or below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state decimal threshold 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10.1 meets or exceeds 1.5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state equal threshold=5 alertValue=5 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 equals 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state integer threshold rounds value 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 11 meets or exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts ALERT state not_equal threshold=5 alertValue=10 1`] = `"🚨 Alert for "Test Chart" in "My Dashboard" - 10 does not equal 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) above threshold=5 okValue=3 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 3 falls below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 3 falls to or below 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) below threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 meets or exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 exceeds 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) equal threshold=5 okValue=10 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 10 does not equal 5"`; + +exports[`buildAlertMessageTemplateTitle tile alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = `"✅ Alert for "Test Chart" in "My Dashboard" - 5 equals 5"`; + +exports[`renderAlertTemplate saved search alerts ALERT state above threshold=5 alertValue=10 1`] = ` +" +10 lines found, which meets or exceeds the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = ` +" +10 lines found, which exceeds the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state below threshold=5 alertValue=2 1`] = ` +" +2 lines found, which falls below the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = ` +" +3 lines found, which falls to or below the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state equal threshold=5 alertValue=5 1`] = ` +" +5 lines found, which equals the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state not_equal threshold=5 alertValue=10 1`] = ` +" +10 lines found, which does not equal the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts ALERT state with group 1`] = ` +"Group: "http" +10 lines found, which meets or exceeds the threshold of 5 lines +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) + +\`\`\` +"2023-03-17 22:14:01","error","Failed to connect to database" +"2023-03-17 22:13:45","error","Connection timeout after 30s" +"2023-03-17 22:12:30","error","Retry limit exceeded" +\`\`\`" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) above threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) below threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate saved search alerts OK state (resolved) with group 1`] = ` +"Group: "http" - The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state above threshold=5 alertValue=10 1`] = ` +" +10 meets or exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state above_exclusive threshold=5 alertValue=10 1`] = ` +" +10 exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state below threshold=5 alertValue=2 1`] = ` +" +2 falls below 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state below_or_equal threshold=5 alertValue=3 1`] = ` +" +3 falls to or below 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state decimal threshold 1`] = ` +" +10.1 meets or exceeds 1.5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state equal threshold=5 alertValue=5 1`] = ` +" +5 equals 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state integer threshold rounds value 1`] = ` +" +11 meets or exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state not_equal threshold=5 alertValue=10 1`] = ` +" +10 does not equal 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts ALERT state with group 1`] = ` +"Group: "us-east-1" +10 meets or exceeds 5 +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) above threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) above_exclusive threshold=5 okValue=3 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) below threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) below_or_equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) equal threshold=5 okValue=10 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) not_equal threshold=5 okValue=5 1`] = ` +"The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; + +exports[`renderAlertTemplate tile alerts OK state (resolved) with group 1`] = ` +"Group: "us-east-1" - The alert has been resolved. +Time Range (UTC): [Mar 17 10:10:00 PM - Mar 17 10:15:00 PM) +" +`; diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts index 55bcade0c1..49305c68a3 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.test.ts @@ -1,6 +1,7 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { AlertState, + AlertThresholdType, SourceKind, Tile, WebhookService, @@ -23,7 +24,7 @@ import { RAW_SQL_ALERT_TEMPLATE, RAW_SQL_NUMBER_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { AlertSource, AlertThresholdType } from '@/models/alert'; +import Alert, { AlertSource } from '@/models/alert'; import AlertHistory from '@/models/alertHistory'; import Connection, { IConnection } from '@/models/connection'; import Dashboard, { IDashboard } from '@/models/dashboard'; @@ -125,6 +126,205 @@ describe('checkAlerts', () => { false, ); }); + + // ABOVE_EXCLUSIVE (>) tests + it('should return true when value is strictly above ABOVE_EXCLUSIVE threshold', () => { + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 10, 11), + ).toBe(true); + }); + + it('should return false when value equals ABOVE_EXCLUSIVE threshold', () => { + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 10, 10), + ).toBe(false); + }); + + it('should return false when value is below ABOVE_EXCLUSIVE threshold', () => { + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 10, 9), + ).toBe(false); + }); + + it('should handle zero values correctly for ABOVE_EXCLUSIVE', () => { + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 0, 1), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 0, 0), + ).toBe(false); + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 0, -1), + ).toBe(false); + }); + + it('should handle negative values correctly for ABOVE_EXCLUSIVE', () => { + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, -5, -3), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, -5, -5), + ).toBe(false); + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, -5, -7), + ).toBe(false); + }); + + it('should handle decimal values correctly for ABOVE_EXCLUSIVE', () => { + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 10.5, 11.0), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 10.5, 10.5), + ).toBe(false); + expect( + doesExceedThreshold(AlertThresholdType.ABOVE_EXCLUSIVE, 10.5, 10.0), + ).toBe(false); + }); + + // BELOW_OR_EQUAL (<=) tests + it('should return true when value is below BELOW_OR_EQUAL threshold', () => { + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 10, 9), + ).toBe(true); + }); + + it('should return true when value equals BELOW_OR_EQUAL threshold', () => { + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 10, 10), + ).toBe(true); + }); + + it('should return false when value is above BELOW_OR_EQUAL threshold', () => { + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 10, 11), + ).toBe(false); + }); + + it('should handle zero values correctly for BELOW_OR_EQUAL', () => { + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 0, -1), + ).toBe(true); + expect(doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 0, 0)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 0, 1)).toBe( + false, + ); + }); + + it('should handle negative values correctly for BELOW_OR_EQUAL', () => { + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, -5, -7), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, -5, -5), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, -5, -3), + ).toBe(false); + }); + + it('should handle decimal values correctly for BELOW_OR_EQUAL', () => { + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 10.5, 10.0), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 10.5, 10.5), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.BELOW_OR_EQUAL, 10.5, 11.0), + ).toBe(false); + }); + + // EQUAL (=) tests + it('should return true when value equals EQUAL threshold', () => { + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 10, 10)).toBe(true); + }); + + it('should return false when value is above EQUAL threshold', () => { + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 10, 11)).toBe(false); + }); + + it('should return false when value is below EQUAL threshold', () => { + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 10, 9)).toBe(false); + }); + + it('should handle zero values correctly for EQUAL', () => { + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 0, 0)).toBe(true); + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 0, 1)).toBe(false); + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 0, -1)).toBe(false); + }); + + it('should handle negative values correctly for EQUAL', () => { + expect(doesExceedThreshold(AlertThresholdType.EQUAL, -5, -5)).toBe(true); + expect(doesExceedThreshold(AlertThresholdType.EQUAL, -5, -3)).toBe(false); + expect(doesExceedThreshold(AlertThresholdType.EQUAL, -5, -7)).toBe(false); + }); + + it('should handle decimal values correctly for EQUAL', () => { + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 10.5, 10.5)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 10.5, 10.0)).toBe( + false, + ); + expect(doesExceedThreshold(AlertThresholdType.EQUAL, 10.5, 11.0)).toBe( + false, + ); + }); + + // NOT_EQUAL (≠) tests + it('should return true when value does not equal NOT_EQUAL threshold', () => { + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 10, 11)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 10, 9)).toBe( + true, + ); + }); + + it('should return false when value equals NOT_EQUAL threshold', () => { + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 10, 10)).toBe( + false, + ); + }); + + it('should handle zero values correctly for NOT_EQUAL', () => { + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 0, 1)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 0, -1)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 0, 0)).toBe( + false, + ); + }); + + it('should handle negative values correctly for NOT_EQUAL', () => { + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, -5, -3)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, -5, -7)).toBe( + true, + ); + expect(doesExceedThreshold(AlertThresholdType.NOT_EQUAL, -5, -5)).toBe( + false, + ); + }); + + it('should handle decimal values correctly for NOT_EQUAL', () => { + expect( + doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 10.5, 11.0), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 10.5, 10.0), + ).toBe(true); + expect( + doesExceedThreshold(AlertThresholdType.NOT_EQUAL, 10.5, 10.5), + ).toBe(false); + }); }); describe('getScheduledWindowStart', () => { @@ -494,7 +694,7 @@ describe('checkAlerts', () => { view: defaultChartView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 exceeds 1"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 meets or exceeds 1"`, ); }); @@ -512,7 +712,7 @@ describe('checkAlerts', () => { state: AlertState.ALERT, }), ).toMatchInlineSnapshot( - `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 exceeds 1"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 5 meets or exceeds 1"`, ); // Test OK state (should have ✅ emoji) @@ -528,7 +728,7 @@ describe('checkAlerts', () => { state: AlertState.OK, }), ).toMatchInlineSnapshot( - `"✅ Alert for "Test Chart" in "My Dashboard" - 5 exceeds 1"`, + `"✅ Alert for "Test Chart" in "My Dashboard" - 5 meets or exceeds 1"`, ); }); @@ -548,7 +748,7 @@ describe('checkAlerts', () => { view: decimalChartView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for "Test Chart" in "My Dashboard" - 1111.1 exceeds 1.5"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 1111.1 meets or exceeds 1.5"`, ); // Test with multiple decimal places @@ -566,7 +766,7 @@ describe('checkAlerts', () => { view: multiDecimalChartView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for "Test Chart" in "My Dashboard" - 1.1235 exceeds 0.1234"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 1.1235 meets or exceeds 0.1234"`, ); // Test with integer value and decimal threshold @@ -584,7 +784,7 @@ describe('checkAlerts', () => { view: integerValueView, }), ).toMatchInlineSnapshot( - `"🚨 Alert for "Test Chart" in "My Dashboard" - 10.00 exceeds 0.12"`, + `"🚨 Alert for "Test Chart" in "My Dashboard" - 10.00 meets or exceeds 0.12"`, ); }); @@ -752,7 +952,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', 'Custom body ', '```', @@ -812,7 +1012,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', 'Custom body ', '```', @@ -919,7 +1119,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', '', ' Runbook URL: https://example.com', @@ -948,7 +1148,7 @@ describe('checkAlerts', () => { text: [ '**', 'Group: "http"', - '10 lines found, expected less than 1 lines', + '10 lines found, which meets or exceeds the threshold of 1 lines', 'Time Range (UTC): [Mar 17 10:13:03 PM - Mar 17 10:13:59 PM)', '', ' Runbook URL: https://example.com', @@ -1624,14 +1824,14 @@ describe('checkAlerts', () => { 1, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1', + text: '🚨 Alert for "Logs Count" in "My Dashboard" - 3 meets or exceeds 1', blocks: [ { text: { text: [ - `**`, + `**`, '', - '3 exceeds 1', + '3 meets or exceeds 1', 'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)', '', ].join('\n'), @@ -1816,7 +2016,7 @@ describe('checkAlerts', () => { expect(fetchMock).toHaveBeenCalledWith('https://webhook.site/123', { method: 'POST', body: JSON.stringify({ - text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 exceeds 1`, + text: `http://app:8080/dashboards/${dashboard.id}?from=1700170200000&granularity=5+minute&to=1700174700000 | 🚨 Alert for "Logs Count" in "My Dashboard" - 3 meets or exceeds 1`, }), headers: { 'Content-Type': 'application/json', @@ -3611,14 +3811,14 @@ describe('checkAlerts', () => { 1, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "CPU" in "My Dashboard" - 6 exceeds 1', + text: '🚨 Alert for "CPU" in "My Dashboard" - 6 meets or exceeds 1', blocks: [ { text: { text: [ - `**`, + `**`, '', - '6 exceeds 1', + '6 meets or exceeds 1', 'Time Range (UTC): [Nov 16 10:05:00 PM - Nov 16 10:10:00 PM)', '', ].join('\n'), @@ -3794,7 +3994,7 @@ describe('checkAlerts', () => { for (const msg of messages) { expect(msg.text).toContain('CPU by Service'); expect(msg.text).toContain('My Dashboard'); - expect(msg.text).toContain('exceeds 1'); + expect(msg.text).toContain('meets or exceeds 1'); } // Body should contain Group: "ServiceName:service-a" or "ServiceName:service-b" @@ -5606,6 +5806,896 @@ describe('checkAlerts', () => { expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); }); + + // --------------------------------------------------------------- + // Integration tests for threshold types + // Each test follows ALERT → Resolve flow with boundary-condition + // values in the resolve period. + // --------------------------------------------------------------- + + it('SAVED_SEARCH alert with ABOVE_EXCLUSIVE threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, ABOVE_EXCLUSIVE means value > 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: 3 error logs (should ALERT since 3 > 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + // Period 2: exactly 2 error logs (should resolve since 2 is NOT > 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 3 logs, threshold is > 2, should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, threshold is > 2, 2 is NOT > 2 so should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with ABOVE_EXCLUSIVE threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 3 logs (ALERT: 3 > 2) + // Period 2: exactly 2 logs (OK: 2 is NOT > 2, boundary value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-above-excl', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find( + (t: any) => t.id === 'tile-above-excl', + ); + if (!tile) throw new Error('tile not found'); + + // threshold = 2, ABOVE_EXCLUSIVE means > 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 2, + dashboardId: dashboard.id, + tileId: 'tile-above-excl', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 3 logs, 3 > 2 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, 2 is NOT > 2 (boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('SAVED_SEARCH alert with BELOW_OR_EQUAL threshold - should alert then resolve', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, BELOW_OR_EQUAL means value <= 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: exactly 2 error logs (should ALERT since 2 <= 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + // Period 2: 3 error logs (should resolve since 3 is NOT <= 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 2 logs, threshold is <= 2, should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 3 logs, threshold is <= 2, should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with BELOW_OR_EQUAL threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 1 log (ALERT: 1 <= 1, boundary) + // Period 2: 2 logs (OK: 2 is NOT <= 1, nearest non-matching value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-below-eq', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'tile-below-eq'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'tile-below-eq', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 1 log, 1 <= 1 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, 2 is NOT <= 1 (near-boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('SAVED_SEARCH alert with EQUAL threshold - should alert then resolve', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, EQUAL means value == 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.EQUAL, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: exactly 2 error logs (should ALERT since 2 == 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + // Period 2: 3 error logs (should resolve since 3 != 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 2 logs, threshold is == 2, should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 3 logs, threshold is == 2, 3 != 2 so should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with EQUAL threshold - should alert then resolve at near-boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 3 logs (ALERT: 3 == 3) + // Period 2: 2 logs (OK: 2 != 3, nearest non-matching value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-equal', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'tile-equal'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.EQUAL, + threshold: 3, + dashboardId: dashboard.id, + tileId: 'tile-equal', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 3 logs, 3 == 3 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, 2 != 3 (near-boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('SAVED_SEARCH alert with NOT_EQUAL threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // threshold = 2, NOT_EQUAL means value != 2 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.NOT_EQUAL, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + await bulkInsertLogs([ + // Period 1: 3 error logs (should ALERT since 3 != 2) + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 3', + }, + // Period 2: exactly 2 error logs (should resolve since 2 == 2) + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + // Period 1: 3 logs, threshold is != 2, 3 != 2 so should ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 2 logs, threshold is != 2, 2 == 2 so should resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); + + it('TILE alert with NOT_EQUAL threshold - should alert then resolve at boundary', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const period1Start = new Date('2023-11-16T22:05:00.000Z'); + const period2Start = new Date(period1Start.getTime() + ms('5m')); + + // Period 1: 2 logs (ALERT: 2 != 3) + // Period 2: exactly 3 logs (OK: 3 == 3, boundary value) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 1', + }, + { + ServiceName: 'api', + Timestamp: period1Start, + SeverityText: 'error', + Body: 'error 2', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 3', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 4', + }, + { + ServiceName: 'api', + Timestamp: period2Start, + SeverityText: 'error', + Body: 'error 5', + }, + ]); + + const dashboard = await new Dashboard({ + name: 'Test Dashboard', + team: team._id, + tiles: [ + { + id: 'tile-not-equal', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + name: 'Error Count', + select: [ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + valueExpression: '', + aggConditionLanguage: 'lucene', + }, + ], + where: '', + displayType: 'line', + granularity: 'auto', + source: source.id, + groupBy: '', + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'tile-not-equal'); + if (!tile) throw new Error('tile not found'); + + // threshold = 3, NOT_EQUAL means != 3 + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.NOT_EQUAL, + threshold: 3, + dashboardId: dashboard.id, + tileId: 'tile-not-equal', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + // Period 1: 2 logs, 2 != 3 → ALERT + const firstRunTime = new Date(period1Start.getTime() + ms('5m')); + await processAlertAtTime( + firstRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + + // Period 2: 3 logs, 3 == 3 (boundary) → resolve to OK + const secondRunTime = new Date(period2Start.getTime() + ms('5m')); + await processAlertAtTime( + secondRunTime, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); + }); }); describe('processAlert with materialized views', () => { diff --git a/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.test.ts new file mode 100644 index 0000000000..07d8f496aa --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.test.ts @@ -0,0 +1,402 @@ +import { + AlertState, + AlertThresholdType, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; +import mongoose from 'mongoose'; + +import { makeTile } from '@/fixtures'; +import { AlertSource } from '@/models/alert'; +import { loadProvider } from '@/tasks/checkAlerts/providers'; +import { + AlertMessageTemplateDefaultView, + buildAlertMessageTemplateTitle, + renderAlertTemplate, +} from '@/tasks/checkAlerts/template'; + +let alertProvider: any; + +beforeAll(async () => { + alertProvider = await loadProvider(); +}); + +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +const mockMetadata = { + getColumn: jest.fn().mockImplementation(({ column }) => { + const columnMap = { + Timestamp: { name: 'Timestamp', type: 'DateTime' }, + Body: { name: 'Body', type: 'String' }, + SeverityText: { name: 'SeverityText', type: 'String' }, + ServiceName: { name: 'ServiceName', type: 'String' }, + }; + return Promise.resolve(columnMap[column]); + }), + getColumns: jest.fn().mockResolvedValue([]), + getMapKeys: jest.fn().mockResolvedValue([]), + getMapValues: jest.fn().mockResolvedValue([]), + getAllFields: jest.fn().mockResolvedValue([]), + getTableMetadata: jest.fn().mockResolvedValue({}), + getClickHouseSettings: jest.fn().mockReturnValue({}), + setClickHouseSettings: jest.fn(), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), +} as any; + +const sampleLogsCsv = [ + '"2023-03-17 22:14:01","error","Failed to connect to database"', + '"2023-03-17 22:13:45","error","Connection timeout after 30s"', + '"2023-03-17 22:12:30","error","Retry limit exceeded"', +].join('\n'); + +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion +const mockClickhouseClient = { + query: jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ data: [] }), + text: jest.fn().mockResolvedValue(sampleLogsCsv), + }), +} as any; + +const startTime = new Date('2023-03-17T22:10:00.000Z'); +const endTime = new Date('2023-03-17T22:15:00.000Z'); + +const makeSearchView = ( + overrides: Partial & { + thresholdType?: AlertThresholdType; + threshold?: number; + value?: number; + group?: string; + } = {}, +): AlertMessageTemplateDefaultView => ({ + alert: { + thresholdType: overrides.thresholdType ?? AlertThresholdType.ABOVE, + threshold: overrides.threshold ?? 5, + source: AlertSource.SAVED_SEARCH, + channel: { type: null }, + interval: '1m', + }, + source: { + id: 'fake-source-id', + kind: SourceKind.Log, + team: 'team-123', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: 'connection-123', + name: 'Logs', + defaultTableSelectExpression: 'Timestamp, Body', + }, + savedSearch: { + _id: 'fake-saved-search-id' as any, + team: 'team-123' as any, + id: 'fake-saved-search-id', + name: 'My Search', + select: 'Body', + where: 'Body: "error"', + whereLanguage: 'lucene', + orderBy: 'timestamp', + source: 'fake-source-id' as any, + tags: ['test'], + createdAt: new Date(), + updatedAt: new Date(), + }, + attributes: {}, + granularity: '1m', + group: overrides.group, + isGroupedAlert: false, + startTime, + endTime, + value: overrides.value ?? 10, +}); + +const testTile = makeTile({ id: 'test-tile-id' }); +const makeTileView = ( + overrides: Partial & { + thresholdType?: AlertThresholdType; + threshold?: number; + value?: number; + group?: string; + } = {}, +): AlertMessageTemplateDefaultView => ({ + alert: { + thresholdType: overrides.thresholdType ?? AlertThresholdType.ABOVE, + threshold: overrides.threshold ?? 5, + source: AlertSource.TILE, + channel: { type: null }, + interval: '1m', + tileId: 'test-tile-id', + }, + dashboard: { + _id: new mongoose.Types.ObjectId(), + id: 'id-123', + name: 'My Dashboard', + tiles: [testTile], + team: 'team-123' as any, + tags: ['test'], + createdAt: new Date(), + updatedAt: new Date(), + }, + attributes: {}, + granularity: '5 minute', + group: overrides.group, + isGroupedAlert: false, + startTime, + endTime, + value: overrides.value ?? 10, +}); + +const render = (view: AlertMessageTemplateDefaultView, state: AlertState) => + renderAlertTemplate({ + alertProvider, + clickhouseClient: mockClickhouseClient, + metadata: mockMetadata, + state, + template: null, + title: 'Test Alert Title', + view, + teamWebhooksById: new Map(), + }); + +interface AlertCase { + thresholdType: AlertThresholdType; + threshold: number; + alertValue: number; // value that would trigger the alert + okValue: number; // value that would resolve the alert +} + +const alertCases: AlertCase[] = [ + { + thresholdType: AlertThresholdType.ABOVE, + threshold: 5, + alertValue: 10, + okValue: 3, + }, + { + thresholdType: AlertThresholdType.ABOVE_EXCLUSIVE, + threshold: 5, + alertValue: 10, + okValue: 3, + }, + { + thresholdType: AlertThresholdType.BELOW, + threshold: 5, + alertValue: 2, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.BELOW_OR_EQUAL, + threshold: 5, + alertValue: 3, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.EQUAL, + threshold: 5, + alertValue: 5, + okValue: 10, + }, + { + thresholdType: AlertThresholdType.NOT_EQUAL, + threshold: 5, + alertValue: 10, + okValue: 5, + }, +]; + +describe('renderAlertTemplate', () => { + describe('saved search alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + async ({ thresholdType, threshold, alertValue }) => { + const result = await render( + makeSearchView({ thresholdType, threshold, value: alertValue }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeSearchView({ group: 'http' }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + async ({ thresholdType, threshold, okValue }) => { + const result = await render( + makeSearchView({ thresholdType, threshold, value: okValue }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeSearchView({ group: 'http' }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }); + }); + }); + + describe('tile alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + async ({ thresholdType, threshold, alertValue }) => { + const result = await render( + makeTileView({ thresholdType, threshold, value: alertValue }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeTileView({ group: 'us-east-1' }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + + it('decimal threshold', async () => { + const result = await render( + makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 1.5, + value: 10.123, + }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + + it('integer threshold rounds value', async () => { + const result = await render( + makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 5, + value: 10.789, + }), + AlertState.ALERT, + ); + expect(result).toMatchSnapshot(); + }); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + async ({ thresholdType, threshold, okValue }) => { + const result = await render( + makeTileView({ thresholdType, threshold, value: okValue }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }, + ); + + it('with group', async () => { + const result = await render( + makeTileView({ group: 'us-east-1' }), + AlertState.OK, + ); + expect(result).toMatchSnapshot(); + }); + }); + }); +}); + +describe('buildAlertMessageTemplateTitle', () => { + describe('saved search alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + ({ thresholdType, threshold, alertValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeSearchView({ + thresholdType, + threshold, + value: alertValue, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }, + ); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + ({ thresholdType, threshold, okValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeSearchView({ thresholdType, threshold, value: okValue }), + state: AlertState.OK, + }); + expect(result).toMatchSnapshot(); + }, + ); + }); + }); + + describe('tile alerts', () => { + describe('ALERT state', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold alertValue=$alertValue', + ({ thresholdType, threshold, alertValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ thresholdType, threshold, value: alertValue }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }, + ); + + it('decimal threshold', () => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 1.5, + value: 10.123, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }); + + it('integer threshold rounds value', () => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ + thresholdType: AlertThresholdType.ABOVE, + threshold: 5, + value: 10.789, + }), + state: AlertState.ALERT, + }); + expect(result).toMatchSnapshot(); + }); + }); + + describe('OK state (resolved)', () => { + it.each(alertCases)( + '$thresholdType threshold=$threshold okValue=$okValue', + ({ thresholdType, threshold, okValue }) => { + const result = buildAlertMessageTemplateTitle({ + view: makeTileView({ thresholdType, threshold, value: okValue }), + state: AlertState.OK, + }); + expect(result).toMatchSnapshot(); + }, + ); + }); + }); +}); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts index 6118e451ae..02d5f2bc34 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/singleInvocationAlert.test.ts @@ -248,7 +248,7 @@ describe('Single Invocation Alert Test', () => { // Verify the message body contains the search link const messageBody = webhookPayload.blocks[0].text.text; expect(messageBody).toContain('lines found'); - expect(messageBody).toContain('expected less than 1 lines'); + expect(messageBody).toContain('meets or exceeds the threshold of 1 lines'); expect(messageBody).toContain('http://app:8080/search/'); expect(messageBody).toContain('from='); expect(messageBody).toContain('to='); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 17e602ca15..a49f374cfb 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -27,6 +27,7 @@ import { isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { + AlertThresholdType, BuilderChartConfigWithOptDateRange, ChartConfigWithOptDateRange, DisplayType, @@ -42,7 +43,7 @@ import ms from 'ms'; import { serializeError } from 'serialize-error'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; -import { AlertState, AlertThresholdType, IAlert } from '@/models/alert'; +import { AlertState, IAlert } from '@/models/alert'; import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; @@ -141,13 +142,20 @@ export const doesExceedThreshold = ( threshold: number, value: number, ) => { - const isThresholdTypeAbove = thresholdType === AlertThresholdType.ABOVE; - if (isThresholdTypeAbove && value >= threshold) { - return true; - } else if (!isThresholdTypeAbove && value < threshold) { - return true; + switch (thresholdType) { + case AlertThresholdType.ABOVE: + return value >= threshold; + case AlertThresholdType.BELOW: + return value < threshold; + case AlertThresholdType.ABOVE_EXCLUSIVE: + return value > threshold; + case AlertThresholdType.BELOW_OR_EQUAL: + return value <= threshold; + case AlertThresholdType.EQUAL: + return value === threshold; + case AlertThresholdType.NOT_EQUAL: + return value !== threshold; } - return false; }; const normalizeScheduleOffsetMinutes = ({ @@ -643,6 +651,9 @@ const parseAlertData = ( for (const [k, v] of Object.entries(data)) { if (meta.valueColumnNames.has(k)) { + // Due to output_format_json_quote_64bit_integers=1, 64-bit integers will be returned as strings. + // Parse them as integers to ensure correct threshold comparison. + // Floats are not returned as strings (unless output_format_json_quote_64bit_floats=1, which is not the default). value = isString(v) ? parseInt(v) : v; } else if (meta.type !== 'time_series' || k !== meta.timestampColumnName) { extraFields.push(`${k}:${v}`); diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index f5494ba633..9587ba2268 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -8,6 +8,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { AlertChannelType, + AlertThresholdType, ChartConfigWithOptDateRange, DisplayType, pickSampleWeightExpressionProps, @@ -24,7 +25,7 @@ import { z } from 'zod'; import * as config from '@/config'; import { AlertInput } from '@/controllers/alerts'; -import { AlertSource, AlertState, AlertThresholdType } from '@/models/alert'; +import { AlertSource, AlertState } from '@/models/alert'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; @@ -42,6 +43,44 @@ import { truncateString } from '@/utils/common'; import logger from '@/utils/logger'; import * as slack from '@/utils/slack'; +const describeThresholdViolation = ( + thresholdType: AlertThresholdType, +): string => { + switch (thresholdType) { + case AlertThresholdType.ABOVE: + return 'meets or exceeds'; + case AlertThresholdType.ABOVE_EXCLUSIVE: + return 'exceeds'; + case AlertThresholdType.BELOW: + return 'falls below'; + case AlertThresholdType.BELOW_OR_EQUAL: + return 'falls to or below'; + case AlertThresholdType.EQUAL: + return 'equals'; + case AlertThresholdType.NOT_EQUAL: + return 'does not equal'; + } +}; + +const describeThresholdResolution = ( + thresholdType: AlertThresholdType, +): string => { + switch (thresholdType) { + case AlertThresholdType.ABOVE: + return 'falls below'; + case AlertThresholdType.ABOVE_EXCLUSIVE: + return 'falls to or below'; + case AlertThresholdType.BELOW: + return 'meets or exceeds'; + case AlertThresholdType.BELOW_OR_EQUAL: + return 'exceeds'; + case AlertThresholdType.EQUAL: + return 'does not equal'; + case AlertThresholdType.NOT_EQUAL: + return 'equals'; + } +}; + const MAX_MESSAGE_LENGTH = 500; const NOTIFY_FN_NAME = '__hdx_notify_channel__'; const IS_MATCH_FN_NAME = 'is_match'; @@ -377,12 +416,8 @@ export const buildAlertMessageTemplateTitle = ({ ? handlebars.compile(template)(view) : `Alert for "${tile.config.name}" in "${dashboard.name}" - ${formattedValue} ${ doesExceedThreshold(alert.thresholdType, alert.threshold, value) - ? alert.thresholdType === AlertThresholdType.ABOVE - ? 'exceeds' - : 'falls below' - : alert.thresholdType === AlertThresholdType.ABOVE - ? 'falls below' - : 'exceeds' + ? describeThresholdViolation(alert.thresholdType) + : describeThresholdResolution(alert.thresholdType) } ${alert.threshold}`; return `${emoji}${baseTitle}`; } @@ -649,11 +684,7 @@ ${targetTemplate}`; } rawTemplateBody = `${group ? `Group: "${group}"` : ''} -${value} lines found, expected ${ - alert.thresholdType === AlertThresholdType.ABOVE - ? 'less than' - : 'greater than' - } ${alert.threshold} lines\n${timeRangeMessage} +${value} lines found, which ${describeThresholdViolation(alert.thresholdType)} the threshold of ${alert.threshold} lines\n${timeRangeMessage} ${targetTemplate} \`\`\` ${truncatedResults} @@ -666,12 +697,8 @@ ${truncatedResults} rawTemplateBody = `${group ? `Group: "${group}"` : ''} ${formattedValue} ${ doesExceedThreshold(alert.thresholdType, alert.threshold, value) - ? alert.thresholdType === AlertThresholdType.ABOVE - ? 'exceeds' - : 'falls below' - : alert.thresholdType === AlertThresholdType.ABOVE - ? 'falls below' - : 'exceeds' + ? describeThresholdViolation(alert.thresholdType) + : describeThresholdResolution(alert.thresholdType) } ${alert.threshold}\n${timeRangeMessage} ${targetTemplate}`; } diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 721e4ad4f3..525c9b05ff 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -1,4 +1,5 @@ import { + AlertThresholdType, BuilderSavedChartConfig, DashboardFilter, DisplayType, @@ -12,7 +13,6 @@ import { AlertDocument, AlertInterval, AlertState, - AlertThresholdType, } from '@/models/alert'; import type { DashboardDocument } from '@/models/dashboard'; import { SeriesTile } from '@/routers/external-api/v2/utils/dashboards'; diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 2ddf02d027..34d283a9c3 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1,5 +1,6 @@ import { AggregateFunctionSchema, + AlertThresholdType, DashboardFilterSchema, MetricsDataType, NumberFormatSchema, @@ -11,7 +12,7 @@ import { import { Types } from 'mongoose'; import { z } from 'zod'; -import { AlertSource, AlertThresholdType } from '@/models/alert'; +import { AlertSource } from '@/models/alert'; export const objectIdSchema = z.string().refine(val => { return Types.ObjectId.isValid(val); diff --git a/packages/app/src/DBSearchPageAlertModal.tsx b/packages/app/src/DBSearchPageAlertModal.tsx index f653cc87b7..d4203b8ccf 100644 --- a/packages/app/src/DBSearchPageAlertModal.tsx +++ b/packages/app/src/DBSearchPageAlertModal.tsx @@ -244,16 +244,33 @@ const AlertForm = ({ Send to - {groupBy && thresholdType === AlertThresholdType.BELOW && ( + {groupBy && + (thresholdType === AlertThresholdType.BELOW || + thresholdType === AlertThresholdType.BELOW_OR_EQUAL || + thresholdType === AlertThresholdType.EQUAL || + thresholdType === AlertThresholdType.NOT_EQUAL) && ( + } + color="gray" + py="xs" + > + + Warning: Alerts with this threshold type and a "grouped + by" value will not alert for periods with no data for a + group. + + + )} + {(thresholdType === AlertThresholdType.EQUAL || + thresholdType === AlertThresholdType.NOT_EQUAL) && ( } - bg="dark" + color="gray" py="xs" > - Warning: Alerts with a "Below (<)" threshold and a - "grouped by" value will not alert for periods with no - data for a group. + Note: Floating-point query results are not rounded during + equality comparison. )} diff --git a/packages/app/src/components/AlertPreviewChart.tsx b/packages/app/src/components/AlertPreviewChart.tsx index 029d0959a2..f220965a46 100644 --- a/packages/app/src/components/AlertPreviewChart.tsx +++ b/packages/app/src/components/AlertPreviewChart.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils'; import { AlertInterval, + AlertThresholdType, Filter, getSampleWeightExpression, isLogSource, @@ -25,7 +26,7 @@ type AlertPreviewChartProps = { filters?: Filter[] | null; interval: AlertInterval; groupBy?: string; - thresholdType: 'above' | 'below'; + thresholdType: AlertThresholdType; threshold: number; select?: string | null; }; diff --git a/packages/app/src/components/Alerts.tsx b/packages/app/src/components/Alerts.tsx index 45a8728da3..77ba95ba44 100644 --- a/packages/app/src/components/Alerts.tsx +++ b/packages/app/src/components/Alerts.tsx @@ -9,6 +9,7 @@ import { import { Label, ReferenceArea, ReferenceLine } from 'recharts'; import { type AlertChannelType, + AlertThresholdType, WebhookService, } from '@hyperdx/common-utils/dist/types'; import { Button, ComboboxData, Group, Modal, Select } from '@mantine/core'; @@ -145,10 +146,16 @@ export const getAlertReferenceLines = ({ threshold, // TODO: zScore }: { - thresholdType: 'above' | 'below'; + thresholdType: AlertThresholdType; threshold: number; }) => { - if (threshold != null && thresholdType === 'below') { + if (threshold == null) { + return null; + } + if ( + thresholdType === AlertThresholdType.BELOW || + thresholdType === AlertThresholdType.BELOW_OR_EQUAL + ) { return ( ); } - if (threshold != null && thresholdType === 'above') { + if ( + thresholdType === AlertThresholdType.ABOVE || + thresholdType === AlertThresholdType.ABOVE_EXCLUSIVE + ) { return ( ); } - if (threshold != null) { - return ( - - } - stroke="red" - strokeDasharray="3 3" - /> - ); - } - return null; + // For 'equal' and 'not_equal', show a reference line at the threshold + return ( + + } + stroke="red" + strokeDasharray="3 3" + /> + ); }; diff --git a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx index 19335df33d..e419443528 100644 --- a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx @@ -4,8 +4,10 @@ import { UseFormSetValue, useWatch, } from 'react-hook-form'; +import { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, + Alert, Badge, Box, Collapse, @@ -21,6 +23,7 @@ import { useDisclosure } from '@mantine/hooks'; import { IconChevronDown, IconHelpCircle, + IconInfoCircleFilled, IconTrash, } from '@tabler/icons-react'; @@ -58,6 +61,7 @@ export function TileAlertEditor({ const [opened, { toggle }] = useDisclosure(true); const alertChannelType = useWatch({ control, name: 'alert.channel.type' }); + const alertThresholdType = useWatch({ control, name: 'alert.thresholdType' }); const alertScheduleOffsetMinutes = useWatch({ control, name: 'alert.scheduleOffsetMinutes', @@ -210,6 +214,18 @@ export function TileAlertEditor({ type={alertChannelType} namePrefix="alert." /> + {(alertThresholdType === AlertThresholdType.EQUAL || + alertThresholdType === AlertThresholdType.NOT_EQUAL) && ( + } + color="gray" + py="xs" + mt="md" + > + Note: Floating-point query results are not rounded during equality + comparison. + + )} diff --git a/packages/app/src/utils/alerts.ts b/packages/app/src/utils/alerts.ts index 412d730574..c447777850 100644 --- a/packages/app/src/utils/alerts.ts +++ b/packages/app/src/utils/alerts.ts @@ -83,11 +83,19 @@ export function extendDateRangeToInterval( export const ALERT_THRESHOLD_TYPE_OPTIONS: Record = { above: 'At least (≥)', below: 'Below (<)', + above_exclusive: 'Above (>)', + below_or_equal: 'At most (≤)', + equal: 'Equal to (=)', + not_equal: 'Not equal to (≠)', }; export const TILE_ALERT_THRESHOLD_TYPE_OPTIONS: Record = { above: 'is at least (≥)', below: 'falls below (<)', + above_exclusive: 'is above (>)', + below_or_equal: 'is at most (≤)', + equal: 'equals (=)', + not_equal: 'does not equal (≠)', }; export const ALERT_INTERVAL_OPTIONS: Record = { diff --git a/packages/cli/src/api/client.ts b/packages/cli/src/api/client.ts index 0832e755d8..04def1e570 100644 --- a/packages/cli/src/api/client.ts +++ b/packages/cli/src/api/client.ts @@ -25,6 +25,7 @@ import { } from '@hyperdx/common-utils/dist/core/metadata'; import { loadSession, saveSession, clearSession } from '@/utils/config'; +import { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; // ------------------------------------------------------------------ // API Client (session management + REST calls) @@ -418,7 +419,7 @@ export interface AlertItem { scheduleOffsetMinutes?: number; scheduleStartAt?: string | null; threshold: number; - thresholdType: 'above' | 'below'; + thresholdType: AlertThresholdType; channel: { type?: string | null }; state?: 'ALERT' | 'OK' | 'INSUFFICIENT_DATA' | 'DISABLED'; source?: 'saved_search' | 'tile'; diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 6dc350a789..cffdecb55c 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -278,6 +278,10 @@ export type WebhookApiData = Omit; export enum AlertThresholdType { ABOVE = 'above', BELOW = 'below', + ABOVE_EXCLUSIVE = 'above_exclusive', + BELOW_OR_EQUAL = 'below_or_equal', + EQUAL = 'equal', + NOT_EQUAL = 'not_equal', } export enum AlertState {