[9.1] Add event based telemetry for detected gaps (#231287) (#232340)

nkhristinin · kibanamachine · elasticmachine · web-flow · commit d3a6ae8f6edf · 2025-08-20T12:28:13.000+02:00
# Backport This will backport the following commits from `main` to `9.1`: - [Add event based telemetry for detected gaps (#231287)](#231287)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts
@@ -48,6 +48,7 @@ import { TIMESTAMP_RUNTIME_FIELD } from './constants';
 import { buildTimestampRuntimeMapping } from './utils/build_timestamp_runtime_mapping';
 import { alertsFieldMap, rulesFieldMap } from '../../../../common/field_maps';
 import { sendAlertSuppressionTelemetryEvent } from './utils/telemetry/send_alert_suppression_telemetry_event';
+import { sendGapDetectedTelemetryEvent } from './utils/telemetry/send_gap_detected_telemetry_event';
 import type { RuleParams } from '../rule_schema';
 import {
   SECURITY_FROM,
@@ -380,6 +381,8 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
             remainingGap,
             warningStatusMessage: rangeTuplesWarningMessage,
             gap,
+            originalFrom,
+            originalTo,
           } = await getRuleRangeTuples({
             startedAt,
             previousStartedAt,
@@ -399,6 +402,16 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
           if (remainingGap.asMilliseconds() > 0) {
             const gapDuration = `${remainingGap.humanize()} (${remainingGap.asMilliseconds()}ms)`;
             const gapErrorMessage = `${gapDuration} were not queried between this rule execution and the last execution, so signals may have been missed. Consider increasing your look behind time or adding more Kibana instances`;
+            if (analytics) {
+              sendGapDetectedTelemetryEvent({
+                analytics,
+                interval,
+                gapDuration: remainingGap,
+                originalFrom,
+                originalTo,
+                ruleParams: params,
+              });
+            }
             wrapperErrors.push(gapErrorMessage);
             await ruleExecutionLogger.logStatusChange({
               newStatus: RuleExecutionStatusEnum.failed,
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/telemetry/send_gap_detected_telemetry_event.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/telemetry/send_gap_detected_telemetry_event.test.ts
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { coreMock } from '@kbn/core/server/mocks';
+import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server';
+import moment from 'moment';
+import { GAP_DETECTED_EVENT } from '../../../../telemetry/event_based/events';
+
+import { sendGapDetectedTelemetryEvent } from './send_gap_detected_telemetry_event';
+import type { RuleParams } from '../../../rule_schema';
+
+describe('sendGapDetectedTelemetryEvent', () => {
+  let mockAnalytics: jest.Mocked<AnalyticsServiceSetup>;
+  let mockCore: ReturnType<typeof coreMock.createSetup>;
+
+  beforeEach(() => {
+    mockCore = coreMock.createSetup();
+    mockAnalytics = mockCore.analytics;
+  });
+
+  it('should report correct event data with valid parameters', () => {
+    const interval = '5m';
+    const gapDuration = moment.duration(10, 'minutes');
+    const originalFrom = moment('2023-01-01T00:00:00Z');
+    const originalTo = moment('2023-01-01T01:00:00Z');
+    const ruleParams = {
+      type: 'query',
+      ruleSource: { type: 'external', isCustomized: true },
+    } as unknown as RuleParams;
+
+    sendGapDetectedTelemetryEvent({
+      analytics: mockAnalytics,
+      interval,
+      gapDuration,
+      originalFrom,
+      originalTo,
+      ruleParams,
+    });
+
+    expect(mockAnalytics.reportEvent).toHaveBeenCalledWith(GAP_DETECTED_EVENT.eventType, {
+      gapDuration: 600000, // 10 minutes in milliseconds
+      intervalDuration: 300000, // 5 minutes in milliseconds
+      intervalAndLookbackDuration: 3600000, // 1 hour in milliseconds
+      ruleType: 'query',
+      ruleSource: 'external',
+      isCustomized: true,
+    });
+  });
+
+  it('should not report event when interval parsing fails', () => {
+    const invalidInterval = 'invalid-interval';
+    const gapDuration = moment.duration(10, 'minutes');
+    const originalFrom = moment('2023-01-01T00:00:00Z');
+    const originalTo = moment('2023-01-01T01:00:00Z');
+    const ruleParams = { type: 'query' } as unknown as RuleParams;
+
+    sendGapDetectedTelemetryEvent({
+      analytics: mockAnalytics,
+      interval: invalidInterval,
+      gapDuration,
+      originalFrom,
+      originalTo,
+      ruleParams,
+    });
+
+    expect(mockAnalytics.reportEvent).not.toHaveBeenCalled();
+  });
+});
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/telemetry/send_gap_detected_telemetry_event.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/telemetry/send_gap_detected_telemetry_event.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server';
+import moment from 'moment';
+import { GAP_DETECTED_EVENT } from '../../../../telemetry/event_based/events';
+import { parseInterval } from '../utils';
+import type { RuleParams } from '../../../rule_schema';
+
+export const sendGapDetectedTelemetryEvent = ({
+  analytics,
+  interval,
+  gapDuration,
+  originalFrom,
+  originalTo,
+  ruleParams,
+}: {
+  analytics: AnalyticsServiceSetup;
+  interval: string;
+  gapDuration: moment.Duration;
+  originalFrom: moment.Moment;
+  originalTo: moment.Moment;
+  ruleParams: RuleParams;
+}) => {
+  const intervalDuration = parseInterval(interval);
+
+  if (!intervalDuration) {
+    return;
+  }
+
+  const ruleType = ruleParams.type;
+  const ruleSource = ruleParams.ruleSource;
+  const isCustomized = ruleSource?.type === 'external' ? ruleSource.isCustomized : false;
+
+  analytics.reportEvent(GAP_DETECTED_EVENT.eventType, {
+    gapDuration: gapDuration.asMilliseconds(),
+    intervalDuration: intervalDuration.asMilliseconds(),
+    intervalAndLookbackDuration: moment.duration(originalTo.diff(originalFrom)).asMilliseconds(),
+    ruleType,
+    ruleSource: ruleSource?.type,
+    isCustomized,
+  });
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.test.ts
@@ -351,6 +351,23 @@ describe('utils', () => {
       expect(tuples.length).toEqual(1);
       expect(warningStatusMessage).toEqual(undefined);
     });
+
+    test('should return originalFrom and originalTo in response', async () => {
+      const { originalFrom, originalTo } = await getRuleRangeTuples({
+        previousStartedAt: moment().subtract(30, 's').toDate(),
+        startedAt: moment().toDate(),
+        interval: '30s',
+        from: 'now-30s',
+        to: 'now',
+        maxSignals: 20,
+        ruleExecutionLogger,
+        alerting,
+      });
+
+      expect(originalFrom).toBeDefined();
+      expect(originalTo).toBeDefined();
+      expect(moment.duration(originalTo.diff(originalFrom)).asSeconds()).toEqual(30);
+    });
   });
 
   describe('calculateFromValue', () => {
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts
@@ -427,7 +427,13 @@ export const getRuleRangeTuples = async ({
         interval
       )}"`
     );
-    return { tuples, remainingGap: moment.duration(0), warningStatusMessage };
+    return {
+      tuples,
+      remainingGap: moment.duration(0),
+      warningStatusMessage,
+      originalFrom,
+      originalTo,
+    };
   }
 
   const gap = getGapBetweenRuns({
@@ -469,6 +475,8 @@ export const getRuleRangeTuples = async ({
     remainingGap: moment.duration(remainingGapMilliseconds),
     warningStatusMessage,
     gap: gapRange,
+    originalFrom,
+    originalTo,
   };
 };
 
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts
@@ -1567,6 +1567,55 @@ export const ENDPOINT_WORKFLOW_INSIGHTS_REMEDIATED_EVENT: EventTypeOpts<{
   },
 };
 
+export const GAP_DETECTED_EVENT: EventTypeOpts<{
+  gapDuration: number;
+  intervalDuration: number;
+  intervalAndLookbackDuration: number;
+  ruleType: string;
+  ruleSource: string;
+  isCustomized: boolean;
+}> = {
+  eventType: 'gap_detected_event',
+  schema: {
+    gapDuration: {
+      type: 'long',
+      _meta: {
+        description: 'The duration of the gap',
+      },
+    },
+    intervalDuration: {
+      type: 'long',
+      _meta: {
+        description: 'The duration of the interval',
+      },
+    },
+    intervalAndLookbackDuration: {
+      type: 'long',
+      _meta: {
+        description: 'The duration of the interval and lookback',
+      },
+    },
+    ruleType: {
+      type: 'keyword',
+      _meta: {
+        description: 'The type of the rule',
+      },
+    },
+    ruleSource: {
+      type: 'keyword',
+      _meta: {
+        description: 'The source of the rule',
+      },
+    },
+    isCustomized: {
+      type: 'boolean',
+      _meta: {
+        description: 'Whether the prebuilt rule is customized',
+      },
+    },
+  },
+};
+
 export const events = [
   RISK_SCORE_EXECUTION_SUCCESS_EVENT,
   RISK_SCORE_EXECUTION_ERROR_EVENT,
@@ -1600,4 +1649,5 @@ export const events = [
   SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE,
   SIEM_MIGRATIONS_PREBUILT_RULES_MATCH,
   SIEM_MIGRATIONS_INTEGRATIONS_MATCH,
+  GAP_DETECTED_EVENT,
 ];
