♻️ manage metric sampling inside telemetry

- allow to apply a telemetry sample rate to telemetry events emitted before the configuration is applied
- centralize telemetry sampling draws in telemetry.ts which now exposes `enabledMetrics`
- expose replay metrics from recorderApi

Author: bcaudan

packages/core/src/domain/telemetry/index.ts

@@ -1,4 +1,4 @@
-export type { Telemetry } from './telemetry'
+export type { Telemetry, SampleRateByMetric } from './telemetry'
 export {
   TelemetryService,
   addTelemetryDebug,

packages/core/src/domain/telemetry/telemetry.spec.ts

@@ -20,14 +20,15 @@ import {
   startTelemetryCollection,
   addTelemetryMetrics,
   addTelemetryDebug,
+  type SampleRateByMetric,
 } from './telemetry'
 import type { TelemetryEvent } from './telemetryEvent.types'
 import { StatusType, TelemetryType } from './rawTelemetryEvent.types'
 
 const NETWORK_METRICS_KIND = 'Network metrics'
 const PERFORMANCE_METRICS_KIND = 'Performance metrics'
 
-function startAndSpyTelemetry(configuration?: Partial<Configuration>) {
+function startAndSpyTelemetry(configuration?: Partial<Configuration>, sampleRateByMetric: SampleRateByMetric = {}) {
   const observable = new Observable<TelemetryEvent & Context>()
 
   const events: TelemetryEvent[] = []
@@ -42,7 +43,8 @@ function startAndSpyTelemetry(configuration?: Partial<Configuration>) {
       ...configuration,
     } as Configuration,
     hooks,
-    observable
+    observable,
+    sampleRateByMetric
   )
 
   return {
@@ -157,7 +159,10 @@ describe('telemetry', () => {
 
   describe('addTelemetryMetrics', () => {
     it('should collect metrics when sampled', async () => {
-      const { getTelemetryEvents } = startAndSpyTelemetry({ telemetrySampleRate: 100 })
+      const { getTelemetryEvents } = startAndSpyTelemetry(
+        { telemetrySampleRate: 100 },
+        { [PERFORMANCE_METRICS_KIND]: 100 }
+      )
 
       addTelemetryMetrics(PERFORMANCE_METRICS_KIND, { speed: 1000 })
 
@@ -172,8 +177,22 @@ describe('telemetry', () => {
       ])
     })
 
-    it('should not notify metrics when not sampled', async () => {
-      const { getTelemetryEvents } = startAndSpyTelemetry({ telemetrySampleRate: 0 })
+    it('should not notify metrics when telemetry not sampled', async () => {
+      const { getTelemetryEvents } = startAndSpyTelemetry(
+        { telemetrySampleRate: 0 },
+        { [PERFORMANCE_METRICS_KIND]: 100 }
+      )
+
+      addTelemetryMetrics(PERFORMANCE_METRICS_KIND, { speed: 1000 })
+
+      expect(await getTelemetryEvents()).toEqual([])
+    })
+
+    it('should not notify metrics when metric not sampled', async () => {
+      const { getTelemetryEvents } = startAndSpyTelemetry(
+        { telemetrySampleRate: 100 },
+        { [PERFORMANCE_METRICS_KIND]: 0 }
+      )
 
       addTelemetryMetrics(PERFORMANCE_METRICS_KIND, { speed: 1000 })
 

packages/core/src/domain/telemetry/telemetry.ts

@@ -59,6 +59,11 @@ export const enum TelemetryService {
 export interface Telemetry {
   stop: () => void
   enabled: boolean
+  enabledMetrics: { [metricName: string]: boolean }
+}
+
+export interface SampleRateByMetric {
+  [metricName: string]: number
 }
 
 const TELEMETRY_EXCLUDED_SITES: string[] = [INTAKE_SITE_US1_FED]
@@ -78,25 +83,34 @@ export function startTelemetry(
   hooks: AbstractHooks,
   reportError: (error: RawError) => void,
   pageMayExitObservable: Observable<PageMayExitEvent>,
-  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder,
+  sampleRateByMetric: SampleRateByMetric = {}
 ): Telemetry {
   const observable = new Observable<TelemetryEvent & Context>()
 
   const { stop } = startTelemetryTransport(configuration, reportError, pageMayExitObservable, createEncoder, observable)
 
-  const { enabled } = startTelemetryCollection(telemetryService, configuration, hooks, observable)
+  const { enabled, enabledMetrics } = startTelemetryCollection(
+    telemetryService,
+    configuration,
+    hooks,
+    observable,
+    sampleRateByMetric
+  )
 
   return {
     stop,
     enabled,
+    enabledMetrics,
   }
 }
 
 export function startTelemetryCollection(
   telemetryService: TelemetryService,
   configuration: Configuration,
   hooks: AbstractHooks,
-  observable: Observable<TelemetryEvent & Context>
+  observable: Observable<TelemetryEvent & Context>,
+  sampleRateByMetric: SampleRateByMetric
 ) {
   const alreadySentEventsByKind: Record<string, Set<string>> = {}
 
@@ -109,10 +123,18 @@ export function startTelemetryCollection(
     [TelemetryType.USAGE]: telemetryEnabled && performDraw(configuration.telemetryUsageSampleRate),
   }
 
+  const telemetryEnabledPerMetrics: { [metricName: string]: boolean } = {}
+  Object.keys(sampleRateByMetric).forEach((metricName) => {
+    telemetryEnabledPerMetrics[metricName] = telemetryEnabled && performDraw(sampleRateByMetric[metricName])
+  })
+
   const runtimeEnvInfo = getRuntimeEnvInfo()
   const telemetryObservable = getTelemetryObservable()
   telemetryObservable.subscribe(({ rawEvent, kind }) => {
-    if (!telemetryEnabledPerType[rawEvent.type!]) {
+    if (
+      !telemetryEnabledPerType[rawEvent.type!] ||
+      (kind in telemetryEnabledPerMetrics && !telemetryEnabledPerMetrics[kind])
+    ) {
       return
     }
 
@@ -154,6 +176,7 @@ export function startTelemetryCollection(
 
   return {
     enabled: telemetryEnabled,
+    enabledMetrics: telemetryEnabledPerMetrics,
   }
 
   function toTelemetryEvent(

packages/core/src/index.ts

@@ -40,6 +40,7 @@ export type {
   TelemetryUsageEvent,
   RawTelemetryUsage,
   RawTelemetryUsageFeature,
+  SampleRateByMetric,
 } from './domain/telemetry'
 export {
   startTelemetry,

packages/rum-core/src/boot/rumPublicApi.ts

@@ -13,6 +13,7 @@ import type {
   Account,
   RumInternalContext,
   Telemetry,
+  SampleRateByMetric,
 } from '@datadog/browser-core'
 import {
   ContextManagerMethod,
@@ -443,6 +444,7 @@ export interface RecorderApi {
   isRecording: () => boolean
   getReplayStats: (viewId: string) => ReplayStats | undefined
   getSessionReplayLink: () => string | undefined
+  getTelemetrySampleRateByMetric: (configuration: RumConfiguration) => SampleRateByMetric | undefined
 }
 
 export interface ProfilerApi {

packages/rum-core/src/boot/startRum.ts

@@ -37,7 +37,7 @@ import { createLocationChangeObservable } from '../browser/locationChangeObserva
 import type { RumConfiguration } from '../domain/configuration'
 import type { ViewOptions } from '../domain/view/trackViews'
 import { startFeatureFlagContexts } from '../domain/contexts/featureFlagContext'
-import { startCustomerDataTelemetry } from '../domain/startCustomerDataTelemetry'
+import { startCustomerDataTelemetry, CUSTOMER_DATA_METRIC_NAME } from '../domain/startCustomerDataTelemetry'
 import type { PageStateHistory } from '../domain/contexts/pageStateHistory'
 import { startPageStateHistory } from '../domain/contexts/pageStateHistory'
 import { startDisplayContext } from '../domain/contexts/displayContext'
@@ -94,13 +94,19 @@ export function startRum(
   })
   cleanupTasks.push(() => pageMayExitSubscription.unsubscribe())
 
+  const sampleRateByMetric = {
+    [CUSTOMER_DATA_METRIC_NAME]: configuration.customerDataTelemetrySampleRate,
+    ...recorderApi.getTelemetrySampleRateByMetric(configuration),
+  }
+
   const telemetry = startTelemetry(
     TelemetryService.RUM,
     configuration,
     hooks,
     reportError,
     pageMayExitObservable,
-    createEncoder
+    createEncoder,
+    sampleRateByMetric
   )
   cleanupTasks.push(telemetry.stop)
 
@@ -118,7 +124,7 @@ export function startRum(
       createEncoder
     )
     cleanupTasks.push(() => batch.stop())
-    startCustomerDataTelemetry(configuration, telemetry, lifeCycle, batch.flushController.flushObservable)
+    startCustomerDataTelemetry(telemetry, lifeCycle, batch.flushController.flushObservable)
   } else {
     startRumEventBridge(lifeCycle)
   }

packages/rum-core/src/domain/startCustomerDataTelemetry.spec.ts

@@ -2,12 +2,14 @@ import type { FlushEvent, Context, Telemetry } from '@datadog/browser-core'
 import { Observable, resetExperimentalFeatures } from '@datadog/browser-core'
 import type { Clock, MockTelemetry } from '@datadog/browser-core/test'
 import { mockClock, startMockTelemetry } from '@datadog/browser-core/test'
-import { mockRumConfiguration } from '../../test'
 import { RumEventType } from '../rawRumEvent.types'
 import type { RumEvent } from '../rumEvent.types'
 import { LifeCycle, LifeCycleEventType } from './lifeCycle'
-import { MEASURES_PERIOD_DURATION, startCustomerDataTelemetry } from './startCustomerDataTelemetry'
-import type { RumConfiguration } from './configuration'
+import {
+  MEASURES_PERIOD_DURATION,
+  startCustomerDataTelemetry,
+  CUSTOMER_DATA_METRIC_NAME,
+} from './startCustomerDataTelemetry'
 
 describe('customerDataTelemetry', () => {
   let clock: Clock
@@ -17,12 +19,6 @@ describe('customerDataTelemetry', () => {
   let lifeCycle: LifeCycle
   const viewEvent = { type: RumEventType.VIEW } as RumEvent & Context
 
-  const config: Partial<RumConfiguration> = {
-    telemetrySampleRate: 100,
-    customerDataTelemetrySampleRate: 100,
-    maxTelemetryEventsPerPage: 2,
-  }
-
   function generateBatch({
     eventNumber,
     batchBytesCount = 1,
@@ -46,15 +42,18 @@ describe('customerDataTelemetry', () => {
     })
   }
 
-  function setupCustomerTelemetryCollection(partialConfig: Partial<RumConfiguration> = config) {
-    const configuration = mockRumConfiguration(partialConfig)
+  function setupCustomerTelemetryCollection(telemetryEnabled: boolean = true) {
     batchFlushObservable = new Observable()
     lifeCycle = new LifeCycle()
     fakeContextBytesCount = 1
 
     telemetry = startMockTelemetry()
 
-    startCustomerDataTelemetry(configuration, { enabled: true } as Telemetry, lifeCycle, batchFlushObservable)
+    startCustomerDataTelemetry(
+      { enabledMetrics: { [CUSTOMER_DATA_METRIC_NAME]: telemetryEnabled } } as unknown as Telemetry,
+      lifeCycle,
+      batchFlushObservable
+    )
   }
 
   beforeEach(() => {
@@ -132,10 +131,7 @@ describe('customerDataTelemetry', () => {
   })
 
   it('should not collect customer data telemetry when telemetry disabled', async () => {
-    setupCustomerTelemetryCollection({
-      telemetrySampleRate: 100,
-      customerDataTelemetrySampleRate: 0,
-    })
+    setupCustomerTelemetryCollection(false)
 
     generateBatch({ eventNumber: 1 })
     clock.tick(MEASURES_PERIOD_DURATION)

packages/rum-core/src/domain/startCustomerDataTelemetry.ts

@@ -1,10 +1,10 @@
 import type { Context, FlushEvent, Observable, Telemetry } from '@datadog/browser-core'
-import { performDraw, ONE_SECOND, addTelemetryMetrics, setInterval } from '@datadog/browser-core'
-import type { RumConfiguration } from './configuration'
+import { ONE_SECOND, addTelemetryMetrics, setInterval } from '@datadog/browser-core'
 import type { LifeCycle } from './lifeCycle'
 import { LifeCycleEventType } from './lifeCycle'
 
 export const MEASURES_PERIOD_DURATION = 10 * ONE_SECOND
+export const CUSTOMER_DATA_METRIC_NAME = 'Customer data measures'
 
 interface Measure extends Context {
   min: number
@@ -22,12 +22,11 @@ let currentPeriodMeasures: CurrentPeriodMeasures
 let batchHasRumEvent: boolean
 
 export function startCustomerDataTelemetry(
-  configuration: RumConfiguration,
   telemetry: Telemetry,
   lifeCycle: LifeCycle,
   batchFlushObservable: Observable<FlushEvent>
 ) {
-  const customerDataTelemetryEnabled = telemetry.enabled && performDraw(configuration.customerDataTelemetrySampleRate)
+  const customerDataTelemetryEnabled = telemetry.enabledMetrics[CUSTOMER_DATA_METRIC_NAME]
   if (!customerDataTelemetryEnabled) {
     return
   }
@@ -61,7 +60,7 @@ function sendCurrentPeriodMeasures() {
     return
   }
 
-  addTelemetryMetrics('Customer data measures', currentPeriodMeasures)
+  addTelemetryMetrics(CUSTOMER_DATA_METRIC_NAME, currentPeriodMeasures)
   initCurrentPeriodMeasures()
 }
 

packages/rum-core/test/noopRecorderApi.ts

@@ -8,4 +8,5 @@ export const noopRecorderApi: RecorderApi = {
   onRumStart: noop,
   getReplayStats: () => undefined,
   getSessionReplayLink: () => undefined,
+  getTelemetrySampleRateByMetric: () => undefined,
 }

packages/rum-slim/src/boot/stubRecorderApi.ts

@@ -13,5 +13,6 @@ export function makeRecorderApiStub() {
     isRecording: () => false,
     getReplayStats: () => undefined,
     getSessionReplayLink: () => getSessionReplayLinkStrategy(),
+    getTelemetrySampleRateByMetric: () => undefined,
   }
 }