✨⚗️ [RUM-10146] implement early request collection (#3740)

Co-authored-by: bcaudan <[email protected]>

Author: BenoitZugmeyer

packages/core/src/tools/experimentalFeatures.ts

@@ -16,6 +16,7 @@ import { objectHasValue } from './utils/objectUtils'
 export enum ExperimentalFeature {
   TRACK_INTAKE_REQUESTS = 'track_intake_requests',
   WRITABLE_RESOURCE_GRAPHQL = 'writable_resource_graphql',
+  EARLY_REQUEST_COLLECTION = 'early_request_collection',
   WATCH_COOKIE_WITHOUT_LOCK = 'watch_cookie_without_lock',
   USE_TREE_WALKER_FOR_ACTION_NAME = 'use_tree_walker_for_action_name',
 }

packages/rum-core/src/domain/resource/requestRegistry.spec.ts

@@ -0,0 +1,83 @@
+import type { RelativeTime } from '@datadog/browser-core'
+import { createPerformanceEntry } from '../../../test'
+import { RumPerformanceEntryType } from '../../browser/performanceObservable'
+import { LifeCycle, LifeCycleEventType } from '../lifeCycle'
+import type { RequestCompleteEvent } from '../requestCollection'
+import { createRequestRegistry, MAX_REQUESTS } from './requestRegistry'
+
+describe('RequestRegistry', () => {
+  const URL = 'https://example.com/resource'
+  it('returns the closest preceding request', () => {
+    const lifeCycle = new LifeCycle()
+
+    const requestRegistry = createRequestRegistry(lifeCycle)
+    const request1 = createRequestCompleteEvent({ startTime: 1 })
+    const request2 = createRequestCompleteEvent({ startTime: 2 })
+    const request3 = createRequestCompleteEvent({ startTime: 3 })
+    const request4 = createRequestCompleteEvent({ startTime: 100 })
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request1)
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request2)
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request3)
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request4)
+
+    expect(requestRegistry.getMatchingRequest(createResourceEntry({ startTime: 99 }))).toBe(request3)
+  })
+
+  it('ignores requests that have a different URL', () => {
+    const lifeCycle = new LifeCycle()
+
+    const requestRegistry = createRequestRegistry(lifeCycle)
+
+    const request1 = createRequestCompleteEvent({ startTime: 1, url: URL })
+    const request2 = createRequestCompleteEvent({ startTime: 2, url: 'https://another-url.com/resource' })
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request1)
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request2)
+
+    expect(requestRegistry.getMatchingRequest(createResourceEntry({ startTime: 3 }))).toBe(request1)
+  })
+
+  it('does not return the same request twice', () => {
+    const lifeCycle = new LifeCycle()
+
+    const requestRegistry = createRequestRegistry(lifeCycle)
+
+    const request1 = createRequestCompleteEvent({ startTime: 1 })
+    const request2 = createRequestCompleteEvent({ startTime: 2 })
+
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request1)
+    lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, request2)
+
+    expect(requestRegistry.getMatchingRequest(createResourceEntry({ startTime: 2 }))).toBe(request2)
+    expect(requestRegistry.getMatchingRequest(createResourceEntry({ startTime: 2 }))).toBe(request1)
+    expect(requestRegistry.getMatchingRequest(createResourceEntry({ startTime: 2 }))).toBeUndefined()
+  })
+
+  it('is limited to a maximum number of requests', () => {
+    const lifeCycle = new LifeCycle()
+    const requestRegistry = createRequestRegistry(lifeCycle)
+    for (let i = 0; i < MAX_REQUESTS + 1; i++) {
+      lifeCycle.notify(LifeCycleEventType.REQUEST_COMPLETED, createRequestCompleteEvent({ startTime: i }))
+    }
+    expect(requestRegistry.getMatchingRequest(createResourceEntry({ startTime: 0 }))).toBeUndefined()
+  })
+
+  function createRequestCompleteEvent({
+    startTime,
+    url = URL,
+  }: {
+    startTime: number
+    url?: string
+  }): RequestCompleteEvent {
+    return {
+      startClocks: { relative: startTime as RelativeTime },
+      url,
+    } as RequestCompleteEvent
+  }
+
+  function createResourceEntry({ startTime }: { startTime: number }) {
+    return createPerformanceEntry(RumPerformanceEntryType.RESOURCE, {
+      startTime: startTime as RelativeTime,
+      name: URL,
+    })
+  }
+})

packages/rum-core/src/domain/resource/requestRegistry.ts

@@ -0,0 +1,51 @@
+import { addTelemetryDebug } from '@datadog/browser-core'
+import type { RumPerformanceResourceTiming } from '../../browser/performanceObservable'
+import type { LifeCycle } from '../lifeCycle'
+import { LifeCycleEventType } from '../lifeCycle'
+import type { RequestCompleteEvent } from '../requestCollection'
+
+export interface RequestRegistry {
+  getMatchingRequest(entry: RumPerformanceResourceTiming): RequestCompleteEvent | undefined
+  stop(): void
+}
+
+// Maximum number of requests to keep in the registry. Requests should be removed quite quickly in
+// general, this is just a safety limit to avoid memory leaks in case of a bug.
+export const MAX_REQUESTS = 1000
+
+export function createRequestRegistry(lifeCycle: LifeCycle): RequestRegistry {
+  const requests = new Set<RequestCompleteEvent>()
+
+  const subscription = lifeCycle.subscribe(LifeCycleEventType.REQUEST_COMPLETED, (request) => {
+    requests.add(request)
+    if (requests.size > MAX_REQUESTS) {
+      addTelemetryDebug('Too many requests')
+      requests.delete(requests.values().next().value!)
+    }
+  })
+
+  return {
+    getMatchingRequest(entry) {
+      // Returns the closest request object that happened before the entry
+      let minTimeDifference = Infinity
+      let closestRequest: RequestCompleteEvent | undefined
+      for (const request of requests) {
+        const timeDifference = entry.startTime - request.startClocks.relative
+        if (0 <= timeDifference && timeDifference < minTimeDifference && request.url === entry.name) {
+          minTimeDifference = Math.abs(timeDifference)
+          closestRequest = request
+        }
+      }
+
+      if (closestRequest) {
+        requests.delete(closestRequest)
+      }
+
+      return closestRequest
+    },
+
+    stop() {
+      subscription.unsubscribe()
+    },
+  }
+}