diff --git a/packages/core/src/browser/fetchObservable.ts b/packages/core/src/browser/fetchObservable.ts index c819956ff5..49ff0b4335 100644 --- a/packages/core/src/browser/fetchObservable.ts +++ b/packages/core/src/browser/fetchObservable.ts @@ -9,6 +9,8 @@ import type { GlobalObject } from '../tools/globalObject' import { globalObject } from '../tools/globalObject' import { readBytesFromStream } from '../tools/readBytesFromStream' import { tryToClone } from '../tools/utils/responseUtils' +import type { NormalizedHeaders } from '../tools/headers' +import { normalizeFetchResponseHeaders, normalizeRequestInitHeaders } from '../tools/headers' interface FetchContextBase { method: string @@ -17,6 +19,7 @@ interface FetchContextBase { init?: RequestInit url: string handlingStack?: string + requestHeaders?: NormalizedHeaders } export interface FetchStartContext extends FetchContextBase { @@ -31,6 +34,7 @@ export interface FetchResolveContext extends FetchContextBase { responseType?: string isAborted: boolean error?: Error + responseHeaders?: NormalizedHeaders } export type FetchContext = FetchStartContext | FetchResolveContext @@ -104,6 +108,7 @@ function beforeSend( startClocks, url, handlingStack, + requestHeaders: normalizeRequestInitHeaders(init), } observable.notify(context) @@ -142,6 +147,7 @@ async function afterSend( context.status = response.status context.responseType = response.type context.isAborted = false + context.responseHeaders = normalizeFetchResponseHeaders(response) const responseBodyCondition = responseBodyActionGetters.reduce( (action, getter) => Math.max(action, getter(context)), diff --git a/packages/core/src/browser/xhrObservable.ts b/packages/core/src/browser/xhrObservable.ts index e1e8b24a51..18f7ad3280 100644 --- a/packages/core/src/browser/xhrObservable.ts +++ b/packages/core/src/browser/xhrObservable.ts @@ -5,9 +5,10 @@ import type { Duration, ClocksState } from '../tools/utils/timeUtils' import { elapsed, clocksNow, timeStampNow } from '../tools/utils/timeUtils' import { normalizeUrl } from '../tools/utils/urlPolyfill' import { shallowClone } from '../tools/utils/objectUtils' +import type { NormalizedHeaders } from '../tools/headers' +import { normalizeXhrResponseHeaders, normalizeXhrRequestHeaders } from '../tools/headers' import type { Configuration } from '../domain/configuration' import { addEventListener } from './addEventListener' - export interface XhrOpenContext { state: 'open' method: string @@ -21,6 +22,8 @@ export interface XhrStartContext extends Omit { xhr: XMLHttpRequest handlingStack?: string requestBody?: unknown + body?: unknown + requestHeaders?: NormalizedHeaders // XHR does not expose request headers reliably; keep undefined for now } export interface XhrCompleteContext extends Omit { @@ -28,6 +31,7 @@ export interface XhrCompleteContext extends Omit { duration: Duration status: number responseBody?: string + responseHeaders?: NormalizedHeaders } export type XhrContext = XhrOpenContext | XhrStartContext | XhrCompleteContext @@ -90,6 +94,18 @@ function sendXhr( startContext.xhr = xhr startContext.handlingStack = handlingStack startContext.requestBody = body + const requestHeaderRecord: Record = {} + + const { stop: stopInstrumentingSetHeader } = instrumentMethod(xhr, 'setRequestHeader', ({ parameters }) => { + try { + const [key, value] = parameters as unknown as [string, string] + if (typeof key === 'string' && typeof value === 'string') { + requestHeaderRecord[key] = value + } + } catch { + // display warning + } + }) let hasBeenReported = false @@ -106,6 +122,7 @@ function sendXhr( const onEnd = () => { unsubscribeLoadEndListener() stopInstrumentingOnReadyStateChange() + stopInstrumentingSetHeader() if (hasBeenReported) { return } @@ -118,6 +135,8 @@ function sendXhr( if (typeof xhr.response === 'string') { completeContext.responseBody = xhr.response } + completeContext.requestHeaders = normalizeXhrRequestHeaders(requestHeaderRecord) + completeContext.responseHeaders = normalizeXhrResponseHeaders(xhr) observable.notify(shallowClone(completeContext)) } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index afe4af85c9..4323eb5b94 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -154,6 +154,7 @@ export * from './tools/serialisation/jsonStringify' export * from './tools/mergeInto' export * from './tools/utils/stringUtils' export * from './tools/matchOption' +export * from './tools/headers' export * from './tools/utils/responseUtils' export * from './tools/utils/typeUtils' export type { RawError, RawErrorCause, ErrorWithCause, Csp } from './domain/error/error.types' diff --git a/packages/core/src/tools/headers.ts b/packages/core/src/tools/headers.ts new file mode 100644 index 0000000000..ddbb09f3b4 --- /dev/null +++ b/packages/core/src/tools/headers.ts @@ -0,0 +1,266 @@ +import type { MatchOption } from './matchOption' +import { safeTruncate } from './utils/stringUtils' + +export interface NormalizedHeaders { + skipped?: string[] + [k: string]: string | string[] | undefined +} + +export interface HeaderMatchOption { + name: MatchOption + extractor?: MatchOption +} + +export interface HeaderCaptureOption { + match: MatchOption + request?: boolean | MatchOption[] | HeaderMatchOption[] + response?: boolean | MatchOption[] | HeaderMatchOption[] +} + +// Maximum number of headers to capture +const MAX_HEADER_COUNT = 100 + +// Maximum bytes per header value (UTF-8 encoding assumed) +const MAX_HEADER_VALUE_BYTES = 128 + +// Regex pattern to detect sensitive header names +const SENSITIVE_HEADER_PATTERN = /(token|cookie|secret|authorization|(api|secret|access|app).?key)$/i + +const DEFAULT_ALLOWED_HEADER_NAMES = new Set([ + 'cache-control', + 'etag', + 'age', + 'expires', + 'content-type', + 'content-encoding', + 'vary', + 'content-length', + 'server-timing', + 'x-cache', +]) + +/** + * Normalizes headers from various input formats into a consistent structure. + * Handles Headers object, [key,value] array, Record, and XHR getAllResponseHeaders() string. + */ +function normalizeHeaders(input: unknown): NormalizedHeaders | undefined { + if (!input) { + return undefined + } + + const map: Record = {} + const skipped: string[] = [] + let headerCount = 0 + + try { + if (input instanceof Headers) { + input.forEach((value, key) => { + if (headerCount < MAX_HEADER_COUNT) { + assignOrSkip(map, skipped, key, value) + headerCount++ + } + }) + } else if (Array.isArray(input)) { + input.forEach(([key, value]) => { + if (headerCount < MAX_HEADER_COUNT) { + assignOrSkip(map, skipped, key, String(value)) + headerCount++ + } + }) + } else if (typeof input === 'string') { + input + .split(/\r?\n/) + .map((l) => l.trim()) + .filter(Boolean) + .forEach((line) => { + if (headerCount < MAX_HEADER_COUNT) { + const idx = line.indexOf(':') + if (idx > -1) { + const key = line.slice(0, idx).trim() + const value = line.slice(idx + 1).trim() + assignOrSkip(map, skipped, key, value) + headerCount++ + } + } + }) + } else if (typeof input === 'object' && input !== null) { + Object.entries(input as Record).forEach(([key, value]) => { + if (headerCount < MAX_HEADER_COUNT) { + assignOrSkip(map, skipped, key, String(value)) + headerCount++ + } + }) + } + } catch { + return undefined + } + + return finalize(map, skipped) +} + +export function normalizeRequestInitHeaders(init?: RequestInit): NormalizedHeaders | undefined { + if (!init || !init.headers) { + return undefined + } + return normalizeHeaders(init.headers) +} + +export function normalizeFetchResponseHeaders(response?: Response): NormalizedHeaders | undefined { + if (!response || !('headers' in response) || !response.headers) { + return undefined + } + return normalizeHeaders(response.headers) +} + +export function normalizeXhrResponseHeaders(xhr?: XMLHttpRequest): NormalizedHeaders | undefined { + if (!xhr) { + return undefined + } + const headerBlock = xhr.getAllResponseHeaders && xhr.getAllResponseHeaders() + if (!headerBlock || typeof headerBlock !== 'string') { + return undefined + } + return normalizeHeaders(headerBlock) +} + +export function normalizeXhrRequestHeaders(record?: Record): NormalizedHeaders | undefined { + return normalizeHeaders(record) +} + +function assignOrSkip(map: Record, skipped: string[], key: string, value: string) { + const lower = key.toLowerCase() + + if (SENSITIVE_HEADER_PATTERN.test(lower)) { + skipped.push(lower) + return + } + + if (DEFAULT_ALLOWED_HEADER_NAMES.has(lower)) { + map[lower] = safeTruncate(value, MAX_HEADER_VALUE_BYTES) + } else { + skipped.push(lower) + } +} + +function finalize(map: Record, skipped: string[]): NormalizedHeaders | undefined { + if (Object.keys(map).length === 0 && skipped.length === 0) { + return undefined + } + const result: NormalizedHeaders = { ...map } + if (skipped.length) { + result.skipped = skipped + } + return result +} + +export function filterHeaders( + headers: NormalizedHeaders | undefined, + url: string, + direction: 'request' | 'response', + config: HeaderCaptureOption[] +): NormalizedHeaders | undefined { + if (!headers) { + return undefined + } + + const matchingConfig = config.find((option) => { + if (typeof option.match === 'string') { + return option.match === url + } else if (option.match instanceof RegExp) { + return option.match.test(url) + } else if (typeof option.match === 'function') { + return option.match(url) + } + return false + }) + + if (!matchingConfig) { + return undefined + } + + const directive = matchingConfig[direction] + + if (directive === false || directive === undefined) { + return undefined + } + + const filtered: Record = {} + let headerCount = 0 + + for (const [key, value] of Object.entries(headers)) { + if (key === 'skipped' || typeof value !== 'string') { + continue + } + + if (headerCount >= MAX_HEADER_COUNT) { + break + } + + const shouldInclude = directive === true || shouldIncludeHeader(key, directive) + + if (shouldInclude) { + const extractedValue = directive === true ? value : extractHeaderValue(key, value, directive) + filtered[key] = safeTruncate(extractedValue, MAX_HEADER_VALUE_BYTES) + headerCount++ + } + } + + if (Object.keys(filtered).length === 0) { + return undefined + } + + const result: NormalizedHeaders = filtered + if (directive === true && headers.skipped && headers.skipped.length > 0) { + result.skipped = headers.skipped + } + + return result +} + +function shouldIncludeHeader( + headerName: string, + directive: Array boolean) | HeaderMatchOption> +): boolean { + return directive.some((item) => { + if (typeof item === 'object' && 'name' in item) { + return matchHeaderName(headerName, item.name) + } + return matchHeaderName(headerName, item) + }) +} + +function matchHeaderName(headerName: string, match: string | RegExp | ((value: string) => boolean)): boolean { + if (typeof match === 'string') { + return match.toLowerCase() === headerName.toLowerCase() + } else if (match instanceof RegExp) { + return match.test(headerName) + } else if (typeof match === 'function') { + return match(headerName) + } + return false +} + +function extractHeaderValue( + headerName: string, + headerValue: string, + directive: Array boolean) | HeaderMatchOption> +): string { + for (const item of directive) { + if (typeof item === 'object' && 'name' in item && item.extractor) { + if (matchHeaderName(headerName, item.name)) { + return applyExtractor(headerValue, item.extractor) + } + } + } + return headerValue +} + +function applyExtractor(value: string, extractor: string | RegExp | ((value: string) => boolean)): string { + if (extractor instanceof RegExp) { + const match = value.match(extractor) + if (match && match[1]) { + return match[1] + } + } + return value +} diff --git a/packages/rum-core/src/domain/configuration/configuration.ts b/packages/rum-core/src/domain/configuration/configuration.ts index f41e76b0ca..28ef2107db 100644 --- a/packages/rum-core/src/domain/configuration/configuration.ts +++ b/packages/rum-core/src/domain/configuration/configuration.ts @@ -1,4 +1,11 @@ -import type { Configuration, InitConfiguration, MatchOption, RawTelemetryConfiguration } from '@datadog/browser-core' +import type { + Configuration, + InitConfiguration, + MatchOption, + RawTelemetryConfiguration, + HeaderCaptureOption, + HeaderMatchOption, +} from '@datadog/browser-core' import { getType, isMatchOption, @@ -271,6 +278,15 @@ export interface RumInitConfiguration extends InitConfiguration { * @category Data Collection */ allowedGraphQlUrls?: Array | undefined + + /** + * Enables collection of HTTP request and response headers for resources. + * When set to true, captures default headers (Cache-Control, ETag, Age, Expires, Content-Type, Content-Encoding, Vary, Content-Length, Server-Timing, X-Cache) for all URLs. + * When set to an array, allows fine-grained control over which headers to capture for specific URLs. + * + * @category Data Collection + */ + trackResourceHeaders?: boolean | Array | undefined } export type HybridInitConfiguration = Omit @@ -283,6 +299,8 @@ export interface GraphQlUrlOption { trackResponseErrors?: boolean } +export type { HeaderMatchOption, HeaderCaptureOption } from '@datadog/browser-core' + export interface RumConfiguration extends Configuration { // Built from init configuration actionNameAttribute: string | undefined @@ -310,6 +328,7 @@ export interface RumConfiguration extends Configuration { profilingSampleRate: number propagateTraceBaggage: boolean allowedGraphQlUrls: GraphQlUrlOption[] + trackResourceHeaders: HeaderCaptureOption[] } export function validateAndBuildRumConfiguration( @@ -348,6 +367,7 @@ export function validateAndBuildRumConfiguration( const baseConfiguration = validateAndBuildConfiguration(initConfiguration, errorStack) const allowedGraphQlUrls = validateAndBuildGraphQlOptions(initConfiguration) + const trackResourceHeaders = validateAndBuildHeaderCaptureOptions(initConfiguration) if (!baseConfiguration) { return @@ -388,6 +408,7 @@ export function validateAndBuildRumConfiguration( profilingSampleRate: initConfiguration.profilingSampleRate ?? 0, propagateTraceBaggage: !!initConfiguration.propagateTraceBaggage, allowedGraphQlUrls, + trackResourceHeaders, ...baseConfiguration, } } @@ -499,6 +520,50 @@ function hasGraphQlResponseErrorsTracking(allowedGraphQlUrls: RumInitConfigurati ) } +function validateAndBuildHeaderCaptureOptions(initConfiguration: RumInitConfiguration): HeaderCaptureOption[] { + const trackResourceHeaders = initConfiguration.trackResourceHeaders + + if (!trackResourceHeaders) { + return [] + } + + if (trackResourceHeaders === true) { + return [{ match: () => true, request: true, response: true }] + } + + if (!Array.isArray(trackResourceHeaders)) { + display.warn('trackResourceHeaders should be a boolean or an array') + return [] + } + + const headerCaptureOptions: HeaderCaptureOption[] = [] + + trackResourceHeaders.forEach((option) => { + if (isMatchOption(option)) { + headerCaptureOptions.push({ match: option, request: true, response: true }) + } else if (option && typeof option === 'object' && 'match' in option && isMatchOption(option.match)) { + const normalizedRequest = normalizeHeaderDirective(option.request) + const normalizedResponse = normalizeHeaderDirective(option.response) + headerCaptureOptions.push({ + match: option.match, + request: normalizedRequest !== undefined ? normalizedRequest : true, + response: normalizedResponse !== undefined ? normalizedResponse : true, + }) + } + }) + + return headerCaptureOptions +} + +function normalizeHeaderDirective( + directive: boolean | MatchOption[] | HeaderMatchOption[] | undefined +): boolean | MatchOption[] | HeaderMatchOption[] | undefined { + if (directive === undefined || typeof directive === 'boolean' || Array.isArray(directive)) { + return directive + } + return undefined +} + export function serializeRumConfiguration(configuration: RumInitConfiguration) { const baseSerializedConfiguration = serializeConfiguration(configuration) diff --git a/packages/rum-core/src/domain/requestCollection.ts b/packages/rum-core/src/domain/requestCollection.ts index de6b6d7ceb..3f73be57a5 100644 --- a/packages/rum-core/src/domain/requestCollection.ts +++ b/packages/rum-core/src/domain/requestCollection.ts @@ -7,6 +7,7 @@ import type { FetchResolveContext, ContextManager, } from '@datadog/browser-core' +import type { NormalizedHeaders } from '@datadog/browser-core/src/tools/headers' import { RequestType, ResponseBodyAction, @@ -54,8 +55,10 @@ export interface RequestCompleteEvent { traceSampled?: boolean xhr?: XMLHttpRequest response?: Response + responseHeaders?: NormalizedHeaders input?: unknown init?: RequestInit + requestHeaders?: NormalizedHeaders error?: Error isAborted: boolean handlingStack?: string @@ -108,6 +111,7 @@ export function trackXhr(lifeCycle: LifeCycle, configuration: RumConfiguration, type: RequestType.XHR, url: context.url, xhr: context.xhr, + responseHeaders: context.responseHeaders, isAborted: context.isAborted, handlingStack: context.handlingStack, requestBody: context.requestBody, @@ -165,6 +169,8 @@ export function trackFetch(lifeCycle: LifeCycle, configuration: RumConfiguration handlingStack: context.handlingStack, requestBody: context.init?.body, responseBody: context.responseBody, + responseHeaders: context.responseHeaders, + requestHeaders: context.requestHeaders, }) break } diff --git a/packages/rum-core/src/domain/resource/resourceCollection.ts b/packages/rum-core/src/domain/resource/resourceCollection.ts index 6f41d430ca..4f5811be9d 100644 --- a/packages/rum-core/src/domain/resource/resourceCollection.ts +++ b/packages/rum-core/src/domain/resource/resourceCollection.ts @@ -7,6 +7,7 @@ import { toServerDuration, relativeToClocks, createTaskQueue, + filterHeaders, } from '@datadog/browser-core' import type { RumConfiguration } from '../configuration' import type { RumPerformanceResourceTiming } from '../../browser/performanceObservable' @@ -135,6 +136,7 @@ function assembleResource( : computeRequestDuration(pageStateHistory, startClocks, request!.duration) const graphql = request && computeGraphQlMetaData(request, configuration) + const headers = computeHeaders(request, configuration) const resourceEvent = combine( { @@ -154,6 +156,7 @@ function assembleResource( protocol: entry && computeResourceEntryProtocol(entry), delivery_type: entry && computeResourceEntryDeliveryType(entry), graphql, + ...(headers && { headers }), }, type: RumEventType.RESOURCE, _dd: { @@ -172,6 +175,34 @@ function assembleResource( } } +function computeHeaders(request: RequestCompleteEvent | undefined, configuration: RumConfiguration) { + if (!request || configuration.trackResourceHeaders.length === 0) { + return undefined + } + + const requestHeaders = filterHeaders( + request.requestHeaders, + request.url, + 'request', + configuration.trackResourceHeaders + ) + const responseHeaders = filterHeaders( + request.responseHeaders, + request.url, + 'response', + configuration.trackResourceHeaders + ) + + if (!requestHeaders && !responseHeaders) { + return undefined + } + + return { + request: requestHeaders, + response: responseHeaders, + } +} + function computeGraphQlMetaData( request: RequestCompleteEvent, configuration: RumConfiguration diff --git a/packages/rum-core/src/rawRumEvent.types.ts b/packages/rum-core/src/rawRumEvent.types.ts index 9229ae390b..b8d10ed975 100644 --- a/packages/rum-core/src/rawRumEvent.types.ts +++ b/packages/rum-core/src/rawRumEvent.types.ts @@ -59,6 +59,10 @@ export interface RawRumResourceEvent { url: string method?: string status_code?: number + headers?: { + request?: NetworkHeaders + response?: NetworkHeaders + } size?: number encoded_body_size?: number decoded_body_size?: number @@ -84,6 +88,11 @@ export interface RawRumResourceEvent { } } +export interface NetworkHeaders { + skipped?: string[] + [k: string]: string | string[] | undefined +} + export interface ResourceEntryDetailsElement { duration: ServerDuration start: ServerDuration