⚗️ [poc] Network headers capture for bundle size estimation

packages/core/src/browser/fetchObservable.ts

@@ -9,6 +9,8 @@ import type { GlobalObject } from '../tools/globalObject'
 import { globalObject } from '../tools/globalObject'
 import { readBytesFromStream } from '../tools/readBytesFromStream'
 import { tryToClone } from '../tools/utils/responseUtils'
+import type { NormalizedHeaders } from '../tools/headers'
+import { normalizeFetchResponseHeaders, normalizeRequestInitHeaders } from '../tools/headers'
 
 interface FetchContextBase {
   method: string
@@ -17,6 +19,7 @@ interface FetchContextBase {
   init?: RequestInit
   url: string
   handlingStack?: string
+  requestHeaders?: NormalizedHeaders
 }
 
 export interface FetchStartContext extends FetchContextBase {
@@ -31,6 +34,7 @@ export interface FetchResolveContext extends FetchContextBase {
   responseType?: string
   isAborted: boolean
   error?: Error
+  responseHeaders?: NormalizedHeaders
 }
 
 export type FetchContext = FetchStartContext | FetchResolveContext
@@ -104,6 +108,7 @@ function beforeSend(
     startClocks,
     url,
     handlingStack,
+    requestHeaders: normalizeRequestInitHeaders(init),
   }
 
   observable.notify(context)
@@ -142,6 +147,7 @@ async function afterSend(
   context.status = response.status
   context.responseType = response.type
   context.isAborted = false
+  context.responseHeaders = normalizeFetchResponseHeaders(response)
 
   const responseBodyCondition = responseBodyActionGetters.reduce(
     (action, getter) => Math.max(action, getter(context)),

packages/core/src/browser/xhrObservable.ts

@@ -5,9 +5,10 @@ import type { Duration, ClocksState } from '../tools/utils/timeUtils'
 import { elapsed, clocksNow, timeStampNow } from '../tools/utils/timeUtils'
 import { normalizeUrl } from '../tools/utils/urlPolyfill'
 import { shallowClone } from '../tools/utils/objectUtils'
+import type { NormalizedHeaders } from '../tools/headers'
+import { normalizeXhrResponseHeaders, normalizeXhrRequestHeaders } from '../tools/headers'
 import type { Configuration } from '../domain/configuration'
 import { addEventListener } from './addEventListener'
-
 export interface XhrOpenContext {
   state: 'open'
   method: string
@@ -21,13 +22,16 @@ export interface XhrStartContext extends Omit<XhrOpenContext, 'state'> {
   xhr: XMLHttpRequest
   handlingStack?: string
   requestBody?: unknown
+  body?: unknown
+  requestHeaders?: NormalizedHeaders // XHR does not expose request headers reliably; keep undefined for now
 }
 
 export interface XhrCompleteContext extends Omit<XhrStartContext, 'state'> {
   state: 'complete'
   duration: Duration
   status: number
   responseBody?: string
+  responseHeaders?: NormalizedHeaders
 }
 
 export type XhrContext = XhrOpenContext | XhrStartContext | XhrCompleteContext
@@ -90,6 +94,18 @@ function sendXhr(
   startContext.xhr = xhr
   startContext.handlingStack = handlingStack
   startContext.requestBody = body
+  const requestHeaderRecord: Record<string, string> = {}
+
+  const { stop: stopInstrumentingSetHeader } = instrumentMethod(xhr, 'setRequestHeader', ({ parameters }) => {
+    try {
+      const [key, value] = parameters as unknown as [string, string]
+      if (typeof key === 'string' && typeof value === 'string') {
+        requestHeaderRecord[key] = value
+      }
+    } catch {
+      // display warning
+    }
+  })
 
   let hasBeenReported = false
 
@@ -106,6 +122,7 @@ function sendXhr(
   const onEnd = () => {
     unsubscribeLoadEndListener()
     stopInstrumentingOnReadyStateChange()
+    stopInstrumentingSetHeader()
     if (hasBeenReported) {
       return
     }
@@ -118,6 +135,8 @@ function sendXhr(
     if (typeof xhr.response === 'string') {
       completeContext.responseBody = xhr.response
     }
+    completeContext.requestHeaders = normalizeXhrRequestHeaders(requestHeaderRecord)
+    completeContext.responseHeaders = normalizeXhrResponseHeaders(xhr)
     observable.notify(shallowClone(completeContext))
   }
 

packages/core/src/index.ts

@@ -154,6 +154,7 @@ export * from './tools/serialisation/jsonStringify'
 export * from './tools/mergeInto'
 export * from './tools/utils/stringUtils'
 export * from './tools/matchOption'
+export * from './tools/headers'
 export * from './tools/utils/responseUtils'
 export * from './tools/utils/typeUtils'
 export type { RawError, RawErrorCause, ErrorWithCause, Csp } from './domain/error/error.types'

packages/core/src/tools/headers.ts

@@ -0,0 +1,266 @@
+import type { MatchOption } from './matchOption'
+import { safeTruncate } from './utils/stringUtils'
+
+export interface NormalizedHeaders {
+  skipped?: string[]
+  [k: string]: string | string[] | undefined
+}
+
+export interface HeaderMatchOption {
+  name: MatchOption
+  extractor?: MatchOption
+}
+
+export interface HeaderCaptureOption {
+  match: MatchOption
+  request?: boolean | MatchOption[] | HeaderMatchOption[]
+  response?: boolean | MatchOption[] | HeaderMatchOption[]
+}
+
+// Maximum number of headers to capture
+const MAX_HEADER_COUNT = 100
+
+// Maximum bytes per header value (UTF-8 encoding assumed)
+const MAX_HEADER_VALUE_BYTES = 128
+
+// Regex pattern to detect sensitive header names
+const SENSITIVE_HEADER_PATTERN = /(token|cookie|secret|authorization|(api|secret|access|app).?key)$/i
+
+const DEFAULT_ALLOWED_HEADER_NAMES = new Set<string>([
+  'cache-control',
+  'etag',
+  'age',
+  'expires',
+  'content-type',
+  'content-encoding',
+  'vary',
+  'content-length',
+  'server-timing',
+  'x-cache',
+])
+
+/**
+ * Normalizes headers from various input formats into a consistent structure.
+ * Handles Headers object, [key,value] array, Record<string,string>, and XHR getAllResponseHeaders() string.
+ */
+function normalizeHeaders(input: unknown): NormalizedHeaders | undefined {
+  if (!input) {
+    return undefined
+  }
+
+  const map: Record<string, string> = {}
+  const skipped: string[] = []
+  let headerCount = 0
+
+  try {
+    if (input instanceof Headers) {
+      input.forEach((value, key) => {
+        if (headerCount < MAX_HEADER_COUNT) {
+          assignOrSkip(map, skipped, key, value)
+          headerCount++
+        }
+      })
+    } else if (Array.isArray(input)) {
+      input.forEach(([key, value]) => {
+        if (headerCount < MAX_HEADER_COUNT) {
+          assignOrSkip(map, skipped, key, String(value))
+          headerCount++
+        }
+      })
+    } else if (typeof input === 'string') {
+      input
+        .split(/\r?\n/)
+        .map((l) => l.trim())
+        .filter(Boolean)
+        .forEach((line) => {
+          if (headerCount < MAX_HEADER_COUNT) {
+            const idx = line.indexOf(':')
+            if (idx > -1) {
+              const key = line.slice(0, idx).trim()
+              const value = line.slice(idx + 1).trim()
+              assignOrSkip(map, skipped, key, value)
+              headerCount++
+            }
+          }
+        })
+    } else if (typeof input === 'object' && input !== null) {
+      Object.entries(input as Record<string, unknown>).forEach(([key, value]) => {
+        if (headerCount < MAX_HEADER_COUNT) {
+          assignOrSkip(map, skipped, key, String(value))
+          headerCount++
+        }
+      })
+    }
+  } catch {
+    return undefined
+  }
+
+  return finalize(map, skipped)
+}
+
+export function normalizeRequestInitHeaders(init?: RequestInit): NormalizedHeaders | undefined {
+  if (!init || !init.headers) {
+    return undefined
+  }
+  return normalizeHeaders(init.headers)
+}
+
+export function normalizeFetchResponseHeaders(response?: Response): NormalizedHeaders | undefined {
+  if (!response || !('headers' in response) || !response.headers) {
+    return undefined
+  }
+  return normalizeHeaders(response.headers)
+}
+
+export function normalizeXhrResponseHeaders(xhr?: XMLHttpRequest): NormalizedHeaders | undefined {
+  if (!xhr) {
+    return undefined
+  }
+  const headerBlock = xhr.getAllResponseHeaders && xhr.getAllResponseHeaders() 
+  if (!headerBlock || typeof headerBlock !== 'string') {
+    return undefined
+  }
+  return normalizeHeaders(headerBlock)
+}
+
+export function normalizeXhrRequestHeaders(record?: Record<string, string>): NormalizedHeaders | undefined {
+  return normalizeHeaders(record)
+}
+
+function assignOrSkip(map: Record<string, string>, skipped: string[], key: string, value: string) {
+  const lower = key.toLowerCase()
+
+  if (SENSITIVE_HEADER_PATTERN.test(lower)) {
+    skipped.push(lower)
+    return
+  }
+
+  if (DEFAULT_ALLOWED_HEADER_NAMES.has(lower)) {
+    map[lower] = safeTruncate(value, MAX_HEADER_VALUE_BYTES)
+  } else {
+    skipped.push(lower)
+  }
+}
+
+function finalize(map: Record<string, string>, skipped: string[]): NormalizedHeaders | undefined {
+  if (Object.keys(map).length === 0 && skipped.length === 0) {
+    return undefined
+  }
+  const result: NormalizedHeaders = { ...map }
+  if (skipped.length) {
+    result.skipped = skipped
+  }
+  return result
+}
+
+export function filterHeaders(
+  headers: NormalizedHeaders | undefined,
+  url: string,
+  direction: 'request' | 'response',
+  config: HeaderCaptureOption[]
+): NormalizedHeaders | undefined {
+  if (!headers) {
+    return undefined
+  }
+
+  const matchingConfig = config.find((option) => {
+    if (typeof option.match === 'string') {
+      return option.match === url
+    } else if (option.match instanceof RegExp) {
+      return option.match.test(url)
+    } else if (typeof option.match === 'function') {
+      return option.match(url)
+    }
+    return false
+  })
+
+  if (!matchingConfig) {
+    return undefined
+  }
+
+  const directive = matchingConfig[direction]
+
+  if (directive === false || directive === undefined) {
+    return undefined
+  }
+
+  const filtered: Record<string, string> = {}
+  let headerCount = 0
+
+  for (const [key, value] of Object.entries(headers)) {
+    if (key === 'skipped' || typeof value !== 'string') {
+      continue
+    }
+
+    if (headerCount >= MAX_HEADER_COUNT) {
+      break
+    }
+
+    const shouldInclude = directive === true || shouldIncludeHeader(key, directive)
+
+    if (shouldInclude) {
+      const extractedValue = directive === true ? value : extractHeaderValue(key, value, directive)
+      filtered[key] = safeTruncate(extractedValue, MAX_HEADER_VALUE_BYTES)
+      headerCount++
+    }
+  }
+
+  if (Object.keys(filtered).length === 0) {
+    return undefined
+  }
+
+  const result: NormalizedHeaders = filtered
+  if (directive === true && headers.skipped && headers.skipped.length > 0) {
+    result.skipped = headers.skipped
+  }
+
+  return result
+}
+
+function shouldIncludeHeader(
+  headerName: string,
+  directive: Array<string | RegExp | ((value: string) => boolean) | HeaderMatchOption>
+): boolean {
+  return directive.some((item) => {
+    if (typeof item === 'object' && 'name' in item) {
+      return matchHeaderName(headerName, item.name)
+    }
+    return matchHeaderName(headerName, item)
+  })
+}
+
+function matchHeaderName(headerName: string, match: string | RegExp | ((value: string) => boolean)): boolean {
+  if (typeof match === 'string') {
+    return match.toLowerCase() === headerName.toLowerCase()
+  } else if (match instanceof RegExp) {
+    return match.test(headerName)
+  } else if (typeof match === 'function') {
+    return match(headerName)
+  }
+  return false
+}
+
+function extractHeaderValue(
+  headerName: string,
+  headerValue: string,
+  directive: Array<string | RegExp | ((value: string) => boolean) | HeaderMatchOption>
+): string {
+  for (const item of directive) {
+    if (typeof item === 'object' && 'name' in item && item.extractor) {
+      if (matchHeaderName(headerName, item.name)) {
+        return applyExtractor(headerValue, item.extractor)
+      }
+    }
+  }
+  return headerValue
+}
+
+function applyExtractor(value: string, extractor: string | RegExp | ((value: string) => boolean)): string {
+  if (extractor instanceof RegExp) {
+    const match = value.match(extractor)
+    if (match && match[1]) {
+      return match[1]
+    }
+  }
+  return value
+}