add normalized request data in http

Author: mydea

packages/core/src/integrations/requestdata.ts

@@ -1,5 +1,6 @@
 import type { IntegrationFn } from '@sentry/types';
 import type { AddRequestDataToEventOptions, TransactionNamingScheme } from '@sentry/utils';
+import { addNormalizedRequestDataToEvent } from '@sentry/utils';
 import { addRequestDataToEvent } from '@sentry/utils';
 import { defineIntegration } from '../integration';
 
@@ -73,15 +74,24 @@ const _requestDataIntegration = ((options: RequestDataIntegrationOptions = {}) =
       // that's happened, it will be easier to add this logic in without worrying about unexpected side effects.)
 
       const { sdkProcessingMetadata = {} } = event;
-      const req = sdkProcessingMetadata.request;
+      const { request, normalizedRequest } = sdkProcessingMetadata;
 
-      if (!req) {
-        return event;
+      const addRequestDataOptions = convertReqDataIntegrationOptsToAddReqDataOpts(_options);
+
+      // If this is set, it takes precedence over the plain request object
+      if (normalizedRequest) {
+        // Some other data is not available in standard HTTP requests, but can sometimes be augmented by e.g. Express or Next.js
+        const ipAddress = request ? request.ip || (request.socket && request.socket.remoteAddress) : undefined;
+        const user = request ? request.user : undefined;
+
+        return addNormalizedRequestDataToEvent(event, normalizedRequest, { ipAddress, user }, addRequestDataOptions);
       }
 
-      const addRequestDataOptions = convertReqDataIntegrationOptsToAddReqDataOpts(_options);
+      if (!request) {
+        return event;
+      }
 
-      return addRequestDataToEvent(event, req, addRequestDataOptions);
+      return addRequestDataToEvent(event, request, addRequestDataOptions);
     },
   };
 }) satisfies IntegrationFn;

packages/node/src/integrations/http.ts

@@ -13,9 +13,10 @@ import {
   setCapturedScopesOnSpan,
 } from '@sentry/core';
 import { getClient } from '@sentry/opentelemetry';
-import type { IntegrationFn, SanitizedRequestData } from '@sentry/types';
+import type { IntegrationFn, PolymorphicRequest, Request, SanitizedRequestData } from '@sentry/types';
 
-import { getSanitizedUrlString, parseUrl, stripUrlQueryAndFragment } from '@sentry/utils';
+import { getSanitizedUrlString, logger, parseUrl, stripUrlQueryAndFragment } from '@sentry/utils';
+import { DEBUG_BUILD } from '../debug-build';
 import type { NodeClient } from '../sdk/client';
 import { setIsolationScope } from '../sdk/scope';
 import type { HTTPModuleRequestIncomingMessage } from '../transports/http-module';
@@ -148,7 +149,27 @@ export const instrumentHttp = Object.assign(
         const isolationScope = (scopes.isolationScope || getIsolationScope()).clone();
         const scope = scopes.scope || getCurrentScope();
 
+        const headers = req.headers;
+        const host = headers.host || '<no host>';
+        const protocol = req.socket && (req.socket as { encrypted?: boolean }).encrypted ? 'https' : 'http';
+        const originalUrl = req.url || '';
+        const absoluteUrl = originalUrl.startsWith(protocol) ? originalUrl : `${protocol}://${host}${originalUrl}`;
+
+        // This is non-standard, but may be set on e.g. Next.js or Express requests
+        const cookies = (req as PolymorphicRequest).cookies;
+
+        const normalizedRequest: Request = {
+          url: absoluteUrl,
+          method: req.method,
+          query_string: extractQueryParams(req),
+          headers: headersToDict(req.headers),
+          cookies,
+        };
+
+        patchRequestToCaptureBody(req, normalizedRequest);
+
         // Update the isolation scope, isolate this request
+        isolationScope.setSDKProcessingMetadata({ normalizedRequest });
         isolationScope.setSDKProcessingMetadata({ request: req });
 
         const client = getClient<NodeClient>();
@@ -301,3 +322,128 @@ function isKnownPrefetchRequest(req: HTTPModuleRequestIncomingMessage): boolean
   // Currently only handles Next.js prefetch requests but may check other frameworks in the future.
   return req.headers['next-router-prefetch'] === '1';
 }
+
+/**
+ * This method patches the request object to capture the body.
+ * Instead of actually consuming the streamed body ourselves, which has potential side effects,
+ * we monkey patch `req.on('data')` to intercept the body chunks.
+ * This way, we only read the body if the user also consumes the body, ensuring we do not change any behavior in unexpected ways.
+ */
+function patchRequestToCaptureBody(req: HTTPModuleRequestIncomingMessage, normalizedRequest: Request): void {
+  const chunks: Buffer[] = [];
+
+  /**
+   * We need to keep track of the original callbacks, in order to be able to remove listeners again.
+   * Since `off` depends on having the exact same function reference passed in, we need to be able to map
+   * original listeners to our wrapped ones.
+   */
+  const callbackMap = new WeakMap();
+
+  try {
+    // eslint-disable-next-line @typescript-eslint/unbound-method
+    req.on = new Proxy(req.on, {
+      apply: (target, thisArg, args: Parameters<typeof req.on>) => {
+        const [event, listener] = args;
+
+        if (event === 'data') {
+          const callback = new Proxy(listener, {
+            apply: (target, thisArg, args: Parameters<typeof listener>) => {
+              const chunk = args[0];
+              try {
+                chunks.push(chunk);
+              } catch {
+                // ignore errors here...
+              }
+              return Reflect.apply(target, thisArg, args);
+            },
+          });
+
+          callbackMap.set(listener, callback);
+
+          return Reflect.apply(target, thisArg, [event, callback]);
+        }
+
+        if (event === 'end') {
+          const callback = new Proxy(listener, {
+            apply: (target, thisArg, args) => {
+              const body = Buffer.concat(chunks).toString('utf-8');
+
+              // We mutate the passed in normalizedRequest and add the body to it
+              if (body) {
+                normalizedRequest.data = body;
+              }
+
+              return Reflect.apply(target, thisArg, args);
+            },
+          });
+
+          callbackMap.set(listener, callback);
+
+          return Reflect.apply(target, thisArg, [event, callback]);
+        }
+
+        return Reflect.apply(target, thisArg, args);
+      },
+    });
+
+    // Ensure we also remove callbacks correctly
+    // eslint-disable-next-line @typescript-eslint/unbound-method
+    req.off = new Proxy(req.off, {
+      apply: (target, thisArg, args: Parameters<typeof req.off>) => {
+        const [, listener] = args;
+
+        const callback = callbackMap.get(listener);
+        if (callback) {
+          callbackMap.delete(listener);
+
+          const modifiedArgs = args.slice();
+          modifiedArgs[1] = callback;
+          return Reflect.apply(target, thisArg, modifiedArgs);
+        }
+
+        return Reflect.apply(target, thisArg, args);
+      },
+    });
+  } catch {
+    // ignore errors if we can't patch stuff
+  }
+}
+
+function extractQueryParams(req: IncomingMessage): string | undefined {
+  // url (including path and query string):
+  let originalUrl = req.url || '';
+
+  if (!originalUrl) {
+    return;
+  }
+
+  // The `URL` constructor can't handle internal URLs of the form `/some/path/here`, so stick a dummy protocol and
+  // hostname on the beginning. Since the point here is just to grab the query string, it doesn't matter what we use.
+  if (originalUrl.startsWith('/')) {
+    originalUrl = `http://dogs.are.great${originalUrl}`;
+  }
+
+  try {
+    const queryParams = new URL(originalUrl).search.slice(1);
+    return queryParams.length ? queryParams : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function headersToDict(reqHeaders: Record<string, string | string[] | undefined>): Record<string, string> {
+  const headers: Record<string, string> = {};
+
+  try {
+    Object.entries(reqHeaders).forEach(([key, value]) => {
+      if (typeof value === 'string') {
+        headers[key] = value;
+      }
+    });
+  } catch (e) {
+    DEBUG_BUILD &&
+      logger.warn('Sentry failed extracting headers from a request object. If you see this, please file an issue.');
+  }
+
+  return headers;
+}

packages/node/src/transports/http-module.ts

@@ -10,7 +10,8 @@ export type HTTPModuleRequestOptions = HTTPRequestOptions | HTTPSRequestOptions
 export interface HTTPModuleRequestIncomingMessage {
   headers: IncomingHttpHeaders;
   statusCode?: number;
-  on(event: 'data' | 'end', listener: () => void): void;
+  on(event: 'data' | 'end', listener: (chunk: Buffer) => void): void;
+  off(event: 'data' | 'end', listener: (chunk: Buffer) => void): void;
   setEncoding(encoding: string): void;
 }
 

packages/types/src/envelope.ts

@@ -101,7 +101,7 @@ export type ProfileItem = BaseEnvelopeItem<ProfileItemHeaders, Profile>;
 export type ProfileChunkItem = BaseEnvelopeItem<ProfileChunkItemHeaders, ProfileChunk>;
 export type SpanItem = BaseEnvelopeItem<SpanItemHeaders, Partial<SpanJSON>>;
 
-export type EventEnvelopeHeaders = { event_id: string; sent_at: string; trace?: DynamicSamplingContext };
+export type EventEnvelopeHeaders = { event_id: string; sent_at: string; trace?: Partial<DynamicSamplingContext> };
 type SessionEnvelopeHeaders = { sent_at: string };
 type CheckInEnvelopeHeaders = { trace?: DynamicSamplingContext };
 type ClientReportEnvelopeHeaders = BaseEnvelopeHeaders;

packages/types/src/event.ts

@@ -2,13 +2,15 @@ import type { Attachment } from './attachment';
 import type { Breadcrumb } from './breadcrumb';
 import type { Contexts } from './context';
 import type { DebugMeta } from './debugMeta';
+import type { DynamicSamplingContext } from './envelope';
 import type { Exception } from './exception';
 import type { Extras } from './extra';
 import type { Measurements } from './measurement';
 import type { Mechanism } from './mechanism';
 import type { Primitive } from './misc';
+import type { PolymorphicRequest } from './polymorphics';
 import type { Request } from './request';
-import type { CaptureContext } from './scope';
+import type { CaptureContext, Scope } from './scope';
 import type { SdkInfo } from './sdkinfo';
 import type { SeverityLevel } from './severity';
 import type { MetricSummary, SpanJSON } from './span';
@@ -51,7 +53,15 @@ export interface Event {
   measurements?: Measurements;
   debug_meta?: DebugMeta;
   // A place to stash data which is needed at some point in the SDK's event processing pipeline but which shouldn't get sent to Sentry
-  sdkProcessingMetadata?: { [key: string]: any };
+  // Note: This is considered internal and is subject to change in minors
+  sdkProcessingMetadata?: { [key: string]: unknown } & {
+    request?: PolymorphicRequest;
+    normalizedRequest?: Request;
+    dynamicSamplingContext?: Partial<DynamicSamplingContext>;
+    capturedSpanScope?: Scope;
+    capturedSpanIsolationScope?: Scope;
+    spanCountBeforeProcessing?: number;
+  };
   transaction_info?: {
     source: TransactionSource;
   };