feat: impl fetch (#542)

- keep urllib:request, urllib:response channel message
- add urllib:fetch:request, urllib:fetch:response channel message

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

- **New Features**
- Introduced a `FetchFactory` class for enhanced HTTP request management
with diagnostics.
  - Added a new `FetchOpaque` type for tracking request metadata.
  - Enhanced `HttpClient` with new timing and diagnostics capabilities.
  
- **Bug Fixes**
  - Improved error handling for socket-related issues in `HttpClient`.

- **Documentation**
- Expanded public exports for better usability, including new interfaces
and constants.

- **Tests**
- Added unit tests for the new `fetch` functionality to ensure
reliability.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: killagu

src/FetchOpaqueInterceptor.ts

@@ -0,0 +1,41 @@
+// const { AsyncLocalStorage } = require('node:async_hooks');
+import { AsyncLocalStorage } from 'node:async_hooks';
+import symbols from './symbols.js';
+import { Dispatcher } from 'undici';
+
+// const RedirectHandler = require('../handler/redirect-handler')
+
+export interface FetchOpaque {
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  [symbols.kRequestId]: number;
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  [symbols.kRequestStartTime]: number;
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  [symbols.kEnableRequestTiming]: number;
+}
+
+// const internalOpaque = {
+//   [symbols.kRequestId]: requestId,
+//   [symbols.kRequestStartTime]: requestStartTime,
+//   [symbols.kEnableRequestTiming]: !!(init.timing ?? true),
+//   [symbols.kRequestTiming]: timing,
+//   // [symbols.kRequestOriginalOpaque]: originalOpaque,
+// };
+
+export interface OpaqueInterceptorOptions {
+  opaqueLocalStorage: AsyncLocalStorage<FetchOpaque>;
+}
+
+export function fetchOpaqueInterceptor(opts: OpaqueInterceptorOptions) {
+  const opaqueLocalStorage = opts?.opaqueLocalStorage;
+  return (dispatch: Dispatcher['dispatch']): Dispatcher['dispatch'] => {
+    return function redirectInterceptor(opts: Dispatcher.DispatchOptions, handler: Dispatcher.DispatchHandlers) {
+      const opaque = opaqueLocalStorage?.getStore();
+      (handler as any).opaque = opaque;
+      return dispatch(opts, handler);
+    };
+  };
+}

src/HttpAgent.ts

@@ -8,12 +8,12 @@ import {
 
 export type CheckAddressFunction = (ip: string, family: number | string, hostname: string) => boolean;
 
-export type HttpAgentOptions = {
+export interface HttpAgentOptions extends Agent.Options {
   lookup?: LookupFunction;
   checkAddress?: CheckAddressFunction;
   connect?: buildConnector.BuildOptions,
   allowH2?: boolean;
-};
+}
 
 class IllegalAddressError extends Error {
   hostname: string;
@@ -36,9 +36,10 @@ export class HttpAgent extends Agent {
 
   constructor(options: HttpAgentOptions) {
     /* eslint node/prefer-promises/dns: off*/
-    const _lookup = options.lookup ?? dns.lookup;
-    const lookup: LookupFunction = (hostname, dnsOptions, callback) => {
-      _lookup(hostname, dnsOptions, (err, ...args: any[]) => {
+    const { lookup = dns.lookup, ...baseOpts } = options;
+
+    const lookupFunction: LookupFunction = (hostname, dnsOptions, callback) => {
+      lookup(hostname, dnsOptions, (err, ...args: any[]) => {
         // address will be array on Node.js >= 20
         const address = args[0];
         const family = args[1];
@@ -63,7 +64,8 @@ export class HttpAgent extends Agent {
       });
     };
     super({
-      connect: { ...options.connect, lookup, allowH2: options.allowH2 },
+      ...baseOpts,
+      connect: { ...options.connect, lookup: lookupFunction, allowH2: options.allowH2 },
     });
     this.#checkAddress = options.checkAddress;
   }

src/HttpClient.ts

@@ -37,7 +37,7 @@ import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
 import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
 import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
-import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
+import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable, updateSocketInfo } from './utils.js';
 import symbols from './symbols.js';
 import { initDiagnosticsChannel } from './diagnosticsChannel.js';
 import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
@@ -47,7 +47,28 @@ type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
 type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
 type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;
 
-const PROTO_RE = /^https?:\/\//i;
+export const PROTO_RE = /^https?:\/\//i;
+
+export interface UnidiciTimingInfo {
+  startTime: number;
+  redirectStartTime: number;
+  redirectEndTime: number;
+  postRedirectStartTime: number;
+  finalServiceWorkerStartTime: number;
+  finalNetworkResponseStartTime: number;
+  finalNetworkRequestStartTime: number;
+  endTime: number;
+  encodedBodySize: number;
+  decodedBodySize: number;
+  finalConnectionTimingInfo: {
+    domainLookupStartTime: number;
+    domainLookupEndTime: number;
+    connectionStartTime: number;
+    connectionEndTime: number;
+    secureConnectionStartTime: number;
+    // ALPNNegotiatedProtocol: undefined
+  };
+}
 
 function noop() {
   // noop
@@ -137,9 +158,11 @@ export type RequestContext = {
   requestStartTime?: number;
 };
 
-const channels = {
+export const channels = {
   request: diagnosticsChannel.channel('urllib:request'),
   response: diagnosticsChannel.channel('urllib:response'),
+  fetchRequest: diagnosticsChannel.channel('urllib:fetch:request'),
+  fetchResponse: diagnosticsChannel.channel('urllib:fetch:response'),
 };
 
 export type RequestDiagnosticsMessage = {
@@ -631,7 +654,7 @@ export class HttpClient extends EventEmitter {
       }
       res.rt = performanceTime(requestStartTime);
       // get real socket info from internalOpaque
-      this.#updateSocketInfo(socketInfo, internalOpaque);
+      updateSocketInfo(socketInfo, internalOpaque);
 
       const clientResponse: HttpClientResponse = {
         opaque: originalOpaque,
@@ -707,7 +730,7 @@ export class HttpClient extends EventEmitter {
         res.requestUrls.push(requestUrl.href);
       }
       res.rt = performanceTime(requestStartTime);
-      this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
+      updateSocketInfo(socketInfo, internalOpaque, rawError);
 
       channels.response.publish({
         request: reqMeta,
@@ -729,40 +752,4 @@ export class HttpClient extends EventEmitter {
       throw err;
     }
   }
-
-  #updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
-    const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
-    if (socket) {
-      socketInfo.id = socket[symbols.kSocketId];
-      socketInfo.handledRequests = socket[symbols.kHandledRequests];
-      socketInfo.handledResponses = socket[symbols.kHandledResponses];
-      if (socket[symbols.kSocketLocalAddress]) {
-        socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
-        socketInfo.localPort = socket[symbols.kSocketLocalPort];
-      }
-      if (socket.remoteAddress) {
-        socketInfo.remoteAddress = socket.remoteAddress;
-        socketInfo.remotePort = socket.remotePort;
-        socketInfo.remoteFamily = socket.remoteFamily;
-      }
-      socketInfo.bytesRead = socket.bytesRead;
-      socketInfo.bytesWritten = socket.bytesWritten;
-      if (socket[symbols.kSocketConnectErrorTime]) {
-        socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
-        if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
-          socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
-        }
-        socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
-        socketInfo.connectHost = socket[symbols.kSocketConnectHost];
-        socketInfo.connectPort = socket[symbols.kSocketConnectPort];
-      }
-      if (socket[symbols.kSocketConnectedTime]) {
-        socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
-      }
-      if (socket[symbols.kSocketRequestEndTime]) {
-        socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
-      }
-      socket[symbols.kSocketRequestEndTime] = new Date();
-    }
-  }
 }

src/Request.ts

@@ -3,6 +3,7 @@ import type { EventEmitter } from 'node:events';
 import type { Dispatcher } from 'undici';
 import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 import type { HttpClientResponse } from './Response.js';
+import { Request } from 'undici';
 
 export type HttpMethod = Dispatcher.HttpMethod;
 
@@ -161,3 +162,8 @@ export type RequestMeta = {
   ctx?: unknown;
   retries: number;
 };
+
+export type FetchMeta = {
+  requestId: number;
+  request: Request,
+};