feat!: TCP/TLS wrap-based interception

.github/workflows/ci.yml

@@ -10,8 +10,9 @@ jobs:
   build:
     runs-on: macos-latest
     strategy:
+      fail-fast: false
       matrix:
-        node: [20, 22]
+        node: [22, 24]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -31,16 +32,16 @@ jobs:
         run: pnpm install
 
       - name: Unit tests
-        run: pnpm test:unit
+        run: pnpm test -- --project=unit
 
       - name: Build
         run: pnpm build
 
       - name: Node.js tests
-        run: pnpm test:node
+        run: pnpm test -- --project=node
 
       - name: Install Playwright browsers
         run: npx playwright install chromium --with-deps --only-shell
 
       - name: Browser tests
-        run: pnpm test:browser
+        run: pnpm test -- --project=browser

.gitignore

@@ -7,3 +7,4 @@ package-lock.json
 .idea
 /test-results
 /tmp
+__screenshots__

README.md

@@ -104,21 +104,6 @@ You can respond to the intercepted HTTP request by constructing a Fetch API Resp
 - Does **not** provide any request matching logic;
 - Does **not** handle requests by default.
 
-## Limitations
-
-- Interceptors will hang indefinitely if you call `req.end()` in the `connect` event listener of the respective `socket`:
-
-```ts
-req.on('socket', (socket) => {
-  socket.on('connect', () => {
-    // ❌ While this is allowed in Node.js, this cannot be handled in Interceptors.
-    req.end()
-  })
-})
-```
-
-> This limitation is intrinsic to the interception algorithm used by the library. In order for it to emit the `connect` event on the socket, the library must know if you've handled the request in any way (e.g. responded with a mocked response or errored it). For that, it emits the `request` event on the interceptor where you can handle the request. Since you can consume the request stream in the `request` event, it waits until the request body stream is complete (i.e. until `req.end()` is called). This creates a catch 22 that causes this limitation.
-
 ## Getting started
 
 ```bash
@@ -297,10 +282,12 @@ Note that a single request _can only be handled once_. You may want to introduce
 Requests must be responded to within the same tick as the request listener. This means you cannot respond to a request using `setTimeout`, as this will delegate the callback to the next tick. If you wish to introduce asynchronous side-effects in the listener, consider making it an `async` function, awaiting any side-effects you need.
 
 ```js
+import { setTimeout } from 'node:timers/promises'
+
 // Respond to all requests with a 500 response
 // delayed by 500ms.
 interceptor.on('request', async ({ controller }) => {
-  await sleep(500)
+  await setTimeout(500)
   controller.respondWith(new Response(null, { status: 500 }))
 })
 ```

XMLHttpRequest/package.json

@@ -1,12 +1,20 @@
 {
-  "main": "../lib/node/interceptors/XMLHttpRequest/index.cjs",
-  "module": "../lib/node/interceptors/XMLHttpRequest/index.mjs",
-  "browser": "../lib/browser/interceptors/XMLHttpRequest/index.mjs",
+  "main": "../lib/node/interceptors/XMLHttpRequest/node.cjs",
+  "module": "../lib/node/interceptors/XMLHttpRequest/node.mjs",
+  "browser": "../lib/browser/interceptors/XMLHttpRequest/web.mjs",
   "exports": {
     ".": {
-      "browser": "./../lib/browser/interceptors/XMLHttpRequest/index.mjs",
-      "import": "./../lib/node/interceptors/XMLHttpRequest/index.mjs",
-      "default": "./../lib/node/interceptors/XMLHttpRequest/index.cjs"
+      "browser": "./../lib/browser/interceptors/XMLHttpRequest/web.mjs",
+      "import": "./../lib/node/interceptors/XMLHttpRequest/node.mjs",
+      "default": "./../lib/node/interceptors/XMLHttpRequest/node.cjs"
+    },
+    "./node": {
+      "import": "./../lib/node/interceptors/XMLHttpRequest/node.mjs",
+      "default": "./../lib/node/interceptors/XMLHttpRequest/node.cjs"
+    },
+    "./web": {
+      "import": "./../lib/browser/interceptors/XMLHttpRequest/web.mjs",
+      "default": "./../lib/browser/interceptors/XMLHttpRequest/web.cjs"
     }
   }
 }

_http_common.d.ts

@@ -1,6 +1,8 @@
 import type { Socket } from 'node:net'
 import type { IncomingMessage, OutgoingMessage } from 'node:http'
 
+declare var methods: Array<string>
+
 declare var HTTPParser: {
   new (): HTTPParser<number>
   REQUEST: 0
@@ -21,15 +23,15 @@ declare var HTTPParser: {
 export interface HTTPParser<ParserType extends number> {
   new (): HTTPParser<ParserType>
 
-  [HTTPParser.kOnMessageBegin]: (() => void) | null
-  [HTTPParser.kOnHeaders]: HeadersCallback | null
-  [HTTPParser.kOnHeadersComplete]: ParserType extends 0
+  [HTTPParser.kOnMessageBegin]?: (() => void) | null
+  [HTTPParser.kOnHeaders]?: HeadersCallback
+  [HTTPParser.kOnHeadersComplete]?: ParserType extends 0
     ? RequestHeadersCompleteCallback | null
     : ResponseHeadersCompleteCallback | null
-  [HTTPParser.kOnBody]: ((chunk: Buffer) => void) | null
-  [HTTPParser.kOnMessageComplete]: (() => void) | null
-  [HTTPParser.kOnExecute]: (() => void) | null
-  [HTTPParser.kOnTimeout]: (() => void) | null
+  [HTTPParser.kOnBody]?: ((chunk: Buffer) => void) | null
+  [HTTPParser.kOnMessageComplete]?: (() => void) | null
+  [HTTPParser.kOnExecute]?: (() => void) | null
+  [HTTPParser.kOnTimeout]?: (() => void) | null
 
   initialize(type: ParserType, asyncResource: object): void
   execute(buffer: Buffer): void

discoveries/http.mdx

@@ -0,0 +1,57 @@
+# General
+
+## Reusing sockets
+
+By default, `Agent` will try to reuse any sockets that were not explicitly closed via `Connection: close` request header. Here's what happens when a socket gets reused:
+
+1. `free` is emitted on the socket to notify that it's done with the previous request.
+2. `connect` is _not_ emitted on this socket anymore since the connection has already been established. No connection attempt will be made whatsoever (`socket._handle.connect` won't be called).
+3. The next request's HTTP message is written into the socket.
+
+> With this in mind, `socket.on('connect', () => request.end())` is a potentially faulty logic. A reused socket will never emit that event and the request will pend indefinitely.
+
+## `CONNECT` requests
+
+While forbidden by the Fetch API specification, `CONNECT` requests are possible in Node.js. Here's what's special about connect requests:
+
+- Are sent to the _running_ server (`options.host` + `options.port)` to notify it about the intent to connect somewhere esle.
+- Do _not_ actually establish any connections. Instead, the server handles the same socket instance to establish that connection.
+- Never recieve the response (`response` is never emitted.)
+
+```ts
+const client = http.request({
+  // Actual server handling the "CONNECT" request.
+  host: '127.0.0.1',
+  port: 1337,
+  // The (proxy) authority in a "host:port" format.
+  path: 'www.example.com:80'
+})
+
+client.on('connect', (request, socket) => {
+  // Use "socket" to write to the authority...
+})
+```
+
+# HTTPS/TLS
+
+### `TLSWrap`
+
+Regular `net.Socket` connections finalize in the `afterConnect` callback [attached](https://github.com/nodejs/node/blob/bdc8131fa78089b81b74dbff467365afb6536e6a/lib/net.js#L1142) to the `TCPWrap` request. A TLS sockets\, while extending `net.Socket` and still relying on `afterConnect` being called to transition the socket into the connected state, _do not_ rely on `_handle.oncomplete`. **It is never called**. 
+
+A TLS socket wraps `net.Socket._handle` in a `TLSWrap`, which becomes responsible for the connection. It calls the `afterConnect` callback _internally_, in the C++ code, and there's no public method to trigger it.
+
+> TLS wrap has its own methods, like `onhandshakedone` and `verifyError`.
+
+### Connection event sequence
+
+> For brevity: "wrap" means the TCP socket TLS extends, "socket" means the TLS socket.
+
+1. `tls.connect()`.
+2. `new TLSSocket(options.socket)`.
+3. `TLSSocket.prototype._wrapHandle` (wraps TCPWrap in TLSWrap).
+4. Calls `socket._handle.start()` (or schedules it until wrap emits "connect").
+5. Wrap emits "connect".
+6. Socket emits "connect".
+7. Socket validates the connection (handshake, certificate validation).
+7. Socket emits "secureConnect".
+7. Socket emits "secure".

discoveries/undici.mdx

@@ -0,0 +1,22 @@
+# Undici
+
+## `connect` and writing the HTTP message
+
+Undici's `fetch` waits for the underlying socket to emit the `connect` event _before_ it writes the HTTP message to it. That is different than in `http.request()`, where the HTTP message is written _immediately_ but gets buffered (`socket._pendinData`) and flushed once `connect` is emitted.
+
+```
+# Node.js v22
+- dispatch() (lib/web/fetch/index.js:2129)
+  - agent.dispatch()
+    - Client.dispatch()
+      - _resume()
+        - connect()
+          - client[kConnector](options, cb*)
+            - Client.connect()
+              - net.connect(options) // no callback here
+                - socket.once('connect', cb*) // or "secureConnect"
+```
+
+> Note that the `cb` called on `connect` is _internal_. It's not provided as the connection callback to `net.connect()`, which makes it invisible to the interceptor.
+
+Then `connectH1()` calls `writeH1()` to write the HTTP message.

http/package.json

@@ -0,0 +1,11 @@
+{
+  "main": "../lib/node/interceptors/node/index.cjs",
+  "module": "../lib/node/interceptors/node/index.mjs",
+  "browser": null,
+  "exports": {
+    ".": {
+      "import": "./../lib/node/interceptors/node/index.mjs",
+      "default": "./../lib/node/interceptors/node/index.cjs"
+    }
+  }
+}

net-architecture.md

@@ -0,0 +1,28 @@
+## Net-based Interception
+
+Your task is to implement the network packet interception on the `net.Socket` level in Node.js. The underlying `SocketInterceptor` is already implemented at `src/interceptors/net/index.ts`. Below, are the outline of the necessary pieces.
+
+## `SocketInterceptor`
+
+- Responsible for patching `net.createConnection` and `net.connect`. Once patched, any created connection receive a mock socket instance.
+- The mock socket instance pretends to successfully establish the connection with whatever connection options provided. This is so connecting to the non-existing hosts works in the mock-first scenario.
+- The mock socket is wrapped in a `controller`. Controller is used by a higher-level interceptors to control the underlying socket. Since the data sent over socket is ambiguous, the controller only allows for `.connect()` (to mock the connection), `.passthrough()` (establish the connection as-is), and `.errorWith()` (abort the socket connection).
+- Only for the user, a special socket reference is created and exposed as the `socket` argument in the `connection` listener. That special socket is meant to represent the intercepted socket instance _from the server's perspective_. This means that `socket.write()` on the actual socket (e.g. the one created by `net.connect` and then used via `http.ClientRequest`) are translated to the "data" events being emitted on the special server-side `socket`. The server-side socket is used ONLY for this purpose: to observe what the client writes _and_ write data _to_ the client from the `connection` listener as if it has been sent from the "server".
+
+## `HttpRequestInterceptor`
+
+- `src/interceptors/http/index.ts`
+- This is a higher-level interceptor that relies on `SocketInterceptor` and routes all the intercepted socket packets through the HTTP request parser (we're using Node's parser).
+- If the request parser tells us that the sent packet is an HTTP request message header, the interceptor then proceeds with handling that request. It emits the `request` event for the consumer, and uses the established `RequestController` to control the request flow (`.respondWith`, `.errorWith`, etc). These APIs are already created and functional, they need no change.
+- The missing parts in the `HttpRequestInterceptor` is establishing the passthrough connection properly. For that, the socket controller accepted the `createConnection` option that constructs the bypassed socket. That works. But the socket hangs forever.
+
+## Requirements
+
+- Feel free to improve the existing classes but stay strictily within each class' responsibilities. Those must not leak.
+- The overall architecture must compose: MockSocket -> SocketController -> Higher-level interceptors.
+- Introduce absolutely no workarounds, zero.
+- You will need to reference Node.js internals, particularly `net` and `http` modules to implement this functionality properly.
+- Feel free to rely on Node.js internals knowledge and be clever, but not hacky.
+- Feel free to `@ts-ignore` property access to Node.js internals, such as `socket.parser`.
+- Do not comment your code.
+- You can verify your changes via `pnpm test:node test/modules/http/response/http-response-delay.test.ts`. All the test cases must pass there.

net-outline.md

@@ -0,0 +1,76 @@
+## High-level architecture overview
+
+> The `net.Socket` constructor doesn't have all the context of `net.connect()` to be a viable layer of interception. All the network requests in Node.js go through `net.connect()` (HTTP, SMTP, etc).
+
+```
+http.request
+  Agent.createConnection()
+      net.Socket
+```
+
+```
+net.connect()
+  net.Socket
+```
+
+## Pieces
+
+- `SocketInterceptor`. A regular interceptor, patches `net.connect()` and emits the "connection" event to the consumers.
+- `MockSocket` returned from the patched `net.connect()`. Always emulates a successful connection, allows spying on the data sent by the client (`socket.write()`) and pushing data to it from the interceptor.
+- `SocketController` that, similar to `RequestController`, lets the user decide what to do with the intercepted socket connection. Unlike requests, there's no single `respondWith()` as the nature of data sent via the socket is ambiguous and is for the higher-level interceptors to determine and appropriately handle. There are still, however, methods to error the intercepted connection (`SocketController.errorWith()`) and perform it as-is (`SocketController.passthrough()`).
+- A special `net.Socket` representation exposed in the "connection" listener. The `MockSocket` placeholder is _client-side_. To work with the intercepted socket from the server's perspective in the "connection" listener, another, special socket instance has to be provided. It acts as a mirror and allows the user to write data _to_ the client socket via `socket.write()` and listen to the data _sent_ from the client via `socket.on('data')`. This won't be possible with the client placeholder as `socket.on('data')` would represent the data received from the _server_ (and must still be emitted when the interceptor sends mock data to the socket).
+
+This pretty much covers the mock-first scenarios. But when it comes to passthrough, it gets tricky. The passthrough socket will usually be created after some time the connection has been intercepted. During that time, the intercepted socket might have been acted upon (e.g. written to, changed via methods). The passthrough socket would have to be put _in the exact same state_ as the intercepted socket at the moment of calling `SocketController.passthrough()`.
+
+This is where I think about employing an _object recorder_. It will use `Proxy` to record any changes done with the intercepted socket instance and then allow us to replay those changes on the passthrough socket.
+
+## Problems
+
+### Problem 1: Passthrough socket state
+
+Node.js sockets are quite intricate. Even with deduped method/property recording via `AsyncLocalStorage`, it still arrives at the state where a change on the intercepted socket is attempted to be recorded when the recorder should've stopped altogether after `passthrough()`.
+
+### Problem 2: The placeholder `MockSocket`
+
+Even when the passthrough connection is established, the consumers, like `http.Agent`, have already received and stored the placeholder `MockSocket` as their socket. This means that for passthrough scenarios, the `MockSocket` instance would have to become a _proxy socket_, forwarding whatever data or changes from the passthrough socket.
+
+What makes it more complex is that the consumers might still _act_ on the placeholder socket even after passthrough. Those acts would also have to be forwarded to the passthrough socket. It seems that the object recorder over the placeholder socket mustn't stop after passthrough, after all. Instead, it should replay the actions on the passthrough socket immediately if passthrough has been established.
+
+```ts
+this.#recorder = new ObjectRecorder(socket, {
+  filter(entry) {
+    if (this.#passthroughSocket) {
+      entry.replay(this.#passthroughSocket)
+      return false
+    }
+  }
+})
+
+// ...later on.
+public passthrough(): net.Socket {
+  this.#passthroughSocket = this.options.createConnection()
+  this.#recorder.pause()
+  this.#recorder.replay(this.#passthroughSocket)
+  this.#recorder.resume()
+}
+```
+
+> Above, I'm using the `filter()` capabilities of the object recorder to immediately replay the recorded action on the passthrough socket, if it exists.
+
+Considering that the recorder should continue recording and replaying events, it looks like I need to implement some sort of _entry buffering_. While the previously recorded changes are being replayed on the passthrough socket, the consumers might produce _new changes_ to the placeholder socket. With the approach above, those changes will be lost. So the recorder have to be put in a buffering state until the replay is done, and then replay any buffered events immediately after that.
+
+> Note: I would love to forego the object recording altogether in favor of something more elegant. I just don't know what that something might be. The actual connection cannot be established as it would error.
+
+## Alternative to object recording
+
+Alternatively, instead of having two separate sockets (placeholder and passthrough), only a _single_ passthrough socket can be used. With this approach, all that I have to do is this:
+
+- Record and silence any errors occurring on the socket. Those are crucial to be replayed if the user decides to passthrough.
+- Buffer the data sent from the client instead of sending it to the server. Any writes are still translated to the special `socket` for the "connection" listener.
+
+Then, if passthrough is established, the socket would have to:
+
+- Replay the errors, if any. This is for passing through to non-existing hosts.
+- If no errors were emitted, replay all the writes that occurred so the original socket will receive them.
+
+> The danger here are the write callbacks: `socket.write(chunk, encoding, callback)`. Those callbacks would have to be called for the mock-first scenario as consumer logic can and will depend on those callbacks. But calling them _again_ in passthrough will be a mistake.